From f40aedc3bfe0da51ac8a1eaef0535bf5cec1fc4e Mon Sep 17 00:00:00 2001 From: Manoel Aranda Neto Date: Mon, 29 Jun 2026 14:11:35 +0200 Subject: [PATCH] feat: add gzip batch compression --- .changeset/brave-tigers-compress.md | 6 ++ lib/posthog/client.rb | 4 +- lib/posthog/send_worker.rb | 7 +- lib/posthog/transport.rb | 28 +++++++- posthog-rails/lib/posthog/rails/railtie.rb | 6 ++ public_api_snapshot.txt | 1 + spec/posthog/client_spec.rb | 10 ++- spec/posthog/transport_spec.rb | 80 +++++++++++++++++++++- 8 files changed, 134 insertions(+), 8 deletions(-) create mode 100644 .changeset/brave-tigers-compress.md diff --git a/.changeset/brave-tigers-compress.md b/.changeset/brave-tigers-compress.md new file mode 100644 index 0000000..8bfbb57 --- /dev/null +++ b/.changeset/brave-tigers-compress.md @@ -0,0 +1,6 @@ +--- +'posthog-ruby': minor +'posthog-rails': minor +--- + +Enable gzip compression for batch uploads by default. diff --git a/lib/posthog/client.rb b/lib/posthog/client.rb index c87f23f..9d84742 100644 --- a/lib/posthog/client.rb +++ b/lib/posthog/client.rb @@ -76,6 +76,7 @@ def _decrement_instance_count(api_key) # the same API key. Use only when you intentionally need multiple clients. Defaults to +false+. # @option opts [Boolean] :skip_ssl_verification +true+ to disable SSL certificate verification for requests. # Intended only for local development or custom deployments. + # @option opts [Boolean] :compress_request Set to +false+ to disable gzip compression for batch uploads. # @option opts [Object] :flag_definition_cache_provider An object implementing the {FlagDefinitionCacheProvider} # interface for distributed flag definition caching. # @option opts [Boolean] :is_server +true+ to stamp captured events with `$is_server => true` so PostHog @@ -109,7 +110,8 @@ def initialize(opts = {}) @transport = Transport.new( api_host: opts[:host], skip_ssl_verification: opts[:skip_ssl_verification], - retries: 3 + retries: 3, + compress_request: opts[:compress_request] ) @sync_lock = Mutex.new end diff --git a/lib/posthog/send_worker.rb b/lib/posthog/send_worker.rb index 3218d37..50e5338 100644 --- a/lib/posthog/send_worker.rb +++ b/lib/posthog/send_worker.rb @@ -27,6 +27,7 @@ class SendWorker # @option options [Proc] :on_error Callback invoked as `on_error.call(status, error)`. # @option options [String] :host PostHog API host URL. # @option options [Boolean] :skip_ssl_verification Disable SSL certificate verification. + # @option options [Boolean] :compress_request Set to +false+ to disable gzip batch request bodies. def initialize(queue, api_key, options = {}) symbolize_keys! options @queue = queue @@ -42,7 +43,11 @@ def initialize(queue, api_key, options = {}) @flush_requested = false @shutdown = false @pid = Process.pid - @transport_options = { api_host: options[:host], skip_ssl_verification: options[:skip_ssl_verification] } + @transport_options = { + api_host: options[:host], + skip_ssl_verification: options[:skip_ssl_verification], + compress_request: options[:compress_request] + } @transport = Transport.new(@transport_options) end diff --git a/lib/posthog/transport.rb b/lib/posthog/transport.rb index d7d26ac..c48070c 100644 --- a/lib/posthog/transport.rb +++ b/lib/posthog/transport.rb @@ -8,6 +8,7 @@ require 'net/http' require 'net/https' require 'json' +require 'zlib' module PostHog # HTTP transport used by the SDK workers. @@ -28,6 +29,7 @@ class Transport # @option options [Integer] :retries Number of retry attempts for retryable failures. # @option options [PostHog::BackoffPolicy] :backoff_policy Backoff policy used between retries. # @option options [Boolean] :skip_ssl_verification Disable SSL certificate verification. + # @option options [Boolean] :compress_request Whether to gzip batch request bodies. Defaults to +true+. def initialize(options = {}) if options[:api_host] uri = URI.parse(options[:api_host]) @@ -44,6 +46,7 @@ def initialize(options = {}) @path = options[:path] || PATH @retries = options[:retries] || RETRIES @backoff_policy = options[:backoff_policy] || PostHog::BackoffPolicy.new + @compress_request = options[:compress_request] != false http = Net::HTTP.new(options[:host], options[:port]) http.use_ssl = options[:ssl] @@ -144,22 +147,41 @@ def retry_with_backoff(retries_remaining, &block) def send_request(api_key, batch) payload = JSON.generate(api_key: api_key, batch: batch) - request = Net::HTTP::Post.new(@path, @headers) + request_path, request_headers, request_payload = build_request(@path, @headers, payload) + request = Net::HTTP::Post.new(request_path, request_headers) if self.class.stub - logger.debug "stubbed request to #{@path}: " \ + logger.debug "stubbed request to #{request_path}: " \ "api key = #{api_key}, batch = #{JSON.generate(batch)}" [200, '{}'] else @http_mutex.synchronize do @http.start unless @http.started? # Maintain a persistent connection - response = @http.request(request, payload) + response = @http.request(request, request_payload) [response.code.to_i, response.body] end end end + def build_request(path, headers, payload) + return [path, headers, payload] unless @compress_request + + compressed_payload = gzip_payload(payload) + return [path, headers, payload] unless compressed_payload + + compressed_headers = headers.merge('Content-Encoding' => 'gzip') + + [path, compressed_headers, compressed_payload] + end + + def gzip_payload(payload) + Zlib.gzip(payload) + rescue Zlib::Error => e + logger.warn("gzip compression failed; sending uncompressed - #{e.message}") + nil + end + class << self attr_writer :stub diff --git a/posthog-rails/lib/posthog/rails/railtie.rb b/posthog-rails/lib/posthog/rails/railtie.rb index 245578b..e06dbc4 100644 --- a/posthog-rails/lib/posthog/rails/railtie.rb +++ b/posthog-rails/lib/posthog/rails/railtie.rb @@ -307,6 +307,12 @@ def sync_mode=(value) @base_options[:sync_mode] = value end + # @param value [Boolean] + # @return [Boolean] + def compress_request=(value) + @base_options[:compress_request] = value + end + # @param value [Proc] # @return [Proc] def on_error=(value) diff --git a/public_api_snapshot.txt b/public_api_snapshot.txt index ad0eff5..2d65f9d 100644 --- a/public_api_snapshot.txt +++ b/public_api_snapshot.txt @@ -184,6 +184,7 @@ constant PostHog::Rails::IN_WEB_REQUEST_KEY: Symbol class PostHog::Rails::InitConfig instance_method PostHog::Rails::InitConfig#api_key=(value) instance_method PostHog::Rails::InitConfig#before_send=(value) +instance_method PostHog::Rails::InitConfig#compress_request=(value) instance_method PostHog::Rails::InitConfig#feature_flag_request_timeout_seconds=(value) instance_method PostHog::Rails::InitConfig#feature_flags_polling_interval=(value) instance_method PostHog::Rails::InitConfig#host=(value) diff --git a/spec/posthog/client_spec.rb b/spec/posthog/client_spec.rb index 553f236..ce83868 100644 --- a/spec/posthog/client_spec.rb +++ b/spec/posthog/client_spec.rb @@ -103,10 +103,18 @@ module PostHog it 'handles skip_ssl_verification' do expect(PostHog::Transport).to receive(:new).with({ api_host: 'https://us.i.posthog.com', - skip_ssl_verification: true }) + skip_ssl_verification: true, + compress_request: nil }) expect { Client.new api_key: API_KEY, skip_ssl_verification: true }.to_not raise_error end + it 'passes compress_request false to the transport' do + expect(PostHog::Transport).to receive(:new).with({ api_host: 'https://us.i.posthog.com', + skip_ssl_verification: nil, + compress_request: false }) + expect { Client.new api_key: API_KEY, compress_request: false }.to_not raise_error + end + it 'trims whitespace-sensitive options' do client = Client.new( api_key: " \n#{API_KEY}\t ", diff --git a/spec/posthog/transport_spec.rb b/spec/posthog/transport_spec.rb index 93fe80d..1e845d5 100644 --- a/spec/posthog/transport_spec.rb +++ b/spec/posthog/transport_spec.rb @@ -8,6 +8,7 @@ module PostHog # Try and keep debug statements out of tests allow(subject.logger).to receive(:error) allow(subject.logger).to receive(:debug) + allow(subject.logger).to receive(:warn) end describe '#initialize' do @@ -50,6 +51,11 @@ module PostHog expect(backoff_policy).to be_a(PostHog::BackoffPolicy) end + it 'compresses requests by default' do + compress_request = subject.instance_variable_get(:@compress_request) + expect(compress_request).to eq(true) + end + it 'uses the default verify mode' do expect(net_http).to_not receive(:verify_mode=) described_class.new @@ -69,6 +75,7 @@ module PostHog let(:retries) { 1234 } let(:backoff_policy) { FakeBackoffPolicy.new([1, 2, 3]) } let(:skip_ssl_verification) { true } + let(:compress_request) { false } let(:host) { 'http://www.example.com' } let(:port) { 8080 } let(:options) do @@ -77,6 +84,7 @@ module PostHog retries: retries, backoff_policy: backoff_policy, skip_ssl_verification: skip_ssl_verification, + compress_request: compress_request, host: host, port: port } @@ -98,6 +106,10 @@ module PostHog ) end + it 'sets passed in compression option' do + expect(subject.instance_variable_get(:@compress_request)).to eq(false) + end + it 'skips SSL verification if passed' do expect(net_http).to receive(:verify_mode=).with(OpenSSL::SSL::VERIFY_NONE) described_class.new(options) @@ -137,12 +149,13 @@ module PostHog allow(response).to receive(:body) { response_body } end - it 'initalizes a new Net::HTTP::Post with path and default headers' do + it 'initalizes a new Net::HTTP::Post with path and gzip header by default' do path = subject.instance_variable_get(:@path) default_headers = { 'Content-Type' => 'application/json', 'Accept' => 'application/json', - 'User-Agent' => "posthog-ruby/#{PostHog::VERSION}" + 'User-Agent' => "posthog-ruby/#{PostHog::VERSION}", + 'Content-Encoding' => 'gzip' } expect(Net::HTTP::Post).to receive(:new) .with(path, default_headers) @@ -215,6 +228,69 @@ module PostHog end end + context 'with default compression' do + let(:batch) { [{ event: 'compression-test' }] } + let(:raw_payload) { JSON.generate(api_key: api_key, batch: batch) } + + it 'gzips the request body and sets the content encoding header' do + http = subject.instance_variable_get(:@http) + expect(http).to receive(:request) do |request, payload| + expect(request.path).to eq('/batch/') + expect(request['Content-Encoding']).to eq('gzip') + expect(Zlib.gunzip(payload)).to eq(raw_payload) + response + end + + subject.send(api_key, batch) + end + + it 'falls back to the original body when gzip compression fails' do + allow(Zlib).to receive(:gzip).and_raise(Zlib::Error.new('boom')) + expect(subject.logger).to receive(:warn).with('gzip compression failed; sending uncompressed - boom') + + http = subject.instance_variable_get(:@http) + expect(http).to receive(:request) do |request, payload| + expect(request.path).to eq('/batch/') + expect(request['Content-Encoding']).to be_nil + expect(payload).to eq(raw_payload) + response + end + + subject.send(api_key, batch) + end + + it 'does not fall back for non-zlib errors' do + allow(Zlib).to receive(:gzip).and_raise(TypeError, 'bad payload') + subject.instance_variable_set(:@retries, 1) + + http = subject.instance_variable_get(:@http) + expect(http).not_to receive(:request) + + res = subject.send(api_key, batch) + expect(res.status).to eq(-1) + expect(res.error).to include('bad payload') + end + end + + context 'with compression disabled' do + subject { described_class.new(compress_request: false) } + + let(:batch) { [{ event: 'compression-test' }] } + let(:raw_payload) { JSON.generate(api_key: api_key, batch: batch) } + + it 'sends the original body without the content encoding header' do + http = subject.instance_variable_get(:@http) + expect(http).to receive(:request) do |request, payload| + expect(request.path).to eq('/batch/') + expect(request['Content-Encoding']).to be_nil + expect(payload).to eq(raw_payload) + response + end + + subject.send(api_key, batch) + end + end + context 'request results in errorful response' do let(:error) { 'this is an error' } let(:response_body) { { error: error }.to_json }