feat: add gzip batch compression

.changeset/brave-tigers-compress.md

@@ -0,0 +1,6 @@
+---
+'posthog-ruby': minor
+'posthog-rails': minor
+---
+
+Enable gzip compression for batch uploads by default.

lib/posthog/client.rb

@@ -76,6 +76,7 @@ def _decrement_instance_count(api_key)
     #   the same API key. Use only when you intentionally need multiple clients. Defaults to +false+.
     # @option opts [Boolean] :skip_ssl_verification +true+ to disable SSL certificate verification for requests.
     #   Intended only for local development or custom deployments.
+    # @option opts [Boolean] :compress_request Set to +false+ to disable gzip compression for batch uploads.
     # @option opts [Object] :flag_definition_cache_provider An object implementing the {FlagDefinitionCacheProvider}
     #   interface for distributed flag definition caching.
     # @option opts [Boolean] :is_server +true+ to stamp captured events with `$is_server => true` so PostHog
@@ -109,7 +110,8 @@ def initialize(opts = {})
         @transport = Transport.new(
           api_host: opts[:host],
           skip_ssl_verification: opts[:skip_ssl_verification],
-          retries: 3
+          retries: 3,
+          compress_request: opts[:compress_request]
         )
         @sync_lock = Mutex.new
       end

lib/posthog/send_worker.rb

@@ -27,6 +27,7 @@ class SendWorker
     # @option options [Proc] :on_error Callback invoked as `on_error.call(status, error)`.
     # @option options [String] :host PostHog API host URL.
     # @option options [Boolean] :skip_ssl_verification Disable SSL certificate verification.
+    # @option options [Boolean] :compress_request Set to +false+ to disable gzip batch request bodies.
     def initialize(queue, api_key, options = {})
       symbolize_keys! options
       @queue = queue
@@ -42,7 +43,11 @@ def initialize(queue, api_key, options = {})
       @flush_requested = false
       @shutdown = false
       @pid = Process.pid
-      @transport_options = { api_host: options[:host], skip_ssl_verification: options[:skip_ssl_verification] }
+      @transport_options = {
+        api_host: options[:host],
+        skip_ssl_verification: options[:skip_ssl_verification],
+        compress_request: options[:compress_request]
+      }
       @transport = Transport.new(@transport_options)
     end
 

lib/posthog/transport.rb

@@ -8,6 +8,7 @@
 require 'net/http'
 require 'net/https'
 require 'json'
+require 'zlib'
 
 module PostHog
   # HTTP transport used by the SDK workers.
@@ -28,6 +29,7 @@ class Transport
     # @option options [Integer] :retries Number of retry attempts for retryable failures.
     # @option options [PostHog::BackoffPolicy] :backoff_policy Backoff policy used between retries.
     # @option options [Boolean] :skip_ssl_verification Disable SSL certificate verification.
+    # @option options [Boolean] :compress_request Whether to gzip batch request bodies. Defaults to +true+.
     def initialize(options = {})
       if options[:api_host]
         uri = URI.parse(options[:api_host])
@@ -44,6 +46,7 @@ def initialize(options = {})
       @path = options[:path] || PATH
       @retries = options[:retries] || RETRIES
       @backoff_policy = options[:backoff_policy] || PostHog::BackoffPolicy.new
+      @compress_request = options[:compress_request] != false
 
       http = Net::HTTP.new(options[:host], options[:port])
       http.use_ssl = options[:ssl]
@@ -144,22 +147,41 @@ def retry_with_backoff(retries_remaining, &block)
     def send_request(api_key, batch)
       payload = JSON.generate(api_key: api_key, batch: batch)
 
-      request = Net::HTTP::Post.new(@path, @headers)
+      request_path, request_headers, request_payload = build_request(@path, @headers, payload)
+      request = Net::HTTP::Post.new(request_path, request_headers)
 
       if self.class.stub
-        logger.debug "stubbed request to #{@path}: " \
+        logger.debug "stubbed request to #{request_path}: " \
                      "api key = #{api_key}, batch = #{JSON.generate(batch)}"
 
         [200, '{}']
       else
         @http_mutex.synchronize do
           @http.start unless @http.started? # Maintain a persistent connection
-          response = @http.request(request, payload)
+          response = @http.request(request, request_payload)
           [response.code.to_i, response.body]
         end
       end
     end
 
+    def build_request(path, headers, payload)
+      return [path, headers, payload] unless @compress_request
+
+      compressed_payload = gzip_payload(payload)
+      return [path, headers, payload] unless compressed_payload
+
+      compressed_headers = headers.merge('Content-Encoding' => 'gzip')
+
+      [path, compressed_headers, compressed_payload]
+    end
+
+    def gzip_payload(payload)
+      Zlib.gzip(payload)
+    rescue Zlib::Error => e
+      logger.warn("gzip compression failed; sending uncompressed - #{e.message}")
+      nil
+    end
+
     class << self
       attr_writer :stub
 

posthog-rails/lib/posthog/rails/railtie.rb

@@ -307,6 +307,12 @@ def sync_mode=(value)
         @base_options[:sync_mode] = value
       end
 
+      # @param value [Boolean]
+      # @return [Boolean]
+      def compress_request=(value)
+        @base_options[:compress_request] = value
+      end
+
       # @param value [Proc]
       # @return [Proc]
       def on_error=(value)

public_api_snapshot.txt

@@ -184,6 +184,7 @@ constant PostHog::Rails::IN_WEB_REQUEST_KEY: Symbol
 class PostHog::Rails::InitConfig
 instance_method PostHog::Rails::InitConfig#api_key=(value)
 instance_method PostHog::Rails::InitConfig#before_send=(value)
+instance_method PostHog::Rails::InitConfig#compress_request=(value)
 instance_method PostHog::Rails::InitConfig#feature_flag_request_timeout_seconds=(value)
 instance_method PostHog::Rails::InitConfig#feature_flags_polling_interval=(value)
 instance_method PostHog::Rails::InitConfig#host=(value)

spec/posthog/client_spec.rb

@@ -103,10 +103,18 @@ module PostHog
 
       it 'handles skip_ssl_verification' do
         expect(PostHog::Transport).to receive(:new).with({ api_host: 'https://us.i.posthog.com',
-                                                           skip_ssl_verification: true })
+                                                           skip_ssl_verification: true,
+                                                           compress_request: nil })
         expect { Client.new api_key: API_KEY, skip_ssl_verification: true }.to_not raise_error
       end
 
+      it 'passes compress_request false to the transport' do
+        expect(PostHog::Transport).to receive(:new).with({ api_host: 'https://us.i.posthog.com',
+                                                           skip_ssl_verification: nil,
+                                                           compress_request: false })
+        expect { Client.new api_key: API_KEY, compress_request: false }.to_not raise_error
+      end
+
       it 'trims whitespace-sensitive options' do
         client = Client.new(
           api_key: " \n#{API_KEY}\t ",

spec/posthog/transport_spec.rb

@@ -8,6 +8,7 @@ module PostHog
       # Try and keep debug statements out of tests
       allow(subject.logger).to receive(:error)
       allow(subject.logger).to receive(:debug)
+      allow(subject.logger).to receive(:warn)
     end
 
     describe '#initialize' do
@@ -50,6 +51,11 @@ module PostHog
           expect(backoff_policy).to be_a(PostHog::BackoffPolicy)
         end
 
+        it 'compresses requests by default' do
+          compress_request = subject.instance_variable_get(:@compress_request)
+          expect(compress_request).to eq(true)
+        end
+
         it 'uses the default verify mode' do
           expect(net_http).to_not receive(:verify_mode=)
           described_class.new
@@ -69,6 +75,7 @@ module PostHog
         let(:retries) { 1234 }
         let(:backoff_policy) { FakeBackoffPolicy.new([1, 2, 3]) }
         let(:skip_ssl_verification) { true }
+        let(:compress_request) { false }
         let(:host) { 'http://www.example.com' }
         let(:port) { 8080 }
         let(:options) do
@@ -77,6 +84,7 @@ module PostHog
             retries: retries,
             backoff_policy: backoff_policy,
             skip_ssl_verification: skip_ssl_verification,
+            compress_request: compress_request,
             host: host,
             port: port
           }
@@ -98,6 +106,10 @@ module PostHog
           )
         end
 
+        it 'sets passed in compression option' do
+          expect(subject.instance_variable_get(:@compress_request)).to eq(false)
+        end
+
         it 'skips SSL verification if passed' do
           expect(net_http).to receive(:verify_mode=).with(OpenSSL::SSL::VERIFY_NONE)
           described_class.new(options)
@@ -137,12 +149,13 @@ module PostHog
         allow(response).to receive(:body) { response_body }
       end
 
-      it 'initalizes a new Net::HTTP::Post with path and default headers' do
+      it 'initalizes a new Net::HTTP::Post with path and gzip header by default' do
         path = subject.instance_variable_get(:@path)
         default_headers = {
           'Content-Type' => 'application/json',
           'Accept' => 'application/json',
-          'User-Agent' => "posthog-ruby/#{PostHog::VERSION}"
+          'User-Agent' => "posthog-ruby/#{PostHog::VERSION}",
+          'Content-Encoding' => 'gzip'
         }
         expect(Net::HTTP::Post).to receive(:new)
           .with(path, default_headers)
@@ -215,6 +228,69 @@ module PostHog
           end
         end
 
+        context 'with default compression' do
+          let(:batch) { [{ event: 'compression-test' }] }
+          let(:raw_payload) { JSON.generate(api_key: api_key, batch: batch) }
+
+          it 'gzips the request body and sets the content encoding header' do
+            http = subject.instance_variable_get(:@http)
+            expect(http).to receive(:request) do |request, payload|
+              expect(request.path).to eq('/batch/')
+              expect(request['Content-Encoding']).to eq('gzip')
+              expect(Zlib.gunzip(payload)).to eq(raw_payload)
+              response
+            end
+
+            subject.send(api_key, batch)
+          end
+
+          it 'falls back to the original body when gzip compression fails' do
+            allow(Zlib).to receive(:gzip).and_raise(Zlib::Error.new('boom'))
+            expect(subject.logger).to receive(:warn).with('gzip compression failed; sending uncompressed - boom')
+
+            http = subject.instance_variable_get(:@http)
+            expect(http).to receive(:request) do |request, payload|
+              expect(request.path).to eq('/batch/')
+              expect(request['Content-Encoding']).to be_nil
+              expect(payload).to eq(raw_payload)
+              response
+            end
+
+            subject.send(api_key, batch)
+          end
+
+          it 'does not fall back for non-zlib errors' do
+            allow(Zlib).to receive(:gzip).and_raise(TypeError, 'bad payload')
+            subject.instance_variable_set(:@retries, 1)
+
+            http = subject.instance_variable_get(:@http)
+            expect(http).not_to receive(:request)
+
+            res = subject.send(api_key, batch)
+            expect(res.status).to eq(-1)
+            expect(res.error).to include('bad payload')
+          end
+        end
+
+        context 'with compression disabled' do
+          subject { described_class.new(compress_request: false) }
+
+          let(:batch) { [{ event: 'compression-test' }] }
+          let(:raw_payload) { JSON.generate(api_key: api_key, batch: batch) }
+
+          it 'sends the original body without the content encoding header' do
+            http = subject.instance_variable_get(:@http)
+            expect(http).to receive(:request) do |request, payload|
+              expect(request.path).to eq('/batch/')
+              expect(request['Content-Encoding']).to be_nil
+              expect(payload).to eq(raw_payload)
+              response
+            end
+
+            subject.send(api_key, batch)
+          end
+        end
+
         context 'request results in errorful response' do
           let(:error) { 'this is an error' }
           let(:response_body) { { error: error }.to_json }