From ff782e0f1e9a2a005fcf7b32128d9ca7795dc45d Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 8 Jul 2026 14:46:47 +0530 Subject: [PATCH 001/198] Quantize auction transport timeouts to stabilize Fastly backend names Fastly dynamic backend names embed the first-byte and between-bytes timeouts so a registration can never be silently reused with a different transport configuration. Deriving those timeouts from the remaining wall-clock auction budget minted a new backend name on nearly every request, defeating cross-request TCP/TLS connection reuse (Fastly pools connections per backend name) and accumulating registrations toward the per-service dynamic backend limit. Compute the effective transport timeout from the configured provider timeout verbatim when the budget allows, floor the budget-bound value to 250ms buckets otherwise, and pass sub-quantum remainders through exactly so publishers with sub-250ms configured budgets keep launching. The quantized value feeds both the backend name and the registered configuration, so they cannot diverge. Rounding down never extends a transport cap past the auction deadline, which the mediator and dispatched-collect paths rely on to bound the hold. Also add a Fastly platform test pinning predict_name == ensure for the same spec, since the orchestrator maps responses back to providers by predicted backend name. Fixes #847 --- .../src/platform.rs | 31 + .../src/auction/orchestrator.rs | 628 ++++++++++++++++-- 2 files changed, 617 insertions(+), 42 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index 106ced787..c5bb60b9c 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -736,6 +736,37 @@ mod tests { ); } + #[test] + fn predict_name_matches_ensured_backend_name() { + // The auction orchestrator maps responses back to providers by the + // predicted backend name, so predict_name and ensure must return the + // identical string for the same spec — a divergence would make + // responses land in the "unknown backend" branch and drop bids + // silently. + let backend = FastlyPlatformBackend; + let spec = PlatformBackendSpec { + scheme: "https".to_string(), + host: "consistency.example.com".to_string(), + port: None, + host_header_override: None, + certificate_check: true, + first_byte_timeout: Duration::from_millis(750), + between_bytes_timeout: Duration::from_millis(750), + }; + + let predicted = backend + .predict_name(&spec) + .expect("should predict backend name"); + let ensured = backend + .ensure(&spec) + .expect("should register backend for valid spec"); + + assert_eq!( + predicted, ensured, + "predicted backend name should match the registered backend name" + ); + } + // --- FastlyPlatformHttpClient ------------------------------------------- #[test] diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 145059d9e..d884a0220 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -157,6 +157,64 @@ fn remaining_budget_ms(start: Instant, timeout_ms: u32) -> u32 { timeout_ms.saturating_sub(elapsed) } +/// Transport-timeout quantum for auction backends. +/// +/// See [`quantize_transport_timeout_ms`] for why provider transport timeouts +/// are rounded to this granularity. +const TRANSPORT_TIMEOUT_QUANTUM_MS: u32 = 250; + +/// Round a transport timeout down to a [`TRANSPORT_TIMEOUT_QUANTUM_MS`] multiple. +/// +/// The Fastly adapter embeds the first-byte and between-bytes timeouts in the +/// dynamic backend name so a registration can never be silently reused with a +/// different transport configuration. Deriving those timeouts from the +/// remaining wall-clock budget minted a new backend name on nearly every +/// request, which defeated cross-request TCP/TLS connection reuse (Fastly +/// pools connections per backend name) and accumulated registrations toward +/// the per-service dynamic backend limit. +/// +/// Quantizing the value — not just the name — keeps the registered backend +/// configuration aligned with its name. Rounding down never extends a +/// transport cap past the auction deadline, which matters on the mediator and +/// dispatched-collect paths where the backend timeouts (not a select-loop +/// deadline check) bound the `` hold. +#[inline] +fn quantize_transport_timeout_ms(timeout_ms: u32) -> u32 { + (timeout_ms / TRANSPORT_TIMEOUT_QUANTUM_MS) * TRANSPORT_TIMEOUT_QUANTUM_MS +} + +/// Compute the transport timeout for a provider launch from the remaining +/// auction budget and the provider's configured timeout. +/// +/// The configured timeout is a per-provider constant, so using it verbatim +/// already yields a stable backend name — including configured values below +/// one quantum, which must not be rounded away or the provider could never +/// launch. Only when the remaining budget is the binding constraint does the +/// wall-clock-derived value enter the name, and that value is quantized via +/// [`quantize_transport_timeout_ms`] so it cannot mint a new backend name on +/// every request. +/// +/// A remaining budget below one quantum is passed through exactly rather +/// than rounded to zero: rounding up would extend the transport cap past the +/// deadline, and rounding down would skip the launch and hard-fail auctions +/// whose configured budget is under one quantum. Name churn in this regime +/// is bounded to sub-quantum values and matches the pre-quantization +/// behavior. The result never exceeds `remaining_ms` and is zero only when +/// `remaining_ms` or `configured_ms` is zero, which callers treat as +/// "budget exhausted — skip the launch". +#[inline] +fn effective_transport_timeout_ms(remaining_ms: u32, configured_ms: u32) -> u32 { + if remaining_ms >= configured_ms { + return configured_ms; + } + let quantized = quantize_transport_timeout_ms(remaining_ms); + if quantized == 0 { + remaining_ms + } else { + quantized + } +} + /// Manages auction execution across multiple providers. pub struct AuctionOrchestrator { config: AuctionConfig, @@ -279,10 +337,14 @@ impl AuctionOrchestrator { // Give the mediator only the remaining time from the auction // deadline, not the full timeout — the bidding phase already - // consumed part of it. + // consumed part of it, and the mediator has no select-loop + // deadline backstop. Quantized for backend-name stability (see + // effective_transport_timeout_ms). let remaining_ms = remaining_budget_ms(mediation_start, context.timeout_ms); + let mediator_timeout = + effective_transport_timeout_ms(remaining_ms, mediator.timeout_ms()); - if remaining_ms == 0 { + if mediator_timeout == 0 { log::warn!("Auction timeout exhausted during bidding phase; skipping mediator"); let winning = self.select_winning_bids(&provider_responses, &floor_prices); return Ok(OrchestrationResult { @@ -297,9 +359,7 @@ impl AuctionOrchestrator { let mediator_context = AuctionContext { settings: context.settings, request: context.request, - // Bound by both the remaining auction budget and the mediator's - // own configured timeout, matching the dispatched collect path. - timeout_ms: remaining_ms.min(mediator.timeout_ms()), + timeout_ms: mediator_timeout, provider_responses: Some(&provider_responses), services: context.services, }; @@ -465,10 +525,11 @@ impl AuctionOrchestrator { // Give each provider only the remaining time from the auction // deadline so that backend transport timeouts do not extend past - // the overall budget. Also respect the provider's own configured - // timeout when it is tighter than the remaining budget. + // the overall budget, quantized for backend-name stability (see + // effective_transport_timeout_ms). let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); - let effective_timeout = remaining_ms.min(provider.timeout_ms()); + let effective_timeout = + effective_transport_timeout_ms(remaining_ms, provider.timeout_ms()); if effective_timeout == 0 { log::warn!("Auction timeout exhausted before launching provider request; skipping"); @@ -876,8 +937,11 @@ impl AuctionOrchestrator { continue; } + // Remaining budget quantized for backend-name stability (see + // effective_transport_timeout_ms). let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); - let effective_timeout = remaining_ms.min(provider.timeout_ms()); + let effective_timeout = + effective_transport_timeout_ms(remaining_ms, provider.timeout_ms()); if effective_timeout == 0 { log::warn!( @@ -1132,8 +1196,15 @@ impl AuctionOrchestrator { // timeout) at dispatch time, so they cannot run past A_deadline // independently. Giving the mediator an uncapped timeout lets it run // past A_deadline, violating the bounded hold invariant. + // The mediator's only time bound on this path is its + // backend transport timeout, so the effective value must + // never exceed the remaining budget. Quantized for + // backend-name stability (see + // effective_transport_timeout_ms). let remaining = remaining_budget_ms(auction_start, timeout_ms); - if remaining == 0 { + let mediator_timeout = + effective_transport_timeout_ms(remaining, mediator.timeout_ms()); + if mediator_timeout == 0 { log::warn!( "A_deadline exhausted before mediator '{}' — returning {} SSP bids without mediation", mediator.provider_name(), @@ -1148,7 +1219,6 @@ impl AuctionOrchestrator { metadata: HashMap::new(), }; } - let mediator_timeout = remaining.min(mediator.timeout_ms()); let mediator_start = Instant::now(); log::info!( "Running mediator '{}' with {}ms budget (A_deadline remaining: {}ms, configured: {}ms)", @@ -1334,7 +1404,7 @@ mod tests { use crate::test_support::tests::crate_test_settings_str; use error_stack::{Report, ResultExt}; use std::collections::{HashMap, HashSet}; - use std::sync::Arc; + use std::sync::{Arc, Mutex}; use super::AuctionOrchestrator; @@ -1342,9 +1412,49 @@ mod tests { // Minimal test double for AuctionProvider // --------------------------------------------------------------------------- + /// Minimal stub provider. Optionally records every transport timeout it + /// observes — the value passed to `backend_name` and the + /// `context.timeout_ms` handed to `request_bids` — so tests can assert + /// the orchestrator quantizes them. struct StubAuctionProvider { name: &'static str, backend: &'static str, + configured_timeout_ms: u32, + observed_timeouts: Option>>>, + } + + impl StubAuctionProvider { + fn new(name: &'static str, backend: &'static str) -> Self { + Self { + name, + backend, + configured_timeout_ms: 2000, + observed_timeouts: None, + } + } + + fn recording( + name: &'static str, + backend: &'static str, + configured_timeout_ms: u32, + observed_timeouts: Arc>>, + ) -> Self { + Self { + name, + backend, + configured_timeout_ms, + observed_timeouts: Some(observed_timeouts), + } + } + + fn record(&self, timeout_ms: u32) { + if let Some(observed) = &self.observed_timeouts { + observed + .lock() + .expect("should lock observed timeouts") + .push(timeout_ms); + } + } } #[async_trait::async_trait(?Send)] @@ -1358,6 +1468,7 @@ mod tests { _request: &AuctionRequest, context: &AuctionContext<'_>, ) -> Result> { + self.record(context.timeout_ms); let req = PlatformHttpRequest::new( http::Request::builder() .method("POST") @@ -1389,10 +1500,11 @@ mod tests { } fn timeout_ms(&self) -> u32 { - 2000 + self.configured_timeout_ms } - fn backend_name(&self, _services: &RuntimeServices, _timeout_ms: u32) -> Option { + fn backend_name(&self, _services: &RuntimeServices, timeout_ms: u32) -> Option { + self.record(timeout_ms); Some(self.backend.to_string()) } } @@ -1509,10 +1621,10 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "bidder", - backend: "bidder-backend", - })); + orchestrator.register_provider(Arc::new(StubAuctionProvider::new( + "bidder", + "bidder-backend", + ))); orchestrator.register_provider(Arc::new(CacheRestoringMediator)); let request = create_test_auction_request(); @@ -1926,6 +2038,438 @@ mod tests { ); } + #[test] + fn quantize_transport_timeout_floors_to_quantum() { + assert_eq!( + super::quantize_transport_timeout_ms(0), + 0, + "should keep zero at zero" + ); + assert_eq!( + super::quantize_transport_timeout_ms(249), + 0, + "should floor a sub-quantum budget to zero" + ); + assert_eq!( + super::quantize_transport_timeout_ms(250), + 250, + "should keep an exact quantum multiple unchanged" + ); + assert_eq!( + super::quantize_transport_timeout_ms(999), + 750, + "should floor to the next-lower quantum multiple" + ); + assert_eq!( + super::quantize_transport_timeout_ms(2000), + 2000, + "should keep a larger exact quantum multiple unchanged" + ); + } + + #[test] + fn effective_transport_timeout_prefers_configured_constant() { + assert_eq!( + super::effective_transport_timeout_ms(2000, 1000), + 1000, + "should use the configured timeout verbatim when the budget allows" + ); + assert_eq!( + super::effective_transport_timeout_ms(2000, 100), + 100, + "should preserve a sub-quantum configured timeout — quantizing it away would permanently disable the provider" + ); + assert_eq!( + super::effective_transport_timeout_ms(999, 2000), + 750, + "should quantize the budget-bound value down to the 750ms bucket" + ); + assert_eq!( + super::effective_transport_timeout_ms(300, 2000), + 250, + "should quantize a tight budget down to one quantum" + ); + assert_eq!( + super::effective_transport_timeout_ms(200, 2000), + 200, + "should pass a sub-quantum budget through exactly instead of rounding to zero" + ); + assert_eq!( + super::effective_transport_timeout_ms(50, 100), + 50, + "should pass through when the budget is below both the quantum and the configured timeout" + ); + assert_eq!( + super::effective_transport_timeout_ms(0, 1000), + 0, + "should return zero for an exhausted budget so the launch is skipped" + ); + assert_eq!( + super::effective_transport_timeout_ms(100, 0), + 0, + "should return zero for a zero configured timeout so the launch is skipped" + ); + } + + #[test] + fn sub_quantum_configured_timeout_still_launches_provider() { + futures::executor::block_on(async { + // A provider whose configured timeout is below one quantum must + // still launch with its exact configured value: the constant is + // name-stable on its own, so only budget-derived values are + // quantized. + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); + let services = build_services_with_http_client(stub); + // SAFETY: `Box::leak` creates a `'static` reference for test use only. + // The leaked allocation is bounded to the test process lifetime. + let services: &'static RuntimeServices = Box::leak(Box::new(services)); + + let observed = Arc::new(Mutex::new(Vec::new())); + let config = AuctionConfig { + enabled: true, + providers: vec!["bidder".to_string()], + timeout_ms: 2000, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(StubAuctionProvider::recording( + "bidder", + "bidder-backend", + 100, + Arc::clone(&observed), + ))); + + let request = create_test_auction_request(); + let settings = create_test_settings(); + let req = http::Request::builder() + .method(http::Method::GET) + .uri("https://example.com/test") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let context = AuctionContext { + settings: &settings, + request: &req, + timeout_ms: 2000, + provider_responses: None, + services, + }; + + orchestrator + .run_auction(&request, &context) + .await + .expect("should complete auction"); + + let observed = observed.lock().expect("should lock observed timeouts"); + assert!( + !observed.is_empty(), + "should launch the sub-quantum-configured provider" + ); + for timeout in observed.iter() { + assert_eq!( + *timeout, 100, + "should pass the configured 100ms timeout through unchanged" + ); + } + }); + } + + #[test] + fn parallel_path_quantizes_provider_transport_timeout() { + futures::executor::block_on(async { + // A 999ms budget must reach the provider as the 750ms quantum + // bucket — both in backend_name (which derives the Fastly backend + // name) and in context.timeout_ms (which configures the backend + // and payload deadlines) — so the backend name stays stable + // across requests with slightly different remaining budgets. + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); + let services = build_services_with_http_client(stub); + // SAFETY: `Box::leak` creates a `'static` reference for test use only. + // The leaked allocation is bounded to the test process lifetime. + let services: &'static RuntimeServices = Box::leak(Box::new(services)); + + let observed = Arc::new(Mutex::new(Vec::new())); + let config = AuctionConfig { + enabled: true, + providers: vec!["bidder".to_string()], + timeout_ms: 999, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(StubAuctionProvider::recording( + "bidder", + "bidder-backend", + 2000, + Arc::clone(&observed), + ))); + + let request = create_test_auction_request(); + let settings = create_test_settings(); + let req = http::Request::builder() + .method(http::Method::GET) + .uri("https://example.com/test") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let context = AuctionContext { + settings: &settings, + request: &req, + timeout_ms: 999, + provider_responses: None, + services, + }; + + orchestrator + .run_auction(&request, &context) + .await + .expect("should complete auction"); + + let observed = observed.lock().expect("should lock observed timeouts"); + assert!( + !observed.is_empty(), + "should record provider transport timeouts" + ); + for timeout in observed.iter() { + assert!( + *timeout % super::TRANSPORT_TIMEOUT_QUANTUM_MS == 0 + && *timeout > 0 + && *timeout <= 750, + "should floor the 999ms budget to a quantum bucket at or below 750ms, got {timeout}ms" + ); + } + }); + } + + #[test] + fn sub_quantum_budget_launches_with_exact_remaining_timeout() { + futures::executor::block_on(async { + // A configured auction budget below one quantum must still launch + // providers with the exact remaining budget — rounding it to zero + // would hard-fail every auction for publishers with sub-250ms + // budgets. + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); + let services = build_services_with_http_client(stub); + // SAFETY: `Box::leak` creates a `'static` reference for test use only. + // The leaked allocation is bounded to the test process lifetime. + let services: &'static RuntimeServices = Box::leak(Box::new(services)); + + let observed = Arc::new(Mutex::new(Vec::new())); + let config = AuctionConfig { + enabled: true, + providers: vec!["bidder".to_string()], + timeout_ms: 200, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(StubAuctionProvider::recording( + "bidder", + "bidder-backend", + 2000, + Arc::clone(&observed), + ))); + + let request = create_test_auction_request(); + let settings = create_test_settings(); + let req = http::Request::builder() + .method(http::Method::GET) + .uri("https://example.com/test") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let context = AuctionContext { + settings: &settings, + request: &req, + timeout_ms: 200, + provider_responses: None, + services, + }; + + let result = orchestrator + .run_auction(&request, &context) + .await + .expect("should complete auction with a sub-quantum budget"); + + assert_eq!( + result.provider_responses.len(), + 1, + "should launch the provider despite the sub-quantum budget" + ); + let observed = observed.lock().expect("should lock observed timeouts"); + assert!( + !observed.is_empty(), + "should record provider transport timeouts" + ); + for timeout in observed.iter() { + assert!( + *timeout > 0 && *timeout <= 200, + "should pass the exact sub-quantum remaining budget through, got {timeout}ms" + ); + } + }); + } + + #[test] + fn synchronous_mediation_quantizes_mediator_timeout() { + futures::executor::block_on(async { + // The mediator has no select-loop deadline backstop, so its + // transport timeout must be quantized by rounding down: a + // quantum-aligned value no larger than the remaining budget. + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); // bidder send_async + stub.push_response(200, b"{}".to_vec()); // mediator send_async + let services = build_services_with_http_client(stub); + // SAFETY: `Box::leak` creates a `'static` reference for test use only. + // The leaked allocation is bounded to the test process lifetime. + let services: &'static RuntimeServices = Box::leak(Box::new(services)); + + let observed = Arc::new(Mutex::new(Vec::new())); + let config = AuctionConfig { + enabled: true, + providers: vec!["bidder".to_string()], + mediator: Some("mediator".to_string()), + timeout_ms: 999, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(StubAuctionProvider::new( + "bidder", + "bidder-backend", + ))); + orchestrator.register_provider(Arc::new(StubAuctionProvider::recording( + "mediator", + "mediator-backend", + 2000, + Arc::clone(&observed), + ))); + + let request = create_test_auction_request(); + let settings = create_test_settings(); + let req = http::Request::builder() + .method(http::Method::GET) + .uri("https://example.com/test") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let context = AuctionContext { + settings: &settings, + request: &req, + timeout_ms: 999, + provider_responses: None, + services, + }; + + orchestrator + .run_auction(&request, &context) + .await + .expect("should complete mediated auction"); + + let observed = observed.lock().expect("should lock observed timeouts"); + assert!(!observed.is_empty(), "should run the mediator"); + for timeout in observed.iter() { + assert!( + *timeout % super::TRANSPORT_TIMEOUT_QUANTUM_MS == 0, + "mediator timeout {timeout}ms should be quantum-aligned" + ); + assert!( + *timeout > 0 && *timeout <= 750, + "mediator timeout {timeout}ms should be positive and floored below the 999ms budget" + ); + } + }); + } + + #[test] + fn dispatched_collect_quantizes_mediator_timeout() { + futures::executor::block_on(async { + // Same invariant as the synchronous path, on the split + // dispatch/collect path used by publisher page rendering. + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); // bidder send_async + stub.push_response(200, b"{}".to_vec()); // mediator send_async + let services = build_services_with_http_client(stub); + // SAFETY: `Box::leak` creates a `'static` reference for test use only. + // The leaked allocation is bounded to the test process lifetime. + let services: &'static RuntimeServices = Box::leak(Box::new(services)); + + let observed_bidder = Arc::new(Mutex::new(Vec::new())); + let observed_mediator = Arc::new(Mutex::new(Vec::new())); + let config = AuctionConfig { + enabled: true, + providers: vec!["bidder".to_string()], + mediator: Some("mediator".to_string()), + timeout_ms: 999, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(StubAuctionProvider::recording( + "bidder", + "bidder-backend", + 2000, + Arc::clone(&observed_bidder), + ))); + orchestrator.register_provider(Arc::new(StubAuctionProvider::recording( + "mediator", + "mediator-backend", + 2000, + Arc::clone(&observed_mediator), + ))); + + let request = create_test_auction_request(); + let settings = create_test_settings(); + let req = http::Request::builder() + .method(http::Method::GET) + .uri("https://example.com/test") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let context = AuctionContext { + settings: &settings, + request: &req, + timeout_ms: 999, + provider_responses: None, + services, + }; + + let dispatched = match orchestrator.dispatch_auction(&request, &context).await { + DispatchAuctionOutcome::Dispatched(dispatched) => dispatched, + _ => panic!("should dispatch the bidder request"), + }; + orchestrator + .collect_dispatched_auction(dispatched, services, &context) + .await; + + let observed_bidder = observed_bidder.lock().expect("should lock bidder timeouts"); + assert!( + !observed_bidder.is_empty(), + "should record dispatched bidder timeouts" + ); + for timeout in observed_bidder.iter() { + assert!( + *timeout % super::TRANSPORT_TIMEOUT_QUANTUM_MS == 0 + && *timeout > 0 + && *timeout <= 750, + "dispatched bidder timeout should floor 999ms to a quantum bucket at or below 750ms, got {timeout}ms" + ); + } + + let observed_mediator = observed_mediator + .lock() + .expect("should lock mediator timeouts"); + assert!(!observed_mediator.is_empty(), "should run the mediator"); + for timeout in observed_mediator.iter() { + assert!( + *timeout % super::TRANSPORT_TIMEOUT_QUANTUM_MS == 0, + "mediator timeout {timeout}ms should be quantum-aligned" + ); + assert!( + *timeout > 0 && *timeout <= 750, + "mediator timeout {timeout}ms should be positive and floored below the 999ms budget" + ); + } + }); + } + #[test] fn select_error_is_attributed_to_correct_provider() { futures::executor::block_on(async { @@ -1950,14 +2494,14 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", - backend: "backend-a", - })); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", - backend: "backend-b", - })); + orchestrator.register_provider(Arc::new(StubAuctionProvider::new( + "provider-a", + "backend-a", + ))); + orchestrator.register_provider(Arc::new(StubAuctionProvider::new( + "provider-b", + "backend-b", + ))); let request = create_test_auction_request(); let settings = create_test_settings(); @@ -2033,14 +2577,14 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", - backend: "backend-a", - })); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", - backend: "backend-b", - })); + orchestrator.register_provider(Arc::new(StubAuctionProvider::new( + "provider-a", + "backend-a", + ))); + orchestrator.register_provider(Arc::new(StubAuctionProvider::new( + "provider-b", + "backend-b", + ))); let request = create_test_auction_request(); let settings = create_test_settings(); @@ -2098,14 +2642,14 @@ mod tests { ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-a", - backend: "backend-a", - })); - orchestrator.register_provider(Arc::new(StubAuctionProvider { - name: "provider-b", - backend: "backend-b", - })); + orchestrator.register_provider(Arc::new(StubAuctionProvider::new( + "provider-a", + "backend-a", + ))); + orchestrator.register_provider(Arc::new(StubAuctionProvider::new( + "provider-b", + "backend-b", + ))); let request = create_test_auction_request(); let settings = create_test_settings(); From 35e872bf621962c814d13866fac26b5c75f4d44c Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 8 Jul 2026 20:56:59 +0530 Subject: [PATCH 002/198] Stream publisher origin bodies end-to-end on Fastly Publisher pages were fully buffered before the first byte reached the client: the platform client materialized the origin body (10 MiB cap), the rewrite pipeline ran over an in-memory cursor, and the EdgeZero finalize buffered the assembled response while awaiting auction collection. TTFB therefore tracked full origin transfer plus the auction instead of origin first byte. - Add supports_streaming_responses() to PlatformHttpClient (default false, Fastly true) and request with_stream_response() on the publisher origin fetch only where honored - Teach the pipeline to consume Body::Stream asynchronously: BodyChunkSource (cumulative raw-byte cap via publisher.max_buffered_body_bytes), push-style BodyStreamDecoder/BodyStreamEncoder in streaming_processor - Replace the Fastly buffered finalize with publisher_response_into_streaming_response: a lazy Body::Stream that commits headers at origin first byte, streams rewritten chunks, and holds only the tail for auction collection; bids still inject before body close - Share one hold implementation (hold_step_decoded_chunk / hold_finish_segments) between the lazy body and the writer-driven loop so the paths cannot drift; collect_non_html_auction dedupes the collect-before-stream path - Finalize brotli decode with close() so truncated origin streams error instead of silently truncating; decode failures emit stream_decode_error telemetry - Guard bodiless (HEAD/204/304) responses and log wasted auction dispatch, matching the buffered finalizer Local A/B on a 183 KB gzip publisher page with a live 3-slot auction (release builds, 20 interleaved rounds): TTFB median 741 ms buffered vs 161 ms streamed (-78%); guest wall time and wasm heap unchanged. --- Cargo.lock | 1 + Cargo.toml | 1 + .../trusted-server-adapter-fastly/src/app.rs | 43 +- .../trusted-server-adapter-fastly/src/main.rs | 11 +- .../src/platform.rs | 4 + crates/trusted-server-core/Cargo.toml | 1 + .../trusted-server-core/src/platform/http.rs | 11 + .../src/platform/test_support.rs | 15 + crates/trusted-server-core/src/proxy.rs | 9 +- crates/trusted-server-core/src/publisher.rs | 1915 +++++++++++++++-- crates/trusted-server-core/src/settings.rs | 15 +- .../src/streaming_processor.rs | 191 ++ ...2026-07-08-true-origin-streaming-fastly.md | 1039 +++++++++ 13 files changed, 3078 insertions(+), 178 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-08-true-origin-streaming-fastly.md diff --git a/Cargo.lock b/Cargo.lock index a19be7abb..cd2227882 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5277,6 +5277,7 @@ dependencies = [ name = "trusted-server-core" version = "0.1.0" dependencies = [ + "async-stream", "async-trait", "base64", "brotli", diff --git a/Cargo.toml b/Cargo.toml index 27411acbd..0512371e7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ debug = 1 [workspace.dependencies] anyhow = "1" +async-stream = "0.3" async-trait = "0.1" axum = "0.8" base64 = "0.22" diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index e56498b10..d5e37f91a 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -65,10 +65,10 @@ //! run on these responses. Legacy ran EC finalization on its own auth //! challenges. Like the 401 geo-skip, this is privacy-conservative: no EC //! cookies are issued to unauthenticated callers. -//! - **Publisher responses** are buffered (bounded by -//! `publisher.max_buffered_body_bytes`) instead of streamed to the client. -//! Asset responses are streamed straight to the client (see -//! [`dispatch_asset_fallback`]), matching legacy. +//! - **Publisher responses** keep Fastly origin bodies streaming through the +//! `EdgeZero` response body when the body is processable or pass-through. +//! Adapters without streaming-body support still use the bounded buffered +//! finalizer. //! - **Router-level 405s** (unregistered verbs) skip EC finalization along //! with the middleware chain; the entry point still adds TS headers. //! @@ -116,8 +116,8 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, AssetProxyCachePolicy, }; use trusted_server_core::publisher::{ - buffer_publisher_response_async, handle_page_bids, handle_publisher_request, - handle_tsjs_dynamic, page_bids_preflight_denied, AuctionDispatch, + handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, + publisher_response_into_streaming_response, AuctionDispatch, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -721,10 +721,9 @@ async fn dispatch_fallback( let result = if uses_dynamic_tsjs_fallback(&method, &path) { handle_tsjs_dynamic(&req, &state.registry) } else if state.registry.has_route(&method, &path) { - // Integration-proxy responses are not bounded by publisher.max_buffered_body_bytes. - // Only the handle_publisher_request branch below routes through - // buffer_publisher_response_async. Integration responses are small in practice - // and the EdgeZero flag is off by default; extend the cap here if that changes. + // Integration-proxy responses are not bounded by + // publisher.max_buffered_body_bytes. Publisher fallback below uses the + // publisher-specific streaming finalizer instead. state .registry .handle_proxy(ProxyDispatchInput { @@ -773,9 +772,8 @@ async fn dispatch_fallback( match runtime_services_for_consent_route(&state.settings, services) { Ok(publisher_services) => { // Run the server-side auction with the configured creative- - // opportunity slots and collect the dispatched bids in the - // buffered finalize (`buffer_publisher_response_async`), matching - // the legacy streaming path. `handle_publisher_request` matches the + // opportunity slots and collect dispatched bids from the lazy + // publisher body stream. `handle_publisher_request` matches the // slots against the request path. The partner registry plus the // EC identity-graph KV (`ec.kv_graph`) enrich the bid request with // server-side EIDs, same as the legacy auction. @@ -797,17 +795,14 @@ async fn dispatch_fallback( ) .await { - Ok(pub_response) => { - buffer_publisher_response_async( - pub_response, - &method, - &state.settings, - &state.registry, - &state.orchestrator, - &publisher_services, - ) - .await - } + Ok(pub_response) => publisher_response_into_streaming_response( + pub_response, + &method, + Arc::clone(&state.settings), + state.registry.as_ref(), + Arc::clone(&state.orchestrator), + publisher_services.clone(), + ), Err(e) => Err(e), } } diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index d20de533d..963686cb8 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -321,10 +321,9 @@ fn run_edgezero_pull_sync_after_send( /// Sends a finalized `EdgeZero` response to the client. /// -/// Asset streams commit headers first, then pipe the origin body chunk by chunk -/// so large responses do not materialize in the Wasm heap. Publisher responses -/// are buffered by the server-side auction path so bids can be injected into the -/// document, and are sent in one shot along with all other responses. +/// Streaming `EdgeZero` bodies commit headers first, then pipe chunks to Fastly's +/// client stream so large asset and publisher-origin responses do not +/// materialize in the Wasm heap. fn send_edgezero_response( mut response: HttpResponse, request_filter_effects: Option<&RequestFilterEffects>, @@ -350,11 +349,11 @@ fn send_edgezero_response( match futures::executor::block_on(stream_asset_body(body, &mut streaming_body)) { Ok(()) => { if let Err(e) = streaming_body.finish() { - log::error!("failed to finish EdgeZero asset streaming body: {e}"); + log::error!("failed to finish EdgeZero streaming body: {e}"); } } Err(e) => { - log::error!("EdgeZero asset streaming failed: {e:?}"); + log::error!("EdgeZero streaming failed: {e:?}"); drop(streaming_body); } } diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index 106ced787..65d0f4b0d 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -426,6 +426,10 @@ pub struct FastlyPlatformHttpClient; #[async_trait::async_trait(?Send)] impl PlatformHttpClient for FastlyPlatformHttpClient { + fn supports_streaming_responses(&self) -> bool { + true + } + async fn send( &self, request: PlatformHttpRequest, diff --git a/crates/trusted-server-core/Cargo.toml b/crates/trusted-server-core/Cargo.toml index ba88f361f..bedefc327 100644 --- a/crates/trusted-server-core/Cargo.toml +++ b/crates/trusted-server-core/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] async-trait = { workspace = true } +async-stream = { workspace = true } base64 = { workspace = true } brotli = { workspace = true } bytes = { workspace = true } diff --git a/crates/trusted-server-core/src/platform/http.rs b/crates/trusted-server-core/src/platform/http.rs index 80bb23121..9e9337edd 100644 --- a/crates/trusted-server-core/src/platform/http.rs +++ b/crates/trusted-server-core/src/platform/http.rs @@ -276,6 +276,17 @@ pub trait PlatformHttpClient: Send + Sync { true } + /// Whether [`send`](Self::send) can preserve upstream response bodies as + /// [`Body::Stream`](edgezero_core::body::Body::Stream) when requested via + /// [`PlatformHttpRequest::with_stream_response`]. + /// + /// Adapters that cannot preserve streaming response bodies must keep the + /// default `false` so callers do not request a contract the adapter will + /// reject or silently buffer. + fn supports_streaming_responses(&self) -> bool { + false + } + /// Wait for one of the in-flight requests to complete. /// /// # Errors diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index ee7201fb8..0c86b9594 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -224,6 +224,9 @@ pub(crate) struct StubHttpClient { // Reported by supports_concurrent_fanout(); set false to emulate // platforms whose send_async executes eagerly (e.g. Cloudflare Workers). concurrent_fanout: std::sync::atomic::AtomicBool, + // Reported by supports_streaming_responses(); set true to emulate Fastly's + // streaming response support. + streaming_responses_supported: std::sync::atomic::AtomicBool, image_optimizer_options: Mutex>>, stream_response_flags: Mutex>, request_methods: Mutex>, @@ -246,6 +249,7 @@ impl StubHttpClient { request_headers: Mutex::new(Vec::new()), select_errors: Mutex::new(VecDeque::new()), concurrent_fanout: std::sync::atomic::AtomicBool::new(true), + streaming_responses_supported: std::sync::atomic::AtomicBool::new(false), image_optimizer_options: Mutex::new(Vec::new()), stream_response_flags: Mutex::new(Vec::new()), request_methods: Mutex::new(Vec::new()), @@ -260,6 +264,12 @@ impl StubHttpClient { .store(supported, std::sync::atomic::Ordering::Relaxed); } + /// Make `supports_streaming_responses()` report the given value. + pub fn set_streaming_responses_supported(&self, supported: bool) { + self.streaming_responses_supported + .store(supported, std::sync::atomic::Ordering::Relaxed); + } + /// Queue a canned response by status code and body bytes. pub fn push_response(&self, status: u16, body: Vec) { self.push_response_with_headers(status, body, Vec::<(String, String)>::new()); @@ -363,6 +373,11 @@ impl PlatformHttpClient for StubHttpClient { .load(std::sync::atomic::Ordering::Relaxed) } + fn supports_streaming_responses(&self) -> bool { + self.streaming_responses_supported + .load(std::sync::atomic::Ordering::Relaxed) + } + async fn send( &self, request: PlatformHttpRequest, diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 19c10a80d..00444b384 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -246,7 +246,10 @@ fn platform_response_to_fastly_asset(platform_resp: PlatformResponse) -> AssetPr } } -/// Stream an asset response body directly to a writable client stream. +/// Stream a platform response body directly to a writable client stream. +/// +/// Asset routes and Fastly `EdgeZero` publisher fallback both use this bridge +/// after headers have been committed through `stream_to_client()`. /// /// # Errors /// @@ -261,7 +264,7 @@ pub async fn stream_asset_body( output .write_all(bytes.as_ref()) .change_context(TrustedServerError::Proxy { - message: "failed to write buffered asset response body".to_string(), + message: "failed to write buffered platform response body".to_string(), })?; } EdgeBody::Stream(mut stream) => { @@ -274,7 +277,7 @@ pub async fn stream_asset_body( output .write_all(chunk.as_ref()) .change_context(TrustedServerError::Proxy { - message: "failed to write streaming asset response body".to_string(), + message: "failed to write streaming platform response body".to_string(), })?; } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 6b0ea3a5d..c10ac0b39 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -22,9 +22,15 @@ use std::io::Write; use std::sync::{Arc, Mutex}; use std::time::Duration; +use brotli::enc::writer::CompressorWriter; +use brotli::enc::BrotliEncoderParams; +use brotli::Decompressor; use cookie::CookieJar; use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; +use flate2::read::{GzDecoder, ZlibDecoder}; +use flate2::write::{GzEncoder, ZlibEncoder}; +use futures::StreamExt as _; use http::{header, HeaderValue, Method, Request, Response, StatusCode, Uri}; use crate::auction::endpoints::{ @@ -53,19 +59,134 @@ use crate::platform::{GeoInfo, PlatformBackendSpec, PlatformHttpRequest, Runtime use crate::price_bucket::{price_bucket, PriceGranularity}; use crate::rsc_flight::RscFlightUrlRewriter; use crate::settings::Settings; -use crate::streaming_processor::{Compression, PipelineConfig, StreamProcessor, StreamingPipeline}; +use crate::streaming_processor::{ + BodyStreamDecoder, BodyStreamEncoder, Compression, PipelineConfig, StreamProcessor, + StreamingPipeline, STREAM_CHUNK_SIZE, +}; use crate::streaming_replacer::create_url_replacer; const SUPPORTED_ENCODING_VALUES: [&str; 3] = ["gzip", "deflate", "br"]; const DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT: Duration = Duration::from_secs(15); -/// Read buffer size for streaming body processing and brotli internal buffers. -/// Both the `Decompressor` and `CompressorWriter` use this value so all -/// brotli I/O layers operate on consistently-sized chunks. -const STREAM_CHUNK_SIZE: usize = 8192; +fn body_as_reader( + body: EdgeBody, +) -> Result, Report> { + let bytes = body.into_bytes().ok_or_else(|| { + Report::new(TrustedServerError::Proxy { + message: "streaming body cannot be processed by sync publisher pipeline".to_string(), + }) + })?; + Ok(std::io::Cursor::new(bytes)) +} + +struct BodyChunkSource { + body: Option, + chunk_size: usize, + max_bytes: usize, + bytes_seen: usize, + once_offset: usize, +} + +impl BodyChunkSource { + fn new(body: EdgeBody, chunk_size: usize) -> Self { + Self { + body: Some(body), + chunk_size, + max_bytes: usize::MAX, + bytes_seen: 0, + once_offset: 0, + } + } + + fn with_max_bytes(mut self, max_bytes: usize) -> Self { + self.max_bytes = max_bytes; + self + } + + async fn next_chunk(&mut self) -> Result, Report> { + let Some(body) = self.body.take() else { + return Ok(None); + }; + + let chunk = match body { + EdgeBody::Once(bytes) => { + if self.once_offset >= bytes.len() { + None + } else { + let end = (self.once_offset + self.chunk_size).min(bytes.len()); + let chunk = bytes.slice(self.once_offset..end); + self.once_offset = end; + if self.once_offset < bytes.len() { + self.body = Some(EdgeBody::Once(bytes)); + } + Some(chunk) + } + } + EdgeBody::Stream(mut stream) => match stream.next().await { + Some(Ok(chunk)) => { + self.body = Some(EdgeBody::Stream(stream)); + Some(chunk) + } + Some(Err(err)) => { + return Err(Report::new(TrustedServerError::Proxy { + message: format!("Failed to read publisher origin body stream: {err}"), + })); + } + None => None, + }, + }; + + let Some(chunk) = chunk else { + return Ok(None); + }; + + self.bytes_seen = self.bytes_seen.checked_add(chunk.len()).ok_or_else(|| { + Report::new(TrustedServerError::Proxy { + message: "publisher origin body byte count overflowed".to_string(), + }) + })?; + if self.bytes_seen > self.max_bytes { + return Err(Report::new(TrustedServerError::Proxy { + message: format!( + "publisher origin body exceeded {}-byte streaming limit", + self.max_bytes + ), + })); + } + + Ok(Some(chunk)) + } +} + +fn process_and_encode_chunk( + processor: &mut P, + encoder: &mut BodyStreamEncoder, + chunk: &[u8], + is_last: bool, + process_error: &str, +) -> Result, Report> { + let processed = + processor + .process_chunk(chunk, is_last) + .change_context(TrustedServerError::Proxy { + message: process_error.to_string(), + })?; + if processed.is_empty() { + return Ok(None); + } + let encoded = encoder.encode_chunk(&processed)?; + if encoded.is_empty() { + return Ok(None); + } + Ok(Some(bytes::Bytes::from(encoded))) +} -fn body_as_reader(body: EdgeBody) -> std::io::Cursor { - std::io::Cursor::new(body.into_bytes().unwrap_or_default()) +fn publisher_stream_error(err: Report) -> std::io::Error { + let message = format!("{err:?}"); + // Consume the report so clippy's needless_pass_by_value accepts the + // by-value signature that `map_err(publisher_stream_error)` requires. + drop(err); + std::io::Error::other(message) } fn not_found_response() -> Response { @@ -233,6 +354,55 @@ struct ProcessResponseParams<'a> { ad_bids_state: &'a Arc>>, } +struct PublisherBodyProcessor { + inner: Box, +} + +impl PublisherBodyProcessor { + fn new( + params: &OwnedProcessResponseParams, + settings: &Settings, + integration_registry: &IntegrationRegistry, + ) -> Result> { + let is_html = is_html_content_type(¶ms.content_type); + let is_rsc_flight = + content_type_contains_ascii_case_insensitive(¶ms.content_type, "text/x-component"); + let inner: Box = if is_html { + Box::new(create_html_stream_processor( + ¶ms.origin_host, + ¶ms.request_host, + ¶ms.request_scheme, + settings, + integration_registry, + params.ad_slots_script.as_deref().map(str::to_string), + Arc::clone(¶ms.ad_bids_state), + )?) + } else if is_rsc_flight { + Box::new(RscFlightUrlRewriter::new( + ¶ms.origin_host, + ¶ms.origin_url, + ¶ms.request_host, + ¶ms.request_scheme, + )) + } else { + Box::new(create_url_replacer( + ¶ms.origin_host, + ¶ms.origin_url, + ¶ms.request_host, + ¶ms.request_scheme, + )) + }; + + Ok(Self { inner }) + } +} + +impl StreamProcessor for PublisherBodyProcessor { + fn process_chunk(&mut self, chunk: &[u8], is_last: bool) -> Result, std::io::Error> { + self.inner.process_chunk(chunk, is_last) + } +} + /// Process response body through the streaming pipeline. /// /// Selects the appropriate processor based on content type (HTML rewriter, @@ -276,7 +446,7 @@ fn process_response_streaming( params.ad_slots_script.map(str::to_string), params.ad_bids_state.clone(), )?; - StreamingPipeline::new(config, processor).process(body_as_reader(body), output)?; + StreamingPipeline::new(config, processor).process(body_as_reader(body)?, output)?; } else if is_rsc_flight { // RSC Flight responses are length-prefixed (T rows). A naive string replacement will // corrupt the stream by changing byte lengths without updating the prefixes. @@ -286,7 +456,7 @@ fn process_response_streaming( params.request_host, params.request_scheme, ); - StreamingPipeline::new(config, processor).process(body_as_reader(body), output)?; + StreamingPipeline::new(config, processor).process(body_as_reader(body)?, output)?; } else { let replacer = create_url_replacer( params.origin_host, @@ -294,12 +464,352 @@ fn process_response_streaming( params.request_host, params.request_scheme, ); - StreamingPipeline::new(config, replacer).process(body_as_reader(body), output)?; + StreamingPipeline::new(config, replacer).process(body_as_reader(body)?, output)?; } Ok(()) } +async fn process_response_streaming_async( + body: EdgeBody, + output: &mut W, + params: &ProcessResponseParams<'_>, + max_raw_body_bytes: usize, +) -> Result<(), Report> { + let is_html = is_html_content_type(params.content_type); + let is_rsc_flight = + content_type_contains_ascii_case_insensitive(params.content_type, "text/x-component"); + log::debug!( + "process_response_streaming_async: content_type={}, content_encoding={}, is_html={}, is_rsc_flight={}", + params.content_type, + params.content_encoding, + is_html, + is_rsc_flight + ); + + let compression = Compression::from_content_encoding(params.content_encoding); + + if is_html { + let mut processor = create_html_stream_processor( + params.origin_host, + params.request_host, + params.request_scheme, + params.settings, + params.integration_registry, + params.ad_slots_script.map(str::to_string), + params.ad_bids_state.clone(), + )?; + process_body_chunks_async( + body, + output, + &mut processor, + compression, + max_raw_body_bytes, + ) + .await + } else if is_rsc_flight { + let mut processor = RscFlightUrlRewriter::new( + params.origin_host, + params.origin_url, + params.request_host, + params.request_scheme, + ); + process_body_chunks_async( + body, + output, + &mut processor, + compression, + max_raw_body_bytes, + ) + .await + } else { + let mut replacer = create_url_replacer( + params.origin_host, + params.origin_url, + params.request_host, + params.request_scheme, + ); + process_body_chunks_async(body, output, &mut replacer, compression, max_raw_body_bytes) + .await + } +} + +async fn process_body_chunks_async( + body: EdgeBody, + writer: &mut W, + processor: &mut P, + compression: Compression, + max_raw_body_bytes: usize, +) -> Result<(), Report> { + let mut decoder = BodyStreamDecoder::new(compression); + let mut encoder = BodyStreamEncoder::new(compression); + let mut source = + BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(max_raw_body_bytes); + + while let Some(chunk) = source.next_chunk().await? { + let decoded = decoder.decode_chunk(&chunk)?; + if decoded.is_empty() { + continue; + } + if let Some(encoded) = process_and_encode_chunk( + processor, + &mut encoder, + &decoded, + false, + "Failed to process chunk", + )? { + write_encoded_segment(writer, &encoded)?; + } + } + + for encoded in passthrough_finish_segments(processor, &mut decoder, &mut encoder)? { + write_encoded_segment(writer, &encoded)?; + } + writer.flush().change_context(TrustedServerError::Proxy { + message: "Failed to flush output".to_string(), + })?; + + Ok(()) +} + +/// Write one encoded output segment produced by the chunk pipeline. +fn write_encoded_segment( + writer: &mut W, + encoded: &[u8], +) -> Result<(), Report> { + writer + .write_all(encoded) + .change_context(TrustedServerError::Proxy { + message: "Failed to write encoded chunk".to_string(), + }) +} + +/// Finalize a no-hold chunk pipeline: drain the decoder tail through the +/// processor, signal end-of-stream to the processor, and emit the encoder +/// trailer. Returns the encoded segments for the caller to emit. +fn passthrough_finish_segments( + processor: &mut P, + decoder: &mut BodyStreamDecoder, + encoder: &mut BodyStreamEncoder, +) -> Result, Report> { + let mut segments = Vec::new(); + let decoded_tail = decoder.finish()?; + if !decoded_tail.is_empty() { + if let Some(encoded) = process_and_encode_chunk( + processor, + encoder, + &decoded_tail, + false, + "Failed to process decoded tail", + )? { + segments.push(encoded); + } + } + if let Some(encoded) = process_and_encode_chunk( + processor, + encoder, + &[], + true, + "Failed to finalize processor", + )? { + segments.push(encoded); + } + let trailer = encoder.finish()?; + if !trailer.is_empty() { + segments.push(bytes::Bytes::from(trailer)); + } + Ok(segments) +} + +/// Mutable auction-hold state threaded through the streaming hold pipeline. +struct AuctionHoldState { + hold: Option, + dispatched: Option, + telemetry: AuctionTelemetryCarry, +} + +impl AuctionHoldState { + fn new(dispatched: DispatchedAuction, telemetry: AuctionTelemetryCarry) -> Self { + Self { + hold: Some(BodyCloseHoldBuffer::new()), + dispatched: Some(dispatched), + telemetry, + } + } +} + +/// Abandon the in-flight auction (if still pending) with the given telemetry +/// reason. No-op once the auction has been collected or already abandoned. +async fn abandon_hold_auction( + state: &mut AuctionHoldState, + services: &RuntimeServices, + reason: &'static str, +) { + if let Some(dispatched) = state.dispatched.take() { + emit_abandoned_auction( + services, + state.telemetry.observation.take(), + dispatched, + reason, + ) + .await; + } +} + +/// Feed one decoded chunk through the close-body hold and processor. +/// +/// Returns the encoded output segments for the caller to emit — written to a +/// client stream by [`body_close_hold_loop_stream`], yielded from the lazy +/// body by [`publisher_response_into_streaming_response`]. Both async hold +/// paths share this function so their behavior cannot drift apart. +/// +/// When the raw `( + processor: &mut P, + encoder: &mut BodyStreamEncoder, + chunk: &[u8], + state: &mut AuctionHoldState, + collect_refs: &AuctionHoldCollectRefs<'_>, +) -> Result, Report> { + let mut segments = Vec::new(); + if let Some(hold_buffer) = state.hold.as_mut() { + let ready = hold_buffer.push(chunk); + match process_and_encode_chunk(processor, encoder, &ready, false, "Failed to process chunk") + { + Ok(Some(encoded)) => segments.push(encoded), + Ok(None) => {} + Err(err) => { + abandon_hold_auction(state, collect_refs.services, "stream_process_error").await; + return Err(err); + } + } + + if state + .hold + .as_ref() + .is_some_and(BodyCloseHoldBuffer::found_close) + { + let dispatched = state + .dispatched + .take() + .expect("should have dispatched auction to collect"); + collect_stream_auction( + dispatched, + state.telemetry.take(), + collect_refs.price_granularity, + collect_refs.ad_bids_state, + collect_refs.orchestrator, + collect_refs.services, + collect_refs.settings, + ) + .await; + + let held = state + .hold + .take() + .expect("should have close-body hold buffer") + .finish(); + if let Some(encoded) = process_and_encode_chunk( + processor, + encoder, + &held, + false, + "Failed to process held body close", + )? { + segments.push(encoded); + } + } + } else { + match process_and_encode_chunk(processor, encoder, chunk, false, "Failed to process chunk") + { + Ok(Some(encoded)) => segments.push(encoded), + Ok(None) => {} + Err(err) => { + abandon_hold_auction(state, collect_refs.services, "stream_process_error").await; + return Err(err); + } + } + } + Ok(segments) +} + +/// Finalize the close-body hold pipeline at end of the origin stream. +/// +/// Drains the decoder tail through the hold (or straight through when the +/// hold was already released mid-stream), collects the auction if the +/// close-body tag never streamed, processes the held tail plus the +/// processor's final chunk, and emits the encoder trailer. Returns the +/// encoded segments for the caller to emit. On decoder failure the pending +/// auction is abandoned before the error is returned. +async fn hold_finish_segments( + processor: &mut P, + decoder: &mut BodyStreamDecoder, + encoder: &mut BodyStreamEncoder, + state: &mut AuctionHoldState, + collect_refs: &AuctionHoldCollectRefs<'_>, +) -> Result, Report> { + let mut segments = Vec::new(); + + let decoded_tail = match decoder.finish() { + Ok(decoded_tail) => decoded_tail, + Err(err) => { + abandon_hold_auction(state, collect_refs.services, "stream_decode_error").await; + return Err(err); + } + }; + if !decoded_tail.is_empty() { + segments.extend( + hold_step_decoded_chunk(processor, encoder, &decoded_tail, state, collect_refs).await?, + ); + } + + if let Some(hold) = state.hold.take() { + let dispatched = state + .dispatched + .take() + .expect("should have dispatched auction to collect"); + collect_stream_auction( + dispatched, + state.telemetry.take(), + collect_refs.price_granularity, + collect_refs.ad_bids_state, + collect_refs.orchestrator, + collect_refs.services, + collect_refs.settings, + ) + .await; + + let held = hold.finish(); + if let Some(encoded) = process_and_encode_chunk( + processor, + encoder, + &held, + false, + "Failed to process held body close", + )? { + segments.push(encoded); + } + } + + if let Some(encoded) = process_and_encode_chunk( + processor, + encoder, + &[], + true, + "Failed to finalize processor", + )? { + segments.push(encoded); + } + let trailer = encoder.finish()?; + if !trailer.is_empty() { + segments.push(bytes::Bytes::from(trailer)); + } + Ok(segments) +} + /// Create a unified HTML stream processor. /// /// Builds the config via [`HtmlProcessorConfig::from_settings`] and then @@ -339,16 +849,13 @@ pub enum PublisherResponse { /// content on any status (2xx or non-2xx — e.g., branded 404/500 HTML and /// error JSON still get URL rewriting) where the encoding is supported. /// Post-processors run inside the streaming processor, so processable HTML - /// is streamed regardless of whether any are registered. The caller must: - /// 1. Call `finalize_response()` on the response - /// 2. Call `response.stream_to_client()` to get a `StreamingBody` - /// 3. Call `stream_publisher_body()` with the body and streaming writer - /// 4. Call `StreamingBody::finish()` + /// is streamed regardless of whether any are registered. /// - /// **Interim (PR 15):** `body` has already been fully materialised into - /// WASM heap by the platform HTTP client. `stream_publisher_body` reads - /// from an in-memory buffer, not a live origin stream. The origin-side - /// peak is bounded by `MAX_PLATFORM_RESPONSE_BODY_BYTES`. + /// Adapters with platform streaming support preserve `body` as + /// [`EdgeBody::Stream`] and attach a lazy processed stream via + /// [`publisher_response_into_streaming_response`]. Buffered adapters use + /// [`buffer_publisher_response_async`] and are bounded by + /// `settings.publisher.max_buffered_body_bytes`. Stream { /// Response with all headers set (EC ID, cookies, etc.) /// but body not yet written. `Content-Length` already removed. @@ -363,12 +870,9 @@ pub enum PublisherResponse { /// `finalize_response()` and `send_to_client()` are applied at the outer /// response-dispatch level, not in this arm. /// - /// `Content-Length` is preserved — the body is unmodified. - /// - /// **Interim (PR 15):** `body` has been fully materialised into WASM heap. - /// Previously, binary assets streamed lazily from origin with no WASM - /// buffering. This path is now bounded by `MAX_PLATFORM_RESPONSE_BODY_BYTES`; - /// assets exceeding that limit return an error instead of exhausting heap. + /// `Content-Length` is preserved — the body is unmodified. Streaming + /// adapters reattach the origin body directly so non-processable 2xx bodies + /// can pass through without materializing in WASM memory. PassThrough { /// Response with all headers set but body not yet written. response: Response, @@ -465,7 +969,7 @@ pub struct OwnedProcessResponseParams { /// statuses (204, 304) carry no body but may advertise the `GET` representation's /// length, so they skip the buffer and length rewrite. /// -/// Every adapter (Axum, Cloudflare, Spin, and the Fastly `EdgeZero` path) calls +/// Buffered adapters (Axum, Cloudflare, Spin, and non-streaming fallbacks) call /// this: it drives /// [`stream_publisher_body_async`], which awaits /// [`AuctionOrchestrator::collect_dispatched_auction`], writes the winning bids @@ -530,48 +1034,225 @@ pub async fn buffer_publisher_response_async( } } -/// Returns `true` when a buffered publisher response should carry a body and a -/// recomputed `Content-Length`. +/// Convert a [`PublisherResponse`] into a response that preserves streaming +/// bodies where possible. /// -/// `HEAD` responses and bodiless statuses (204, 304) carry no body; rewriting -/// their `Content-Length` to the (empty) buffered length would mislead clients -/// and caches, so the origin metadata is preserved instead. -fn response_carries_body(method: &Method, status: StatusCode) -> bool { - *method != Method::HEAD - && status != StatusCode::NO_CONTENT - && status != StatusCode::NOT_MODIFIED -} - -/// A [`Write`] sink that buffers into a `Vec` but fails once the configured -/// byte limit would be exceeded. +/// Buffered adapters should keep using [`buffer_publisher_response_async`]. +/// Fastly uses this helper before the entry point commits headers, allowing the +/// response body to be pulled lazily by `stream_to_client()`. /// -/// Used to bound in-WASM-heap buffering of decoded/re-written publisher bodies. -/// A highly-compressible origin response can sit under the platform raw-body cap -/// yet expand past a safe heap size after decode and post-processing; this writer -/// turns that into a recoverable error instead of an out-of-memory abort. -pub struct BoundedWriter { - inner: Vec, - limit: usize, -} - -impl BoundedWriter { - /// Creates a writer that accepts at most `limit` bytes before erroring. - #[must_use] - pub fn new(limit: usize) -> Self { - Self { - inner: Vec::new(), - limit, +/// # Errors +/// +/// Returns an error if processor construction fails before the streaming body is +/// created. +pub fn publisher_response_into_streaming_response( + publisher_response: PublisherResponse, + method: &Method, + settings: Arc, + integration_registry: &IntegrationRegistry, + orchestrator: Arc, + services: RuntimeServices, +) -> Result, Report> { + match publisher_response { + PublisherResponse::Buffered(response) => Ok(response), + PublisherResponse::PassThrough { mut response, body } => { + if response_carries_body(method, response.status()) { + *response.body_mut() = body; + } + Ok(response) } - } - - /// Consumes the writer and returns the buffered bytes. - #[must_use] - pub fn into_inner(self) -> Vec { - self.inner - } -} - -impl Write for BoundedWriter { + PublisherResponse::Stream { + mut response, + body, + params, + } => { + if !response_carries_body(method, response.status()) { + if params.dispatched_auction.is_some() { + // A bodiless response (HEAD navigation, 204/304) has no + // `` to inject bids into, so the dispatched SSP + // requests are wasted — surface it for quota observability, + // matching the buffered finalizer. + log::warn!( + "Server-side auction dispatched but response is bodiless (method: {}, status: {}); in-flight SSP bid requests will not be collected", + method, + response.status(), + ); + } + return Ok(response); + } + + response.headers_mut().remove(header::CONTENT_LENGTH); + let mut params = *params; + let mut processor = + PublisherBodyProcessor::new(¶ms, &settings, integration_registry)?; + let stream = async_stream::try_stream! { + let compression = Compression::from_content_encoding(¶ms.content_encoding); + let mut decoder = BodyStreamDecoder::new(compression); + let mut encoder = BodyStreamEncoder::new(compression); + let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE) + .with_max_bytes(settings.publisher.max_buffered_body_bytes); + + // HTML rides the close-body hold so bids land before ``; + // non-HTML has no injection point, so its auction is collected + // before any byte streams (matching the buffered finalizer). + let mut hold_auction = None; + if let Some(dispatched) = params.dispatched_auction.take() { + let telemetry = AuctionTelemetryCarry { + observation: params.auction_observation.take(), + auction_request: params.auction_request.take(), + }; + if is_html_content_type(¶ms.content_type) { + hold_auction = Some((dispatched, telemetry)); + } else { + collect_non_html_auction( + dispatched, + telemetry, + ¶ms, + &orchestrator, + &services, + &settings, + ) + .await; + } + } + + if let Some((dispatched, telemetry)) = hold_auction { + let mut state = AuctionHoldState::new(dispatched, telemetry); + let collect_refs = AuctionHoldCollectRefs { + price_granularity: params.price_granularity, + ad_bids_state: ¶ms.ad_bids_state, + orchestrator: &orchestrator, + services: &services, + settings: &settings, + }; + + loop { + let raw_chunk = match source.next_chunk().await { + Ok(Some(chunk)) => chunk, + Ok(None) => break, + Err(err) => { + abandon_hold_auction(&mut state, &services, "stream_read_error") + .await; + Err(publisher_stream_error(err))?; + unreachable!("error should have returned"); + } + }; + let decoded = match decoder.decode_chunk(&raw_chunk) { + Ok(decoded) => decoded, + Err(err) => { + abandon_hold_auction(&mut state, &services, "stream_decode_error") + .await; + Err(publisher_stream_error(err))?; + unreachable!("error should have returned"); + } + }; + if decoded.is_empty() { + continue; + } + for encoded in hold_step_decoded_chunk( + &mut processor, + &mut encoder, + &decoded, + &mut state, + &collect_refs, + ) + .await + .map_err(publisher_stream_error)? + { + yield encoded; + } + } + + for encoded in hold_finish_segments( + &mut processor, + &mut decoder, + &mut encoder, + &mut state, + &collect_refs, + ) + .await + .map_err(publisher_stream_error)? + { + yield encoded; + } + } else { + while let Some(raw_chunk) = + source.next_chunk().await.map_err(publisher_stream_error)? + { + let decoded = decoder + .decode_chunk(&raw_chunk) + .map_err(publisher_stream_error)?; + if decoded.is_empty() { + continue; + } + if let Some(encoded) = process_and_encode_chunk( + &mut processor, + &mut encoder, + &decoded, + false, + "Failed to process chunk", + ) + .map_err(publisher_stream_error)? + { + yield encoded; + } + } + for encoded in + passthrough_finish_segments(&mut processor, &mut decoder, &mut encoder) + .map_err(publisher_stream_error)? + { + yield encoded; + } + } + }; + *response.body_mut() = EdgeBody::from_stream::<_, std::io::Error>(stream); + Ok(response) + } + } +} + +/// Returns `true` when a buffered publisher response should carry a body and a +/// recomputed `Content-Length`. +/// +/// `HEAD` responses and bodiless statuses (204, 304) carry no body; rewriting +/// their `Content-Length` to the (empty) buffered length would mislead clients +/// and caches, so the origin metadata is preserved instead. +fn response_carries_body(method: &Method, status: StatusCode) -> bool { + *method != Method::HEAD + && status != StatusCode::NO_CONTENT + && status != StatusCode::NOT_MODIFIED +} + +/// A [`Write`] sink that buffers into a `Vec` but fails once the configured +/// byte limit would be exceeded. +/// +/// Used to bound in-WASM-heap buffering of decoded/re-written publisher bodies. +/// A highly-compressible origin response can sit under the platform raw-body cap +/// yet expand past a safe heap size after decode and post-processing; this writer +/// turns that into a recoverable error instead of an out-of-memory abort. +pub struct BoundedWriter { + inner: Vec, + limit: usize, +} + +impl BoundedWriter { + /// Creates a writer that accepts at most `limit` bytes before erroring. + #[must_use] + pub fn new(limit: usize) -> Self { + Self { + inner: Vec::new(), + limit, + } + } + + /// Consumes the writer and returns the buffered bytes. + #[must_use] + pub fn into_inner(self) -> Vec { + self.inner + } +} + +impl Write for BoundedWriter { fn write(&mut self, buf: &[u8]) -> std::io::Result { if self.inner.len() + buf.len() > self.limit { return Err(std::io::Error::other( @@ -652,7 +1333,29 @@ pub async fn stream_publisher_body_async( services: &RuntimeServices, ) -> Result<(), Report> { let Some(dispatched) = params.dispatched_auction.take() else { - // No auction — use the existing sync pipeline unchanged. + if body.is_stream() { + let borrowed = ProcessResponseParams { + content_encoding: ¶ms.content_encoding, + origin_host: ¶ms.origin_host, + origin_url: ¶ms.origin_url, + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, + settings, + content_type: ¶ms.content_type, + integration_registry, + ad_slots_script: params.ad_slots_script.as_deref(), + ad_bids_state: ¶ms.ad_bids_state, + }; + return process_response_streaming_async( + body, + output, + &borrowed, + settings.publisher.max_buffered_body_bytes, + ) + .await; + } + + // No auction and already-buffered body — keep the existing sync pipeline. return stream_publisher_body(body, output, params, settings, integration_registry); }; let telemetry = AuctionTelemetryCarry { @@ -665,35 +1368,36 @@ pub async fn stream_publisher_body_async( if !is_html { // Non-HTML: collect auction first, then stream. There is no // to hold, so delaying the entire body until collection is acceptable. - let placeholder = mediator_placeholder_request(); - let result = orchestrator - .collect_dispatched_auction( - dispatched, - services, - &make_collect_context(settings, services, &placeholder), + collect_non_html_auction( + dispatched, + telemetry, + params, + orchestrator, + services, + settings, + ) + .await; + if body.is_stream() { + let borrowed = ProcessResponseParams { + content_encoding: ¶ms.content_encoding, + origin_host: ¶ms.origin_host, + origin_url: ¶ms.origin_url, + request_host: ¶ms.request_host, + request_scheme: ¶ms.request_scheme, + settings, + content_type: ¶ms.content_type, + integration_registry, + ad_slots_script: params.ad_slots_script.as_deref(), + ad_bids_state: ¶ms.ad_bids_state, + }; + return process_response_streaming_async( + body, + output, + &borrowed, + settings.publisher.max_buffered_body_bytes, ) .await; - if let (Some(observation), Some(auction_request)) = - (telemetry.observation, telemetry.auction_request.as_ref()) - { - emit_auction_events_best_effort_lazy(services, || { - build_auction_events( - observation, - AuctionTerminalOutcome::Completed { - request: auction_request, - result: &result, - }, - ) - }) - .await; } - - write_bids_to_state( - &result.winning_bids, - params.price_granularity, - ¶ms.ad_bids_state, - settings.debug.inject_adm_for_testing, - ); return stream_publisher_body(body, output, params, settings, integration_registry); } @@ -917,6 +1621,14 @@ struct AuctionCollectCtx<'a> { settings: &'a Settings, } +struct AuctionHoldCollectRefs<'a> { + price_granularity: PriceGranularity, + ad_bids_state: &'a Arc>>, + orchestrator: &'a AuctionOrchestrator, + services: &'a RuntimeServices, + settings: &'a Settings, +} + /// Run the close-body hold loop for HTML bodies, collecting the auction before /// the raw `( @@ -926,13 +1638,20 @@ async fn stream_html_with_auction_hold( compression: Compression, ctx: AuctionCollectCtx<'_>, ) -> Result<(), Report> { - use brotli::enc::writer::CompressorWriter; - use brotli::enc::BrotliEncoderParams; - use brotli::Decompressor; - use flate2::read::{GzDecoder, ZlibDecoder}; - use flate2::write::{GzEncoder, ZlibEncoder}; + if body.is_stream() { + let max_raw_body_bytes = ctx.settings.publisher.max_buffered_body_bytes; + return body_close_hold_loop_stream( + body, + output, + processor, + compression, + ctx, + max_raw_body_bytes, + ) + .await; + } - let body = body_as_reader(body); + let body = body_as_reader(body)?; match compression { Compression::None => body_close_hold_loop(body, output, processor, ctx).await, Compression::Gzip => { @@ -969,6 +1688,85 @@ async fn stream_html_with_auction_hold( } } +/// Async-pull variant of [`body_close_hold_loop`] for live origin streams. +/// +/// Shares [`hold_step_decoded_chunk`] and [`hold_finish_segments`] with the +/// lazy streaming body built by [`publisher_response_into_streaming_response`], +/// so the two async hold paths cannot drift apart. +async fn body_close_hold_loop_stream( + body: EdgeBody, + writer: &mut W, + processor: &mut P, + compression: Compression, + ctx: AuctionCollectCtx<'_>, + max_raw_body_bytes: usize, +) -> Result<(), Report> { + let AuctionCollectCtx { + dispatched, + telemetry, + price_granularity, + ad_bids_state, + orchestrator, + services, + settings, + } = ctx; + let mut decoder = BodyStreamDecoder::new(compression); + let mut encoder = BodyStreamEncoder::new(compression); + let mut source = + BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(max_raw_body_bytes); + let mut state = AuctionHoldState::new(dispatched, telemetry); + let collect_refs = AuctionHoldCollectRefs { + price_granularity, + ad_bids_state, + orchestrator, + services, + settings, + }; + + loop { + let raw_chunk = match source.next_chunk().await { + Ok(Some(chunk)) => chunk, + Ok(None) => break, + Err(err) => { + abandon_hold_auction(&mut state, services, "stream_read_error").await; + return Err(err); + } + }; + let decoded = match decoder.decode_chunk(&raw_chunk) { + Ok(decoded) => decoded, + Err(err) => { + abandon_hold_auction(&mut state, services, "stream_decode_error").await; + return Err(err); + } + }; + if decoded.is_empty() { + continue; + } + for encoded in + hold_step_decoded_chunk(processor, &mut encoder, &decoded, &mut state, &collect_refs) + .await? + { + write_encoded_segment(writer, &encoded)?; + } + } + + for encoded in hold_finish_segments( + processor, + &mut decoder, + &mut encoder, + &mut state, + &collect_refs, + ) + .await? + { + write_encoded_segment(writer, &encoded)?; + } + writer.flush().change_context(TrustedServerError::Proxy { + message: "Failed to flush output".to_string(), + })?; + Ok(()) +} + const BODY_CLOSE_PREFIX: &[u8] = b"` to inject into, so bids are written to state up front and the +/// auction telemetry completes immediately. +async fn collect_non_html_auction( + dispatched: DispatchedAuction, + telemetry: AuctionTelemetryCarry, + params: &OwnedProcessResponseParams, + orchestrator: &AuctionOrchestrator, + services: &RuntimeServices, + settings: &Settings, +) { + let placeholder = mediator_placeholder_request(); + let result = orchestrator + .collect_dispatched_auction( + dispatched, + services, + &make_collect_context(settings, services, &placeholder), + ) + .await; + if let (Some(observation), Some(auction_request)) = + (telemetry.observation, telemetry.auction_request.as_ref()) + { + emit_auction_events_best_effort_lazy(services, || { + build_auction_events( + observation, + AuctionTerminalOutcome::Completed { + request: auction_request, + result: &result, + }, + ) + }) + .await; + } + write_bids_to_state( + &result.winning_bids, + params.price_granularity, + ¶ms.ad_bids_state, + settings.debug.inject_adm_for_testing, + ); +} + async fn collect_stream_auction( dispatched: DispatchedAuction, telemetry: AuctionTelemetryCarry, @@ -1600,11 +2439,12 @@ pub async fn handle_publisher_request( // SSP requests are already racing through the platform HTTP client, so // origin TTFB tracks origin latency rather than the auction timeout. - let mut response = match services - .http_client() - .send(PlatformHttpRequest::new(req, backend_name)) - .await - { + let mut platform_request = PlatformHttpRequest::new(req, backend_name); + if services.http_client().supports_streaming_responses() { + platform_request = platform_request.with_stream_response(); + } + + let mut response = match services.http_client().send(platform_request).await { Ok(platform_response) => platform_response.response, Err(err) => { if let Some(dispatched) = dispatched_auction.take() { @@ -2505,6 +3345,24 @@ mod tests { output } + fn deflate_encode(input: &[u8]) -> Vec { + let mut encoder = + flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(input) + .expect("should write deflate test input"); + encoder.finish().expect("should finish deflate encoding") + } + + fn deflate_decode(input: &[u8]) -> Vec { + let mut decoder = flate2::read::ZlibDecoder::new(input); + let mut output = Vec::new(); + decoder + .read_to_end(&mut output) + .expect("should decode deflate test output"); + output + } + fn brotli_encode(input: &[u8]) -> Vec { let mut encoder = CompressorWriter::new(Vec::new(), 4096, 5, 22); encoder @@ -2734,50 +3592,107 @@ mod tests { } #[tokio::test] - async fn handle_publisher_request_does_not_self_generate_ec() { - // EC generation is the adapter's real-browser-gated responsibility. This - // handler must never mint an EC ID on its own: for a navigation from a - // client the adapter did not pre-generate for (e.g. a non-real browser), - // `ec_value` must stay `None` so no IP-derived identifier reaches the - // auction. Consent allows EC creation and a client IP is present here — - // exactly the conditions under which the old inline call would have - // generated one. + async fn publisher_origin_fetch_leaves_stream_response_disabled_when_unsupported() { let settings = create_test_settings(); let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"ok".to_vec()); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); let services = build_services_with_http_client( Arc::clone(&stub) as Arc ); - - let consent = crate::consent::ConsentContext { - jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, - ..Default::default() - }; - let mut ec_context = - EcContext::new_for_test_with_ip(None, consent, Some("203.0.113.7".to_string())); - assert!( - ec_context.ec_allowed(), - "test precondition: consent must allow EC creation" - ); - - let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let req = HttpRequest::builder() .method(Method::GET) - .uri("https://publisher.example/article") + .uri("https://publisher.example/page") .header(header::HOST, "publisher.example") - .header("sec-fetch-dest", "document") .body(EdgeBody::empty()) .expect("should build request"); - let _ = handle_publisher_request( - &settings, - &services, - None, - &mut ec_context, - AuctionDispatch { - orchestrator: &orchestrator, - slots: &[], - registry: None, + let _ = run_publisher_proxy(&settings, &services, req).await; + + assert_eq!( + stub.recorded_stream_response_flags(), + vec![false], + "publisher origin fetch must not request streams when the platform does not support them" + ); + } + + #[tokio::test] + async fn publisher_origin_fetch_sets_stream_response_when_supported() { + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.set_streaming_responses_supported(true); + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![("content-type", "text/html; charset=utf-8")], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/page") + .header(header::HOST, "publisher.example") + .body(EdgeBody::empty()) + .expect("should build request"); + + let _ = run_publisher_proxy(&settings, &services, req).await; + + assert_eq!( + stub.recorded_stream_response_flags(), + vec![true], + "publisher origin fetch should request streams when the platform supports them" + ); + } + + #[tokio::test] + async fn handle_publisher_request_does_not_self_generate_ec() { + // EC generation is the adapter's real-browser-gated responsibility. This + // handler must never mint an EC ID on its own: for a navigation from a + // client the adapter did not pre-generate for (e.g. a non-real browser), + // `ec_value` must stay `None` so no IP-derived identifier reaches the + // auction. Consent allows EC creation and a client IP is present here — + // exactly the conditions under which the old inline call would have + // generated one. + let settings = create_test_settings(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"ok".to_vec()); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = + EcContext::new_for_test_with_ip(None, consent, Some("203.0.113.7".to_string())); + assert!( + ec_context.ec_allowed(), + "test precondition: consent must allow EC creation" + ); + + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let req = HttpRequest::builder() + .method(Method::GET) + .uri("https://publisher.example/article") + .header(header::HOST, "publisher.example") + .header("sec-fetch-dest", "document") + .body(EdgeBody::empty()) + .expect("should build request"); + + let _ = handle_publisher_request( + &settings, + &services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots: &[], + registry: None, }, req, ) @@ -3659,6 +4574,734 @@ mod tests { ); } + #[test] + fn stream_publisher_body_rejects_stream_body_in_sync_path() { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let params = OwnedProcessResponseParams { + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "text/html; charset=utf-8".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(Mutex::new(None)), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; + let body = EdgeBody::from_stream(futures::stream::iter(vec![Ok::<_, io::Error>( + bytes::Bytes::from_static(b"live"), + )])); + let mut output = Vec::new(); + + let err = stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) + .expect_err("should reject stream body in sync path"); + + assert!( + format!("{err:?}").contains("streaming body"), + "should explain that Body::Stream is not supported by the sync path: {err:?}" + ); + } + + #[test] + fn body_chunk_source_yields_once_body_in_chunks() { + futures::executor::block_on(async { + let body = EdgeBody::from_bytes(bytes::Bytes::from_static(b"abcdef")); + let mut source = BodyChunkSource::new(body, 3).with_max_bytes(16); + + assert_eq!( + source.next_chunk().await.expect("should read").as_deref(), + Some(&b"abc"[..]), + "should yield the first chunk" + ); + assert_eq!( + source.next_chunk().await.expect("should read").as_deref(), + Some(&b"def"[..]), + "should yield the second chunk" + ); + assert!( + source.next_chunk().await.expect("should read").is_none(), + "should end after buffered bytes are exhausted" + ); + }); + } + + #[test] + fn body_chunk_source_preserves_stream_chunks() { + futures::executor::block_on(async { + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::from_static(b"first"), + bytes::Bytes::from_static(b"second"), + ])); + let mut source = BodyChunkSource::new(body, 3).with_max_bytes(16); + + assert_eq!( + source.next_chunk().await.expect("should read").as_deref(), + Some(&b"first"[..]), + "stream chunks should pass through without re-chunking" + ); + assert_eq!( + source.next_chunk().await.expect("should read").as_deref(), + Some(&b"second"[..]), + "stream chunks should preserve upstream boundaries" + ); + assert!( + source.next_chunk().await.expect("should read").is_none(), + "should end after stream is exhausted" + ); + }); + } + + #[test] + fn body_chunk_source_enforces_cumulative_raw_cap() { + futures::executor::block_on(async { + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::from_static(b"1234"), + bytes::Bytes::from_static(b"5678"), + ])); + let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(6); + + assert!( + source + .next_chunk() + .await + .expect("first chunk should pass") + .is_some(), + "first chunk should stay under cap" + ); + let err = source + .next_chunk() + .await + .expect_err("second chunk should exceed cap"); + + assert!( + format!("{err:?}").contains("publisher origin body exceeded"), + "should report cumulative cap: {err:?}" + ); + }); + } + + #[test] + fn stream_publisher_body_async_processes_stream_without_auction() { + futures::executor::block_on(async { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let mut params = OwnedProcessResponseParams { + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "text/css".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(Mutex::new(None)), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::from_static(b"body{background:url('https://origin.example.com/"), + bytes::Bytes::from_static(b"asset.png')}"), + ])); + let mut output = Vec::new(); + + stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("stream body should process on async path"); + + let css = String::from_utf8(output).expect("should be valid UTF-8"); + assert!( + css.contains("proxy.example.com"), + "should rewrite origin host while streaming. Got: {css}" + ); + assert!( + !css.contains("origin.example.com"), + "should not leave origin host after rewrite. Got: {css}" + ); + }); + } + + #[test] + fn stream_publisher_body_async_processes_gzip_stream_without_auction() { + futures::executor::block_on(async { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let mut params = OwnedProcessResponseParams { + content_encoding: "gzip".to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "text/css".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(Mutex::new(None)), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; + let compressed = + gzip_encode(b"body{background:url('https://origin.example.com/asset.png')}"); + let split_at = compressed.len() / 2; + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::copy_from_slice(&compressed[..split_at]), + bytes::Bytes::copy_from_slice(&compressed[split_at..]), + ])); + let mut output = Vec::new(); + + stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("gzip stream body should process on async path"); + + let css = String::from_utf8(gzip_decode(&output)).expect("should be valid UTF-8"); + assert!( + css.contains("proxy.example.com"), + "should rewrite origin host while streaming gzip. Got: {css}" + ); + assert!( + !css.contains("origin.example.com"), + "should not leave origin host after gzip rewrite. Got: {css}" + ); + }); + } + + #[test] + fn stream_publisher_body_async_processes_deflate_stream_without_auction() { + futures::executor::block_on(async { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let mut params = OwnedProcessResponseParams { + content_encoding: "deflate".to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "text/css".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(Mutex::new(None)), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; + let compressed = + deflate_encode(b"body{background:url('https://origin.example.com/asset.png')}"); + let split_at = compressed.len() / 2; + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::copy_from_slice(&compressed[..split_at]), + bytes::Bytes::copy_from_slice(&compressed[split_at..]), + ])); + let mut output = Vec::new(); + + stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("deflate stream body should process on async path"); + + let css = String::from_utf8(deflate_decode(&output)).expect("should be valid UTF-8"); + assert!( + css.contains("proxy.example.com"), + "should rewrite origin host while streaming deflate. Got: {css}" + ); + assert!( + !css.contains("origin.example.com"), + "should not leave origin host after deflate rewrite. Got: {css}" + ); + }); + } + + #[test] + fn stream_publisher_body_async_processes_brotli_stream_without_auction() { + futures::executor::block_on(async { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let mut params = OwnedProcessResponseParams { + content_encoding: "br".to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "text/css".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(Mutex::new(None)), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; + let compressed = + brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); + let split_at = compressed.len() / 2; + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::copy_from_slice(&compressed[..split_at]), + bytes::Bytes::copy_from_slice(&compressed[split_at..]), + ])); + let mut output = Vec::new(); + + stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("brotli stream body should process on async path"); + + let css = String::from_utf8(brotli_decode(&output)).expect("should be valid UTF-8"); + assert!( + css.contains("proxy.example.com"), + "should rewrite origin host while streaming brotli. Got: {css}" + ); + assert!( + !css.contains("origin.example.com"), + "should not leave origin host after brotli rewrite. Got: {css}" + ); + }); + } + + #[test] + fn stream_publisher_body_async_rejects_truncated_brotli_stream() { + futures::executor::block_on(async { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let mut params = OwnedProcessResponseParams { + content_encoding: "br".to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "text/css".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(Mutex::new(None)), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; + let compressed = + brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); + let truncated = &compressed[..compressed.len() - 3]; + let body = + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::copy_from_slice( + truncated, + )])); + let mut output = Vec::new(); + + let err = stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect_err("truncated brotli stream must fail instead of truncating silently"); + + assert!( + format!("{err:?}").contains("brotli"), + "should surface the brotli finalization failure: {err:?}" + ); + }); + } + + #[test] + fn stream_publisher_body_async_processes_stream_with_auction_hold() { + futures::executor::block_on(async { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let state = Arc::new(Mutex::new(None)); + let mut params = OwnedProcessResponseParams { + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "text/html; charset=utf-8".to_string(), + ad_slots_script: Some( + r#""# + .to_string(), + ), + ad_bids_state: state, + auction_observation: None, + auction_request: Some(test_auction_request()), + dispatched_auction: Some(DispatchedAuction::empty_for_test( + test_auction_request(), + 10, + )), + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; + let body = EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::from_static(b"hello"), + bytes::Bytes::from_static(b""), + ])); + let mut output = Vec::new(); + + stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("stream body with auction should process on async path"); + + let html = String::from_utf8(output).expect("should be valid UTF-8"); + assert!( + html.contains("hello"), + "should preserve streamed HTML content. Got: {html}" + ); + assert!( + html.contains(".adSlots=JSON.parse"), + "should still inject ad slots. Got: {html}" + ); + assert!( + html.contains(".bids=JSON.parse"), + "should collect auction and inject bids before body close. Got: {html}" + ); + }); + } + + #[test] + fn stream_publisher_body_async_processes_non_html_stream_after_auction_collect() { + futures::executor::block_on(async { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let mut params = OwnedProcessResponseParams { + content_encoding: String::new(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "text/css".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(Mutex::new(None)), + auction_observation: None, + auction_request: Some(test_auction_request()), + dispatched_auction: Some(DispatchedAuction::empty_for_test( + test_auction_request(), + 10, + )), + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; + let body = EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( + b"body{background:url('https://origin.example.com/asset.png')}", + )])); + let mut output = Vec::new(); + + stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect("non-html stream body should process after auction collection"); + + let css = String::from_utf8(output).expect("should be valid UTF-8"); + assert!( + css.contains("proxy.example.com"), + "should rewrite non-html stream after auction collection. Got: {css}" + ); + assert!( + !css.contains("origin.example.com"), + "should not leave origin host after rewrite. Got: {css}" + ); + }); + } + + fn drain_streaming_finalize_body(content_encoding: &str, body: EdgeBody) -> Vec { + let settings = Arc::new(create_test_settings()); + let registry = Arc::new( + IntegrationRegistry::new(&settings).expect("should create integration registry"), + ); + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let services = noop_services(); + let response = Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/css") + .body(EdgeBody::empty()) + .expect("should build response"); + let params = OwnedProcessResponseParams { + content_encoding: content_encoding.to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "text/css".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(Mutex::new(None)), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; + let publisher_response = PublisherResponse::Stream { + response, + body, + params: Box::new(params), + }; + + let response = publisher_response_into_streaming_response( + publisher_response, + &Method::GET, + Arc::clone(&settings), + registry.as_ref(), + orchestrator, + services, + ) + .expect("should build streaming response"); + + assert!( + matches!(response.body(), EdgeBody::Stream(_)), + "streaming finalize should keep a lazy Body::Stream" + ); + + futures::executor::block_on( + response + .into_body() + .into_bytes_bounded(settings.publisher.max_buffered_body_bytes), + ) + .expect("streaming body should drain") + .to_vec() + } + + #[test] + fn publisher_response_streaming_finalize_keeps_stream_body_lazy() { + let body_bytes = drain_streaming_finalize_body( + "", + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( + b"body{background:url('https://origin.example.com/asset.png')}", + )])), + ); + let css = String::from_utf8(body_bytes).expect("should be valid UTF-8"); + assert!( + css.contains("proxy.example.com"), + "streaming response body should still run publisher rewriting. Got: {css}" + ); + assert!( + !css.contains("origin.example.com"), + "streaming response body should not leave origin URLs unrewritten. Got: {css}" + ); + } + + #[test] + fn publisher_response_streaming_finalize_processes_gzip_stream() { + let compressed = + gzip_encode(b"body{background:url('https://origin.example.com/asset.png')}"); + let split_at = compressed.len() / 2; + let output = drain_streaming_finalize_body( + "gzip", + EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::copy_from_slice(&compressed[..split_at]), + bytes::Bytes::copy_from_slice(&compressed[split_at..]), + ])), + ); + + let css = String::from_utf8(gzip_decode(&output)).expect("should be valid UTF-8"); + assert!( + css.contains("proxy.example.com"), + "streaming response finalize should rewrite gzip body. Got: {css}" + ); + assert!( + !css.contains("origin.example.com"), + "streaming response finalize should not leave gzip origin URLs. Got: {css}" + ); + } + + #[test] + fn publisher_response_streaming_finalize_processes_deflate_stream() { + let compressed = + deflate_encode(b"body{background:url('https://origin.example.com/asset.png')}"); + let split_at = compressed.len() / 2; + let output = drain_streaming_finalize_body( + "deflate", + EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::copy_from_slice(&compressed[..split_at]), + bytes::Bytes::copy_from_slice(&compressed[split_at..]), + ])), + ); + + let css = String::from_utf8(deflate_decode(&output)).expect("should be valid UTF-8"); + assert!( + css.contains("proxy.example.com"), + "streaming response finalize should rewrite deflate body. Got: {css}" + ); + assert!( + !css.contains("origin.example.com"), + "streaming response finalize should not leave deflate origin URLs. Got: {css}" + ); + } + + #[test] + fn publisher_response_streaming_finalize_processes_brotli_stream() { + let compressed = + brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); + let split_at = compressed.len() / 2; + let output = drain_streaming_finalize_body( + "br", + EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::copy_from_slice(&compressed[..split_at]), + bytes::Bytes::copy_from_slice(&compressed[split_at..]), + ])), + ); + + let css = String::from_utf8(brotli_decode(&output)).expect("should be valid UTF-8"); + assert!( + css.contains("proxy.example.com"), + "streaming response finalize should rewrite brotli body. Got: {css}" + ); + assert!( + !css.contains("origin.example.com"), + "streaming response finalize should not leave brotli origin URLs. Got: {css}" + ); + } + + #[test] + fn publisher_response_streaming_finalize_holds_auction_and_keeps_gzip_tail() { + let settings = Arc::new(create_test_settings()); + let registry = Arc::new( + IntegrationRegistry::new(&settings).expect("should create integration registry"), + ); + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + let services = noop_services(); + let response = Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, "text/html; charset=utf-8") + .body(EdgeBody::empty()) + .expect("should build response"); + // The trailing content after `` must exceed the flate2 write + // decoder's 32 KiB internal output buffer: the close-body tag then + // surfaces (and releases the auction hold) mid-stream, while the + // trailing markup only surfaces at decoder finalization. This guards + // against the EOF decoded tail being dropped once the hold is gone. + let trailing_comment = format!("", "trailing-content ".repeat(3 * 1024)); + let page = format!("hello{trailing_comment}"); + let compressed = gzip_encode(page.as_bytes()); + let chunks: Vec = compressed + .chunks(STREAM_CHUNK_SIZE) + .map(bytes::Bytes::copy_from_slice) + .collect(); + let params = OwnedProcessResponseParams { + content_encoding: "gzip".to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "text/html; charset=utf-8".to_string(), + ad_slots_script: Some( + r#""# + .to_string(), + ), + ad_bids_state: Arc::new(Mutex::new(None)), + auction_observation: None, + auction_request: Some(test_auction_request()), + dispatched_auction: Some(DispatchedAuction::empty_for_test( + test_auction_request(), + 10, + )), + price_granularity: crate::price_bucket::PriceGranularity::default(), + }; + let publisher_response = PublisherResponse::Stream { + response, + body: EdgeBody::stream(futures::stream::iter(chunks)), + params: Box::new(params), + }; + + let response = publisher_response_into_streaming_response( + publisher_response, + &Method::GET, + Arc::clone(&settings), + registry.as_ref(), + orchestrator, + services, + ) + .expect("should build streaming response"); + + let output = futures::executor::block_on( + response + .into_body() + .into_bytes_bounded(settings.publisher.max_buffered_body_bytes), + ) + .expect("streaming body should drain") + .to_vec(); + + let html = String::from_utf8(gzip_decode(&output)).expect("should be valid UTF-8"); + assert!( + html.contains(".bids=JSON.parse"), + "should collect the held auction and inject bids. Got tail: {}", + &html[html.len().saturating_sub(200)..] + ); + assert!( + html.contains("trailing-content"), + "should preserve content after the close-body tag" + ); + assert!( + html.trim_end().ends_with(""), + "should not drop the decoded tail once the auction hold is released. Got tail: {}", + &html[html.len().saturating_sub(200)..] + ); + } + #[test] fn stream_publisher_body_treats_mixed_case_html_as_html() { let settings = create_test_settings(); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 9cbb2a546..008b4a2d5 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -50,15 +50,12 @@ pub struct Publisher { /// exceeding it fails the response rather than allocating past the cap. /// Defaults to 16 MiB — a conservative cap that prevents Wasm-heap OOM. /// - /// On Fastly the *effective* ceiling for a publisher page is lower: the - /// platform HTTP client rejects any origin response whose raw (still - /// compressed) body exceeds 10 MiB before this buffer is ever filled, so - /// raising this value only helps highly compressible pages whose decoded - /// size exceeds the 16 MiB default while their compressed origin body stays - /// under 10 MiB. Raising it above ~10 MiB does not lift the platform cap for - /// uncompressed pages. That platform limit is removed once true streaming - /// lands (tracked for PR 15, issue #495), after which this setting becomes - /// the sole ceiling. + /// Fastly origin bodies are preserved as streams on the publisher path, so + /// this setting is also the cumulative raw-byte cap while the streaming + /// processor decodes and rewrites chunks. Buffered adapters keep using it + /// as the post-rewrite output buffer cap. On the streaming path headers + /// are already committed when the cap trips, so the response is truncated + /// mid-body (with the error logged) rather than replaced with a 5xx. /// /// Must be at least 1: a zero-byte cap fails every non-empty buffered /// publisher response at request time, so it is rejected at config diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index 5692118a8..ef1a0bc55 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -349,6 +349,197 @@ impl StreamProcessor for StreamingReplacer { } } +/// Read buffer size for streaming body processing and brotli internal buffers. +/// Both the `Decompressor` and `CompressorWriter` use this value so all +/// brotli I/O layers operate on consistently-sized chunks. +pub(crate) const STREAM_CHUNK_SIZE: usize = 8192; + +/// Incremental push-style decompressor for the async chunk pipeline. +/// +/// Compressed bytes go in via [`Self::decode_chunk`]; decoded bytes drain +/// out of the internal buffer after every push. Write-based decoders are +/// used because the async publisher path cannot wrap a blocking `Read`. +pub(crate) enum BodyStreamDecoder { + None, + Gzip(flate2::write::GzDecoder>), + Deflate(flate2::write::ZlibDecoder>), + Brotli(Box>>), +} + +impl BodyStreamDecoder { + pub(crate) fn new(compression: Compression) -> Self { + match compression { + Compression::None => Self::None, + Compression::Gzip => Self::Gzip(flate2::write::GzDecoder::new(Vec::new())), + Compression::Deflate => Self::Deflate(flate2::write::ZlibDecoder::new(Vec::new())), + Compression::Brotli => Self::Brotli(Box::new(brotli::DecompressorWriter::new( + Vec::new(), + STREAM_CHUNK_SIZE, + ))), + } + } + + pub(crate) fn decode_chunk( + &mut self, + chunk: &[u8], + ) -> Result, Report> { + match self { + Self::None => Ok(chunk.to_vec()), + Self::Gzip(decoder) => { + decoder + .write_all(chunk) + .change_context(TrustedServerError::Proxy { + message: "Failed to decode gzip publisher body chunk".to_string(), + })?; + Ok(std::mem::take(decoder.get_mut())) + } + Self::Deflate(decoder) => { + decoder + .write_all(chunk) + .change_context(TrustedServerError::Proxy { + message: "Failed to decode deflate publisher body chunk".to_string(), + })?; + Ok(std::mem::take(decoder.get_mut())) + } + Self::Brotli(decoder) => { + decoder + .write_all(chunk) + .change_context(TrustedServerError::Proxy { + message: "Failed to decode brotli publisher body chunk".to_string(), + })?; + Ok(std::mem::take(decoder.get_mut())) + } + } + } + + pub(crate) fn finish(&mut self) -> Result, Report> { + match self { + Self::None => Ok(Vec::new()), + Self::Gzip(decoder) => { + decoder + .try_finish() + .change_context(TrustedServerError::Proxy { + message: "Failed to finalize gzip publisher body decoder".to_string(), + })?; + Ok(std::mem::take(decoder.get_mut())) + } + Self::Deflate(decoder) => { + decoder + .try_finish() + .change_context(TrustedServerError::Proxy { + message: "Failed to finalize deflate publisher body decoder".to_string(), + })?; + Ok(std::mem::take(decoder.get_mut())) + } + Self::Brotli(decoder) => { + // `close()` (not `flush()`): flush accepts a truncated brotli + // stream silently, while close validates end-of-stream and + // errors on incomplete input, matching the gzip/deflate arms. + decoder.close().change_context(TrustedServerError::Proxy { + message: "Failed to finalize brotli publisher body decoder".to_string(), + })?; + Ok(std::mem::take(decoder.get_mut())) + } + } + } +} + +/// Incremental push-style compressor mirroring [`BodyStreamDecoder`]. +/// +/// Processed bytes go in via [`Self::encode_chunk`]; encoded bytes drain out +/// after every push, and [`Self::finish`] emits the stream trailer. +pub(crate) enum BodyStreamEncoder { + None, + Gzip(flate2::write::GzEncoder>), + Deflate(flate2::write::ZlibEncoder>), + Brotli(Box>>), +} + +fn new_brotli_vec_encoder() -> brotli::enc::writer::CompressorWriter> { + let params = brotli::enc::BrotliEncoderParams { + quality: 4, + lgwin: 22, + ..Default::default() + }; + brotli::enc::writer::CompressorWriter::with_params(Vec::new(), STREAM_CHUNK_SIZE, ¶ms) +} + +impl BodyStreamEncoder { + pub(crate) fn new(compression: Compression) -> Self { + match compression { + Compression::None => Self::None, + Compression::Gzip => Self::Gzip(flate2::write::GzEncoder::new( + Vec::new(), + flate2::Compression::default(), + )), + Compression::Deflate => Self::Deflate(flate2::write::ZlibEncoder::new( + Vec::new(), + flate2::Compression::default(), + )), + Compression::Brotli => Self::Brotli(Box::new(new_brotli_vec_encoder())), + } + } + + pub(crate) fn encode_chunk( + &mut self, + chunk: &[u8], + ) -> Result, Report> { + match self { + Self::None => Ok(chunk.to_vec()), + Self::Gzip(encoder) => { + encoder + .write_all(chunk) + .change_context(TrustedServerError::Proxy { + message: "Failed to encode gzip publisher body chunk".to_string(), + })?; + Ok(std::mem::take(encoder.get_mut())) + } + Self::Deflate(encoder) => { + encoder + .write_all(chunk) + .change_context(TrustedServerError::Proxy { + message: "Failed to encode deflate publisher body chunk".to_string(), + })?; + Ok(std::mem::take(encoder.get_mut())) + } + Self::Brotli(encoder) => { + encoder + .write_all(chunk) + .change_context(TrustedServerError::Proxy { + message: "Failed to encode brotli publisher body chunk".to_string(), + })?; + Ok(std::mem::take(encoder.get_mut())) + } + } + } + + pub(crate) fn finish(&mut self) -> Result, Report> { + match self { + Self::None => Ok(Vec::new()), + Self::Gzip(encoder) => { + encoder + .try_finish() + .change_context(TrustedServerError::Proxy { + message: "Failed to finalize gzip publisher body encoder".to_string(), + })?; + Ok(std::mem::take(encoder.get_mut())) + } + Self::Deflate(encoder) => { + encoder + .try_finish() + .change_context(TrustedServerError::Proxy { + message: "Failed to finalize deflate publisher body encoder".to_string(), + })?; + Ok(std::mem::take(encoder.get_mut())) + } + Self::Brotli(encoder) => { + let encoder = std::mem::replace(encoder, Box::new(new_brotli_vec_encoder())); + Ok((*encoder).into_inner()) + } + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/docs/superpowers/plans/2026-07-08-true-origin-streaming-fastly.md b/docs/superpowers/plans/2026-07-08-true-origin-streaming-fastly.md new file mode 100644 index 000000000..bc1983a97 --- /dev/null +++ b/docs/superpowers/plans/2026-07-08-true-origin-streaming-fastly.md @@ -0,0 +1,1039 @@ +# True Origin Streaming Fastly Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fix issue #849 for the production Fastly path so publisher HTML origin bodies stream through the rewrite pipeline to the client, with auction collection outside TTFB except for the held `` tail. + +**Architecture:** Keep one cohesive Fastly PR because the core pipeline, Fastly origin fetch, and Fastly finalize path are incomplete in isolation. Convert publisher body processing from sync `Read` over buffered bodies to async chunk-pull over `edgezero_core::body::Body`, then enable Fastly `with_stream_response()` and return a lazy streaming body for publisher responses. Leave Cloudflare, Spin, and Axum streaming as follow-up work. + +**Tech Stack:** Rust 2024, `edgezero_core::body::Body`, `futures::StreamExt`, `error-stack`, `flate2`, `brotli`, Fastly Compute, Viceroy tests. + +--- + +## Scope + +In scope: + +- Publisher HTML and processable publisher responses on the Fastly adapter. +- Core publisher pipeline support for `Body::Stream`. +- Fastly platform capability signaling for streaming origin responses. +- Fastly EdgeZero response delivery that streams publisher bodies to clients. +- Tests proving stream-vs-buffer parity, bodiless handling, stream caps, and Fastly routing behavior. + +Out of scope: + +- Cloudflare origin streaming. Current adapter rejects `PlatformHttpRequest::stream_response`. +- Spin streaming. Current adapter and upstream EdgeZero Spin conversion are buffered/blocking issues. +- Axum client streaming. Axum is dev-only and has `LocalBoxStream`/`Send` constraints. +- Parser-context `` scan fix from issue #850. +- Origin template caching and transformed HTML caching from issue #852. + +## Current Failure Points + +- `crates/trusted-server-adapter-fastly/src/platform.rs`: `fastly_response_to_platform(..., stream_response: false)` uses `take_body_bytes()` and the 10 MiB platform cap for publisher origin responses. +- `crates/trusted-server-core/src/publisher.rs`: `body_as_reader()` calls `body.into_bytes().unwrap_or_default()`, so `Body::Stream` becomes an empty body. +- `crates/trusted-server-core/src/publisher.rs`: `stream_html_with_auction_hold()` and `body_close_hold_loop()` are sync-`Read` based. +- `crates/trusted-server-adapter-fastly/src/app.rs`: publisher route calls `buffer_publisher_response_async()`, buffering all processed output and awaiting auction before any client bytes are sent. +- `crates/trusted-server-adapter-fastly/src/main.rs`: `send_edgezero_response()` already streams `EdgeBody::Stream`, but currently only asset responses reach that arm. + +## File Structure + +- Modify `crates/trusted-server-core/src/platform/http.rs` + - Add `PlatformHttpClient::supports_streaming_responses()` with default `false`. +- Modify adapter platform implementations: + - `crates/trusted-server-adapter-fastly/src/platform.rs`: return `true` for `supports_streaming_responses()`. + - `crates/trusted-server-adapter-cloudflare/src/platform.rs`: inherit default `false`. + - `crates/trusted-server-adapter-spin/src/platform.rs`: inherit default `false`. + - `crates/trusted-server-adapter-axum/src/platform.rs`: inherit default `false`. + - `crates/trusted-server-core/src/platform/test_support.rs`: configurable test support if needed. +- Modify `crates/trusted-server-core/src/streaming_processor.rs` + - Add small push decoder/encoder helpers only if keeping them here reduces duplication. + - Keep the existing `StreamingPipeline::process(Read, Write)` API for existing call sites. +- Modify `crates/trusted-server-core/src/publisher.rs` + - Replace publisher async processing internals with async chunk-pull. + - Keep public `buffer_publisher_response_async()` for buffered adapters. + - Add a streaming response constructor/helper for Fastly to use. + - Make `body_as_reader()` reject `Body::Stream` loudly or remove its use from any stream-capable path. +- Modify `crates/trusted-server-adapter-fastly/src/app.rs` + - Replace publisher `buffer_publisher_response_async()` call with streaming finalize for streamable publisher responses. + - Preserve buffered behavior for `PublisherResponse::Buffered`, pass-through/bodiless responses, and error paths. +- Modify `crates/trusted-server-adapter-fastly/src/main.rs` + - Reuse existing `EdgeBody::Stream` delivery. + - If publisher streaming needs a different log message from asset streaming, split the helper name/log text without changing behavior. +- Tests: + - `crates/trusted-server-core/src/publisher.rs` unit tests. + - `crates/trusted-server-core/src/streaming_processor.rs` unit tests if push codec helpers are introduced there. + - `crates/trusted-server-core/src/platform/test_support.rs` tests for capability behavior. + - `crates/trusted-server-adapter-fastly/src/app.rs` route tests for publisher streaming response shape. + +## Design Decisions + +- Use one PR for the full Fastly production fix. Intermediate merged PRs would create incomplete behavior and review confusion. +- Use existing `publisher.max_buffered_body_bytes` as the publisher body ceiling after streaming. `settings.rs` already documents that this becomes the sole ceiling after true streaming removes the 10 MiB Fastly materialization cap. +- Keep `Content-Length` removed for rewritten stream responses. Streaming output can change size due to URL rewriting and bid injection. +- Preserve bodiless behavior for `HEAD`, `204`, and `304`: do not attach or drive a body stream, and log abandoned/wasted auctions as current code does. +- Do not build a sync `Read` bridge over `Body::Stream`; nested `block_on` can panic on Fastly because the router already runs under `futures::executor::block_on`. +- Avoid adding `async-stream` initially. Use `futures::stream::unfold` or a custom stream type so the PR does not add a dependency unless the implementation becomes materially clearer. + +## Task 1: Baseline Tests for Stream Input Safety + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Add a failing test for `Body::Stream` not becoming empty** + +Add a test near the existing `stream_publisher_body` tests: + +```rust +#[test] +fn stream_publisher_body_rejects_stream_body_in_sync_path() { + let settings = create_test_settings(); + let registry = IntegrationRegistry::new(&settings).expect("should build registry"); + let body = EdgeBody::from_stream(futures::stream::iter(vec![Ok(Bytes::from_static( + b"live", + ))])); + let params = test_process_params("text/html", ""); + let mut output = Vec::new(); + + let err = stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) + .expect_err("should reject stream body in sync path"); + + assert!( + format!("{err:?}").contains("streaming body"), + "should explain that Body::Stream is not supported by the sync path: {err:?}" + ); +} +``` + +- [ ] **Step 2: Run the targeted test and verify it fails** + +Run: + +```bash +cargo test-axum stream_publisher_body_rejects_stream_body_in_sync_path +``` + +Expected: FAIL because current `body_as_reader()` silently returns empty bytes. + +- [ ] **Step 3: Replace `body_as_reader()` with a fallible helper** + +Change `body_as_reader(body: EdgeBody) -> Cursor` to return `Result, Report>` and return a proxy error for `Body::Stream`. + +Minimal shape: + +```rust +fn body_as_reader(body: EdgeBody) -> Result, Report> { + let bytes = body.into_bytes().ok_or_else(|| { + Report::new(TrustedServerError::Proxy { + message: "streaming body cannot be processed by sync publisher pipeline".to_owned(), + }) + })?; + Ok(std::io::Cursor::new(bytes)) +} +``` + +Update existing sync call sites to use `body_as_reader(body)?`. + +- [ ] **Step 4: Run the targeted test and existing publisher sync tests** + +Run: + +```bash +cargo test-axum stream_publisher_body +``` + +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs +git commit -m "Reject publisher stream bodies on sync path" +``` + +## Task 2: Async Chunk Source and Cumulative Cap + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Add tests for async chunk pulling** + +Add focused tests for a private helper that will pull chunks from both body variants: + +```rust +#[test] +fn body_chunk_source_yields_once_body_in_chunks() { + futures::executor::block_on(async { + let body = EdgeBody::from(Bytes::from_static(b"abcdef")); + let mut source = BodyChunkSource::new(body, 3); + + assert_eq!(source.next_chunk().await.expect("should read").as_deref(), Some(&b"abc"[..])); + assert_eq!(source.next_chunk().await.expect("should read").as_deref(), Some(&b"def"[..])); + assert!(source.next_chunk().await.expect("should read").is_none()); + }); +} +``` + +Add a separate test for `Body::Stream` preserving chunk boundaries and surfacing stream errors. + +- [ ] **Step 2: Add a failing test for the cumulative cap** + +Use a stream with two chunks whose total exceeds a small cap: + +```rust +#[test] +fn body_chunk_source_enforces_cumulative_raw_cap() { + futures::executor::block_on(async { + let body = EdgeBody::from_stream(futures::stream::iter(vec![ + Ok(Bytes::from_static(b"1234")), + Ok(Bytes::from_static(b"5678")), + ])); + let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(6); + + assert!(source.next_chunk().await.expect("first chunk should pass").is_some()); + let err = source.next_chunk().await.expect_err("second chunk should exceed cap"); + assert!( + format!("{err:?}").contains("publisher origin body exceeded"), + "should report cumulative cap: {err:?}" + ); + }); +} +``` + +- [ ] **Step 3: Run the new tests and verify they fail** + +Run: + +```bash +cargo test-axum body_chunk_source +``` + +Expected: FAIL because helper does not exist. + +- [ ] **Step 4: Implement `BodyChunkSource`** + +Implement a private helper near `STREAM_CHUNK_SIZE`: + +- Owns `EdgeBody`. +- For `Body::Once`, yields `Bytes` slices up to `chunk_size` without copying more than necessary. +- For `Body::Stream`, awaits `stream.next()`. +- Tracks cumulative raw bytes and errors when total exceeds `max_bytes`. +- Maps stream errors to `TrustedServerError::Proxy`. + +Do not use `block_on` inside the helper. + +- [ ] **Step 5: Run helper tests** + +Run: + +```bash +cargo test-axum body_chunk_source +``` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs +git commit -m "Add async publisher body chunk source" +``` + +## Task 3: Push Compression Helpers + +**Files:** + +- Modify: `crates/trusted-server-core/src/streaming_processor.rs` +- Or modify: `crates/trusted-server-core/src/publisher.rs` if helpers are publisher-only + +- [ ] **Step 1: Add parity tests for compressed chunk processing** + +For each compression mode used by publisher HTML (`gzip`, `deflate`, `br`), add a test that feeds compressed HTML in multiple raw chunks through the future async path and verifies the decompressed/processed/recompressed output decodes to expected HTML. + +Start with gzip: + +```rust +#[test] +fn async_publisher_pipeline_preserves_gzip_html_across_stream_chunks() { + futures::executor::block_on(async { + let compressed = gzip_bytes(b"Hello"); + let body = EdgeBody::from_stream(bytes_to_two_chunk_stream(compressed)); + let output = process_test_body_async(body, "text/html", "gzip") + .await + .expect("should process gzip stream"); + + assert_eq!( + gunzip_bytes(&output), + b"Hello" + ); + }); +} +``` + +Use existing HTML processor expectations rather than inventing new behavior. + +- [ ] **Step 2: Run the gzip test and verify it fails** + +Run: + +```bash +cargo test-axum async_publisher_pipeline_preserves_gzip_html_across_stream_chunks +``` + +Expected: FAIL because async compressed processing does not exist. + +- [ ] **Step 3: Implement write-based push decoders** + +Use write-based APIs: + +- `flate2::write::GzDecoder` +- `flate2::write::ZlibDecoder` +- `brotli::DecompressorWriter` + +The helper should: + +- Accept raw compressed chunks. +- Write decoded bytes into an internal `Vec` sink. +- Return newly decoded bytes after each input chunk. +- Finalize at EOF and return any decoder tail bytes. +- Surface decoder errors as `TrustedServerError::Proxy`. + +Keep this helper private unless tests or other modules need it. + +- [ ] **Step 4: Implement output encoding wrapper** + +Continue to use existing write-based encoders: + +- `flate2::write::GzEncoder` +- `flate2::write::ZlibEncoder` +- `brotli::enc::writer::CompressorWriter` + +The async loop should write processed decoded chunks into the encoder and finalize once. + +- [ ] **Step 5: Add deflate and brotli tests** + +Run: + +```bash +cargo test-axum async_publisher_pipeline_preserves_ +``` + +Expected: gzip, deflate, and brotli async parity tests PASS. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/streaming_processor.rs crates/trusted-server-core/src/publisher.rs +git commit -m "Add push compression support for publisher streams" +``` + +## Task 4: Async Publisher Pipeline Without Auction + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Add stream-vs-once parity tests for no-auction paths** + +Cover: + +- HTML rewrite. +- RSC flight rewrite. +- Generic URL replacement. +- Unsupported stream body cannot reach sync path. + +Example: + +```rust +#[test] +fn stream_publisher_body_async_matches_buffered_html_without_auction() { + futures::executor::block_on(async { + let settings = create_test_settings(); + let registry = IntegrationRegistry::new(&settings).expect("should build registry"); + let html = Bytes::from_static(b"x"); + + let mut once_params = test_process_params("text/html", ""); + let mut once_output = Vec::new(); + stream_publisher_body_async( + EdgeBody::from(html.clone()), + &mut once_output, + &mut once_params, + &settings, + ®istry, + &AuctionOrchestrator::new(settings.auction.clone()), + &noop_services(), + ) + .await + .expect("once body should process"); + + let mut stream_params = test_process_params("text/html", ""); + let mut stream_output = Vec::new(); + stream_publisher_body_async( + EdgeBody::from_stream(futures::stream::iter(vec![Ok(html)])), + &mut stream_output, + &mut stream_params, + &settings, + ®istry, + &AuctionOrchestrator::new(settings.auction.clone()), + &noop_services(), + ) + .await + .expect("stream body should process"); + + assert_eq!(stream_output, once_output); + }); +} +``` + +- [ ] **Step 2: Run parity tests and verify they fail** + +Run: + +```bash +cargo test-axum stream_publisher_body_async_matches_buffered +``` + +Expected: FAIL because no-auction async path still delegates to sync processing. + +- [ ] **Step 3: Refactor `process_response_streaming` into reusable processor construction** + +Extract the shared routing logic into a helper such as: + +```rust +enum PublisherProcessor { + Html(HtmlRewriterAdapter), + Rsc(RscFlightUrlRewriter), + Url(StreamingReplacer), +} +``` + +Or use a generic closure/helper if that fits existing patterns better. The goal is to avoid duplicating content-type routing between sync and async paths. + +- [ ] **Step 4: Drive all `stream_publisher_body_async()` calls through async chunk-pull** + +Even when `params.dispatched_auction` is `None`, build the same processor and use `BodyChunkSource`. This prevents stream bodies from falling into the sync path. + +- [ ] **Step 5: Keep `stream_publisher_body()` for compatibility** + +The sync function should remain for old tests and any current non-stream callers, but it must not be used by the async path once this task is complete. + +- [ ] **Step 6: Run targeted tests** + +Run: + +```bash +cargo test-axum stream_publisher_body_async_matches_buffered +``` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs +git commit -m "Drive publisher async processing from body chunks" +``` + +## Task 5: Async Auction Hold Loop + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Replace reader-based hold-loop test with stream-based test** + +Update or add a test based on `body_close_hold_loop_processes_close_tail_before_reading_post_body_chunks()`: + +- Feed pre-`` chunk. +- Feed held `` chunk. +- Feed post-body chunk. +- Assert the loop collects auction immediately when `` test** + +Verify that auction collection happens at EOF and finalization still calls `processor.process_chunk(&[], true)`. + +- [ ] **Step 3: Add stream error abandonment test** + +Feed a stream error after dispatch and assert telemetry abandonment uses `stream_read_error` or the current expected reason. + +- [ ] **Step 4: Run tests and verify failures** + +Run: + +```bash +cargo test-axum body_close_hold_loop +``` + +Expected: FAIL until the loop consumes `BodyChunkSource`. + +- [ ] **Step 5: Change `body_close_hold_loop` to async chunk-pull** + +Replace: + +```rust +async fn body_close_hold_loop(...) +``` + +with a shape that accepts decoded chunks from an async driver, or accepts `BodyChunkSource` plus codec state. Keep the control flow: + +- Push decoded chunks into `BodyCloseHoldBuffer`. +- Write ready bytes immediately. +- On first ` bool { + false +} +``` + +In `FastlyPlatformHttpClient`: + +```rust +fn supports_streaming_responses(&self) -> bool { + true +} +``` + +In `StubHttpClient`, add a configurable flag if tests need both states. + +- [ ] **Step 4: Enable publisher origin streaming behind the gate** + +At the publisher origin fetch: + +```rust +let mut platform_request = PlatformHttpRequest::new(req, backend_name); +if services.http_client().supports_streaming_responses() { + platform_request = platform_request.with_stream_response(); +} +let mut response = services.http_client().send(platform_request).await?; +``` + +- [ ] **Step 5: Run capability tests** + +Run: + +```bash +cargo test-axum publisher_origin_fetch_sets_stream_response_when_supported +cargo test-axum publisher_origin_fetch_leaves_stream_response_disabled_when_unsupported +``` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/platform/http.rs crates/trusted-server-adapter-fastly/src/platform.rs crates/trusted-server-core/src/platform/test_support.rs crates/trusted-server-core/src/publisher.rs +git commit -m "Gate publisher origin streaming by platform capability" +``` + +## Task 8: Fastly Publisher Streaming Finalize + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `crates/trusted-server-core/src/publisher.rs` if a helper is needed +- Possibly modify: `crates/trusted-server-adapter-fastly/src/main.rs` for logging/helper naming + +- [ ] **Step 1: Add Fastly route test for publisher response body shape** + +In Fastly app tests, configure a publisher HTML origin response and assert the router returns `Body::Stream` for processable publisher responses on `GET`. + +Expected assertion: + +```rust +assert!( + matches!(response.body(), Body::Stream(_)), + "processable publisher response should remain streaming on Fastly" +); +``` + +- [ ] **Step 2: Add bodiless route tests** + +Assert `HEAD`, `204`, and `304` publisher responses do not carry a stream body, preserving existing metadata. + +- [ ] **Step 3: Run tests and verify failure** + +Run: + +```bash +cargo test-fastly publisher_response_streams +``` + +Expected: FAIL because app still calls `buffer_publisher_response_async()`. + +- [ ] **Step 4: Add a core helper to convert `PublisherResponse` to streaming response** + +Preferred shape in `publisher.rs`: + +```rust +pub fn publisher_response_into_streaming_body( + publisher_response: PublisherResponse, + method: &Method, + settings: Arc, + integration_registry: Arc, + orchestrator: Arc, + services: RuntimeServices, +) -> Result, Report> +``` + +The helper should: + +- Return `PublisherResponse::Buffered` unchanged. +- Return `PublisherResponse::PassThrough` with body attached, except bodiless responses. +- For `PublisherResponse::Stream`, build `EdgeBody::from_stream(futures::stream::unfold(...))` or equivalent. +- Move `OwnedProcessResponseParams`, origin body, settings, registry, orchestrator, and services into the stream state. +- Yield processed chunks as they become available. +- On mid-stream processing error, log and end the stream. The client sees a truncated body, matching existing mid-stream error behavior. + +If borrowing/lifetime pressure is high, keep the helper in Fastly `app.rs` and call core `stream_publisher_body_async()` from inside the stream. Prefer core if it avoids Fastly-specific body processing logic. + +- [ ] **Step 5: Replace Fastly buffered finalize for publisher route** + +In `handle_publisher_route`, replace the `buffer_publisher_response_async()` call for Fastly with the streaming helper. Keep non-Fastly adapters on `buffer_publisher_response_async()`. + +- [ ] **Step 6: Preserve entry-point finalization** + +Verify the returned `Response` still carries extensions needed by `main.rs`: + +- `EcFinalizeState` +- `RequestFilterEffects` +- Final cache privacy guard + +Headers must be finalized before `send_edgezero_response()` splits the response and commits headers. + +- [ ] **Step 7: Run Fastly route tests** + +Run: + +```bash +cargo test-fastly publisher_response +``` + +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add crates/trusted-server-adapter-fastly/src/app.rs crates/trusted-server-core/src/publisher.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Stream Fastly publisher responses to clients" +``` + +## Task 9: Pass-Through Publisher Bodies + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` + +- [ ] **Step 1: Add pass-through large body test** + +Verify a non-processable successful publisher response (`image/png`, font, video) is returned as `Body::Stream` when origin streaming is supported and body is not bodiless. + +- [ ] **Step 2: Add pass-through bodiless test** + +Verify `HEAD`, `204`, and `304` pass-through arms preserve headers but do not attach/drain the stream body. + +- [ ] **Step 3: Run targeted tests** + +Run: + +```bash +cargo test-fastly publisher_pass_through +``` + +Expected: FAIL if pass-through still buffers or attaches a body for bodiless responses. + +- [ ] **Step 4: Make pass-through use the same body-carrying guard** + +Mirror `asset_response_carries_body()` semantics in publisher finalize. If `response_carries_body(method, status)` is false, drop the body and return headers only. + +- [ ] **Step 5: Run targeted tests** + +Run: + +```bash +cargo test-fastly publisher_pass_through +``` + +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs crates/trusted-server-adapter-fastly/src/app.rs +git commit -m "Preserve publisher pass-through streaming semantics" +``` + +## Task 10: Headers, Length, and Error Semantics + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/main.rs` if logs are misleading + +- [ ] **Step 1: Add header tests** + +Assert for streamed processed publisher responses: + +- `Content-Length` is absent. +- `Transfer-Encoding` is not manually set. +- `Content-Encoding` is preserved when recompression is used. +- Cache/privacy headers still downgrade when `Set-Cookie` is present. + +- [ ] **Step 2: Add mid-stream cap/error test** + +Use a body stream that exceeds `publisher.max_buffered_body_bytes` after headers would be committed. Assert the stream returns an error/truncates consistently with existing mid-stream asset behavior and logs enough context. + +- [ ] **Step 3: Run tests and verify failures** + +Run: + +```bash +cargo test-fastly publisher_stream +``` + +Expected: FAIL until header/error cleanup is complete. + +- [ ] **Step 4: Clean header handling** + +Ensure the existing `response.headers_mut().remove(header::CONTENT_LENGTH)` remains on `PublisherResponse::Stream`. Do not re-add content length for streaming finalize. + +- [ ] **Step 5: Improve log wording** + +If `main.rs` still logs "asset streaming" for all `EdgeBody::Stream` responses, rename log messages to "EdgeZero streaming body" or split publisher/asset helpers. + +- [ ] **Step 6: Run tests** + +Run: + +```bash +cargo test-fastly publisher_stream +``` + +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs crates/trusted-server-adapter-fastly/src/app.rs crates/trusted-server-adapter-fastly/src/main.rs +git commit -m "Tighten publisher streaming headers and errors" +``` + +## Task 11: End-to-End Regression Coverage + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Modify: existing integration/parity tests if appropriate + +- [ ] **Step 1: Add a slow-origin behavior test if feasible in existing harness** + +Preferred test shape: + +- Origin response body is a stream with first chunk available immediately and second chunk delayed or instrumented. +- Router returns `Body::Stream` without collecting the whole body. +- Pulling the first output chunk does not require pulling the entire origin stream. + +If the harness cannot model time cleanly, use an instrumented stream that panics if polled past the first chunk before the returned response body is consumed. + +- [ ] **Step 2: Add auction timing test** + +Verify response construction does not await `collect_dispatched_auction`; collection happens when the body stream is pulled and reaches `` or EOF. + +- [ ] **Step 3: Run targeted tests** + +Run: + +```bash +cargo test-fastly publisher_streaming_does_not_buffer_origin_before_response +``` + +Expected: PASS after Fastly finalize is lazy. + +- [ ] **Step 4: Commit** + +```bash +git add crates/trusted-server-adapter-fastly/src/app.rs crates/trusted-server-core/src/publisher.rs +git commit -m "Cover lazy publisher streaming behavior" +``` + +## Task 12: Documentation Cleanup + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/settings.rs` +- Modify: `crates/trusted-server-adapter-fastly/src/app.rs` +- Optionally modify: issue/PR description only, not repo docs + +- [ ] **Step 1: Update stale interim comments** + +Remove or rewrite comments saying publisher stream bodies are already materialized into WASM heap. + +Targets: + +- `PublisherResponse::Stream` doc. +- `PublisherResponse::PassThrough` doc if Fastly now preserves stream bodies. +- `settings.rs` comments that reference future true streaming. +- Fastly app module comment that says publisher responses are buffered by `publisher.max_buffered_body_bytes`. + +- [ ] **Step 2: Run doc-related checks locally** + +Run: + +```bash +cargo fmt --all -- --check +``` + +Expected: PASS. + +- [ ] **Step 3: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs crates/trusted-server-core/src/settings.rs crates/trusted-server-adapter-fastly/src/app.rs +git commit -m "Update publisher streaming documentation" +``` + +## Task 13: Full Verification + +**Files:** + +- No source edits unless failures reveal issues. + +- [ ] **Step 1: Run formatting** + +Run: + +```bash +cargo fmt --all -- --check +``` + +Expected: PASS. + +- [ ] **Step 2: Run target checks** + +Run: + +```bash +cargo check-fastly +cargo check-axum +cargo check-cloudflare +``` + +Expected: PASS. + +- [ ] **Step 3: Run target tests** + +Run: + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +``` + +Expected: PASS. + +- [ ] **Step 4: Run Spin if touched by shared trait changes** + +Run: + +```bash +cargo test-spin +``` + +Expected: PASS. + +- [ ] **Step 5: Run clippy gates** + +Run: + +```bash +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +``` + +Expected: PASS. + +- [ ] **Step 6: Run parity suite if available locally** + +Run: + +```bash +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +``` + +Expected: PASS. + +- [ ] **Step 7: Optional local TTFB smoke** + +Run Fastly local serve against an artificially slow publisher origin: + +- Origin sends headers and first HTML chunk immediately. +- Origin delays later body chunks. +- Verify browser/curl receives response headers and first chunk before full origin drain. +- Verify bids still inject before `` when auction completes. + +Expected: TTFB tracks origin first byte, not full origin transfer or auction collection. + +## Review Checklist + +- [ ] No `block_on` inside stream body processing or `Read::read` equivalents. +- [ ] `Body::Stream` never falls through `into_bytes().unwrap_or_default()`. +- [ ] Fastly publisher origin fetch sets `with_stream_response()` only through capability gate. +- [ ] Cloudflare, Spin, and Axum do not start receiving stream-response requests. +- [ ] `HEAD`, `204`, and `304` do not drive or attach response bodies. +- [ ] `Content-Length` is absent on processed streaming responses. +- [ ] Existing buffered adapters still work through `buffer_publisher_response_async()`. +- [ ] Auction telemetry handles completed and abandoned stream cases. +- [ ] Mid-stream errors do not panic; they log and truncate consistently with current streaming behavior. +- [ ] Comments no longer describe publisher streaming as interim/in-memory cursor based. + +## PR Description Skeleton + +```markdown +## Summary + +- convert publisher response processing to async chunk-pull over `Body::Stream` +- enable Fastly publisher origin streaming behind a platform capability gate +- stream Fastly publisher responses to clients instead of buffering and awaiting auction before send + +## Scope + +Fastly production path for issue #849. Cloudflare, Spin, and Axum streaming remain follow-up work. + +## Tests + +- [ ] cargo fmt --all -- --check +- [ ] cargo check-fastly +- [ ] cargo check-axum +- [ ] cargo check-cloudflare +- [ ] cargo test-fastly +- [ ] cargo test-axum +- [ ] cargo test-cloudflare +- [ ] cargo test-spin +- [ ] cargo clippy-fastly +- [ ] cargo clippy-axum +- [ ] cargo clippy-cloudflare +- [ ] cargo clippy-cloudflare-wasm +- [ ] cargo clippy-spin-native +- [ ] cargo clippy-spin-wasm +``` + +## Known Follow-Ups + +- Cloudflare origin streaming once Worker `ReadableStream` is wrapped into `Body::Stream` and response header/set-cookie behavior is verified. +- Spin streaming after upstream EdgeZero Spin response conversion supports incremental body writes. +- Axum streaming only if the dev server needs it enough to justify a `Send` bridge. +- Issue #850 parser-context `` detection. From affb46c9781239ae78695ad5157826ea232e143e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 8 Jul 2026 22:39:26 +0530 Subject: [PATCH 003/198] Harden the streaming publisher pipeline after review Address the deep-review findings on the streaming cutover: - Cap cumulative decoded bytes in BodyStreamDecoder against publisher.max_buffered_body_bytes: the chunk source only bounds raw compressed bytes, so a decompression bomb could expand ~1000x past it and push unbounded decoded volume through the rewrite pipeline - Detect truncated deflate streams: write::ZlibDecoder::try_finish accepts truncated input silently, so the deflate arm now drives flate2::Decompress directly and requires Status::StreamEnd at finalization; trailing bytes after the end marker stay ignored. Add truncated-gzip and truncated-deflate regression tests - Make BodyChunkSource::next_chunk cancellation-safe by polling the body in place instead of moving it out across an await; a cancelled pull no longer turns into a silent EOF - Log dispatched auctions dropped uncollected (client disconnect mid-stream or never-polled body) via DispatchedAuctionGuard; the guard is created before the lazy stream so unpolled drops log too - Share the pull+decode step between the lazy publisher body and the write-sink drivers (hold_step_next_chunk / passthrough_step), removing the unreachable!() error plumbing and the triplicated processor selection; document body_close_hold_loop_stream as groundwork for the buffered adapters' streaming cutover - Pass identity-encoded chunks through zero-copy and finish encoders by consuming them instead of allocating a throwaway replacement - Add a Fastly dispatch test asserting the publisher fallback returns Body::Stream without a stale Content-Length, plus a comment on why the publisher fetch gates streaming on capability while the asset path does not Behavior note: gzip bodies with trailing garbage after the trailer now error mid-stream; the old read-path decoder ignored them. --- .../trusted-server-adapter-fastly/src/app.rs | 34 + crates/trusted-server-core/src/publisher.rs | 639 ++++++++++++------ crates/trusted-server-core/src/settings.rs | 13 +- .../src/streaming_processor.rs | 297 ++++++-- 4 files changed, 685 insertions(+), 298 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index d5e37f91a..ae9a2749b 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -2150,6 +2150,10 @@ mod tests { #[async_trait::async_trait(?Send)] impl PlatformHttpClient for StreamingHttpClient { + fn supports_streaming_responses(&self) -> bool { + true + } + async fn send( &self, request: PlatformHttpRequest, @@ -2260,6 +2264,36 @@ mod tests { ); } + #[test] + fn dispatch_fallback_streams_publisher_body_without_buffering() { + // Regression guard for the publisher streaming cutover (#849): a + // successful publisher origin fetch must hand `edgezero_main` a lazy + // streaming body (`Body::Stream`) so headers commit at origin first + // byte, rather than draining the processed page into a buffered + // `Body::Once`. Core tests cover the rewrite pipeline itself; this + // guards the adapter wiring that could silently re-buffer. + let settings = test_settings(); + let state = build_state_from_settings(settings).expect("should build state"); + let services = streaming_runtime_services(); + let req = empty_request(Method::GET, "/article"); + + let response = block_on(super::dispatch_fallback(&state, &services, req)); + + assert_eq!( + response.status(), + StatusCode::OK, + "publisher proxy should succeed against the streaming origin stub" + ); + assert!( + matches!(response.body(), Body::Stream(_)), + "EdgeZero publisher dispatch must attach the lazy streaming body, not buffer it" + ); + assert!( + !response.headers().contains_key(header::CONTENT_LENGTH), + "processed streaming publisher responses must not carry a stale Content-Length" + ); + } + #[test] fn dispatch_runs_request_filter_and_threads_response_effects() { // Regression guard for the EdgeZero request-filter bypass: the publisher diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index c10ac0b39..fe6467e12 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -104,40 +104,40 @@ impl BodyChunkSource { } async fn next_chunk(&mut self) -> Result, Report> { - let Some(body) = self.body.take() else { - return Ok(None); - }; - - let chunk = match body { - EdgeBody::Once(bytes) => { - if self.once_offset >= bytes.len() { - None + // The body is polled in place (never moved out across an await) so a + // cancelled `next_chunk` future leaves the source resumable instead of + // silently reporting end-of-stream on the next call. + let pulled = match &mut self.body { + None => Ok(None), + Some(EdgeBody::Once(bytes)) => { + let end = (self.once_offset + self.chunk_size).min(bytes.len()); + if self.once_offset >= end { + Ok(None) } else { - let end = (self.once_offset + self.chunk_size).min(bytes.len()); let chunk = bytes.slice(self.once_offset..end); self.once_offset = end; - if self.once_offset < bytes.len() { - self.body = Some(EdgeBody::Once(bytes)); - } - Some(chunk) + Ok(Some(chunk)) } } - EdgeBody::Stream(mut stream) => match stream.next().await { - Some(Ok(chunk)) => { - self.body = Some(EdgeBody::Stream(stream)); - Some(chunk) - } - Some(Err(err)) => { - return Err(Report::new(TrustedServerError::Proxy { - message: format!("Failed to read publisher origin body stream: {err}"), - })); - } - None => None, + Some(EdgeBody::Stream(stream)) => match stream.next().await { + Some(Ok(chunk)) => Ok(Some(chunk)), + Some(Err(err)) => Err(Report::new(TrustedServerError::Proxy { + message: format!("Failed to read publisher origin body stream: {err}"), + })), + None => Ok(None), }, }; - let Some(chunk) = chunk else { - return Ok(None); + let chunk = match pulled { + Ok(Some(chunk)) => chunk, + Ok(None) => { + self.body = None; + return Ok(None); + } + Err(err) => { + self.body = None; + return Err(err); + } }; self.bytes_seen = self.bytes_seen.checked_add(chunk.len()).ok_or_else(|| { @@ -174,19 +174,17 @@ fn process_and_encode_chunk( if processed.is_empty() { return Ok(None); } - let encoded = encoder.encode_chunk(&processed)?; + let encoded = encoder.encode_chunk(processed)?; if encoded.is_empty() { return Ok(None); } Ok(Some(bytes::Bytes::from(encoded))) } +// By-value signature so `map_err(publisher_stream_error)` works directly. +#[allow(clippy::needless_pass_by_value)] fn publisher_stream_error(err: Report) -> std::io::Error { - let message = format!("{err:?}"); - // Consume the report so clippy's needless_pass_by_value accepts the - // by-value signature that `map_err(publisher_stream_error)` requires. - drop(err); - std::io::Error::other(message) + std::io::Error::other(format!("{err:?}")) } fn not_found_response() -> Response { @@ -473,65 +471,58 @@ fn process_response_streaming( async fn process_response_streaming_async( body: EdgeBody, output: &mut W, - params: &ProcessResponseParams<'_>, - max_raw_body_bytes: usize, + params: &OwnedProcessResponseParams, + settings: &Settings, + integration_registry: &IntegrationRegistry, ) -> Result<(), Report> { - let is_html = is_html_content_type(params.content_type); - let is_rsc_flight = - content_type_contains_ascii_case_insensitive(params.content_type, "text/x-component"); log::debug!( - "process_response_streaming_async: content_type={}, content_encoding={}, is_html={}, is_rsc_flight={}", + "process_response_streaming_async: content_type={}, content_encoding={}", params.content_type, - params.content_encoding, - is_html, - is_rsc_flight + params.content_encoding ); - let compression = Compression::from_content_encoding(params.content_encoding); + let compression = Compression::from_content_encoding(¶ms.content_encoding); + let mut processor = PublisherBodyProcessor::new(params, settings, integration_registry)?; + process_body_chunks_async( + body, + output, + &mut processor, + compression, + settings.publisher.max_buffered_body_bytes, + ) + .await +} - if is_html { - let mut processor = create_html_stream_processor( - params.origin_host, - params.request_host, - params.request_scheme, - params.settings, - params.integration_registry, - params.ad_slots_script.map(str::to_string), - params.ad_bids_state.clone(), - )?; - process_body_chunks_async( - body, - output, - &mut processor, - compression, - max_raw_body_bytes, - ) - .await - } else if is_rsc_flight { - let mut processor = RscFlightUrlRewriter::new( - params.origin_host, - params.origin_url, - params.request_host, - params.request_scheme, - ); - process_body_chunks_async( - body, - output, - &mut processor, - compression, - max_raw_body_bytes, - ) - .await - } else { - let mut replacer = create_url_replacer( - params.origin_host, - params.origin_url, - params.request_host, - params.request_scheme, - ); - process_body_chunks_async(body, output, &mut replacer, compression, max_raw_body_bytes) - .await +/// Pull, decode, process, and encode the next chunk of a no-hold pipeline. +/// +/// Returns `Ok(None)` when the source is exhausted; the caller must then emit +/// [`passthrough_finish_segments`]. Shared by the write-sink driver +/// ([`process_body_chunks_async`]) and the lazy publisher body stream so the +/// two no-hold paths cannot drift apart. +async fn passthrough_step( + source: &mut BodyChunkSource, + decoder: &mut BodyStreamDecoder, + encoder: &mut BodyStreamEncoder, + processor: &mut P, +) -> Result>, Report> { + let Some(raw_chunk) = source.next_chunk().await? else { + return Ok(None); + }; + let decoded = decoder.decode_chunk(raw_chunk)?; + if decoded.is_empty() { + return Ok(Some(Vec::new())); } + let mut segments = Vec::new(); + if let Some(encoded) = process_and_encode_chunk( + processor, + encoder, + &decoded, + false, + "Failed to process chunk", + )? { + segments.push(encoded); + } + Ok(Some(segments)) } async fn process_body_chunks_async( @@ -539,25 +530,16 @@ async fn process_body_chunks_async( writer: &mut W, processor: &mut P, compression: Compression, - max_raw_body_bytes: usize, + max_body_bytes: usize, ) -> Result<(), Report> { - let mut decoder = BodyStreamDecoder::new(compression); + let mut decoder = BodyStreamDecoder::new(compression, max_body_bytes); let mut encoder = BodyStreamEncoder::new(compression); - let mut source = - BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(max_raw_body_bytes); + let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(max_body_bytes); - while let Some(chunk) = source.next_chunk().await? { - let decoded = decoder.decode_chunk(&chunk)?; - if decoded.is_empty() { - continue; - } - if let Some(encoded) = process_and_encode_chunk( - processor, - &mut encoder, - &decoded, - false, - "Failed to process chunk", - )? { + while let Some(segments) = + passthrough_step(&mut source, &mut decoder, &mut encoder, processor).await? + { + for encoded in segments { write_encoded_segment(writer, &encoded)?; } } @@ -621,18 +603,51 @@ fn passthrough_finish_segments( Ok(segments) } +/// Owns a [`DispatchedAuction`] and logs if it is dropped uncollected. +/// +/// The lazy publisher body stream can be dropped at any await point — a +/// client disconnect aborts the transfer mid-body, or the response may never +/// be polled at all. Async telemetry cannot run in `Drop`, so the loss is +/// surfaced in logs; the abandoned-auction telemetry event is only emitted on +/// error paths that can still await (see [`abandon_hold_auction`]). +struct DispatchedAuctionGuard { + dispatched: Option, +} + +impl DispatchedAuctionGuard { + fn new(dispatched: DispatchedAuction) -> Self { + Self { + dispatched: Some(dispatched), + } + } + + fn take(&mut self) -> Option { + self.dispatched.take() + } +} + +impl Drop for DispatchedAuctionGuard { + fn drop(&mut self) { + if self.dispatched.is_some() { + log::warn!( + "Dispatched server-side auction dropped without collection; SSP bid responses discarded (publisher body stream aborted or never polled)" + ); + } + } +} + /// Mutable auction-hold state threaded through the streaming hold pipeline. struct AuctionHoldState { hold: Option, - dispatched: Option, + dispatched: DispatchedAuctionGuard, telemetry: AuctionTelemetryCarry, } impl AuctionHoldState { - fn new(dispatched: DispatchedAuction, telemetry: AuctionTelemetryCarry) -> Self { + fn new(dispatched: DispatchedAuctionGuard, telemetry: AuctionTelemetryCarry) -> Self { Self { hold: Some(BodyCloseHoldBuffer::new()), - dispatched: Some(dispatched), + dispatched, telemetry, } } @@ -736,6 +751,45 @@ async fn hold_step_decoded_chunk( Ok(segments) } +/// Pull and decode the next chunk of the close-body hold pipeline, feeding it +/// through [`hold_step_decoded_chunk`]. +/// +/// Returns `Ok(None)` when the source is exhausted; the caller must then emit +/// [`hold_finish_segments`]. On read or decode failure the pending auction is +/// abandoned before the error is returned. Shared by the write-sink driver +/// ([`body_close_hold_loop_stream`]) and the lazy publisher body stream so +/// the two hold paths cannot drift apart. +async fn hold_step_next_chunk( + source: &mut BodyChunkSource, + decoder: &mut BodyStreamDecoder, + encoder: &mut BodyStreamEncoder, + processor: &mut P, + state: &mut AuctionHoldState, + collect_refs: &AuctionHoldCollectRefs<'_>, +) -> Result>, Report> { + let raw_chunk = match source.next_chunk().await { + Ok(Some(chunk)) => chunk, + Ok(None) => return Ok(None), + Err(err) => { + abandon_hold_auction(state, collect_refs.services, "stream_read_error").await; + return Err(err); + } + }; + let decoded = match decoder.decode_chunk(raw_chunk) { + Ok(decoded) => decoded, + Err(err) => { + abandon_hold_auction(state, collect_refs.services, "stream_decode_error").await; + return Err(err); + } + }; + if decoded.is_empty() { + return Ok(Some(Vec::new())); + } + hold_step_decoded_chunk(processor, encoder, &decoded, state, collect_refs) + .await + .map(Some) +} + /// Finalize the close-body hold pipeline at end of the origin stream. /// /// Drains the decoder tail through the hold (or straight through when the @@ -843,7 +897,11 @@ fn create_html_stream_processor( /// Result of publisher request handling, indicating whether the response body /// should be streamed or has already been buffered. pub enum PublisherResponse { - /// Response is fully buffered and ready to send via `send_to_client()`. + /// Response returned unmodified, ready to send via `send_to_client()`. + /// + /// On streaming adapters the unmodified body may still be a live + /// [`EdgeBody::Stream`] (the origin fetch requested streaming before the + /// response was classified); it passes through to the client untouched. Buffered(Response), /// Response headers are ready for a streaming response. Covers processable /// content on any status (2xx or non-2xx — e.g., branded 404/500 HTML and @@ -1085,25 +1143,31 @@ pub fn publisher_response_into_streaming_response( let mut params = *params; let mut processor = PublisherBodyProcessor::new(¶ms, &settings, integration_registry)?; + // The guard is created before the lazy stream so an auction whose + // response body is dropped unpolled still logs the loss. + let dispatched_auction = params.dispatched_auction.take().map(|dispatched| { + let telemetry = AuctionTelemetryCarry { + observation: params.auction_observation.take(), + auction_request: params.auction_request.take(), + }; + (DispatchedAuctionGuard::new(dispatched), telemetry) + }); let stream = async_stream::try_stream! { let compression = Compression::from_content_encoding(¶ms.content_encoding); - let mut decoder = BodyStreamDecoder::new(compression); + let max_body_bytes = settings.publisher.max_buffered_body_bytes; + let mut decoder = BodyStreamDecoder::new(compression, max_body_bytes); let mut encoder = BodyStreamEncoder::new(compression); let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE) - .with_max_bytes(settings.publisher.max_buffered_body_bytes); + .with_max_bytes(max_body_bytes); // HTML rides the close-body hold so bids land before ``; // non-HTML has no injection point, so its auction is collected // before any byte streams (matching the buffered finalizer). let mut hold_auction = None; - if let Some(dispatched) = params.dispatched_auction.take() { - let telemetry = AuctionTelemetryCarry { - observation: params.auction_observation.take(), - auction_request: params.auction_request.take(), - }; + if let Some((mut guard, telemetry)) = dispatched_auction { if is_html_content_type(¶ms.content_type) { - hold_auction = Some((dispatched, telemetry)); - } else { + hold_auction = Some((guard, telemetry)); + } else if let Some(dispatched) = guard.take() { collect_non_html_auction( dispatched, telemetry, @@ -1116,8 +1180,8 @@ pub fn publisher_response_into_streaming_response( } } - if let Some((dispatched, telemetry)) = hold_auction { - let mut state = AuctionHoldState::new(dispatched, telemetry); + if let Some((guard, telemetry)) = hold_auction { + let mut state = AuctionHoldState::new(guard, telemetry); let collect_refs = AuctionHoldCollectRefs { price_granularity: params.price_granularity, ad_bids_state: ¶ms.ad_bids_state, @@ -1126,39 +1190,18 @@ pub fn publisher_response_into_streaming_response( settings: &settings, }; - loop { - let raw_chunk = match source.next_chunk().await { - Ok(Some(chunk)) => chunk, - Ok(None) => break, - Err(err) => { - abandon_hold_auction(&mut state, &services, "stream_read_error") - .await; - Err(publisher_stream_error(err))?; - unreachable!("error should have returned"); - } - }; - let decoded = match decoder.decode_chunk(&raw_chunk) { - Ok(decoded) => decoded, - Err(err) => { - abandon_hold_auction(&mut state, &services, "stream_decode_error") - .await; - Err(publisher_stream_error(err))?; - unreachable!("error should have returned"); - } - }; - if decoded.is_empty() { - continue; - } - for encoded in hold_step_decoded_chunk( - &mut processor, - &mut encoder, - &decoded, - &mut state, - &collect_refs, - ) - .await - .map_err(publisher_stream_error)? - { + while let Some(segments) = hold_step_next_chunk( + &mut source, + &mut decoder, + &mut encoder, + &mut processor, + &mut state, + &collect_refs, + ) + .await + .map_err(publisher_stream_error)? + { + for encoded in segments { yield encoded; } } @@ -1176,24 +1219,16 @@ pub fn publisher_response_into_streaming_response( yield encoded; } } else { - while let Some(raw_chunk) = - source.next_chunk().await.map_err(publisher_stream_error)? + while let Some(segments) = passthrough_step( + &mut source, + &mut decoder, + &mut encoder, + &mut processor, + ) + .await + .map_err(publisher_stream_error)? { - let decoded = decoder - .decode_chunk(&raw_chunk) - .map_err(publisher_stream_error)?; - if decoded.is_empty() { - continue; - } - if let Some(encoded) = process_and_encode_chunk( - &mut processor, - &mut encoder, - &decoded, - false, - "Failed to process chunk", - ) - .map_err(publisher_stream_error)? - { + for encoded in segments { yield encoded; } } @@ -1334,23 +1369,12 @@ pub async fn stream_publisher_body_async( ) -> Result<(), Report> { let Some(dispatched) = params.dispatched_auction.take() else { if body.is_stream() { - let borrowed = ProcessResponseParams { - content_encoding: ¶ms.content_encoding, - origin_host: ¶ms.origin_host, - origin_url: ¶ms.origin_url, - request_host: ¶ms.request_host, - request_scheme: ¶ms.request_scheme, - settings, - content_type: ¶ms.content_type, - integration_registry, - ad_slots_script: params.ad_slots_script.as_deref(), - ad_bids_state: ¶ms.ad_bids_state, - }; return process_response_streaming_async( body, output, - &borrowed, - settings.publisher.max_buffered_body_bytes, + params, + settings, + integration_registry, ) .await; } @@ -1378,23 +1402,12 @@ pub async fn stream_publisher_body_async( ) .await; if body.is_stream() { - let borrowed = ProcessResponseParams { - content_encoding: ¶ms.content_encoding, - origin_host: ¶ms.origin_host, - origin_url: ¶ms.origin_url, - request_host: ¶ms.request_host, - request_scheme: ¶ms.request_scheme, - settings, - content_type: ¶ms.content_type, - integration_registry, - ad_slots_script: params.ad_slots_script.as_deref(), - ad_bids_state: ¶ms.ad_bids_state, - }; return process_response_streaming_async( body, output, - &borrowed, - settings.publisher.max_buffered_body_bytes, + params, + settings, + integration_registry, ) .await; } @@ -1639,14 +1652,14 @@ async fn stream_html_with_auction_hold( ctx: AuctionCollectCtx<'_>, ) -> Result<(), Report> { if body.is_stream() { - let max_raw_body_bytes = ctx.settings.publisher.max_buffered_body_bytes; + let max_body_bytes = ctx.settings.publisher.max_buffered_body_bytes; return body_close_hold_loop_stream( body, output, processor, compression, ctx, - max_raw_body_bytes, + max_body_bytes, ) .await; } @@ -1690,16 +1703,22 @@ async fn stream_html_with_auction_hold( /// Async-pull variant of [`body_close_hold_loop`] for live origin streams. /// -/// Shares [`hold_step_decoded_chunk`] and [`hold_finish_segments`] with the +/// Shares [`hold_step_next_chunk`] and [`hold_finish_segments`] with the /// lazy streaming body built by [`publisher_response_into_streaming_response`], /// so the two async hold paths cannot drift apart. +/// +/// No production caller reaches this today: it is only entered through +/// [`buffer_publisher_response_async`], and the buffered adapters (Axum, +/// Cloudflare, Spin) never produce `Body::Stream` because the publisher fetch +/// is gated on `supports_streaming_responses()`. It is groundwork for those +/// adapters' streaming cutover; Fastly uses the lazy stream instead. async fn body_close_hold_loop_stream( body: EdgeBody, writer: &mut W, processor: &mut P, compression: Compression, ctx: AuctionCollectCtx<'_>, - max_raw_body_bytes: usize, + max_body_bytes: usize, ) -> Result<(), Report> { let AuctionCollectCtx { dispatched, @@ -1710,11 +1729,10 @@ async fn body_close_hold_loop_stream( services, settings, } = ctx; - let mut decoder = BodyStreamDecoder::new(compression); + let mut decoder = BodyStreamDecoder::new(compression, max_body_bytes); let mut encoder = BodyStreamEncoder::new(compression); - let mut source = - BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(max_raw_body_bytes); - let mut state = AuctionHoldState::new(dispatched, telemetry); + let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE).with_max_bytes(max_body_bytes); + let mut state = AuctionHoldState::new(DispatchedAuctionGuard::new(dispatched), telemetry); let collect_refs = AuctionHoldCollectRefs { price_granularity, ad_bids_state, @@ -1723,29 +1741,17 @@ async fn body_close_hold_loop_stream( settings, }; - loop { - let raw_chunk = match source.next_chunk().await { - Ok(Some(chunk)) => chunk, - Ok(None) => break, - Err(err) => { - abandon_hold_auction(&mut state, services, "stream_read_error").await; - return Err(err); - } - }; - let decoded = match decoder.decode_chunk(&raw_chunk) { - Ok(decoded) => decoded, - Err(err) => { - abandon_hold_auction(&mut state, services, "stream_decode_error").await; - return Err(err); - } - }; - if decoded.is_empty() { - continue; - } - for encoded in - hold_step_decoded_chunk(processor, &mut encoder, &decoded, &mut state, &collect_refs) - .await? - { + while let Some(segments) = hold_step_next_chunk( + &mut source, + &mut decoder, + &mut encoder, + processor, + &mut state, + &collect_refs, + ) + .await? + { + for encoded in segments { write_encoded_segment(writer, &encoded)?; } } @@ -2439,6 +2445,11 @@ pub async fn handle_publisher_request( // SSP requests are already racing through the platform HTTP client, so // origin TTFB tracks origin latency rather than the auction timeout. + // + // Streaming is gated on the capability (unlike the asset-proxy path, which + // sets the flag unconditionally and tolerates buffered fallback): adapters + // without streaming support may reject the flag outright rather than + // silently buffering, which would fail every publisher fetch. let mut platform_request = PlatformHttpRequest::new(req, backend_name); if services.http_client().supports_streaming_responses() { platform_request = platform_request.with_stream_response(); @@ -3268,6 +3279,7 @@ pub async fn handle_page_bids( #[cfg(test)] mod tests { + use std::future::Future as _; use std::io::{self, Read as _, Write as _}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -4952,6 +4964,185 @@ mod tests { }); } + fn non_html_stream_params(content_encoding: &str) -> OwnedProcessResponseParams { + OwnedProcessResponseParams { + content_encoding: content_encoding.to_string(), + origin_host: "origin.example.com".to_string(), + origin_url: "https://origin.example.com".to_string(), + request_host: "proxy.example.com".to_string(), + request_scheme: "https".to_string(), + content_type: "text/css".to_string(), + ad_slots_script: None, + ad_bids_state: Arc::new(Mutex::new(None)), + auction_observation: None, + auction_request: None, + dispatched_auction: None, + price_granularity: crate::price_bucket::PriceGranularity::default(), + } + } + + #[test] + fn stream_publisher_body_async_rejects_truncated_gzip_stream() { + futures::executor::block_on(async { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let mut params = non_html_stream_params("gzip"); + let compressed = + gzip_encode(b"body{background:url('https://origin.example.com/asset.png')}"); + let truncated = &compressed[..compressed.len() - 3]; + let body = + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::copy_from_slice( + truncated, + )])); + let mut output = Vec::new(); + + let err = stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect_err("truncated gzip stream must fail instead of truncating silently"); + + assert!( + format!("{err:?}").contains("gzip"), + "should surface the gzip finalization failure: {err:?}" + ); + }); + } + + #[test] + fn stream_publisher_body_async_rejects_truncated_deflate_stream() { + futures::executor::block_on(async { + let settings = create_test_settings(); + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let mut params = non_html_stream_params("deflate"); + let compressed = + deflate_encode(b"body{background:url('https://origin.example.com/asset.png')}"); + // Cut into the deflate data itself, not just the adler32 trailer. + let truncated = &compressed[..compressed.len() / 2]; + let body = + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::copy_from_slice( + truncated, + )])); + let mut output = Vec::new(); + + let err = stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect_err("truncated deflate stream must fail instead of truncating silently"); + + assert!( + format!("{err:?}").contains("deflate"), + "should surface the deflate finalization failure: {err:?}" + ); + }); + } + + #[test] + fn stream_publisher_body_async_enforces_decoded_byte_cap() { + futures::executor::block_on(async { + let mut settings = create_test_settings(); + // Raw compressed input stays tiny (well under the cap); only the + // decoded expansion exceeds it — the decompression-bomb case the + // raw-byte cap alone cannot catch. + settings.publisher.max_buffered_body_bytes = 1024; + let registry = + IntegrationRegistry::new(&settings).expect("should create integration registry"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let services = noop_services(); + let mut params = non_html_stream_params("gzip"); + let compressed = gzip_encode(&vec![b'a'; 64 * 1024]); + assert!( + compressed.len() < 1024, + "test precondition: compressed input must stay under the raw cap" + ); + let body = + EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from(compressed)])); + let mut output = Vec::new(); + + let err = stream_publisher_body_async( + body, + &mut output, + &mut params, + &settings, + ®istry, + &orchestrator, + &services, + ) + .await + .expect_err("decoded expansion past the cap must fail"); + + assert!( + format!("{err:?}").contains("decoded size exceeded"), + "should report the cumulative decoded cap: {err:?}" + ); + }); + } + + #[test] + fn body_chunk_source_resumes_after_cancelled_poll() { + futures::executor::block_on(async { + let mut pending_once = true; + let mut yielded = false; + let stream = futures::stream::poll_fn(move |cx| { + if pending_once { + pending_once = false; + cx.waker().wake_by_ref(); + return std::task::Poll::Pending; + } + if yielded { + return std::task::Poll::Ready(None); + } + yielded = true; + std::task::Poll::Ready(Some(Ok::<_, io::Error>(bytes::Bytes::from_static( + b"chunk", + )))) + }); + let body = EdgeBody::from_stream(stream); + let mut source = BodyChunkSource::new(body, STREAM_CHUNK_SIZE); + + { + // Poll the pull future once (Pending), then drop it — + // simulating a cancelled await (select/timeout wrapper). + let mut pull = Box::pin(source.next_chunk()); + let waker = futures::task::noop_waker(); + let mut context = std::task::Context::from_waker(&waker); + assert!( + pull.as_mut().poll(&mut context).is_pending(), + "first poll should be pending" + ); + } + + let chunk = source + .next_chunk() + .await + .expect("should read after cancelled poll"); + assert_eq!( + chunk.as_deref(), + Some(&b"chunk"[..]), + "cancelled pull must not lose the origin stream" + ); + }); + } + #[test] fn stream_publisher_body_async_processes_stream_with_auction_hold() { futures::executor::block_on(async { diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 008b4a2d5..0a6178a07 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -51,11 +51,14 @@ pub struct Publisher { /// Defaults to 16 MiB — a conservative cap that prevents Wasm-heap OOM. /// /// Fastly origin bodies are preserved as streams on the publisher path, so - /// this setting is also the cumulative raw-byte cap while the streaming - /// processor decodes and rewrites chunks. Buffered adapters keep using it - /// as the post-rewrite output buffer cap. On the streaming path headers - /// are already committed when the cap trips, so the response is truncated - /// mid-body (with the error logged) rather than replaced with a 5xx. + /// this setting also caps the streaming pipeline twice over: cumulative + /// raw (still compressed) bytes pulled from origin, and cumulative decoded + /// bytes emitted by the decompressor — the latter so a decompression bomb + /// cannot push an unbounded decoded volume through the rewrite pipeline. + /// Buffered adapters keep using it as the post-rewrite output buffer cap. + /// On the streaming path headers are already committed when either cap + /// trips, so the response is truncated mid-body (with the error logged) + /// rather than replaced with a 5xx. /// /// Must be at least 1: a zero-byte cap fails every non-empty buffered /// publisher response at request time, so it is rejected at config diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index ef1a0bc55..963c69daa 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -359,88 +359,177 @@ pub(crate) const STREAM_CHUNK_SIZE: usize = 8192; /// Compressed bytes go in via [`Self::decode_chunk`]; decoded bytes drain /// out of the internal buffer after every push. Write-based decoders are /// used because the async publisher path cannot wrap a blocking `Read`. -pub(crate) enum BodyStreamDecoder { +/// +/// Decoded output is capped cumulatively: the chunk source only bounds raw +/// (still compressed) bytes, and a decompression bomb can expand ~1000x past +/// that, so the decoder enforces its own ceiling on the total bytes it emits. +/// +/// Every codec validates end-of-stream at [`Self::finish`] so a truncated +/// origin body errors instead of silently truncating the page: gzip via its +/// trailer checksum, brotli via `close()`, and deflate via an explicit +/// [`flate2::Status::StreamEnd`] check (`write::ZlibDecoder` accepts +/// truncated input silently, so the deflate arm drives [`flate2::Decompress`] +/// directly). +pub(crate) struct BodyStreamDecoder { + codec: BodyStreamDecoderCodec, + decoded_bytes: usize, + max_decoded_bytes: usize, +} + +enum BodyStreamDecoderCodec { None, Gzip(flate2::write::GzDecoder>), - Deflate(flate2::write::ZlibDecoder>), + Deflate(DeflateStreamDecoder), Brotli(Box>>), } +/// Streaming zlib decoder that tracks whether the stream reached its end +/// marker, so truncated deflate bodies fail at finalization. +struct DeflateStreamDecoder { + decompress: flate2::Decompress, + stream_ended: bool, +} + +impl DeflateStreamDecoder { + fn new() -> Self { + Self { + decompress: flate2::Decompress::new(true), + stream_ended: false, + } + } + + fn decode(&mut self, chunk: &[u8]) -> Result, Report> { + let mut output = Vec::with_capacity(STREAM_CHUNK_SIZE); + let mut offset = 0usize; + // Trailing bytes after the zlib end marker are ignored, matching the + // read-based decoder used by the buffered pipeline. + while offset < chunk.len() && !self.stream_ended { + if output.len() == output.capacity() { + output.reserve(STREAM_CHUNK_SIZE); + } + let before_in = self.decompress.total_in(); + let before_out = self.decompress.total_out(); + let status = self + .decompress + .decompress_vec(&chunk[offset..], &mut output, flate2::FlushDecompress::None) + .change_context(TrustedServerError::Proxy { + message: "Failed to decode deflate publisher body chunk".to_string(), + })?; + let consumed = (self.decompress.total_in() - before_in) as usize; + let produced = (self.decompress.total_out() - before_out) as usize; + offset += consumed; + match status { + flate2::Status::StreamEnd => self.stream_ended = true, + flate2::Status::Ok | flate2::Status::BufError => { + if consumed == 0 && produced == 0 && output.len() < output.capacity() { + return Err(Report::new(TrustedServerError::Proxy { + message: "deflate publisher body decoder made no progress".to_string(), + })); + } + } + } + } + Ok(output) + } +} + impl BodyStreamDecoder { - pub(crate) fn new(compression: Compression) -> Self { - match compression { - Compression::None => Self::None, - Compression::Gzip => Self::Gzip(flate2::write::GzDecoder::new(Vec::new())), - Compression::Deflate => Self::Deflate(flate2::write::ZlibDecoder::new(Vec::new())), - Compression::Brotli => Self::Brotli(Box::new(brotli::DecompressorWriter::new( - Vec::new(), - STREAM_CHUNK_SIZE, - ))), + pub(crate) fn new(compression: Compression, max_decoded_bytes: usize) -> Self { + let codec = match compression { + Compression::None => BodyStreamDecoderCodec::None, + Compression::Gzip => { + BodyStreamDecoderCodec::Gzip(flate2::write::GzDecoder::new(Vec::new())) + } + Compression::Deflate => BodyStreamDecoderCodec::Deflate(DeflateStreamDecoder::new()), + Compression::Brotli => BodyStreamDecoderCodec::Brotli(Box::new( + brotli::DecompressorWriter::new(Vec::new(), STREAM_CHUNK_SIZE), + )), + }; + Self { + codec, + decoded_bytes: 0, + max_decoded_bytes, } } pub(crate) fn decode_chunk( &mut self, - chunk: &[u8], - ) -> Result, Report> { - match self { - Self::None => Ok(chunk.to_vec()), - Self::Gzip(decoder) => { + chunk: bytes::Bytes, + ) -> Result> { + let decoded = match &mut self.codec { + BodyStreamDecoderCodec::None => chunk, + BodyStreamDecoderCodec::Gzip(decoder) => { decoder - .write_all(chunk) + .write_all(&chunk) .change_context(TrustedServerError::Proxy { message: "Failed to decode gzip publisher body chunk".to_string(), })?; - Ok(std::mem::take(decoder.get_mut())) + bytes::Bytes::from(std::mem::take(decoder.get_mut())) } - Self::Deflate(decoder) => { + BodyStreamDecoderCodec::Deflate(decoder) => bytes::Bytes::from(decoder.decode(&chunk)?), + BodyStreamDecoderCodec::Brotli(decoder) => { decoder - .write_all(chunk) - .change_context(TrustedServerError::Proxy { - message: "Failed to decode deflate publisher body chunk".to_string(), - })?; - Ok(std::mem::take(decoder.get_mut())) - } - Self::Brotli(decoder) => { - decoder - .write_all(chunk) + .write_all(&chunk) .change_context(TrustedServerError::Proxy { message: "Failed to decode brotli publisher body chunk".to_string(), })?; - Ok(std::mem::take(decoder.get_mut())) + bytes::Bytes::from(std::mem::take(decoder.get_mut())) } - } + }; + self.track_decoded(decoded.len())?; + Ok(decoded) } pub(crate) fn finish(&mut self) -> Result, Report> { - match self { - Self::None => Ok(Vec::new()), - Self::Gzip(decoder) => { + let tail = match &mut self.codec { + BodyStreamDecoderCodec::None => Vec::new(), + BodyStreamDecoderCodec::Gzip(decoder) => { decoder .try_finish() .change_context(TrustedServerError::Proxy { message: "Failed to finalize gzip publisher body decoder".to_string(), })?; - Ok(std::mem::take(decoder.get_mut())) + std::mem::take(decoder.get_mut()) } - Self::Deflate(decoder) => { - decoder - .try_finish() - .change_context(TrustedServerError::Proxy { - message: "Failed to finalize deflate publisher body decoder".to_string(), - })?; - Ok(std::mem::take(decoder.get_mut())) + BodyStreamDecoderCodec::Deflate(decoder) => { + if !decoder.stream_ended { + return Err(Report::new(TrustedServerError::Proxy { + message: + "Failed to finalize deflate publisher body decoder: truncated stream" + .to_string(), + })); + } + Vec::new() } - Self::Brotli(decoder) => { + BodyStreamDecoderCodec::Brotli(decoder) => { // `close()` (not `flush()`): flush accepts a truncated brotli // stream silently, while close validates end-of-stream and // errors on incomplete input, matching the gzip/deflate arms. decoder.close().change_context(TrustedServerError::Proxy { message: "Failed to finalize brotli publisher body decoder".to_string(), })?; - Ok(std::mem::take(decoder.get_mut())) + std::mem::take(decoder.get_mut()) } + }; + self.track_decoded(tail.len())?; + Ok(tail) + } + + fn track_decoded(&mut self, len: usize) -> Result<(), Report> { + self.decoded_bytes = self.decoded_bytes.checked_add(len).ok_or_else(|| { + Report::new(TrustedServerError::Proxy { + message: "publisher origin body decoded byte count overflowed".to_string(), + }) + })?; + if self.decoded_bytes > self.max_decoded_bytes { + return Err(Report::new(TrustedServerError::Proxy { + message: format!( + "publisher origin body decoded size exceeded {}-byte streaming limit", + self.max_decoded_bytes + ), + })); } + Ok(()) } } @@ -482,13 +571,14 @@ impl BodyStreamEncoder { pub(crate) fn encode_chunk( &mut self, - chunk: &[u8], + chunk: Vec, ) -> Result, Report> { match self { - Self::None => Ok(chunk.to_vec()), + // Identity encoding passes the processed chunk through untouched. + Self::None => Ok(chunk), Self::Gzip(encoder) => { encoder - .write_all(chunk) + .write_all(&chunk) .change_context(TrustedServerError::Proxy { message: "Failed to encode gzip publisher body chunk".to_string(), })?; @@ -496,7 +586,7 @@ impl BodyStreamEncoder { } Self::Deflate(encoder) => { encoder - .write_all(chunk) + .write_all(&chunk) .change_context(TrustedServerError::Proxy { message: "Failed to encode deflate publisher body chunk".to_string(), })?; @@ -504,7 +594,7 @@ impl BodyStreamEncoder { } Self::Brotli(encoder) => { encoder - .write_all(chunk) + .write_all(&chunk) .change_context(TrustedServerError::Proxy { message: "Failed to encode brotli publisher body chunk".to_string(), })?; @@ -513,29 +603,18 @@ impl BodyStreamEncoder { } } + /// Emits the encoder trailer. Consumes the codec state (the encoder + /// becomes identity afterwards); terminal — call once at end of stream. pub(crate) fn finish(&mut self) -> Result, Report> { - match self { + match std::mem::replace(self, Self::None) { Self::None => Ok(Vec::new()), - Self::Gzip(encoder) => { - encoder - .try_finish() - .change_context(TrustedServerError::Proxy { - message: "Failed to finalize gzip publisher body encoder".to_string(), - })?; - Ok(std::mem::take(encoder.get_mut())) - } - Self::Deflate(encoder) => { - encoder - .try_finish() - .change_context(TrustedServerError::Proxy { - message: "Failed to finalize deflate publisher body encoder".to_string(), - })?; - Ok(std::mem::take(encoder.get_mut())) - } - Self::Brotli(encoder) => { - let encoder = std::mem::replace(encoder, Box::new(new_brotli_vec_encoder())); - Ok((*encoder).into_inner()) - } + Self::Gzip(encoder) => encoder.finish().change_context(TrustedServerError::Proxy { + message: "Failed to finalize gzip publisher body encoder".to_string(), + }), + Self::Deflate(encoder) => encoder.finish().change_context(TrustedServerError::Proxy { + message: "Failed to finalize deflate publisher body encoder".to_string(), + }), + Self::Brotli(encoder) => Ok((*encoder).into_inner()), } } } @@ -545,6 +624,86 @@ mod tests { use super::*; use crate::streaming_replacer::{Replacement, StreamingReplacer}; + #[test] + fn body_stream_decoder_enforces_cumulative_decoded_cap() { + let compressed = { + let mut encoder = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(&vec![b'a'; 64 * 1024]) + .expect("should write gzip test input"); + encoder.finish().expect("should finish gzip encoding") + }; + assert!( + compressed.len() < 1024, + "test precondition: compressed input must stay small" + ); + let mut decoder = BodyStreamDecoder::new(Compression::Gzip, 1024); + + let err = decoder + .decode_chunk(bytes::Bytes::from(compressed)) + .expect_err("decoded expansion past the cap must fail"); + + assert!( + format!("{err:?}").contains("decoded size exceeded"), + "should report the cumulative decoded cap: {err:?}" + ); + } + + #[test] + fn body_stream_decoder_rejects_truncated_deflate_stream() { + let compressed = { + let mut encoder = + flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(b"deflate payload that spans more than one deflate block boundary") + .expect("should write deflate test input"); + encoder.finish().expect("should finish deflate encoding") + }; + let truncated = &compressed[..compressed.len() / 2]; + let mut decoder = BodyStreamDecoder::new(Compression::Deflate, usize::MAX); + decoder + .decode_chunk(bytes::Bytes::copy_from_slice(truncated)) + .expect("partial deflate input should decode incrementally"); + + let err = decoder + .finish() + .expect_err("truncated deflate stream must fail at finalization"); + + assert!( + format!("{err:?}").contains("truncated stream"), + "should report the missing deflate end marker: {err:?}" + ); + } + + #[test] + fn body_stream_decoder_ignores_deflate_trailing_bytes() { + let compressed = { + let mut encoder = + flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(b"deflate payload") + .expect("should write deflate test input"); + encoder.finish().expect("should finish deflate encoding") + }; + let mut with_trailing = compressed; + with_trailing.extend_from_slice(b"junk"); + let mut decoder = BodyStreamDecoder::new(Compression::Deflate, usize::MAX); + + let decoded = decoder + .decode_chunk(bytes::Bytes::from(with_trailing)) + .expect("complete deflate stream should decode"); + decoder + .finish() + .expect("trailing bytes after the end marker should be ignored"); + + assert_eq!( + decoded.as_ref(), + b"deflate payload", + "should decode the payload and drop trailing junk" + ); + } + /// Verify that `lol_html` fragments text nodes when input chunks split /// mid-text-node. Script rewriters must be fragment-safe — they accumulate /// text fragments internally until `is_last_in_text_node` is true. From 2c64f3c613e4079ac741fab75762376c97573ca1 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 8 Jul 2026 23:02:08 +0530 Subject: [PATCH 004/198] Emit processor_init_error telemetry on streaming finalize failure The buffered finalizer abandons a dispatched auction with processor_init_error telemetry when HTML processor construction fails; the streaming finalizer dropped the in-flight SSP responses silently. Make publisher_response_into_streaming_response async and emit the same abandonment before returning the construction error. --- .../trusted-server-adapter-fastly/src/app.rs | 19 +++++----- crates/trusted-server-core/src/publisher.rs | 35 ++++++++++++++----- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index ae9a2749b..b68897077 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -795,14 +795,17 @@ async fn dispatch_fallback( ) .await { - Ok(pub_response) => publisher_response_into_streaming_response( - pub_response, - &method, - Arc::clone(&state.settings), - state.registry.as_ref(), - Arc::clone(&state.orchestrator), - publisher_services.clone(), - ), + Ok(pub_response) => { + publisher_response_into_streaming_response( + pub_response, + &method, + Arc::clone(&state.settings), + state.registry.as_ref(), + Arc::clone(&state.orchestrator), + publisher_services.clone(), + ) + .await + } Err(e) => Err(e), } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index fe6467e12..4d7e38cf9 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1101,9 +1101,10 @@ pub async fn buffer_publisher_response_async( /// /// # Errors /// -/// Returns an error if processor construction fails before the streaming body is -/// created. -pub fn publisher_response_into_streaming_response( +/// Returns an error if processor construction fails before the streaming body +/// is created; a dispatched auction is abandoned with `processor_init_error` +/// telemetry first, matching the buffered finalizer. +pub async fn publisher_response_into_streaming_response( publisher_response: PublisherResponse, method: &Method, settings: Arc, @@ -1142,7 +1143,25 @@ pub fn publisher_response_into_streaming_response( response.headers_mut().remove(header::CONTENT_LENGTH); let mut params = *params; let mut processor = - PublisherBodyProcessor::new(¶ms, &settings, integration_registry)?; + match PublisherBodyProcessor::new(¶ms, &settings, integration_registry) { + Ok(processor) => processor, + Err(err) => { + // Parity with the buffered finalizer: a processor + // construction failure abandons the dispatched auction + // with telemetry instead of dropping the in-flight SSP + // responses silently. + if let Some(dispatched) = params.dispatched_auction.take() { + emit_abandoned_auction( + &services, + params.auction_observation.take(), + dispatched, + "processor_init_error", + ) + .await; + } + return Err(err); + } + }; // The guard is created before the lazy stream so an auction whose // response body is dropped unpolled still logs the loss. let dispatched_auction = params.dispatched_auction.take().map(|dispatched| { @@ -5292,14 +5311,14 @@ mod tests { params: Box::new(params), }; - let response = publisher_response_into_streaming_response( + let response = futures::executor::block_on(publisher_response_into_streaming_response( publisher_response, &Method::GET, Arc::clone(&settings), registry.as_ref(), orchestrator, services, - ) + )) .expect("should build streaming response"); assert!( @@ -5458,14 +5477,14 @@ mod tests { params: Box::new(params), }; - let response = publisher_response_into_streaming_response( + let response = futures::executor::block_on(publisher_response_into_streaming_response( publisher_response, &Method::GET, Arc::clone(&settings), registry.as_ref(), orchestrator, services, - ) + )) .expect("should build streaming response"); let output = futures::executor::block_on( From b95813ccc9a5c6542eebd38dd9b63a2b3ba6200f Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 10 Jul 2026 09:29:00 -0500 Subject: [PATCH 005/198] Fix Prebid User ID diagnostics Publisher-specific bundles need diagnostics based on the modules they actually contain. Otherwise, missing identity integrations can be hidden by the default preset. Refresh the comparison when auctions begin so late publisher configuration is visible without repeating warnings. Resolves: #886 --- .../lib/build-prebid-external.mjs | 10 +++- .../prebid/_user_ids.generated.ts | 5 +- .../lib/src/integrations/prebid/index.ts | 42 ++++++++--------- .../integrations/prebid/user_id_modules.json | 12 +++++ .../lib/test/build-prebid-external.test.mjs | 12 ++++- .../test/integrations/prebid/index.test.ts | 47 ++++++++++++++++++- .../prebid/user_id_modules.test.ts | 22 ++++++++- 7 files changed, 119 insertions(+), 31 deletions(-) diff --git a/crates/trusted-server-js/lib/build-prebid-external.mjs b/crates/trusted-server-js/lib/build-prebid-external.mjs index e3bb5ab3c..4e89723ed 100644 --- a/crates/trusted-server-js/lib/build-prebid-external.mjs +++ b/crates/trusted-server-js/lib/build-prebid-external.mjs @@ -107,7 +107,7 @@ function validateUserIdImport(entry) { } } -function writeGeneratedModule(filePath, title, moduleNames, imports) { +function writeGeneratedModule(filePath, title, moduleNames, imports, exports = []) { const content = [ '// Auto-generated by build-prebid-external.mjs.', '//', @@ -115,12 +115,17 @@ function writeGeneratedModule(filePath, title, moduleNames, imports) { `// Modules: ${moduleNames.join(', ')}`, '', ...imports, + ...(exports.length > 0 ? ['', ...exports] : []), '', ].join('\n'); fs.writeFileSync(filePath, content); } +export function renderIncludedUserIdModulesExport(moduleNames) { + return `export const INCLUDED_PREBID_USER_ID_MODULES = ${JSON.stringify(moduleNames)};`; +} + function generateAdapterImports(adapterNames, adaptersFile) { const modulesDir = path.join(PREBID_PACKAGE_DIR, 'modules'); const imports = []; @@ -165,7 +170,8 @@ function generateUserIdImports(requestedModules, userIdsFile) { userIdsFile, '// External Prebid bundle User ID module imports.', moduleNames, - imports + imports, + [renderIncludedUserIdModulesExport(moduleNames)] ); return moduleNames; } diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/_user_ids.generated.ts b/crates/trusted-server-js/lib/src/integrations/prebid/_user_ids.generated.ts index e9fa55f7d..e7c0112a9 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/_user_ids.generated.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/_user_ids.generated.ts @@ -1,6 +1,7 @@ // Placeholder for generated Prebid User ID module imports. // // build-prebid-external.mjs aliases this module to a temporary file containing -// publisher-specific imports during external bundle generation. +// publisher-specific imports and the corresponding module-name list during +// external bundle generation. -export {}; +export const INCLUDED_PREBID_USER_ID_MODULES: string[] = []; diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index be839d8fc..c6cf97cfe 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -25,14 +25,14 @@ import 'prebid.js/modules/userId.js'; // shim leaves its bids untouched and the corresponding adapter handles them // natively in the browser. import './_adapters.generated'; -import './_user_ids.generated'; +import { INCLUDED_PREBID_USER_ID_MODULES } from './_user_ids.generated'; import { log } from '../../core/log'; import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; import type { AuctionBid, AuctionEid } from '../../core/auction'; import type { AuctionSlot } from '../../core/types'; -import { DEFAULT_PREBID_USER_ID_MODULES, PREBID_USER_ID_MODULE_REGISTRY } from './user_id_modules'; +import { PREBID_USER_ID_MODULE_REGISTRY } from './user_id_modules'; const ADAPTER_CODE = 'trustedServer'; const BIDDER_PARAMS_KEY = 'bidderParams'; @@ -139,7 +139,7 @@ function recordUserIdModuleDiagnostics(): PrebidUserIdDiagnostics { const configuredUserIdNames = [...new Set(readConfiguredUserIdNames())].sort(); const coveredConfigNames = new Set( PREBID_USER_ID_MODULE_REGISTRY.filter((entry) => - DEFAULT_PREBID_USER_ID_MODULES.includes(entry.moduleName) + INCLUDED_PREBID_USER_ID_MODULES.includes(entry.moduleName) ).flatMap((entry) => entry.configNames) ); const missingConfiguredUserIdNames = configuredUserIdNames.filter( @@ -147,15 +147,20 @@ function recordUserIdModuleDiagnostics(): PrebidUserIdDiagnostics { ); const diagnostics: PrebidUserIdDiagnostics = { - includedModules: [...DEFAULT_PREBID_USER_ID_MODULES], + includedModules: [...INCLUDED_PREBID_USER_ID_MODULES], configuredUserIdNames, missingConfiguredUserIdNames, }; + const previouslyMissingConfiguredUserIdNames = new Set(); if (typeof window !== 'undefined') { const tsjsWindow = window as typeof window & { __tsjs_prebid_diagnostics?: { userIdModules?: PrebidUserIdDiagnostics }; }; + for (const name of tsjsWindow.__tsjs_prebid_diagnostics?.userIdModules + ?.missingConfiguredUserIdNames ?? []) { + previouslyMissingConfiguredUserIdNames.add(name); + } tsjsWindow.__tsjs_prebid_diagnostics = { ...(tsjsWindow.__tsjs_prebid_diagnostics ?? {}), userIdModules: diagnostics, @@ -163,9 +168,11 @@ function recordUserIdModuleDiagnostics(): PrebidUserIdDiagnostics { } for (const name of missingConfiguredUserIdNames) { - log.warn( - `[tsjs-prebid] configured User ID module "${name}" is not included in the external bundle` - ); + if (!previouslyMissingConfiguredUserIdNames.has(name)) { + log.warn( + `[tsjs-prebid] configured User ID module "${name}" is not included in the external bundle` + ); + } } return diagnostics; @@ -559,6 +566,7 @@ export function installPrebidNpm(config?: Partial): typeof pbjs // client-side bidders are left untouched. pbjs.requestBids = function (requestObj?: Parameters[0]) { log.debug('[tsjs-prebid] requestBids called'); + recordUserIdModuleDiagnostics(); const opts = requestObj || {}; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -811,25 +819,13 @@ export function installRefreshHandler(timeoutMs = 1500): void { } /** - * Configure Prebid.js userID modules for identity warm-up. - * - * Runs post-window.load (called from installPrebidNpm after setup). - * Writes identity tokens to 1P cookies so the next server-side request - * can harvest them for EC graph enrichment. + * Configure identity sync behavior for the generated Prebid User ID modules. * - * **Current state:** This function only configures `pbjs.userSync` settings. - * It does NOT import or register any userID modules. Actual module imports - * (ID5, sharedID, LiveRamp ATS, Lockr) must be added to this bundle explicitly - * — there is currently no `_userIdModules.generated.ts` build step. - * Track as Phase B follow-up: add `TSJS_PREBID_USER_ID_MODULES` handling to - * `build-all.mjs` (similar to `TSJS_PREBID_ADAPTERS`) and import generated file. + * The external bundle generator statically imports the selected modules through + * `_user_ids.generated.ts`. This post-window-load configuration controls when + * those modules synchronize identities; it does not select or register modules. */ export function installUserIdModules(): void { - // NOTE: No userID module imports exist yet. `_userIdModules.generated.ts` and - // `TSJS_PREBID_USER_ID_MODULES` handling in `build-all.mjs` are not implemented. - // This function only configures pbjs.userSync settings; actual module registration - // requires the Phase B follow-up described in the docblock above. - // Configure sync behavior so modules will run post-window.load when added. try { pbjs.setConfig({ userSync: { diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.json b/crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.json index 10f244f31..a4fd58dbe 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.json +++ b/crates/trusted-server-js/lib/src/integrations/prebid/user_id_modules.json @@ -58,6 +58,18 @@ "importPath": "prebid.js/modules/liveIntentIdSystem.js", "notes": "Imported through a local ESM shim because the public Prebid wrapper contains CommonJS require()." }, + { + "moduleName": "lockrAIMIdSystem", + "configNames": ["lockrAIMId"], + "eidSources": [], + "importPath": "prebid.js/modules/lockrAIMIdSystem.js" + }, + { + "moduleName": "pairIdSystem", + "configNames": ["pairId"], + "eidSources": ["google.com"], + "importPath": "prebid.js/modules/pairIdSystem.js" + }, { "moduleName": "pubProvidedIdSystem", "configNames": ["pubProvidedId"], diff --git a/crates/trusted-server-js/lib/test/build-prebid-external.test.mjs b/crates/trusted-server-js/lib/test/build-prebid-external.test.mjs index 609bdba91..10702f914 100644 --- a/crates/trusted-server-js/lib/test/build-prebid-external.test.mjs +++ b/crates/trusted-server-js/lib/test/build-prebid-external.test.mjs @@ -3,7 +3,11 @@ import path from 'node:path'; import { describe, expect, it } from 'vitest'; -import { deriveBundleMetadata, parseArgs } from '../build-prebid-external.mjs'; +import { + deriveBundleMetadata, + parseArgs, + renderIncludedUserIdModulesExport, +} from '../build-prebid-external.mjs'; describe('build-prebid-external metadata', () => { it('derives filename, sha256, and SRI from exact bundle bytes', () => { @@ -18,6 +22,12 @@ describe('build-prebid-external metadata', () => { }); }); + it('renders the exact selected User ID modules for runtime diagnostics', () => { + expect(renderIncludedUserIdModulesExport(['liveIntentIdSystem', 'pairIdSystem'])).toBe( + 'export const INCLUDED_PREBID_USER_ID_MODULES = ["liveIntentIdSystem","pairIdSystem"];' + ); + }); + it('resolves relative output paths against the current working directory', () => { const parsed = parseArgs(['--adapters', 'rubicon', '--out', 'dist/prebid']); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 9ad7945c4..e6b5fd354 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -7,6 +7,7 @@ const { mockRequestBids, mockRegisterBidAdapter, mockGetUserIdsAsEids, + mockGetConfig, mockPbjs, mockGetBidAdapter, mockAdapterManager, @@ -19,12 +20,14 @@ const { const mockGetUserIdsAsEids = vi.fn( () => [] as Array<{ source: string; uids?: Array<{ id: string; atype?: number }> }> ); + const mockGetConfig = vi.fn(); const mockPbjs = { setConfig: mockSetConfig, processQueue: mockProcessQueue, requestBids: mockRequestBids, registerBidAdapter: mockRegisterBidAdapter, getUserIdsAsEids: mockGetUserIdsAsEids, + getConfig: mockGetConfig, adUnits: [] as any[], }; const mockAdapterManager = { @@ -36,6 +39,7 @@ const { mockRequestBids, mockRegisterBidAdapter, mockGetUserIdsAsEids, + mockGetConfig, mockPbjs, mockGetBidAdapter, mockAdapterManager, @@ -53,9 +57,11 @@ vi.mock('prebid.js/modules/consentManagementGpp.js', () => ({})); vi.mock('prebid.js/modules/consentManagementUsp.js', () => ({})); vi.mock('prebid.js/modules/userId.js', () => ({})); -// Mock the build-generated side-effect imports (no-op in tests) +// Mock the build-generated imports in tests. vi.mock('../../../src/integrations/prebid/_adapters.generated', () => ({})); -vi.mock('../../../src/integrations/prebid/_user_ids.generated', () => ({})); +vi.mock('../../../src/integrations/prebid/_user_ids.generated', () => ({ + INCLUDED_PREBID_USER_ID_MODULES: ['sharedIdSystem'], +})); import { collectBidders, @@ -65,6 +71,7 @@ import { installRefreshHandler, } from '../../../src/integrations/prebid/index'; import type { AuctionBid } from '../../../src/core/auction'; +import { log } from '../../../src/core/log'; describe('prebid/collectBidders', () => { it('returns empty array for empty ad units', () => { @@ -207,8 +214,14 @@ describe('prebid/installPrebidNpm', () => { mockPbjs.adUnits = []; mockGetUserIdsAsEids.mockReset(); mockGetUserIdsAsEids.mockReturnValue([]); + mockGetConfig.mockReset(); document.cookie = 'ts-eids=; Path=/; Max-Age=0'; delete (window as any).__tsjs_prebid; + delete (window as any).__tsjs_prebid_diagnostics; + }); + + afterEach(() => { + vi.restoreAllMocks(); }); it('registers the trustedServer bid adapter', () => { @@ -251,6 +264,36 @@ describe('prebid/installPrebidNpm', () => { expect(mockProcessQueue).toHaveBeenCalledTimes(1); }); + it('reports the User ID modules selected by the generated bundle', () => { + installPrebidNpm(); + + expect((window as any).__tsjs_prebid_diagnostics.userIdModules).toEqual({ + includedModules: ['sharedIdSystem'], + configuredUserIdNames: [], + missingConfiguredUserIdNames: [], + }); + }); + + it('refreshes late User ID config without repeating missing-module warnings', () => { + installPrebidNpm(); + mockGetConfig.mockImplementation((key?: string) => + key === 'userSync.userIds' ? [{ name: 'sharedId' }, { name: 'pairId' }] : {} + ); + const warnSpy = vi.spyOn(log, 'warn').mockImplementation(() => {}); + + mockPbjs.requestBids({ adUnits: [] }); + mockPbjs.requestBids({ adUnits: [] }); + + expect((window as any).__tsjs_prebid_diagnostics.userIdModules).toEqual({ + includedModules: ['sharedIdSystem'], + configuredUserIdNames: ['pairId', 'sharedId'], + missingConfiguredUserIdNames: ['pairId'], + }); + expect( + warnSpy.mock.calls.filter(([message]) => String(message).includes('"pairId"')) + ).toHaveLength(1); + }); + it('returns the pbjs instance', () => { const result = installPrebidNpm(); expect(result).toBe(mockPbjs); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts index f011f294d..2832a4082 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/user_id_modules.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from 'vitest'; -import { resolvePrebidUserIdModulesFromEids } from '../../../src/integrations/prebid/user_id_modules'; +import { + knownUserIdConfigNames, + resolvePrebidUserIdModulesFromEids, +} from '../../../src/integrations/prebid/user_id_modules'; const sampleEids = [ { source: 'yahoo.com', uids: [{ id: 'connect-id', atype: 3 }] }, @@ -65,6 +68,23 @@ describe('prebid user ID module registry', () => { }); }); + it('exposes config names for modules that do not map EID sources', () => { + expect(knownUserIdConfigNames()).toEqual( + expect.arrayContaining(['lockrAIMId', 'pubProvidedId']) + ); + }); + + it('maps the Google PAIR EID source to pairIdSystem', () => { + const result = resolvePrebidUserIdModulesFromEids([ + { source: 'google.com', uids: [{ id: 'pair-id' }] }, + ]); + + expect(result).toEqual({ + modules: ['userId', 'pairIdSystem'], + missingSources: [], + }); + }); + it('maps unknown LiveIntent provider-backed sources to liveIntentIdSystem', () => { const result = resolvePrebidUserIdModulesFromEids([ { From e94430eb72afe9feffdba612e01ec63de5ed8934 Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 10 Jul 2026 09:50:08 -0500 Subject: [PATCH 006/198] Order Prebid imports for lint --- crates/trusted-server-js/lib/src/integrations/prebid/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index c6cf97cfe..a4b1ccff2 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -25,13 +25,13 @@ import 'prebid.js/modules/userId.js'; // shim leaves its bids untouched and the corresponding adapter handles them // natively in the browser. import './_adapters.generated'; -import { INCLUDED_PREBID_USER_ID_MODULES } from './_user_ids.generated'; import { log } from '../../core/log'; import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; import type { AuctionBid, AuctionEid } from '../../core/auction'; import type { AuctionSlot } from '../../core/types'; +import { INCLUDED_PREBID_USER_ID_MODULES } from './_user_ids.generated'; import { PREBID_USER_ID_MODULE_REGISTRY } from './user_id_modules'; const ADAPTER_CODE = 'trustedServer'; From b56adcb39ade3902ff8c131bb8ac7397fbfff74c Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 10 Jul 2026 10:28:03 -0500 Subject: [PATCH 007/198] Preserve vendor-specific OpenRTB atype values --- .../src/auction/endpoints.rs | 22 +++++++- crates/trusted-server-core/src/ec/eids.rs | 16 +++++- .../trusted-server-core/src/ec/prebid_eids.rs | 48 ++++++++++++++--- crates/trusted-server-core/src/ec/registry.rs | 2 +- .../src/integrations/prebid.rs | 18 ++++++- crates/trusted-server-core/src/openrtb.rs | 23 ++++++++- crates/trusted-server-core/src/settings.rs | 51 +++++++++++++++++-- .../lib/src/integrations/prebid/index.ts | 5 +- .../lib/test/build-prebid-external.test.mjs | 32 ++++++++++++ .../test/integrations/prebid/index.test.ts | 10 +++- trusted-server.example.toml | 1 + 11 files changed, 206 insertions(+), 22 deletions(-) diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 83c7df349..2833e63eb 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -476,7 +476,7 @@ fn parse_client_auction_uid(raw: &JsonValue) -> Option { let atype = uid .get("atype") .and_then(JsonValue::as_u64) - .and_then(|atype| u8::try_from(atype).ok()); + .and_then(|atype| i32::try_from(atype).ok()); let ext = match uid.get("ext") { Some(JsonValue::Object(_)) => uid.get("ext").cloned(), @@ -1057,6 +1057,24 @@ mod tests { assert_eq!(parsed[0].uids[0].id, "valid", "should keep valid UID"); } + #[test] + fn parse_client_auction_eids_preserves_pair_atype() { + let raw = json!([ + { + "source": "google.com", + "uids": [{ "id": "pair-id", "atype": 571187 }] + } + ]); + + let parsed = parse_client_auction_eids(Some(&raw)).expect("should parse PAIR EID"); + + assert_eq!( + parsed[0].uids[0].atype, + Some(571187), + "should preserve PAIR's vendor-specific atype" + ); + } + #[test] fn parse_client_auction_eids_preserves_uid_ext_and_sanitizes_invalid_atype() { let raw = json!([ @@ -1070,7 +1088,7 @@ mod tests { }, { "id": "uid-bad-atype", - "atype": 999, + "atype": 2_147_483_648_u64, "ext": { "keep": true } }, { diff --git a/crates/trusted-server-core/src/ec/eids.rs b/crates/trusted-server-core/src/ec/eids.rs index 1dd8b1795..2c01bf852 100644 --- a/crates/trusted-server-core/src/ec/eids.rs +++ b/crates/trusted-server-core/src/ec/eids.rs @@ -25,7 +25,7 @@ pub struct ResolvedPartnerId { /// The synced user ID value. pub uid: String, /// `OpenRTB` agent type for this partner's identifiers. - pub openrtb_atype: u8, + pub openrtb_atype: i32, } /// Resolves source-domain keyed IDs from a KV entry against the partner registry. @@ -214,17 +214,29 @@ mod tests { source_domain: "id5-sync.com".to_owned(), openrtb_atype: 1, }, + ResolvedPartnerId { + uid: "pair-id".to_owned(), + source_domain: "google.com".to_owned(), + openrtb_atype: 571187, + }, ]; let eids = to_eids(&resolved); - assert_eq!(eids.len(), 2, "should produce one EID per resolved partner"); + assert_eq!(eids.len(), 3, "should produce one EID per resolved partner"); assert_eq!(eids[0].source, "liveramp.com"); assert_eq!(eids[0].uids[0].id, "LR_xyz"); assert_eq!(eids[0].uids[0].atype, Some(3)); assert_eq!(eids[1].source, "id5-sync.com"); assert_eq!(eids[1].uids[0].id, "ID5_abc"); assert_eq!(eids[1].uids[0].atype, Some(1)); + assert_eq!(eids[2].source, "google.com", "should preserve PAIR source"); + assert_eq!(eids[2].uids[0].id, "pair-id", "should preserve PAIR ID"); + assert_eq!( + eids[2].uids[0].atype, + Some(571187), + "should preserve PAIR vendor-specific atype" + ); } #[test] diff --git a/crates/trusted-server-core/src/ec/prebid_eids.rs b/crates/trusted-server-core/src/ec/prebid_eids.rs index 22620e599..dc95be1aa 100644 --- a/crates/trusted-server-core/src/ec/prebid_eids.rs +++ b/crates/trusted-server-core/src/ec/prebid_eids.rs @@ -33,7 +33,7 @@ struct LegacyCookieEid { dead_code, reason = "legacy cookie field is deserialized for compatibility but not emitted" )] - atype: u8, + atype: i32, } /// OpenRTB-style `ts-eids` cookie entry. @@ -48,7 +48,7 @@ struct StructuredCookieEid { struct StructuredCookieUid { id: String, #[serde(default)] - atype: Option, + atype: Option, #[serde(default)] ext: Option, } @@ -318,6 +318,7 @@ fn structured_cookie_uid_to_openrtb(uid: StructuredCookieUid) -> Option { return None; } + let atype = uid.atype.filter(|atype| *atype >= 0); let ext = match uid.ext { Some(JsonValue::Object(_)) => uid.ext, _ => None, @@ -325,7 +326,7 @@ fn structured_cookie_uid_to_openrtb(uid: StructuredCookieUid) -> Option { Some(Uid { id: uid.id, - atype: uid.atype, + atype, ext, }) } @@ -338,7 +339,7 @@ fn legacy_cookie_eids_to_openrtb(entries: Vec) -> Vec { source: entry.source, uids: vec![Uid { id: entry.id, - atype: Some(entry.atype), + atype: (entry.atype >= 0).then_some(entry.atype), ext: None, }], }) @@ -407,15 +408,25 @@ mod tests { let eids = vec![ json!({"source": "id5-sync.com", "id": "ID5_abc", "atype": 1}), json!({"source": "liveramp.com", "id": "LR_xyz", "atype": 3}), + json!({"source": "google.com", "id": "pair-id", "atype": 571187}), ]; let encoded = BASE64.encode(serde_json::to_vec(&eids).expect("should serialize")); let decoded = parse_prebid_eids_cookie(&encoded).expect("should decode valid payload"); - assert_eq!(decoded.len(), 2, "should parse both EIDs"); + assert_eq!(decoded.len(), 3, "should parse all EIDs"); assert_eq!(decoded[0].source, "id5-sync.com"); assert_eq!(decoded[0].uids[0].id, "ID5_abc"); assert_eq!(decoded[1].source, "liveramp.com"); assert_eq!(decoded[1].uids[0].id, "LR_xyz"); + assert_eq!( + decoded[2].source, "google.com", + "should preserve PAIR source" + ); + assert_eq!( + decoded[2].uids[0].atype, + Some(571187), + "should preserve PAIR vendor-specific atype" + ); } #[test] @@ -424,7 +435,8 @@ mod tests { "source": "sharedid.org", "uids": [ {"id": "shared_123", "atype": 3}, - {"id": "shared_456", "ext": {"provider": "example"}} + {"id": "shared_456", "ext": {"provider": "example"}}, + {"id": "shared_invalid", "atype": -1} ] })]; let encoded = BASE64.encode(serde_json::to_vec(&eids).expect("should serialize")); @@ -432,7 +444,7 @@ mod tests { let decoded = parse_prebid_eids_cookie(&encoded).expect("should decode valid payload"); assert_eq!(decoded.len(), 1, "should parse one structured EID entry"); assert_eq!(decoded[0].source, "sharedid.org"); - assert_eq!(decoded[0].uids.len(), 2, "should preserve multiple UIDs"); + assert_eq!(decoded[0].uids.len(), 3, "should preserve multiple UIDs"); assert_eq!(decoded[0].uids[0].id, "shared_123"); assert_eq!(decoded[0].uids[0].atype, Some(3)); assert_eq!( @@ -440,6 +452,28 @@ mod tests { Some(json!({"provider": "example"})), "should preserve UID ext objects" ); + assert_eq!( + decoded[0].uids[2].atype, None, + "should drop negative atype values" + ); + } + + #[test] + fn parse_prebid_eids_cookie_preserves_pair_atype() { + let encoded = encode_json(&json!([ + { + "source": "google.com", + "uids": [{ "id": "pair-id", "atype": 571187 }] + } + ])); + + let decoded = parse_prebid_eids_cookie(&encoded).expect("should decode PAIR EID"); + + assert_eq!( + decoded[0].uids[0].atype, + Some(571187), + "should preserve PAIR's vendor-specific atype" + ); } #[test] diff --git a/crates/trusted-server-core/src/ec/registry.rs b/crates/trusted-server-core/src/ec/registry.rs index 6b688d30e..8532de03b 100644 --- a/crates/trusted-server-core/src/ec/registry.rs +++ b/crates/trusted-server-core/src/ec/registry.rs @@ -25,7 +25,7 @@ pub struct PartnerConfig { /// Canonical `OpenRTB` EID source domain and EC KV `ids` key. pub source_domain: String, /// `OpenRTB` `atype` value. - pub openrtb_atype: u8, + pub openrtb_atype: i32, /// Whether this partner's UIDs appear in auction `user.eids`. pub bidstream_enabled: bool, /// SHA-256 hex of the partner's API token (precomputed at startup). diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 32a1c2a85..acfef5727 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -4654,6 +4654,14 @@ external_bundle_sri = "sha384-AAAA" ext: None, }], }, + crate::openrtb::Eid { + source: "google.com".to_owned(), + uids: vec![crate::openrtb::Uid { + id: "pair-id".to_owned(), + atype: Some(571187), + ext: None, + }], + }, ]); let settings = make_settings(); @@ -4670,7 +4678,7 @@ external_bundle_sri = "sha384-AAAA" let serialized = serde_json::to_value(&openrtb).expect("should serialize OpenRTB request"); let ext_eids = &serialized["user"]["ext"]["eids"]; assert!(ext_eids.is_array(), "should populate user.ext.eids"); - assert_eq!(ext_eids.as_array().unwrap().len(), 2, "should have 2 EIDs"); + assert_eq!(ext_eids.as_array().unwrap().len(), 3, "should have 3 EIDs"); assert_eq!( ext_eids[0]["source"], "liveramp.com", "should include liveramp EID" @@ -4679,6 +4687,14 @@ external_bundle_sri = "sha384-AAAA" ext_eids[1]["source"], "id5-sync.com", "should include id5 EID" ); + assert_eq!( + ext_eids[2]["source"], "google.com", + "should include PAIR EID" + ); + assert_eq!( + ext_eids[2]["uids"][0]["atype"], 571187, + "should preserve PAIR's vendor-specific atype" + ); } #[test] diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index df99f5653..65237ce0e 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -80,9 +80,9 @@ pub struct Eid { pub struct Uid { /// The identifier value. pub id: String, - /// Agent type: 1 = cookie/device, 2 = person, 3 = user-provided. + /// `OpenRTB` agent type, including vendor-specific values such as PAIR's `571187`. #[serde(skip_serializing_if = "Option::is_none")] - pub atype: Option, + pub atype: Option, /// Provider-specific extension data. #[serde(skip_serializing_if = "Option::is_none")] pub ext: Option, @@ -400,4 +400,23 @@ mod tests { "ext should be omitted when None" ); } + + #[test] + fn eid_serializes_vendor_specific_atype() { + let eid = Eid { + source: "google.com".to_owned(), + uids: vec![Uid { + id: "pair-id".to_owned(), + atype: Some(571187), + ext: None, + }], + }; + + let serialized = serde_json::to_value(&eid).expect("should serialize"); + + assert_eq!( + serialized["uids"][0]["atype"], 571187, + "should preserve PAIR's vendor-specific atype" + ); + } } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 9cbb2a546..aaacb29df 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -299,12 +299,13 @@ pub struct EcPartner { /// This normalized domain is also the canonical EC KV `ids` map key. #[validate(custom(function = EcPartner::validate_source_domain))] pub source_domain: String, - /// `OpenRTB` `atype` value (typically 3). + /// `OpenRTB` `atype` value, including vendor-specific values such as PAIR's `571187`. #[serde( default = "EcPartner::default_openrtb_atype", deserialize_with = "from_value_or_str" )] - pub openrtb_atype: u8, + #[validate(range(min = 0, message = "must be a non-negative OpenRTB agent type"))] + pub openrtb_atype: i32, /// Whether this partner's UIDs appear in auction `user.eids`. #[serde(default, deserialize_with = "from_value_or_str")] pub bidstream_enabled: bool, @@ -417,7 +418,7 @@ impl EcPartner { } #[must_use] - pub const fn default_openrtb_atype() -> u8 { + pub const fn default_openrtb_atype() -> i32 { 3 } @@ -2808,6 +2809,46 @@ mod tests { } } + #[test] + fn validate_accepts_vendor_specific_ec_partner_atype() { + let toml_str = format!( + r#"{} + [[ec.partners]] + name = "PAIR Partner" + source_domain = "google.com" + openrtb_atype = 571187 + api_token = "test-vendor-token-32-bytes-minimum" + "#, + crate_test_settings_str(), + ); + + let settings = Settings::from_toml(&toml_str) + .expect("should accept vendor-specific OpenRTB agent type"); + + assert_eq!( + settings.ec.partners[0].openrtb_atype, 571187, + "should preserve PAIR's vendor-specific atype" + ); + } + + #[test] + fn validate_rejects_negative_ec_partner_atype() { + let toml_str = format!( + r#"{} + [[ec.partners]] + name = "Invalid Partner" + source_domain = "partner.example.com" + openrtb_atype = -1 + api_token = "test-vendor-token-32-bytes-minimum" + "#, + crate_test_settings_str(), + ); + + let result = Settings::from_toml(&toml_str); + + assert!(result.is_err(), "should reject negative OpenRTB agent type"); + } + #[test] fn validate_accepts_origin_host_header_override() { let toml_str = crate_test_settings_str().replace( @@ -3647,7 +3688,7 @@ origin_host_header_overide = "www.example.com""#, (origin_key, Some("https://origin.test-publisher.com")), (partner_0_name_key, Some("Env Partner 0")), (partner_0_source_domain_key, Some("envpartner0.example.com")), - (partner_0_openrtb_atype_key, Some("1")), + (partner_0_openrtb_atype_key, Some("571187")), (partner_0_bidstream_enabled_key, Some("true")), (partner_0_api_token_key, Some("env-token-0")), (partner_1_name_key, Some("Env Partner 1")), @@ -3666,7 +3707,7 @@ origin_host_header_overide = "www.example.com""#, settings.ec.partners[0].source_domain, "envpartner0.example.com" ); - assert_eq!(settings.ec.partners[0].openrtb_atype, 1); + assert_eq!(settings.ec.partners[0].openrtb_atype, 571187); assert!(settings.ec.partners[0].bidstream_enabled); assert_eq!(settings.ec.partners[0].api_token.expose(), "env-token-0"); assert_eq!(settings.ec.partners[1].name, "Env Partner 1"); diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index a4b1ccff2..342e4038d 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -35,6 +35,9 @@ import { INCLUDED_PREBID_USER_ID_MODULES } from './_user_ids.generated'; import { PREBID_USER_ID_MODULE_REGISTRY } from './user_id_modules'; const ADAPTER_CODE = 'trustedServer'; +// OpenRTB permits vendor-specific agent types; PAIR uses 571187. +// Keep this range aligned with the signed 32-bit Rust/OpenRTB representation. +const MAX_OPENRTB_ATYPE = 2_147_483_647; const BIDDER_PARAMS_KEY = 'bidderParams'; const ZONE_KEY = 'zone'; const TS_REFRESH_TARGETING_KEYS = [ @@ -281,7 +284,7 @@ function sanitizeAuctionUid(uid: { typeof uid.atype === 'number' && Number.isInteger(uid.atype) && uid.atype >= 0 && - uid.atype <= 255 + uid.atype <= MAX_OPENRTB_ATYPE ) { sanitizedUid.atype = uid.atype; } diff --git a/crates/trusted-server-js/lib/test/build-prebid-external.test.mjs b/crates/trusted-server-js/lib/test/build-prebid-external.test.mjs index 10702f914..a1c77c47d 100644 --- a/crates/trusted-server-js/lib/test/build-prebid-external.test.mjs +++ b/crates/trusted-server-js/lib/test/build-prebid-external.test.mjs @@ -1,10 +1,15 @@ +// @vitest-environment node + import crypto from 'node:crypto'; +import fs from 'node:fs'; +import os from 'node:os'; import path from 'node:path'; import { describe, expect, it } from 'vitest'; import { deriveBundleMetadata, + main, parseArgs, renderIncludedUserIdModulesExport, } from '../build-prebid-external.mjs'; @@ -28,6 +33,33 @@ describe('build-prebid-external metadata', () => { ); }); + it('includes generated User ID metadata in the production external bundle', async () => { + const outputDirectory = fs.mkdtempSync( + path.join(os.tmpdir(), 'trusted-server-prebid-build-test-') + ); + + try { + await main([ + '--adapters', + 'rubicon', + '--user-id-modules', + 'pairIdSystem,lockrAIMIdSystem', + '--out', + outputDirectory, + ]); + + const manifest = JSON.parse( + fs.readFileSync(path.join(outputDirectory, 'manifest.json'), 'utf8') + ); + const bundle = fs.readFileSync(path.join(outputDirectory, manifest.filename), 'utf8'); + + expect(manifest.userIdModules).toEqual(['pairIdSystem', 'lockrAIMIdSystem']); + expect(bundle).toContain('["pairIdSystem","lockrAIMIdSystem"]'); + } finally { + fs.rmSync(outputDirectory, { recursive: true, force: true }); + } + }, 120_000); + it('resolves relative output paths against the current working directory', () => { const parsed = parseArgs(['--adapters', 'rubicon', '--out', 'dist/prebid']); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index e6b5fd354..726f40b49 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -344,6 +344,10 @@ describe('prebid/installPrebidNpm', () => { source: 'sharedid.org', uids: [{ id: 'shared_123' }, { id: 'shared_456', atype: 3 }], }, + { + source: 'google.com', + uids: [{ id: 'pair_123', atype: 571187 }], + }, ]); const result = spec.buildRequests([ @@ -365,6 +369,10 @@ describe('prebid/installPrebidNpm', () => { source: 'sharedid.org', uids: [{ id: 'shared_123' }, { id: 'shared_456', atype: 3 }], }, + { + source: 'google.com', + uids: [{ id: 'pair_123', atype: 571187 }], + }, ]); }); @@ -398,7 +406,7 @@ describe('prebid/installPrebidNpm', () => { }, { id: 'uid-bad-atype', - atype: 999, + atype: 2_147_483_648, ext: { keep: true }, }, { diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 0951b781e..879eb4b91 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -22,6 +22,7 @@ pull_sync_concurrency = 3 # [[ec.partners]] # name = "Example Partner" # source_domain = "partner.example.com" +# OpenRTB agent type; vendor-specific values are supported (PAIR uses 571187). # openrtb_atype = 3 # bidstream_enabled = true # api_token = "replace-with-partner-api-token-32-bytes-minimum" From b52b415755fbffd63935a94ad3d14f8165b82440 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 11 Jul 2026 00:32:35 +0530 Subject: [PATCH 008/198] Dump full SSAT prebid responses in ts-debug auction comment The server-side auction stream path only emitted a summary counter (ssp/mediator/winning/time), so an operator seeing winning=0 could not tell whether prebid returned nothing, errored, or bid below the floor. Serialize the full provider_responses (and mediator_response) into the ts-debug HTML comment so the SSAT surfaces the same prebid server response detail available from the /auction endpoint. Bid creative and metadata are attacker/partner-influenced, so neutralize the '-->' and '--!>' comment terminators before embedding to keep the dump inside the comment and out of the live DOM. --- crates/trusted-server-core/src/publisher.rs | 92 ++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 6b0ea3a5d..c14410121 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -871,8 +871,28 @@ pub(crate) fn prepend_auction_debug_comment( Some(r) => format!("ok({}_bids)", r.bids.len()), None => "none".to_string(), }; + // Full per-provider (and mediator) dump so the operator can see exactly what + // each SSP returned — `status` (nobid vs error vs success), every `bids` + // entry, and `metadata` (which carries PBS `ext.errors` / `ext.debug.httpcalls` + // when prebid `debug=true`) — without needing log access. + // + // `Bid.creative` and provider metadata are attacker/partner-influenced and + // may contain `-->` (or the `--!>` variant), which would terminate the HTML + // comment early and leak the remaining markup into the live DOM. Neutralise + // both terminators before embedding so the dump stays inside the comment. + let neutralise_comment_terminators = + |json: String| -> String { json.replace("-->", "-- >").replace("--!>", "-- !>") }; + let providers_dump = serde_json::to_string_pretty(&result.provider_responses) + .map(neutralise_comment_terminators) + .unwrap_or_else(|e| format!("")); + let mediator_dump = serde_json::to_string_pretty(&result.mediator_response) + .map(neutralise_comment_terminators) + .unwrap_or_else(|e| format!("")); let debug_comment = format!( - "", + "", result.winning_bids.len(), result.total_time_ms, ); @@ -2439,6 +2459,76 @@ mod tests { use super::*; use crate::auction::types::{AdFormat, AdSlot, MediaType}; use crate::integrations::IntegrationRegistry; + + #[test] + fn auction_debug_comment_dumps_provider_status_and_neutralises_terminators() { + use crate::auction::orchestrator::OrchestrationResult; + use crate::auction::types::AuctionResponse; + + // One provider that returned nothing (the `winning=0` case) and one that + // returned a bid whose creative embeds an HTML-comment terminator. + let no_bid = AuctionResponse::no_bid("prebid", 665); + let mut bid = make_test_bid_with_creative("
evil-->break
"); + bid.slot_id = "ad-header-0".to_string(); + let with_bid = AuctionResponse::success("aps", vec![bid], 42); + + let result = OrchestrationResult { + provider_responses: vec![no_bid, with_bid], + mediator_response: None, + winning_bids: std::collections::HashMap::new(), + total_time_ms: 665, + metadata: std::collections::HashMap::new(), + }; + + let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); + prepend_auction_debug_comment("stream", &result, &state); + let comment = state + .lock() + .expect("should lock state") + .clone() + .expect("should have comment"); + + assert!( + comment.contains("\"status\": \"nobid\""), + "should surface the no-bid provider status: {comment}" + ); + assert!( + comment.contains("provider_responses="), + "should dump the provider_responses payload" + ); + // The creative's `-->` must be neutralised so the only comment terminator + // is the trailing one — otherwise embedded markup would leak into the DOM. + assert_eq!( + comment.matches("-->").count(), + 1, + "creative `-->` must be neutralised, leaving only the closing terminator: {comment}" + ); + assert!( + comment.contains("evil-- >break"), + "should retain the creative content with the terminator neutralised" + ); + } + + fn make_test_bid_with_creative(creative: &str) -> Bid { + Bid { + slot_id: "slot".to_string(), + price: Some(1.0), + currency: "USD".to_string(), + creative: Some(creative.to_string()), + adomain: None, + bidder: "seat".to_string(), + width: 300, + height: 250, + nurl: None, + burl: None, + ad_id: None, + cache_id: None, + cache_host: None, + cache_path: None, + metadata: Default::default(), + } + } + use crate::platform::test_support::{ build_services_with_http_client, noop_services, StubHttpClient, }; From 48db8bf6a56dd31fd2a2b7334354b12f882b8fd2 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Sat, 11 Jul 2026 00:51:41 +0530 Subject: [PATCH 009/198] Surface prebid HTTP error status and body in auction dump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When Prebid Server returns a non-2xx status, the parser returned a bare AuctionResponse::error with empty metadata — indistinguishable in the ts-debug dump from a transport, parse, or timeout failure, all of which tag error_type. An operator seeing status=error with metadata={} had no way to know the upstream HTTP code without log access. Attach error_type=http_status, the status code, and a 512-byte body snippet to the error response metadata so the auction dump shows exactly why prebid errored (e.g. a 4xx from a PBS rejecting the request). --- .../src/integrations/prebid.rs | 51 ++++++++++++++++++- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 32a1c2a85..c6fccc2c3 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -2124,14 +2124,23 @@ impl AuctionProvider for PrebidAuctionProvider { if !status.is_success() { log::warn!("Prebid returned non-success status: {}", status,); + let body_preview = String::from_utf8_lossy(&body_bytes); if log::log_enabled!(log::Level::Trace) { - let body_preview = String::from_utf8_lossy(&body_bytes); log::trace!( "Prebid error response body: {}", &body_preview[..body_preview.floor_char_boundary(1000)] ); } - return Ok(AuctionResponse::error("prebid", response_time_ms)); + // Surface the HTTP status and a body snippet on the response metadata + // so the ts-debug auction dump shows *why* prebid errored (e.g. a 4xx + // from a PBS that rejects the unsigned server-side request) without + // needing log access. A bare `AuctionResponse::error` yields empty + // metadata, which is indistinguishable from other failures in the dump. + let body_snippet = body_preview[..body_preview.floor_char_boundary(512)].to_string(); + return Ok(AuctionResponse::error("prebid", response_time_ms) + .with_metadata("error_type", serde_json::json!("http_status")) + .with_metadata("status", serde_json::json!(status.as_u16())) + .with_metadata("body", serde_json::json!(body_snippet))); } let response_json: Json = @@ -2341,6 +2350,44 @@ mod tests { ); } + #[test] + fn parse_response_attaches_status_and_body_metadata_on_http_error() { + use crate::auction::types::BidStatus; + + let provider = PrebidAuctionProvider::new(base_config()); + let response = PlatformResponse::new( + edgezero_core::http::response_builder() + .status(403) + .body(EdgeBody::from(br#"{"error":"missing signature"}"#.to_vec())) + .expect("should build test response"), + ); + + let result = futures::executor::block_on(provider.parse_response(response, 643)) + .expect("should return Ok(error response) for non-success status"); + + assert_eq!( + result.status, + BidStatus::Error, + "non-success HTTP status should map to an error response" + ); + assert_eq!( + result.metadata["error_type"], + json!("http_status"), + "should tag the error path so the auction dump is distinguishable" + ); + assert_eq!( + result.metadata["status"], + json!(403), + "should surface the upstream HTTP status code" + ); + assert!( + result.metadata["body"] + .as_str() + .is_some_and(|body| body.contains("missing signature")), + "should include the response body snippet" + ); + } + fn test_sri(algorithm: &str, digest: &[u8]) -> String { format!("{algorithm}-{}", TEST_BASE64_STANDARD.encode(digest)) } From d5871da84ec21af5c8118c402b5e2243b0731c0f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 13 Jul 2026 13:12:30 +0530 Subject: [PATCH 010/198] Stop leaking prebid error body into the public auction response MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address PR review of the SSAT debug-dump change: - The PBS non-2xx response body was attached to AuctionResponse.metadata, which ProviderSummary clones verbatim into ext.orchestrator.provider_details on the public /auction response — violating the documented invariant in auction/orchestrator.rs. Drop the body from metadata, keep only the numeric status, and log the snippet server-side at warn. - Register ERROR_TYPE_HTTP_STATUS and match it in provider_status so PBS HTTP errors get their own telemetry bucket instead of the transport_error fallback. - Bound the ts-debug dump: compact serialization capped at 256 KiB, and skip the mediator_response line when no mediator ran. - Correct the auction_html_comment and prepend_auction_debug_comment docs to state the comment now embeds raw SSP creative markup (never enable in prod). - Keep the targeted two-replace terminator neutralisation: a single replace("--", ...) re-forms -->/--!> at odd dash-run junctions and is not equivalent. Add a table-driven test over the comment-terminator vectors. - Hoist test-local imports to module scope per CLAUDE.md. --- .../src/auction/orchestrator.rs | 5 + .../src/auction/telemetry.rs | 1 + .../src/integrations/prebid.rs | 58 +++--- crates/trusted-server-core/src/publisher.rs | 189 +++++++++++------- crates/trusted-server-core/src/settings.rs | 8 +- 5 files changed, 165 insertions(+), 96 deletions(-) diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 145059d9e..45d90e761 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -96,6 +96,11 @@ const ERROR_TYPE_PARSE_RESPONSE: &str = "parse_response"; const ERROR_TYPE_LAUNCH_FAILED: &str = "launch_failed"; const ERROR_TYPE_TRANSPORT: &str = "transport"; const ERROR_TYPE_TIMEOUT: &str = "timeout"; +/// A non-2xx HTTP status from an upstream SSP (e.g. a PBS 4xx/5xx). Distinct +/// from [`ERROR_TYPE_TRANSPORT`] (a connection-level failure) so telemetry can +/// bucket it separately. `pub(crate)` so producers such as the prebid provider +/// tag errors with the exact value the telemetry layer recognises. +pub(crate) const ERROR_TYPE_HTTP_STATUS: &str = "http_status"; // SECURITY: the returned string is included verbatim (truncated to // PROVIDER_ERROR_MESSAGE_CHARS) in the public /auction response via diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index 4819cef6c..ac92a98f1 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -800,6 +800,7 @@ fn provider_status(response: &AuctionResponse) -> &'static str { Some("parse_response") => "parse_error", Some("transport") => "transport_error", Some("timeout") => "timeout", + Some("http_status") => "http_status_error", _ => "transport_error", }, BidStatus::Pending => "timeout", diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index c6fccc2c3..22c972f5c 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -2123,24 +2123,25 @@ impl AuctionProvider for PrebidAuctionProvider { })?; if !status.is_success() { - log::warn!("Prebid returned non-success status: {}", status,); let body_preview = String::from_utf8_lossy(&body_bytes); - if log::log_enabled!(log::Level::Trace) { - log::trace!( - "Prebid error response body: {}", - &body_preview[..body_preview.floor_char_boundary(1000)] - ); - } - // Surface the HTTP status and a body snippet on the response metadata - // so the ts-debug auction dump shows *why* prebid errored (e.g. a 4xx - // from a PBS that rejects the unsigned server-side request) without - // needing log access. A bare `AuctionResponse::error` yields empty - // metadata, which is indistinguishable from other failures in the dump. - let body_snippet = body_preview[..body_preview.floor_char_boundary(512)].to_string(); + // SECURITY: the PBS response body is upstream-controlled and may leak + // internal detail (hostnames, stack traces, auth hints). Per the + // invariant documented in `auction/orchestrator.rs`, it MUST NOT reach + // the public `/auction` response, which happens if it lands in + // `AuctionResponse.metadata` (cloned verbatim into + // `ext.orchestrator.provider_details[].metadata`). Log the snippet + // server-side and surface only the numeric HTTP status — enough for an + // operator to tell an error from a no-bid without publishing the body. + log::warn!( + "Prebid returned non-success status {status}: {}", + &body_preview[..body_preview.floor_char_boundary(512)] + ); return Ok(AuctionResponse::error("prebid", response_time_ms) - .with_metadata("error_type", serde_json::json!("http_status")) - .with_metadata("status", serde_json::json!(status.as_u16())) - .with_metadata("body", serde_json::json!(body_snippet))); + .with_metadata( + "error_type", + serde_json::json!(crate::auction::orchestrator::ERROR_TYPE_HTTP_STATUS), + ) + .with_metadata("status", serde_json::json!(status.as_u16()))); } let response_json: Json = @@ -2242,7 +2243,8 @@ mod tests { use super::*; use crate::auction::test_support::create_test_auction_context as shared_test_auction_context; use crate::auction::types::{ - AdFormat, AdSlot, AuctionContext, AuctionRequest, DeviceInfo, PublisherInfo, UserInfo, + AdFormat, AdSlot, AuctionContext, AuctionRequest, BidStatus, DeviceInfo, PublisherInfo, + UserInfo, }; use crate::consent::{ConsentContext, ConsentSource}; @@ -2351,14 +2353,14 @@ mod tests { } #[test] - fn parse_response_attaches_status_and_body_metadata_on_http_error() { - use crate::auction::types::BidStatus; - + fn parse_response_attaches_status_metadata_without_leaking_body_on_http_error() { let provider = PrebidAuctionProvider::new(base_config()); let response = PlatformResponse::new( edgezero_core::http::response_builder() .status(403) - .body(EdgeBody::from(br#"{"error":"missing signature"}"#.to_vec())) + .body(EdgeBody::from( + br#"{"error":"upstream-secret-detail"}"#.to_vec(), + )) .expect("should build test response"), ); @@ -2373,18 +2375,24 @@ mod tests { assert_eq!( result.metadata["error_type"], json!("http_status"), - "should tag the error path so the auction dump is distinguishable" + "should tag the error path so telemetry buckets it as an http status error" ); assert_eq!( result.metadata["status"], json!(403), "should surface the upstream HTTP status code" ); + // SECURITY: the upstream response body must never reach the public + // /auction response via AuctionResponse.metadata. + assert!( + !result.metadata.contains_key("body"), + "upstream response body must not be surfaced on the response metadata" + ); assert!( - result.metadata["body"] + !result.metadata.values().any(|v| v .as_str() - .is_some_and(|body| body.contains("missing signature")), - "should include the response body snippet" + .is_some_and(|s| s.contains("upstream-secret-detail"))), + "no metadata value may contain the upstream body" ); } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index c14410121..56d3c77f6 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -855,12 +855,23 @@ pub(crate) fn write_bids_to_state( *ad_bids_state.lock().expect("should lock bid state") = Some(bids_script); } -/// Prepend an HTML comment summarising the auction result onto the shared -/// `ad_bids_state` so it lands directly before the injected bids `` breakout and `U+2028/2029`. Requirement: a regression test proving a +hostile `adm` containing `` (and `U+2028/2029`) cannot break out of the +injected `` / + `U+2028/2029` `adm` is escaped so `build_bids_script` output stays inside the + `" + ) +} +``` + +- [ ] **Step 4: Update the caller** at `~853/854` to pass `settings.debug.inject_adm_for_testing`; update the empty-bids helper at `~2033` and any test expectations that pin the old script string. + +- [ ] **Step 5: Run — expect PASS.** `cargo test-fastly bids_script_emits_inject_adm_for_testing_flag` + +- [ ] **Step 6: Commit** — `git commit -m "Emit injectAdmForTesting flag on window.tsjs with bids"` + +--- + +## Task 3: Escaping regression — hostile `adm` cannot break out of `` + `U+2028` adm is neutralized in the emitted script. + +```rust +#[test] +fn build_bids_script_escapes_hostile_adm() { + let mut winning = std::collections::HashMap::new(); + let mut bid = /* Bid, price Some(1.0), creative Some("\u{2028}") */; + winning.insert("s".to_string(), bid); + let map = build_bid_map(&winning, PriceGranularity::Dense, true, false); + let script = build_bids_script(&map, false); + // Raw must not survive; U+2028 must be unicode-escaped. + assert!(!script.contains("" - ) -} -``` - -- [ ] **Step 4: Update `build_bids_script` callers:** - - `write_bids_to_state` (`~854`): pass the `inject_adm_for_testing` param threaded in Task 1. - - `build_empty_bids_script` (`~2032`) has **no settings access** — pass `false` (no bids ⇒ no `adm` ⇒ flag inert). Documented tradeoff: an empty *initial* nav on a testing build emits `injectAdmForTesting=false`, so a later SPA-loaded `adm` won't fire the test bypass; acceptable (production is always `false`). - - Fix any test that pins the old `bids` `\u{2028}") */; - winning.insert("s".to_string(), bid); - let map = build_bid_map(&winning, PriceGranularity::Dense, true, false); - let script = build_bids_script(&map, false); - // Raw must not survive; U+2028 must be unicode-escaped. - assert!(!script.contains("\u{2028}"), + ); + let map = build_bid_map(&winning, PriceGranularity::Dense, false); + let script = build_bids_script(&map); + assert!( + !script.contains("` breakout and `U+2028/2029`. Requirement: a regression test proving a -hostile `adm` containing `` (and `U+2028/2029`) cannot break out of the -injected `` breakout and `U+2028/2029`. **This is the guarantee trusted-server +directly provides**, pinned by a hostile-`adm` regression test. + +Frame isolation of the rendered creative is **not** guaranteed by TS on the +bridge path: `injectAdmIntoSlot` sets `sandbox=ADM_IFRAME_SANDBOX`, but the +bridge renderer hands `adm` to the PUC-provided `mkFrame`, which TS neither sets +nor verifies a sandbox on. Bridge isolation therefore depends on the Prebid +Universal Creative implementation, not on TS. ## Components changed | Unit | Change | | --- | --- | -| `build_bid_map` (Rust) | Split `include_adm` → `render_adm` (always) + `debug_bid` (testing). Always insert `adm` for winners. | -| `build_bid_map` callers | Pass `render_adm = true`; `debug_bid = inject_adm_for_testing`. | -| tsjs config injection (Rust→JS) | Surface `injectAdmForTesting` flag. | -| `gpt/index.ts` `injectAdmIntoSlot` call site | Gate on the injected `injectAdmForTesting` flag, not bare `bid.adm`. | +| `build_bid_map` (Rust) | Always insert `adm` when `bid.creative` is `Some`. Rename `include_adm` → `include_debug_bid`, gating only the `debug_bid` blob. | +| `build_bid_map` callers | Pass `include_debug_bid = inject_adm_for_testing`. | +| `gpt/index.ts` `injectAdmIntoSlot` call site | Gate on `bid.adm && bid.debug_bid`. | +| bridge/`ad_init` tests (JS) | Rename "debug adm" → "inline/local adm"; confirm existing coverage. | + +No `build_bids_script` change, no `window.tsjs` flag, no `TsjsApi` change. ## Data flow (after) @@ -110,40 +122,42 @@ SSAT auction → winner (bid.creative held) → build_bid_map inserts adm → build_bids_script (html_escape_for_script) → window.tsjs.bids → hb_pb targeting → GAM competes ├ GAM picks TS line item → PUC "Prebid Request" - │ → bridge replies with local adm → RENDER (no round trip) + │ → bridge replies with local adm → RENDER (no round trip) + beacons │ → (adm absent) → PBS Cache fetch → RENDER (fallback) └ GAM has higher demand → GAM serves its own creative ``` -## Testing - -- **Rust**: `build_bid_map` includes `adm` for winners on the production path; - `debug_bid` present only under the testing flag; a hostile `` / - `U+2028/2029` `adm` is escaped so `build_bids_script` output stays inside the - `` / `U+2028/2029` + `adm` is escaped so `build_bids_script` output stays inside the `\u{2028}"), - ); + let mut bid = make_bid("s", 1.50, "kargo", "abc123", "https://ssp/win", "https://ssp/bill"); + // Both line/paragraph separators — the spec promises escaping for each. + bid.creative = Some("\u{2028}\u{2029}".to_string()); + winning.insert("s".to_string(), bid); let map = build_bid_map(&winning, PriceGranularity::Dense, false); let script = build_bids_script(&map); assert!( @@ -115,8 +119,8 @@ fn build_bids_script_escapes_hostile_adm() { "should not let a hostile adm break out of the script context" ); assert!( - !script.contains('\u{2028}'), - "should unicode-escape U+2028 in the adm" + !script.contains('\u{2028}') && !script.contains('\u{2029}'), + "should unicode-escape both U+2028 and U+2029 in the adm" ); } ``` @@ -135,12 +139,15 @@ Run: `cargo test-fastly build_bids_script_escapes_hostile_adm` - Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts:~599` - Test: `crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts` -- [ ] **Step 1: Write failing vitest** — bypass does NOT fire in production (no `debug_bid`), even with `bid.adm`. +- [ ] **Step 1: Write failing vitest — observable behavior (not a spy).** `injectAdmIntoSlot` is module-private, so assert its *effect* on the DOM: ```ts -// window.tsjs.bids = { 'ad-header-0': { adm: '
x
' } } // no debug_bid -// simulate slotRenderEnded for ad-header-0 -// spy on injectAdmIntoSlot → assert NOT called (the bridge handles render) +// Setup: +// bids['ad-header-0'] = { adm: '' } // NO debug_bid +// place an existing GAM iframe (src="about:blank") in the slot div +// capture the slotRenderEnded listener, fire it for 'ad-header-0' +// Assert (production): the GAM iframe src stays 'about:blank' +// — the bypass did not fire; the render bridge handles it. ``` - [ ] **Step 2: Run — expect FAIL.** `cd crates/trusted-server-js/lib && npx vitest run ad_init` @@ -156,7 +163,7 @@ if (bid.adm && bid.debug_bid) { } ``` -- [ ] **Step 4: Add companion test** — with `bid.debug_bid` present, `injectAdmIntoSlot` IS called. +- [ ] **Step 4: Add companion test (testing mode)** — same setup but with `bid.debug_bid` present. Fire `slotRenderEnded` → assert the slot iframe's `src` **changes to** the creative URL (`https://cdn.example/creative.html`), proving `injectAdmIntoSlot` ran. - [ ] **Step 5: Run — expect PASS.** @@ -194,8 +201,9 @@ concurrency + beacon dedup. Do **not** duplicate them. cargo clippy-spin-wasm ``` - [ ] **Step 4:** `cd crates/trusted-server-js/lib && npx vitest run && npm run format && node build-all.mjs` -- [ ] **Step 5:** Manual: with `[debug].auction_html_comment` off, load a nav page; confirm the winning creative renders **without** a request to `hb_cache_host` (Network tab) and GAM still received `hb_pb`. -- [ ] **Step 6: Commit** any format fixes. +- [ ] **Step 5:** Docs format (these spec/plan docs changed): `cd docs && npm run format` +- [ ] **Step 6:** Manual: with `[debug].auction_html_comment` off, load a nav page; confirm the winning creative renders **without** a request to `hb_cache_host` (Network tab) and GAM still received `hb_pb`. +- [ ] **Step 7: Commit** any format fixes. --- From 6649875643b5c70b2cf218fbf477a9fa9eaae47a Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 13 Jul 2026 22:53:35 +0530 Subject: [PATCH 016/198] Always include adm in bid map; gate only debug_bid blob build_bid_map now always inserts the winning creative as adm so the pbRender bridge can render it locally (no PBS Cache round trip); the verbose debug_bid blob and the GAM-bypass gate stay behind inject_adm_for_testing. Rename the param include_adm -> include_debug_bid and thread it through write_bids_to_state. Reconcile the by-default test to the new behavior, drop the now-redundant debug-only-adm test, and pin script-context escaping for a hostile adm ( + U+2028/U+2029). --- crates/trusted-server-core/src/publisher.rs | 66 ++++++++++++--------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 6b0ea3a5d..f32d32891 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -843,14 +843,14 @@ pub(crate) fn write_bids_to_state( winning_bids: &std::collections::HashMap, price_granularity: PriceGranularity, ad_bids_state: &Arc>>, - inject_adm: bool, + include_debug_bid: bool, ) { log::debug!( "write_bids_to_state: {} winning bid(s): [{}]", winning_bids.len(), winning_bids.keys().cloned().collect::>().join(", ") ); - let bid_map = build_bid_map(winning_bids, price_granularity, inject_adm); + let bid_map = build_bid_map(winning_bids, price_granularity, include_debug_bid); let bids_script = build_bids_script(&bid_map); *ad_bids_state.lock().expect("should lock bid state") = Some(bids_script); } @@ -1933,7 +1933,7 @@ fn html_escape_for_script(s: &str) -> String { pub(crate) fn build_bid_map( winning_bids: &std::collections::HashMap, granularity: crate::price_bucket::PriceGranularity, - include_adm: bool, + include_debug_bid: bool, ) -> serde_json::Map { winning_bids .iter() @@ -1978,12 +1978,16 @@ pub(crate) fn build_bid_map( if let Some(ref burl) = bid.burl { obj.insert("burl".to_string(), serde_json::Value::String(burl.clone())); } - // Include raw creative markup only for explicit debug injection. - // The pbRender bridge can use it while PBS Cache is unavailable. - if include_adm { - if let Some(ref adm) = bid.creative { - obj.insert("adm".to_string(), serde_json::Value::String(adm.clone())); - } + // Always include the winning creative so the pbRender bridge can + // render it locally when GAM serves the Prebid Universal Creative + // — no PBS Cache round trip. The `hb_cache_*` coordinates above + // remain as the fallback for an absent `adm`. + if let Some(ref adm) = bid.creative { + obj.insert("adm".to_string(), serde_json::Value::String(adm.clone())); + } + // Verbose per-bid debug blob only under the testing flag; also + // doubles as the client-side gate for the direct GAM-replace path. + if include_debug_bid { obj.insert( "debug_bid".to_string(), serde_json::json!({ @@ -4071,7 +4075,7 @@ mod tests { } #[test] - fn client_bid_map_omits_adm_by_default() { + fn client_bid_map_includes_adm_and_omits_debug_bid_by_default() { let mut winning_bids = HashMap::new(); let mut bid = make_bid( "atf_sidebar_ad", @@ -4084,6 +4088,9 @@ mod tests { bid.creative = Some("
Creative
".to_string()); winning_bids.insert("atf_sidebar_ad".to_string(), bid); + // Production path (include_debug_bid = false): the creative is always + // included so the bridge can render it locally, but the verbose + // debug_bid blob is not. let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); let obj = map .get("atf_sidebar_ad") @@ -4091,41 +4098,42 @@ mod tests { .as_object() .expect("should be object"); - assert!( - obj.get("adm").is_none(), - "should omit adm when debug injection is disabled" + assert_eq!( + obj.get("adm").and_then(|v| v.as_str()), + Some("
Creative
"), + "should include creative markup for local rendering by default" ); assert!( obj.get("debug_bid").is_none(), - "should omit debug bid when debug injection is disabled" + "should omit the debug_bid blob when debug injection is disabled" ); } #[test] - fn client_bid_map_includes_adm_when_debug_injection_enabled() { + fn build_bids_script_escapes_hostile_adm() { let mut winning_bids = HashMap::new(); let mut bid = make_bid( - "atf_sidebar_ad", + "s", 1.50, "kargo", "abc123", "https://ssp/win", "https://ssp/bill", ); - bid.creative = Some("
Creative
".to_string()); - winning_bids.insert("atf_sidebar_ad".to_string(), bid); - - let map = build_bid_map(&winning_bids, PriceGranularity::Dense, true); - let obj = map - .get("atf_sidebar_ad") - .expect("should have bid entry") - .as_object() - .expect("should be object"); + // A hostile creative that tries to break out of the \u{2028}\u{2029}".to_string()); + winning_bids.insert("s".to_string(), bid); - assert_eq!( - obj.get("adm").and_then(|v| v.as_str()), - Some("
Creative
"), - "should include adm when debug injection is enabled" + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, false); + let script = build_bids_script(&map); + assert!( + !script.contains("` + `U+2028` adm is neutralized in the emitted script. @@ -136,10 +138,11 @@ Run: `cargo test-fastly build_bids_script_escapes_hostile_adm` ## Task 3: Gate the GAM-bypass (`injectAdmIntoSlot`) on `bid.debug_bid` **Files:** + - Modify: `crates/trusted-server-js/lib/src/integrations/gpt/index.ts:~599` - Test: `crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts` -- [ ] **Step 1: Write failing vitest — observable behavior (not a spy).** `injectAdmIntoSlot` is module-private, so assert its *effect* on the DOM: +- [ ] **Step 1: Write failing vitest — observable behavior (not a spy).** `injectAdmIntoSlot` is module-private, so assert its _effect_ on the DOM: ```ts // Setup: @@ -159,7 +162,7 @@ Run: `cargo test-fastly build_bids_script_escapes_hostile_adm` // when inject_adm_for_testing is on, so it doubles as the per-bid gate — no // global flag needed, and it is correct across SPA auction responses. if (bid.adm && bid.debug_bid) { - injectAdmIntoSlot(divId, bid.adm); + injectAdmIntoSlot(divId, bid.adm) } ``` @@ -174,6 +177,7 @@ if (bid.adm && bid.debug_bid) { ## Task 4: Reconcile existing bridge tests (no duplicates) **Files:** + - Modify: `crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts` `ad_init.test.ts` already covers: PBS Cache fetch when `adm` absent; local `adm` @@ -208,7 +212,8 @@ concurrency + beacon dedup. Do **not** duplicate them. --- ## Notes -- Do NOT remove `hb_cache_host`/`hb_cache_path` — they are the fallback for an **absent** `adm`. Render failure *after* `adm` is supplied is not detectable and does not fall back (spec Risks). + +- Do NOT remove `hb_cache_host`/`hb_cache_path` — they are the fallback for an **absent** `adm`. Render failure _after_ `adm` is supplied is not detectable and does not fall back (spec Risks). - Do NOT ship the `debug_bid` blob in production (Task 1 keeps it behind the flag). - No global `window.tsjs` flag, no `TsjsApi` change — the bypass gate is the per-bid `debug_bid`. - Page-weight cost (inline creatives, uncacheable response) accepted per spec; size-capping out of scope. diff --git a/docs/superpowers/specs/2026-07-13-ssat-render-inline-creative-design.md b/docs/superpowers/specs/2026-07-13-ssat-render-inline-creative-design.md index 8c826d632..8dee3f452 100644 --- a/docs/superpowers/specs/2026-07-13-ssat-render-inline-creative-design.md +++ b/docs/superpowers/specs/2026-07-13-ssat-render-inline-creative-design.md @@ -14,7 +14,7 @@ time from PBS Cache: https://?uuid= ``` -This is an extra network round trip *after* the GAM call, even though +This is an extra network round trip _after_ the GAM call, even though trusted-server already holds the winning creative markup (`bid.creative`) from the server-side auction it just ran. The client-side `/auction` flow never does this — Prebid.js renders the winner from the copy it already has in the browser. @@ -27,7 +27,7 @@ while keeping GAM in the loop (the header bid still competes against GAM's own demand via `hb_pb`). Non-goal: bypassing GAM. SSAT winners must still compete in GAM; we only remove -the round trip that happens *after* GAM has already picked the TS line item. +the round trip that happens _after_ GAM has already picked the TS line item. ## Current flow (verified in code) @@ -47,7 +47,7 @@ the round trip that happens *after* GAM has already picked the TS line item. - **fetches from PBS Cache** using `hb_cache_host`/`hb_cache_path` (the round trip we want to remove). 5. A separate consumer, `injectAdmIntoSlot` ([gpt/index.ts:599]), fires on - `if (bid.adm)` and **replaces the GAM creative directly** — a GAM *bypass*. + `if (bid.adm)` and **replaces the GAM creative directly** — a GAM _bypass_. Its "testing only" status is a comment, not an actual gate. ## Design @@ -55,6 +55,7 @@ the round trip that happens *after* GAM has already picked the TS line item. ### 1. Always include the render `adm`; keep `debug_bid` gated `build_bid_map`: + - **Always** insert `adm` (from `bid.creative`) for a winner when present — there is no runtime reason to withhold it, so it is not parameterized. - Insert the verbose `debug_bid` blob **only** when the testing flag is set. The @@ -62,7 +63,7 @@ the round trip that happens *after* GAM has already picked the TS line item. `hb_cache_host`/`hb_cache_path` remain inserted unconditionally. -### 2. Bridge renders local `adm`; cache is the fallback for an *absent* `adm` +### 2. Bridge renders local `adm`; cache is the fallback for an _absent_ `adm` `installTsRenderBridge` already prefers `matchedBid.adm` and falls back to PBS Cache. Once `adm` is present in production, the local render becomes the default @@ -70,7 +71,7 @@ and the round trip disappears. **Fallback scope (corrected):** the bridge posts the markup to the PUC and returns; it receives **no render-success signal**. So the PBS Cache fallback -fires only when `adm` is **absent or empty** — *not* when `adm` is present but +fires only when `adm` is **absent or empty** — _not_ when `adm` is present but fails to render. Render failures after `adm` is supplied are not currently detectable and do not trigger fallback. @@ -84,7 +85,7 @@ the bypass on the per-bid `debug_bid` field, which is already present **iff** ```ts if (bid.adm && bid.debug_bid) { - injectAdmIntoSlot(divId, bid.adm); + injectAdmIntoSlot(divId, bid.adm) } ``` @@ -106,12 +107,12 @@ Universal Creative implementation, not on TS. ## Components changed -| Unit | Change | -| --- | --- | -| `build_bid_map` (Rust) | Always insert `adm` when `bid.creative` is `Some`. Rename `include_adm` → `include_debug_bid`, gating only the `debug_bid` blob. | -| `build_bid_map` callers | Pass `include_debug_bid = inject_adm_for_testing`. | -| `gpt/index.ts` `injectAdmIntoSlot` call site | Gate on `bid.adm && bid.debug_bid`. | -| bridge/`ad_init` tests (JS) | Rename "debug adm" → "inline/local adm"; confirm existing coverage. | +| Unit | Change | +| -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `build_bid_map` (Rust) | Always insert `adm` when `bid.creative` is `Some`. Rename `include_adm` → `include_debug_bid`, gating only the `debug_bid` blob. | +| `build_bid_map` callers | Pass `include_debug_bid = inject_adm_for_testing`. | +| `gpt/index.ts` `injectAdmIntoSlot` call site | Gate on `bid.adm && bid.debug_bid`. | +| bridge/`ad_init` tests (JS) | Rename "debug adm" → "inline/local adm"; confirm existing coverage. | No `build_bids_script` change, no `window.tsjs` flag, no `TsjsApi` change. @@ -129,7 +130,7 @@ SSAT auction → winner (bid.creative held) → build_bid_map inserts adm ## Precondition -This changes only the render bridge's *data source* — local `adm` vs a PBS Cache +This changes only the render bridge's _data source_ — local `adm` vs a PBS Cache fetch — **when GAM's Prebid line item already serves the PUC**. It does not change GAM competition, nor whether the PUC fires. A publisher without Prebid line items in GAM sees no behavioral change. From c8ae53f78014458241753f53db630420ff119132 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 14 Jul 2026 15:49:42 +0530 Subject: [PATCH 019/198] Harden Fastly publisher streaming against review findings Resolve five correctness and resource-safety findings from the PR #867 review of the end-to-end Fastly publisher streaming path. - Drive the deflate decoder to StreamEnd at finalization so a valid stream that exactly fills the internal output buffer is no longer rejected as truncated; the inflater is also drained after all input is consumed within a chunk. - Decode concatenated (multi-member) gzip bodies via MultiGzDecoder on both the streaming decoder and the buffered read pipeline so adapters agree. - Enforce the decoded-body cap during decompression through a bounded sink shared by the gzip and brotli codecs, so a compression bomb errors before its expanded bytes are buffered instead of after a full chunk expands; the deflate codec charges each produced block as it is emitted. - Drop the body of bodiless responses (HEAD, 204, 205, 304) in both the streaming and buffered finalizer Buffered arms, and add RESET_CONTENT to response_carries_body, so a buffered-unmodified stream body is never streamed to the client for a response that must be bodiless. - Keep the dispatched-auction guard armed across the collection await and disarm it only once collection reaches a terminal result, so a body dropped while collection is pending still logs the discarded SSP work. Add regression tests for the deflate output-buffer boundary, multi-member gzip, bodiless buffered stream bodies, and the auction guard sentinel. --- crates/trusted-server-core/src/publisher.rs | 166 +++++++- .../src/streaming_processor.rs | 391 +++++++++++++++--- 2 files changed, 484 insertions(+), 73 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4d7e38cf9..3f9160201 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -612,23 +612,40 @@ fn passthrough_finish_segments( /// error paths that can still await (see [`abandon_hold_auction`]). struct DispatchedAuctionGuard { dispatched: Option, + /// Stays `true` from dispatch until collection (or telemetry-emitting + /// abandonment) reaches a terminal result. [`Self::take`] removes the + /// dispatched auction to hand it to the async collector but deliberately + /// leaves the guard armed, so a drop *while collection is still pending* — + /// a client disconnect at the collection await point — still logs the + /// loss. [`Self::disarm`] clears it only once collection has completed. + armed: bool, } impl DispatchedAuctionGuard { fn new(dispatched: DispatchedAuction) -> Self { Self { dispatched: Some(dispatched), + armed: true, } } + /// Remove the dispatched auction to begin collection. The guard stays armed + /// until [`Self::disarm`] is called, so a drop before collection reaches a + /// terminal result is still reported. fn take(&mut self) -> Option { self.dispatched.take() } + + /// Disarm the drop warning once collection (or telemetry-emitting + /// abandonment) has reached a terminal result. + fn disarm(&mut self) { + self.armed = false; + } } impl Drop for DispatchedAuctionGuard { fn drop(&mut self) { - if self.dispatched.is_some() { + if self.armed { log::warn!( "Dispatched server-side auction dropped without collection; SSP bid responses discarded (publisher body stream aborted or never polled)" ); @@ -668,6 +685,10 @@ async fn abandon_hold_auction( reason, ) .await; + // Abandonment with telemetry is a terminal result, so the drop warning + // is no longer warranted. (A drop *during* the emit above still fires + // it, since the guard stays armed until here.) + state.dispatched.disarm(); } } @@ -721,6 +742,9 @@ async fn hold_step_decoded_chunk( collect_refs.settings, ) .await; + // Collection reached a terminal result; disarm only now so a drop + // while the collect await above was still pending is reported. + state.dispatched.disarm(); let held = state .hold @@ -835,6 +859,9 @@ async fn hold_finish_segments( collect_refs.settings, ) .await; + // Collection reached a terminal result; disarm only now so a drop while + // the collect await above was still pending is reported. + state.dispatched.disarm(); let held = hold.finish(); if let Some(encoded) = process_and_encode_chunk( @@ -1046,7 +1073,17 @@ pub async fn buffer_publisher_response_async( services: &RuntimeServices, ) -> Result, Report> { match publisher_response { - PublisherResponse::Buffered(response) => Ok(response), + PublisherResponse::Buffered(mut response) => { + // A buffered-unmodified response can carry an origin body (a stream + // on streaming-capable adapters). A bodiless response (HEAD, 204, + // 205, 304) must stay bodiless, so drop the body while preserving + // metadata such as `Content-Length`, matching the streaming + // finalizer. + if !response_carries_body(method, response.status()) { + *response.body_mut() = EdgeBody::empty(); + } + Ok(response) + } PublisherResponse::Stream { mut response, body, @@ -1113,7 +1150,18 @@ pub async fn publisher_response_into_streaming_response( services: RuntimeServices, ) -> Result, Report> { match publisher_response { - PublisherResponse::Buffered(response) => Ok(response), + PublisherResponse::Buffered(mut response) => { + // Fastly requests the origin body as a stream before the response is + // classified, so a buffered-unmodified response can still hold an + // `EdgeBody::Stream`. A bodiless response (HEAD, 204, 205, 304) must + // stay bodiless — `send_edgezero_response` streams any + // `EdgeBody::Stream` to the client — so drop the body while + // preserving metadata such as `Content-Length`. + if !response_carries_body(method, response.status()) { + *response.body_mut() = EdgeBody::empty(); + } + Ok(response) + } PublisherResponse::PassThrough { mut response, body } => { if response_carries_body(method, response.status()) { *response.body_mut() = body; @@ -1196,6 +1244,10 @@ pub async fn publisher_response_into_streaming_response( &settings, ) .await; + // Collection reached a terminal result; disarm only now + // so a drop while the collect await above was still + // pending is reported. + guard.disarm(); } } @@ -1268,12 +1320,15 @@ pub async fn publisher_response_into_streaming_response( /// Returns `true` when a buffered publisher response should carry a body and a /// recomputed `Content-Length`. /// -/// `HEAD` responses and bodiless statuses (204, 304) carry no body; rewriting -/// their `Content-Length` to the (empty) buffered length would mislead clients -/// and caches, so the origin metadata is preserved instead. +/// `HEAD` responses and bodiless statuses (204, 205, 304) carry no body; +/// rewriting their `Content-Length` to the (empty) buffered length — or +/// streaming an origin body for them at all — would mislead clients and caches +/// and violate HTTP framing, so the origin metadata is preserved and the body +/// is dropped instead. fn response_carries_body(method: &Method, status: StatusCode) -> bool { *method != Method::HEAD && status != StatusCode::NO_CONTENT + && status != StatusCode::RESET_CONTENT && status != StatusCode::NOT_MODIFIED } @@ -3755,12 +3810,44 @@ mod tests { !super::response_carries_body(&Method::GET, StatusCode::NO_CONTENT), "204 responses must not get a recomputed Content-Length" ); + assert!( + !super::response_carries_body(&Method::GET, StatusCode::RESET_CONTENT), + "205 responses must not get a recomputed Content-Length" + ); assert!( !super::response_carries_body(&Method::GET, StatusCode::NOT_MODIFIED), "304 responses must not get a recomputed Content-Length" ); } + #[test] + fn dispatched_auction_guard_stays_armed_until_collection_completes() { + // `take()` hands the dispatched auction to the async collector, but the + // guard must stay armed across the collection await so a drop while + // collection is still pending (a client disconnect at the await point) + // still logs the loss. Only `disarm()` — called once collection reaches + // a terminal result — clears the warning. + let mut guard = DispatchedAuctionGuard::new(DispatchedAuction::empty_for_test( + test_auction_request(), + 10, + )); + assert!(guard.armed, "a freshly dispatched guard should be armed"); + + let _dispatched = guard + .take() + .expect("guard should yield the dispatched auction for collection"); + assert!( + guard.armed, + "guard must stay armed across the collection await so a drop mid-collection is reported" + ); + + guard.disarm(); + assert!( + !guard.armed, + "guard must disarm once collection reaches a terminal result" + ); + } + fn response_body_string(response: http::Response) -> String { String::from_utf8( response @@ -5354,6 +5441,73 @@ mod tests { ); } + #[test] + fn publisher_response_streaming_finalize_drops_bodiless_buffered_stream_body() { + // Fastly requests the origin body as a stream before classification, so + // a buffered-unmodified response can hold an `EdgeBody::Stream`. The + // adapter streams any `EdgeBody::Stream` to the client, so bodiless + // responses must be normalized to carry no body while keeping metadata. + let settings = Arc::new(create_test_settings()); + let registry = Arc::new( + IntegrationRegistry::new(&settings).expect("should create integration registry"), + ); + let orchestrator = Arc::new(AuctionOrchestrator::new(settings.auction.clone())); + + let cases = [ + (Method::HEAD, StatusCode::OK), + (Method::GET, StatusCode::NO_CONTENT), + (Method::GET, StatusCode::RESET_CONTENT), + (Method::GET, StatusCode::NOT_MODIFIED), + ]; + + for (method, status) in cases { + let response = Response::builder() + .status(status) + .header(header::CONTENT_LENGTH, "42") + .body(EdgeBody::stream(futures::stream::iter(vec![ + bytes::Bytes::from_static(b"origin body bytes that must not reach the client"), + ]))) + .expect("should build response"); + let publisher_response = PublisherResponse::Buffered(response); + + let response = futures::executor::block_on(publisher_response_into_streaming_response( + publisher_response, + &method, + Arc::clone(&settings), + registry.as_ref(), + Arc::clone(&orchestrator), + noop_services(), + )) + .expect("should finalize buffered response"); + + assert!( + !matches!(response.body(), EdgeBody::Stream(_)), + "bodiless {method} {status} must not carry a streaming body" + ); + assert_eq!( + response + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()), + Some("42"), + "bodiless {method} {status} must preserve the origin Content-Length" + ); + + let drained = futures::executor::block_on( + response + .into_body() + .into_bytes_bounded(settings.publisher.max_buffered_body_bytes), + ) + .expect("body should drain") + .to_vec(); + assert!( + drained.is_empty(), + "bodiless {method} {status} must deliver zero body bytes, got {} bytes", + drained.len() + ); + } + } + #[test] fn publisher_response_streaming_finalize_processes_gzip_stream() { let compressed = diff --git a/crates/trusted-server-core/src/streaming_processor.rs b/crates/trusted-server-core/src/streaming_processor.rs index 963c69daa..e7e9a92bb 100644 --- a/crates/trusted-server-core/src/streaming_processor.rs +++ b/crates/trusted-server-core/src/streaming_processor.rs @@ -19,7 +19,7 @@ //! streaming interface. See `crate::platform` module doc for the //! authoritative note. -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::io::{self, Read, Write}; use std::rc::Rc; @@ -27,7 +27,7 @@ use brotli::enc::writer::CompressorWriter; use brotli::enc::BrotliEncoderParams; use brotli::Decompressor; use error_stack::{Report, ResultExt as _}; -use flate2::read::{GzDecoder, ZlibDecoder}; +use flate2::read::{MultiGzDecoder, ZlibDecoder}; use flate2::write::{GzEncoder, ZlibEncoder}; use crate::error::TrustedServerError; @@ -144,7 +144,10 @@ impl StreamingPipeline

{ ) { (Compression::None, Compression::None) => self.process_chunks(input, output), (Compression::Gzip, Compression::Gzip) => { - let decoder = GzDecoder::new(input); + // Multi-member decoder: RFC 1952 permits concatenated gzip + // members, so a single-member reader would stop after the first. + // Matches the streaming `BodyStreamDecoder` gzip codec. + let decoder = MultiGzDecoder::new(input); let mut encoder = GzEncoder::new(output, flate2::Compression::default()); self.process_chunks(decoder, &mut encoder)?; encoder.finish().change_context(TrustedServerError::Proxy { @@ -153,7 +156,7 @@ impl StreamingPipeline

{ Ok(()) } (Compression::Gzip, Compression::None) => { - self.process_chunks(GzDecoder::new(input), output) + self.process_chunks(MultiGzDecoder::new(input), output) } (Compression::Deflate, Compression::Deflate) => { let decoder = ZlibDecoder::new(input); @@ -360,27 +363,106 @@ pub(crate) const STREAM_CHUNK_SIZE: usize = 8192; /// out of the internal buffer after every push. Write-based decoders are /// used because the async publisher path cannot wrap a blocking `Read`. /// -/// Decoded output is capped cumulatively: the chunk source only bounds raw -/// (still compressed) bytes, and a decompression bomb can expand ~1000x past -/// that, so the decoder enforces its own ceiling on the total bytes it emits. +/// Decoded output is capped cumulatively and the cap is enforced *during* +/// decompression, not after: the chunk source only bounds raw (still +/// compressed) bytes, and a decompression bomb can expand ~1000x past that, so +/// a small compressed chunk must not be allowed to fully expand before the +/// ceiling is checked. The gzip and brotli codecs decode into a +/// [`BoundedDecodeSink`] that errors the moment a write would exceed the limit; +/// the deflate codec charges each produced output block as it is emitted. /// /// Every codec validates end-of-stream at [`Self::finish`] so a truncated /// origin body errors instead of silently truncating the page: gzip via its -/// trailer checksum, brotli via `close()`, and deflate via an explicit -/// [`flate2::Status::StreamEnd`] check (`write::ZlibDecoder` accepts -/// truncated input silently, so the deflate arm drives [`flate2::Decompress`] -/// directly). +/// trailer checksum, brotli via `close()`, and deflate by driving +/// [`flate2::Decompress`] to its [`flate2::Status::StreamEnd`] marker (the +/// `write`-based zlib decoder accepts truncated input silently, so the deflate +/// arm drives [`flate2::Decompress`] directly). Concatenated gzip members +/// (RFC 1952) are decoded via [`flate2::write::MultiGzDecoder`]. pub(crate) struct BodyStreamDecoder { codec: BodyStreamDecoderCodec, - decoded_bytes: usize, + /// Cumulative decoded byte count, shared with the codec sinks so the cap is + /// enforced from inside the decompressor writes rather than after them. + decoded_bytes: Rc>, max_decoded_bytes: usize, } enum BodyStreamDecoderCodec { None, - Gzip(flate2::write::GzDecoder>), + Gzip(flate2::write::MultiGzDecoder), Deflate(DeflateStreamDecoder), - Brotli(Box>>), + Brotli(Box>), +} + +/// A [`Write`] sink that buffers decoded bytes while enforcing a shared +/// cumulative decode budget. +/// +/// The gzip and brotli decoders write their decompressed output here as they +/// process input. Rejecting the write as soon as it would push the cumulative +/// decoded total past `max_decoded_bytes` makes the cap a hard ceiling on +/// Wasm-heap growth: a decompression bomb errors before its expanded bytes are +/// buffered, rather than after a full chunk has already expanded. +struct BoundedDecodeSink { + buffer: Vec, + decoded_bytes: Rc>, + max_decoded_bytes: usize, +} + +impl BoundedDecodeSink { + fn new(decoded_bytes: Rc>, max_decoded_bytes: usize) -> Self { + Self { + buffer: Vec::new(), + decoded_bytes, + max_decoded_bytes, + } + } +} + +impl Write for BoundedDecodeSink { + fn write(&mut self, data: &[u8]) -> io::Result { + let next = self + .decoded_bytes + .get() + .checked_add(data.len()) + .ok_or_else(|| { + io::Error::other("publisher origin body decoded byte count overflowed") + })?; + if next > self.max_decoded_bytes { + return Err(io::Error::other(format!( + "publisher origin body decoded size exceeded {}-byte streaming limit", + self.max_decoded_bytes + ))); + } + self.decoded_bytes.set(next); + self.buffer.extend_from_slice(data); + Ok(data.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +/// Charge `len` decoded bytes against `decoded_bytes`, erroring if the +/// cumulative total would exceed `max_decoded_bytes`. +fn charge_decoded( + decoded_bytes: &Cell, + max_decoded_bytes: usize, + len: usize, +) -> Result<(), Report> { + let next = decoded_bytes.get().checked_add(len).ok_or_else(|| { + Report::new(TrustedServerError::Proxy { + message: "publisher origin body decoded byte count overflowed".to_string(), + }) + })?; + if next > max_decoded_bytes { + return Err(Report::new(TrustedServerError::Proxy { + message: format!( + "publisher origin body decoded size exceeded {max_decoded_bytes}-byte streaming limit" + ), + })); + } + decoded_bytes.set(next); + Ok(()) } /// Streaming zlib decoder that tracks whether the stream reached its end @@ -388,22 +470,40 @@ enum BodyStreamDecoderCodec { struct DeflateStreamDecoder { decompress: flate2::Decompress, stream_ended: bool, + decoded_bytes: Rc>, + max_decoded_bytes: usize, } impl DeflateStreamDecoder { - fn new() -> Self { + fn new(decoded_bytes: Rc>, max_decoded_bytes: usize) -> Self { Self { decompress: flate2::Decompress::new(true), stream_ended: false, + decoded_bytes, + max_decoded_bytes, } } + /// Charge `len` decoded bytes against the shared budget. + fn charge(&self, len: usize) -> Result<(), Report> { + charge_decoded(&self.decoded_bytes, self.max_decoded_bytes, len) + } + + /// Decode as much of `chunk` as possible, draining any output the inflater + /// can still produce once all input is consumed. + /// + /// flate2 fills the output buffer up to its capacity, so a chunk that + /// exactly fills the buffer leaves decoded bytes (and possibly the + /// end-of-stream marker) pending with all input already consumed. The loop + /// keeps driving the inflater — reserving more output space — until it + /// makes no further progress, so those pending bytes are never stranded and + /// a valid stream is not mistaken for a truncated one at `finish`. fn decode(&mut self, chunk: &[u8]) -> Result, Report> { let mut output = Vec::with_capacity(STREAM_CHUNK_SIZE); let mut offset = 0usize; // Trailing bytes after the zlib end marker are ignored, matching the // read-based decoder used by the buffered pipeline. - while offset < chunk.len() && !self.stream_ended { + while !self.stream_ended { if output.len() == output.capacity() { output.reserve(STREAM_CHUNK_SIZE); } @@ -418,36 +518,85 @@ impl DeflateStreamDecoder { let consumed = (self.decompress.total_in() - before_in) as usize; let produced = (self.decompress.total_out() - before_out) as usize; offset += consumed; + self.charge(produced)?; match status { flate2::Status::StreamEnd => self.stream_ended = true, flate2::Status::Ok | flate2::Status::BufError => { + // Stop only when the inflater is starved for input: it made + // no progress and there is still spare output capacity, so + // the stall is missing input (arriving in a later chunk, or + // resolved at `finish`), not an exhausted output buffer. if consumed == 0 && produced == 0 && output.len() < output.capacity() { - return Err(Report::new(TrustedServerError::Proxy { - message: "deflate publisher body decoder made no progress".to_string(), - })); + break; } } } } Ok(output) } + + /// Drive the inflater to completion at end of input, draining the final + /// decoded bytes and validating the end-of-stream marker. + /// + /// A valid stream whose last decoded byte exactly filled the previous + /// output buffer still has its end marker pending here; a genuinely + /// truncated stream makes no further progress and errors. + fn finish(&mut self) -> Result, Report> { + let mut output = Vec::new(); + while !self.stream_ended { + if output.len() == output.capacity() { + output.reserve(STREAM_CHUNK_SIZE); + } + let before_out = self.decompress.total_out(); + let status = self + .decompress + .decompress_vec(&[], &mut output, flate2::FlushDecompress::Finish) + .change_context(TrustedServerError::Proxy { + message: "Failed to finalize deflate publisher body decoder".to_string(), + })?; + let produced = (self.decompress.total_out() - before_out) as usize; + self.charge(produced)?; + match status { + flate2::Status::StreamEnd => self.stream_ended = true, + flate2::Status::Ok | flate2::Status::BufError => { + if produced == 0 { + break; + } + } + } + } + if !self.stream_ended { + return Err(Report::new(TrustedServerError::Proxy { + message: "Failed to finalize deflate publisher body decoder: truncated stream" + .to_string(), + })); + } + Ok(output) + } } impl BodyStreamDecoder { pub(crate) fn new(compression: Compression, max_decoded_bytes: usize) -> Self { + let decoded_bytes = Rc::new(Cell::new(0usize)); let codec = match compression { Compression::None => BodyStreamDecoderCodec::None, - Compression::Gzip => { - BodyStreamDecoderCodec::Gzip(flate2::write::GzDecoder::new(Vec::new())) - } - Compression::Deflate => BodyStreamDecoderCodec::Deflate(DeflateStreamDecoder::new()), - Compression::Brotli => BodyStreamDecoderCodec::Brotli(Box::new( - brotli::DecompressorWriter::new(Vec::new(), STREAM_CHUNK_SIZE), + Compression::Gzip => BodyStreamDecoderCodec::Gzip(flate2::write::MultiGzDecoder::new( + BoundedDecodeSink::new(Rc::clone(&decoded_bytes), max_decoded_bytes), + )), + Compression::Deflate => BodyStreamDecoderCodec::Deflate(DeflateStreamDecoder::new( + Rc::clone(&decoded_bytes), + max_decoded_bytes, )), + Compression::Brotli => { + BodyStreamDecoderCodec::Brotli(Box::new(brotli::DecompressorWriter::new( + BoundedDecodeSink::new(Rc::clone(&decoded_bytes), max_decoded_bytes), + STREAM_CHUNK_SIZE, + ))) + } }; Self { codec, - decoded_bytes: 0, + decoded_bytes, max_decoded_bytes, } } @@ -456,51 +605,52 @@ impl BodyStreamDecoder { &mut self, chunk: bytes::Bytes, ) -> Result> { - let decoded = match &mut self.codec { - BodyStreamDecoderCodec::None => chunk, + match &mut self.codec { + BodyStreamDecoderCodec::None => { + // No sink guards the pass-through path, so charge the raw chunk + // directly against the shared budget. + charge_decoded(&self.decoded_bytes, self.max_decoded_bytes, chunk.len())?; + Ok(chunk) + } BodyStreamDecoderCodec::Gzip(decoder) => { decoder .write_all(&chunk) .change_context(TrustedServerError::Proxy { message: "Failed to decode gzip publisher body chunk".to_string(), })?; - bytes::Bytes::from(std::mem::take(decoder.get_mut())) + // The sink charged the decoded bytes during `write_all`. + Ok(bytes::Bytes::from(std::mem::take( + &mut decoder.get_mut().buffer, + ))) + } + BodyStreamDecoderCodec::Deflate(decoder) => { + Ok(bytes::Bytes::from(decoder.decode(&chunk)?)) } - BodyStreamDecoderCodec::Deflate(decoder) => bytes::Bytes::from(decoder.decode(&chunk)?), BodyStreamDecoderCodec::Brotli(decoder) => { decoder .write_all(&chunk) .change_context(TrustedServerError::Proxy { message: "Failed to decode brotli publisher body chunk".to_string(), })?; - bytes::Bytes::from(std::mem::take(decoder.get_mut())) + Ok(bytes::Bytes::from(std::mem::take( + &mut decoder.get_mut().buffer, + ))) } - }; - self.track_decoded(decoded.len())?; - Ok(decoded) + } } pub(crate) fn finish(&mut self) -> Result, Report> { - let tail = match &mut self.codec { - BodyStreamDecoderCodec::None => Vec::new(), + match &mut self.codec { + BodyStreamDecoderCodec::None => Ok(Vec::new()), BodyStreamDecoderCodec::Gzip(decoder) => { decoder .try_finish() .change_context(TrustedServerError::Proxy { message: "Failed to finalize gzip publisher body decoder".to_string(), })?; - std::mem::take(decoder.get_mut()) - } - BodyStreamDecoderCodec::Deflate(decoder) => { - if !decoder.stream_ended { - return Err(Report::new(TrustedServerError::Proxy { - message: - "Failed to finalize deflate publisher body decoder: truncated stream" - .to_string(), - })); - } - Vec::new() + Ok(std::mem::take(&mut decoder.get_mut().buffer)) } + BodyStreamDecoderCodec::Deflate(decoder) => decoder.finish(), BodyStreamDecoderCodec::Brotli(decoder) => { // `close()` (not `flush()`): flush accepts a truncated brotli // stream silently, while close validates end-of-stream and @@ -508,28 +658,9 @@ impl BodyStreamDecoder { decoder.close().change_context(TrustedServerError::Proxy { message: "Failed to finalize brotli publisher body decoder".to_string(), })?; - std::mem::take(decoder.get_mut()) + Ok(std::mem::take(&mut decoder.get_mut().buffer)) } - }; - self.track_decoded(tail.len())?; - Ok(tail) - } - - fn track_decoded(&mut self, len: usize) -> Result<(), Report> { - self.decoded_bytes = self.decoded_bytes.checked_add(len).ok_or_else(|| { - Report::new(TrustedServerError::Proxy { - message: "publisher origin body decoded byte count overflowed".to_string(), - }) - })?; - if self.decoded_bytes > self.max_decoded_bytes { - return Err(Report::new(TrustedServerError::Proxy { - message: format!( - "publisher origin body decoded size exceeded {}-byte streaming limit", - self.max_decoded_bytes - ), - })); } - Ok(()) } } @@ -704,6 +835,132 @@ mod tests { ); } + #[test] + fn body_stream_decoder_decodes_deflate_filling_output_buffer_exactly() { + // A decoded length one byte past the decoder's internal output buffer + // (`STREAM_CHUNK_SIZE`) hits the boundary where flate2 consumes all + // input while exactly filling the output buffer and returns + // `Status::Ok` with the stream-end marker still pending. The decoder + // must drive the inflater to completion instead of reporting a + // truncated stream. + let payload = vec![b'a'; STREAM_CHUNK_SIZE + 1]; + let compressed = { + let mut encoder = + flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(&payload) + .expect("should write deflate test input"); + encoder.finish().expect("should finish deflate encoding") + }; + let mut decoder = BodyStreamDecoder::new(Compression::Deflate, usize::MAX); + + let mut decoded = decoder + .decode_chunk(bytes::Bytes::from(compressed)) + .expect("complete deflate stream should decode") + .to_vec(); + decoded.extend( + decoder + .finish() + .expect("a complete deflate stream must not report truncation"), + ); + + assert_eq!( + decoded, payload, + "should decode the full payload across the output-buffer boundary" + ); + } + + #[test] + fn body_stream_decoder_decodes_deflate_split_across_many_chunks() { + let payload = vec![b'x'; STREAM_CHUNK_SIZE * 3 + 7]; + let compressed = { + let mut encoder = + flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(&payload) + .expect("should write deflate test input"); + encoder.finish().expect("should finish deflate encoding") + }; + let mut decoder = BodyStreamDecoder::new(Compression::Deflate, usize::MAX); + + let mut decoded = Vec::new(); + // Feed the compressed stream a few bytes at a time to exercise many + // input split points, including splits inside the end-of-stream marker. + for piece in compressed.chunks(3) { + decoded.extend( + decoder + .decode_chunk(bytes::Bytes::copy_from_slice(piece)) + .expect("partial deflate input should decode incrementally"), + ); + } + decoded.extend( + decoder + .finish() + .expect("a complete deflate stream must finalize"), + ); + + assert_eq!( + decoded, payload, + "should decode the full payload regardless of input split points" + ); + } + + fn gzip_member(data: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(data) + .expect("should write gzip test input"); + encoder.finish().expect("should finish gzip encoding") + } + + #[test] + fn body_stream_decoder_decodes_multi_member_gzip_single_chunk() { + let mut compressed = gzip_member(b"first member "); + compressed.extend(gzip_member(b"second member")); + let mut decoder = BodyStreamDecoder::new(Compression::Gzip, usize::MAX); + + let mut decoded = decoder + .decode_chunk(bytes::Bytes::from(compressed)) + .expect("a multi-member gzip body must decode all members") + .to_vec(); + decoded.extend( + decoder + .finish() + .expect("a multi-member gzip body must finalize"), + ); + + assert_eq!( + decoded, b"first member second member", + "should concatenate the decoded output of every gzip member" + ); + } + + #[test] + fn body_stream_decoder_decodes_multi_member_gzip_split_across_chunks() { + let mut compressed = gzip_member(b"alpha"); + compressed.extend(gzip_member(b"omega")); + let mut decoder = BodyStreamDecoder::new(Compression::Gzip, usize::MAX); + + let mut decoded = Vec::new(); + for piece in compressed.chunks(4) { + decoded.extend( + decoder + .decode_chunk(bytes::Bytes::copy_from_slice(piece)) + .expect("multi-member gzip should decode across chunk boundaries"), + ); + } + decoded.extend( + decoder + .finish() + .expect("a multi-member gzip body must finalize"), + ); + + assert_eq!( + decoded, b"alphaomega", + "should decode both gzip members split across chunk boundaries" + ); + } + /// Verify that `lol_html` fragments text nodes when input chunks split /// mid-text-node. Script rewriters must be fragment-safe — they accumulate /// text fragments internally until `is_last_in_text_node` is true. From 5512d8507cbfeee9777ddbb09be2f44ba734fc86 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 14 Jul 2026 18:10:20 +0530 Subject: [PATCH 020/198] Address auction transport-timeout review findings Resolve the PR review by making transport-timeout canonicalization a platform capability and hardening auction backend-name correlation. - Move quantization behind PlatformBackend::canonicalize_transport_timeout_ms. Fastly floors budget-derived timeouts to a 250ms quantum with a bounded sub-quantum ladder [200,150,100,50]; other adapters use the exact remaining budget so bidder deadlines (Prebid tmax, APS timeout) are not shortened where no connection-pooling benefit exists. - Bound sub-quantum backend-name cardinality: exact 1-249ms values no longer pass through, capping the budget-derived names a single origin can mint toward the per-service dynamic backend limit. - Add a provider discriminator to PlatformBackendSpec, folded into every adapter's backend name, so two providers sharing one origin no longer collide on the response-correlation key. Reject a duplicate backend_to_provider insertion with an attributed launch failure instead of silently overwriting and misattributing a response. - Make the orchestrator call-site tests deterministic: record predicted and registered transport timeouts separately and assert exact equality via a controllable platform backend, and enumerate the sub-quantum ladder to assert a bounded name cardinality. - Correct the timeout-semantics comments that overstated absolute-deadline enforcement; the Fastly connect/first-byte/between-bytes timeouts bound connection, first-byte, and inactivity, not total response time. A true absolute deadline carried through the platform HTTP API remains follow-up work (#849). --- .../src/platform.rs | 12 +- .../src/platform.rs | 9 +- .../src/backend.rs | 35 +- .../src/platform.rs | 200 +++++ .../src/tinybird.rs | 1 + .../src/platform.rs | 9 +- .../src/auction/orchestrator.rs | 794 ++++++++++-------- .../trusted-server-core/src/ec/pull_sync.rs | 1 + .../src/integrations/datadome/protection.rs | 1 + .../src/integrations/mod.rs | 4 + .../src/platform/test_support.rs | 28 + .../src/platform/traits.rs | 22 + .../trusted-server-core/src/platform/types.rs | 10 + crates/trusted-server-core/src/proxy.rs | 2 + crates/trusted-server-core/src/publisher.rs | 1 + 15 files changed, 772 insertions(+), 357 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/platform.rs b/crates/trusted-server-adapter-axum/src/platform.rs index 461b567d1..a511daab2 100644 --- a/crates/trusted-server-adapter-axum/src/platform.rs +++ b/crates/trusted-server-adapter-axum/src/platform.rs @@ -158,11 +158,19 @@ impl PlatformBackend for AxumPlatformBackend { let port = spec .port .unwrap_or(if spec.scheme == "https" { 443 } else { 80 }); + // Keep two providers that share an origin on distinct names so auction + // response correlation cannot cross providers. + let discriminator = spec + .discriminator + .as_deref() + .map(|d| format!("_p_{}", normalize_env_segment(d))) + .unwrap_or_default(); Ok(format!( - "{}_{}_{}", + "{}_{}_{}{}", normalize_env_segment(&spec.scheme), normalize_env_segment(&spec.host), port, + discriminator, )) } @@ -644,6 +652,7 @@ mod tests { first_byte_timeout: Duration::from_secs(15), between_bytes_timeout: Duration::from_secs(15), host_header_override: None, + discriminator: None, }; let name1 = backend.predict_name(&spec).expect("should return a name"); let name2 = backend @@ -664,6 +673,7 @@ mod tests { first_byte_timeout: Duration::from_secs(15), between_bytes_timeout: Duration::from_secs(15), host_header_override: None, + discriminator: None, }; assert_eq!( backend.predict_name(&spec).expect("should return name"), diff --git a/crates/trusted-server-adapter-cloudflare/src/platform.rs b/crates/trusted-server-adapter-cloudflare/src/platform.rs index a01c4d979..9467abb71 100644 --- a/crates/trusted-server-adapter-cloudflare/src/platform.rs +++ b/crates/trusted-server-adapter-cloudflare/src/platform.rs @@ -71,8 +71,15 @@ impl PlatformBackend for NoopBackend { } else { "_nocert" }; + // Keep two providers that share an origin on distinct names so auction + // response correlation cannot cross providers. + let discriminator = spec + .discriminator + .as_deref() + .map(|d| format!("_p_{d}")) + .unwrap_or_default(); Ok(format!( - "{}_{}_{}_{timeout_ms}ms{cert_suffix}", + "{}_{}_{}_{timeout_ms}ms{cert_suffix}{discriminator}", spec.scheme, spec.host, port )) } diff --git a/crates/trusted-server-adapter-fastly/src/backend.rs b/crates/trusted-server-adapter-fastly/src/backend.rs index 4056c81da..7205a8a00 100644 --- a/crates/trusted-server-adapter-fastly/src/backend.rs +++ b/crates/trusted-server-adapter-fastly/src/backend.rs @@ -64,6 +64,7 @@ pub struct BackendConfig<'a> { first_byte_timeout: Duration, between_bytes_timeout: Duration, host_header_override: Option<&'a str>, + discriminator: Option<&'a str>, } impl<'a> BackendConfig<'a> { @@ -81,6 +82,7 @@ impl<'a> BackendConfig<'a> { first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, between_bytes_timeout: DEFAULT_BETWEEN_BYTES_TIMEOUT, host_header_override: None, + discriminator: None, } } @@ -128,14 +130,30 @@ impl<'a> BackendConfig<'a> { self } + /// Set an optional stable discriminator folded into the backend name. + /// + /// Two callers targeting the same origin with the same transport timeout + /// otherwise share a backend name. Auction response correlation keys on the + /// backend name, so a shared name would let one provider's response be + /// parsed as another's. A per-provider discriminator keeps the names + /// distinct while staying stable across requests. + #[must_use] + pub fn discriminator(mut self, discriminator: Option<&'a str>) -> Self { + self.discriminator = discriminator; + self + } + /// Compute the deterministic backend name and resolved port without /// registering anything. /// - /// The name encodes scheme, host, port, certificate setting, and - /// first-byte timeout so that backends with different configurations - /// never collide. Including the timeout prevents "first-registration-wins" - /// poisoning where a later request for the same origin with a tighter - /// timeout would silently inherit the original registration's value. + /// The name encodes scheme, host, port, certificate setting, optional + /// discriminator, and the first-byte/between-bytes timeouts so that + /// backends with different configurations never collide. Including the + /// timeout prevents "first-registration-wins" poisoning where a later + /// request for the same origin with a tighter timeout would silently + /// inherit the original registration's value. Including the discriminator + /// keeps two callers that target the same origin with the same timeout + /// (e.g. two auction providers behind one gateway) on distinct backends. fn compute_name(&self) -> Result<(String, u16), Report> { if self.host.is_empty() { return Err(Report::new(TrustedServerError::Proxy { @@ -174,13 +192,18 @@ impl<'a> BackendConfig<'a> { } else { "_nocert" }; + let discriminator_suffix = self + .discriminator + .map(|d| format!("_p_{}", sanitize_backend_name_component(d))) + .unwrap_or_default(); let first_byte_timeout_ms = self.first_byte_timeout.as_millis(); let between_bytes_timeout_ms = self.between_bytes_timeout.as_millis(); let backend_name = format!( - "backend_{}{}{}_fb{}_bb{}", + "backend_{}{}{}{}_fb{}_bb{}", sanitize_backend_name_component(&name_base), host_override_suffix, cert_suffix, + discriminator_suffix, first_byte_timeout_ms, between_bytes_timeout_ms ); diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index c5bb60b9c..c89508d36 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -156,6 +156,40 @@ fn backend_config_from_spec(spec: &PlatformBackendSpec) -> BackendConfig<'_> { .certificate_check(spec.certificate_check) .first_byte_timeout(spec.first_byte_timeout) .between_bytes_timeout(spec.between_bytes_timeout) + .discriminator(spec.discriminator.as_deref()) +} + +/// Transport-timeout quantum for auction backends (see +/// [`FastlyPlatformBackend::canonicalize_transport_timeout_ms`]). +const TRANSPORT_TIMEOUT_QUANTUM_MS: u32 = 250; + +/// Coarse rungs for budget-bound transport timeouts below one quantum, +/// ordered high to low. +/// +/// A budget-bound value at or above one quantum is floored to a +/// [`TRANSPORT_TIMEOUT_QUANTUM_MS`] multiple. Below one quantum, passing the +/// exact wall-clock remainder through would mint a distinct backend name for +/// every millisecond in `1..250`, so the near-exhausted tail alone could +/// exceed Fastly's per-service dynamic backend limit. Snapping to this finite +/// ladder instead bounds the number of budget-derived names an origin can +/// produce. Budgets below the smallest rung round to zero, which callers treat +/// as "budget exhausted — skip the launch". +const SUB_QUANTUM_LADDER_MS: [u32; 4] = [200, 150, 100, 50]; + +/// Round a budget-bound transport timeout down to a stable bucket. +/// +/// At or above one quantum, floors to a [`TRANSPORT_TIMEOUT_QUANTUM_MS`] +/// multiple. Below one quantum, snaps down to the greatest +/// [`SUB_QUANTUM_LADDER_MS`] rung no larger than `remaining_ms` (or zero). +fn quantize_transport_timeout_ms(remaining_ms: u32) -> u32 { + let floored = (remaining_ms / TRANSPORT_TIMEOUT_QUANTUM_MS) * TRANSPORT_TIMEOUT_QUANTUM_MS; + if floored > 0 { + return floored; + } + SUB_QUANTUM_LADDER_MS + .into_iter() + .find(|&rung| rung <= remaining_ms) + .unwrap_or(0) } impl PlatformBackend for FastlyPlatformBackend { @@ -170,6 +204,28 @@ impl PlatformBackend for FastlyPlatformBackend { .ensure() .change_context(PlatformError::Backend) } + + /// Quantize the transport timeout so budget-derived values do not mint a + /// new dynamic backend name on every request. + /// + /// Fastly embeds the first-byte and between-bytes timeouts in the dynamic + /// backend name (see [`BackendConfig`]) and pools connections per backend + /// name. A per-request wall-clock budget would otherwise defeat that + /// pooling and accumulate registrations toward the per-service dynamic + /// backend limit. + /// + /// A provider's own configured timeout is a constant, so when it is the + /// binding constraint it is returned verbatim — including sub-quantum + /// configured values, which must not be rounded away or the provider could + /// never launch. Only the budget-bound value is snapped to a stable bucket + /// via [`quantize_transport_timeout_ms`]. Rounding down never extends a + /// transport cap past the remaining budget. + fn canonicalize_transport_timeout_ms(&self, remaining_ms: u32, configured_ms: u32) -> u32 { + if remaining_ms >= configured_ms { + return configured_ms; + } + quantize_transport_timeout_ms(remaining_ms) + } } // --------------------------------------------------------------------------- @@ -637,6 +693,7 @@ mod tests { certificate_check: true, first_byte_timeout: Duration::from_secs(15), between_bytes_timeout: Duration::from_secs(15), + discriminator: None, }; let name = backend @@ -660,6 +717,7 @@ mod tests { certificate_check: true, first_byte_timeout: Duration::from_secs(15), between_bytes_timeout: Duration::from_secs(15), + discriminator: None, }; let name = backend @@ -683,6 +741,7 @@ mod tests { certificate_check: false, first_byte_timeout: Duration::from_secs(15), between_bytes_timeout: Duration::from_secs(15), + discriminator: None, }; let name = backend @@ -706,6 +765,7 @@ mod tests { certificate_check: true, first_byte_timeout: Duration::from_secs(15), between_bytes_timeout: Duration::from_secs(15), + discriminator: None, }; let result = backend.predict_name(&spec); @@ -724,6 +784,7 @@ mod tests { certificate_check: true, first_byte_timeout: Duration::from_millis(2000), between_bytes_timeout: Duration::from_millis(2000), + discriminator: None, }; let name = backend @@ -752,6 +813,7 @@ mod tests { certificate_check: true, first_byte_timeout: Duration::from_millis(750), between_bytes_timeout: Duration::from_millis(750), + discriminator: None, }; let predicted = backend @@ -937,4 +999,142 @@ mod tests { "should describe the unsupported streaming body: {err:?}" ); } + + // --- FastlyPlatformBackend::canonicalize_transport_timeout_ms ----------- + + #[test] + fn canonicalize_prefers_configured_timeout_when_budget_allows() { + let backend = FastlyPlatformBackend; + assert_eq!( + backend.canonicalize_transport_timeout_ms(2000, 1000), + 1000, + "should use the configured timeout verbatim when the budget allows" + ); + assert_eq!( + backend.canonicalize_transport_timeout_ms(2000, 100), + 100, + "should preserve a sub-quantum configured constant — it is name-stable on its own" + ); + } + + #[test] + fn canonicalize_floors_budget_bound_value_to_quantum() { + let backend = FastlyPlatformBackend; + assert_eq!( + backend.canonicalize_transport_timeout_ms(999, 2000), + 750, + "should floor a 999ms budget to the 750ms quantum bucket" + ); + assert_eq!( + backend.canonicalize_transport_timeout_ms(300, 2000), + 250, + "should floor a tight budget down to one quantum" + ); + assert_eq!( + backend.canonicalize_transport_timeout_ms(250, 2000), + 250, + "should keep an exact quantum multiple" + ); + } + + #[test] + fn canonicalize_snaps_sub_quantum_budget_to_bounded_ladder() { + let backend = FastlyPlatformBackend; + // Exact wall-clock values in 1..250 must NOT pass through — that is the + // unbounded-cardinality regression this ladder closes. + assert_eq!( + backend.canonicalize_transport_timeout_ms(249, 2000), + 200, + "should snap a sub-quantum budget down to the greatest ladder rung, not pass 249 through" + ); + assert_eq!(backend.canonicalize_transport_timeout_ms(200, 2000), 200); + assert_eq!(backend.canonicalize_transport_timeout_ms(150, 2000), 150); + assert_eq!(backend.canonicalize_transport_timeout_ms(100, 2000), 100); + assert_eq!(backend.canonicalize_transport_timeout_ms(50, 2000), 50); + assert_eq!( + backend.canonicalize_transport_timeout_ms(49, 2000), + 0, + "a budget below the smallest rung rounds to zero (launch skipped)" + ); + assert_eq!( + backend.canonicalize_transport_timeout_ms(0, 1000), + 0, + "an exhausted budget canonicalizes to zero" + ); + assert_eq!( + backend.canonicalize_transport_timeout_ms(100, 0), + 0, + "a zero configured timeout canonicalizes to zero" + ); + } + + #[test] + fn canonicalize_budget_derived_names_stay_within_a_safe_cardinality() { + // Enumerate every reachable remaining budget for a normal 2000ms + // ceiling and confirm the number of distinct backend-name-bearing + // transport values an origin can mint stays far below Fastly's + // per-service dynamic backend limit (documented default 200). + let backend = FastlyPlatformBackend; + let configured = 2000; + let mut distinct = std::collections::BTreeSet::new(); + for remaining in 0..=configured { + let value = backend.canonicalize_transport_timeout_ms(remaining, configured); + if value > 0 { + distinct.insert(value); + } + // No arbitrary clock-derived value may leak: every canonical value + // is either a quantum multiple or one of the bounded ladder rungs. + assert!( + value == 0 + || value % TRANSPORT_TIMEOUT_QUANTUM_MS == 0 + || SUB_QUANTUM_LADDER_MS.contains(&value), + "canonical value {value}ms (from remaining {remaining}ms) is neither a quantum \ + multiple nor a ladder rung" + ); + } + assert!( + distinct.len() <= 16, + "budget-derived transport values should stay well under the dynamic backend limit, \ + got {} distinct values: {distinct:?}", + distinct.len() + ); + } + + // --- FastlyPlatformBackend::predict_name discriminator ------------------ + + #[test] + fn predict_name_includes_provider_discriminator() { + let backend = FastlyPlatformBackend; + let base = PlatformBackendSpec { + scheme: "https".to_string(), + host: "gateway.example.com".to_string(), + port: None, + host_header_override: None, + certificate_check: true, + first_byte_timeout: Duration::from_millis(750), + between_bytes_timeout: Duration::from_millis(750), + discriminator: Some("prebid".to_string()), + }; + let prebid_name = backend + .predict_name(&base) + .expect("should predict name with discriminator"); + assert!( + prebid_name.contains("_p_prebid"), + "should fold the provider discriminator into the name, got {prebid_name}" + ); + + // Same origin + same transport timeout, different provider → distinct + // backend names, so auction response correlation cannot cross them. + let aps = PlatformBackendSpec { + discriminator: Some("aps".to_string()), + ..base.clone() + }; + let aps_name = backend + .predict_name(&aps) + .expect("should predict name for the second provider"); + assert_ne!( + prebid_name, aps_name, + "two providers on one origin must not share a backend name" + ); + } } diff --git a/crates/trusted-server-adapter-fastly/src/tinybird.rs b/crates/trusted-server-adapter-fastly/src/tinybird.rs index b5d332a65..8df6dbe6e 100644 --- a/crates/trusted-server-adapter-fastly/src/tinybird.rs +++ b/crates/trusted-server-adapter-fastly/src/tinybird.rs @@ -212,6 +212,7 @@ fn tinybird_backend_spec(api_host: &str) -> PlatformBackendSpec { certificate_check: true, first_byte_timeout: TINYBIRD_FIRST_BYTE_TIMEOUT, between_bytes_timeout: TINYBIRD_BETWEEN_BYTES_TIMEOUT, + discriminator: None, } } diff --git a/crates/trusted-server-adapter-spin/src/platform.rs b/crates/trusted-server-adapter-spin/src/platform.rs index 1e13ca300..492f1a518 100644 --- a/crates/trusted-server-adapter-spin/src/platform.rs +++ b/crates/trusted-server-adapter-spin/src/platform.rs @@ -92,8 +92,15 @@ impl PlatformBackend for NoopBackend { } else { "_nocert" }; + // Keep two providers that share an origin on distinct names so auction + // response correlation cannot cross providers. + let discriminator = spec + .discriminator + .as_deref() + .map(|d| format!("_p_{d}")) + .unwrap_or_default(); Ok(format!( - "{}_{}_{}_{timeout_ms}ms{cert_suffix}", + "{}_{}_{}_{timeout_ms}ms{cert_suffix}{discriminator}", spec.scheme, spec.host, port )) } diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index d884a0220..02a87cdb5 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -157,64 +157,6 @@ fn remaining_budget_ms(start: Instant, timeout_ms: u32) -> u32 { timeout_ms.saturating_sub(elapsed) } -/// Transport-timeout quantum for auction backends. -/// -/// See [`quantize_transport_timeout_ms`] for why provider transport timeouts -/// are rounded to this granularity. -const TRANSPORT_TIMEOUT_QUANTUM_MS: u32 = 250; - -/// Round a transport timeout down to a [`TRANSPORT_TIMEOUT_QUANTUM_MS`] multiple. -/// -/// The Fastly adapter embeds the first-byte and between-bytes timeouts in the -/// dynamic backend name so a registration can never be silently reused with a -/// different transport configuration. Deriving those timeouts from the -/// remaining wall-clock budget minted a new backend name on nearly every -/// request, which defeated cross-request TCP/TLS connection reuse (Fastly -/// pools connections per backend name) and accumulated registrations toward -/// the per-service dynamic backend limit. -/// -/// Quantizing the value — not just the name — keeps the registered backend -/// configuration aligned with its name. Rounding down never extends a -/// transport cap past the auction deadline, which matters on the mediator and -/// dispatched-collect paths where the backend timeouts (not a select-loop -/// deadline check) bound the `` hold. -#[inline] -fn quantize_transport_timeout_ms(timeout_ms: u32) -> u32 { - (timeout_ms / TRANSPORT_TIMEOUT_QUANTUM_MS) * TRANSPORT_TIMEOUT_QUANTUM_MS -} - -/// Compute the transport timeout for a provider launch from the remaining -/// auction budget and the provider's configured timeout. -/// -/// The configured timeout is a per-provider constant, so using it verbatim -/// already yields a stable backend name — including configured values below -/// one quantum, which must not be rounded away or the provider could never -/// launch. Only when the remaining budget is the binding constraint does the -/// wall-clock-derived value enter the name, and that value is quantized via -/// [`quantize_transport_timeout_ms`] so it cannot mint a new backend name on -/// every request. -/// -/// A remaining budget below one quantum is passed through exactly rather -/// than rounded to zero: rounding up would extend the transport cap past the -/// deadline, and rounding down would skip the launch and hard-fail auctions -/// whose configured budget is under one quantum. Name churn in this regime -/// is bounded to sub-quantum values and matches the pre-quantization -/// behavior. The result never exceeds `remaining_ms` and is zero only when -/// `remaining_ms` or `configured_ms` is zero, which callers treat as -/// "budget exhausted — skip the launch". -#[inline] -fn effective_transport_timeout_ms(remaining_ms: u32, configured_ms: u32) -> u32 { - if remaining_ms >= configured_ms { - return configured_ms; - } - let quantized = quantize_transport_timeout_ms(remaining_ms); - if quantized == 0 { - remaining_ms - } else { - quantized - } -} - /// Manages auction execution across multiple providers. pub struct AuctionOrchestrator { config: AuctionConfig, @@ -338,11 +280,16 @@ impl AuctionOrchestrator { // Give the mediator only the remaining time from the auction // deadline, not the full timeout — the bidding phase already // consumed part of it, and the mediator has no select-loop - // deadline backstop. Quantized for backend-name stability (see - // effective_transport_timeout_ms). + // deadline backstop. The platform canonicalizes the value for + // backend-name stability (see + // `PlatformBackend::canonicalize_transport_timeout_ms`); it never + // exceeds the remaining budget. See the transport-deadline note on + // `run_providers_parallel` for the limits of this bound. let remaining_ms = remaining_budget_ms(mediation_start, context.timeout_ms); - let mediator_timeout = - effective_transport_timeout_ms(remaining_ms, mediator.timeout_ms()); + let mediator_timeout = context + .services + .backend() + .canonicalize_transport_timeout_ms(remaining_ms, mediator.timeout_ms()); if mediator_timeout == 0 { log::warn!("Auction timeout exhausted during bidding phase; skipping mediator"); @@ -525,11 +472,14 @@ impl AuctionOrchestrator { // Give each provider only the remaining time from the auction // deadline so that backend transport timeouts do not extend past - // the overall budget, quantized for backend-name stability (see - // effective_transport_timeout_ms). + // the overall budget. The platform canonicalizes the value for + // backend-name stability (see + // `PlatformBackend::canonicalize_transport_timeout_ms`). let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); - let effective_timeout = - effective_transport_timeout_ms(remaining_ms, provider.timeout_ms()); + let effective_timeout = context + .services + .backend() + .canonicalize_transport_timeout_ms(remaining_ms, provider.timeout_ms()); if effective_timeout == 0 { log::warn!("Auction timeout exhausted before launching provider request; skipping"); @@ -580,15 +530,35 @@ impl AuctionOrchestrator { ); backend_name.clone() }); - backend_to_provider.insert( - request_backend_name.clone(), - (provider.provider_name(), start_time, provider.as_ref()), - ); - pending_requests.push(pending); - log::debug!( - "Request to '{}' launched successfully", - provider.provider_name() - ); + // Responses are correlated back to providers by backend + // name. If another provider this auction already claimed + // this name (e.g. two providers on one origin whose specs + // canonicalize to the same backend), inserting here would + // silently overwrite the first mapping and misattribute or + // drop a response. Fail this launch attributably instead. + if backend_to_provider.contains_key(&request_backend_name) { + let response_time_ms = start_time.elapsed().as_millis() as u64; + log::warn!( + "Provider '{}' resolved to backend name '{}' already claimed by another \ + provider this auction; skipping launch to avoid response misattribution", + provider.provider_name(), + request_backend_name, + ); + responses.push(provider_launch_failed_response( + provider.provider_name(), + response_time_ms, + )); + } else { + backend_to_provider.insert( + request_backend_name.clone(), + (provider.provider_name(), start_time, provider.as_ref()), + ); + pending_requests.push(pending); + log::debug!( + "Request to '{}' launched successfully", + provider.provider_name() + ); + } } Err(e) => { let response_time_ms = start_time.elapsed().as_millis() as u64; @@ -621,14 +591,25 @@ impl AuctionOrchestrator { ); // Phase 2: Wait for responses using select() to process as they become ready. - // Enforce the auction deadline: after each select() returns, check - // elapsed time and drop remaining requests if the timeout is exceeded. + // After each select() returns, check elapsed time and drop remaining + // requests once the auction deadline passes. // - // NOTE: `select()` blocks until at least one backend responds and, on - // some adapters, buffers the selected response body before returning. - // Hard deadline enforcement therefore depends on every backend's - // first-byte and between-bytes timeouts being set to at most the - // remaining auction budget, which Phase 1 above guarantees. + // TRANSPORT-DEADLINE NOTE: this select loop is the only *absolute* + // wall-clock bound on the parallel path — it drops still-pending + // requests once `auction_start.elapsed()` exceeds the deadline. The + // per-backend transport timeouts set in Phase 1 are a complementary, + // not equivalent, bound: Fastly's connect timeout is a fixed ~1s, the + // first-byte timeout only starts after the connection is established, + // and the between-bytes timeout is an inactivity timer that resets on + // every byte received. A backend that connects slowly or trickles one + // byte just inside the between-bytes window can therefore outlive the + // configured budget. Bounding them to the remaining budget (Phase 1) + // guarantees they never *extend past* the deadline by their own + // configuration, but does not by itself enforce a hard total-response + // deadline. Paths without this select loop (the mediator and the + // dispatched-collect body read) inherit that weaker bound; a true + // absolute deadline carried through the platform HTTP API is tracked + // as follow-up work (see the streaming/deadline effort, #849). let mut remaining = pending_requests; while !remaining.is_empty() { @@ -937,11 +918,13 @@ impl AuctionOrchestrator { continue; } - // Remaining budget quantized for backend-name stability (see - // effective_transport_timeout_ms). + // Remaining budget canonicalized by the platform for backend-name + // stability (see `PlatformBackend::canonicalize_transport_timeout_ms`). let remaining_ms = remaining_budget_ms(auction_start, context.timeout_ms); - let effective_timeout = - effective_transport_timeout_ms(remaining_ms, provider.timeout_ms()); + let effective_timeout = context + .services + .backend() + .canonicalize_transport_timeout_ms(remaining_ms, provider.timeout_ms()); if effective_timeout == 0 { log::warn!( @@ -974,21 +957,39 @@ impl AuctionOrchestrator { let start_time = Instant::now(); match provider.request_bids(request, &provider_context).await { Ok(pending) => { - log::info!( - "Dispatching bid request to '{}' (backend: {}, budget: {}ms)", - provider.provider_name(), - backend_name, - effective_timeout - ); - backend_to_provider.insert( - backend_name.clone(), - ( - provider.provider_name().to_string(), - start_time, - Arc::clone(provider), - ), - ); - pending_requests.push(pending.with_backend_name(backend_name)); + // See the parallel path: a backend name already claimed by + // another provider this auction would misattribute the + // collected response, so fail this launch attributably + // rather than overwrite the mapping. + if backend_to_provider.contains_key(&backend_name) { + let response_time_ms = start_time.elapsed().as_millis() as u64; + log::warn!( + "Provider '{}' resolved to backend name '{}' already claimed by another \ + provider this auction; skipping dispatch to avoid response misattribution", + provider.provider_name(), + backend_name, + ); + launch_responses.push(provider_launch_failed_response( + provider.provider_name(), + response_time_ms, + )); + } else { + log::info!( + "Dispatching bid request to '{}' (backend: {}, budget: {}ms)", + provider.provider_name(), + backend_name, + effective_timeout + ); + backend_to_provider.insert( + backend_name.clone(), + ( + provider.provider_name().to_string(), + start_time, + Arc::clone(provider), + ), + ); + pending_requests.push(pending.with_backend_name(backend_name)); + } } Err(e) => { let response_time_ms = start_time.elapsed().as_millis() as u64; @@ -1189,21 +1190,27 @@ impl AuctionOrchestrator { match self.providers.get(mediator_name.as_str()) { Some(mediator) => { // Cap the mediator at whichever is tighter: its own configured - // timeout or the remaining auction budget (A_deadline). The old - // comment here claimed origin drain could exhaust the budget before - // collection, but SSP backends are given first-byte and between-bytes - // timeouts equal to effective_timeout (capped at their provider - // timeout) at dispatch time, so they cannot run past A_deadline - // independently. Giving the mediator an uncapped timeout lets it run - // past A_deadline, violating the bounded hold invariant. - // The mediator's only time bound on this path is its - // backend transport timeout, so the effective value must - // never exceed the remaining budget. Quantized for - // backend-name stability (see - // effective_transport_timeout_ms). + // timeout or the remaining auction budget (A_deadline). Giving + // the mediator an uncapped timeout would let it hold `` + // well past A_deadline, so the effective value must never + // exceed the remaining budget. + // + // Caveat: unlike the parallel select loop, this path has no + // absolute wall-clock backstop around the mediator call, and a + // backend transport timeout bounds first-byte/inactivity rather + // than total response time (see the transport-deadline note on + // `run_providers_parallel`). Capping the value to the remaining + // budget therefore prevents the mediator from *extending* the + // hold by its own configuration, but a slow-connecting or + // byte-trickling mediator can still overrun; a true absolute + // deadline is tracked as follow-up (#849). + // + // The platform canonicalizes the value for backend-name + // stability (see `PlatformBackend::canonicalize_transport_timeout_ms`). let remaining = remaining_budget_ms(auction_start, timeout_ms); - let mediator_timeout = - effective_transport_timeout_ms(remaining, mediator.timeout_ms()); + let mediator_timeout = services + .backend() + .canonicalize_transport_timeout_ms(remaining, mediator.timeout_ms()); if mediator_timeout == 0 { log::warn!( "A_deadline exhausted before mediator '{}' — returning {} SSP bids without mediation", @@ -1397,9 +1404,13 @@ mod tests { MediaType, PublisherInfo, UserInfo, }; use crate::error::TrustedServerError; - use crate::platform::test_support::{build_services_with_http_client, StubHttpClient}; + use crate::platform::test_support::{ + build_services_with_backend_and_http_client, build_services_with_http_client, + StubHttpClient, + }; use crate::platform::{ - PlatformHttpRequest, PlatformPendingRequest, PlatformResponse, RuntimeServices, + PlatformBackend, PlatformBackendSpec, PlatformError, PlatformHttpRequest, + PlatformPendingRequest, PlatformResponse, RuntimeServices, }; use crate::test_support::tests::crate_test_settings_str; use error_stack::{Report, ResultExt}; @@ -1412,15 +1423,19 @@ mod tests { // Minimal test double for AuctionProvider // --------------------------------------------------------------------------- - /// Minimal stub provider. Optionally records every transport timeout it - /// observes — the value passed to `backend_name` and the - /// `context.timeout_ms` handed to `request_bids` — so tests can assert - /// the orchestrator quantizes them. + /// Minimal stub provider. Optionally records the transport timeouts it + /// observes, keeping the value passed to `backend_name` (which derives the + /// predicted backend name) separate from the `context.timeout_ms` handed to + /// `request_bids` (which configures the registered request). Recording them + /// separately lets tests assert the orchestrator hands the *same* + /// canonicalized value to both — a divergence would land responses in the + /// "unknown backend" branch and drop bids. struct StubAuctionProvider { name: &'static str, backend: &'static str, configured_timeout_ms: u32, - observed_timeouts: Option>>>, + predicted_timeouts: Option>>>, + request_timeouts: Option>>>, } impl StubAuctionProvider { @@ -1429,7 +1444,8 @@ mod tests { name, backend, configured_timeout_ms: 2000, - observed_timeouts: None, + predicted_timeouts: None, + request_timeouts: None, } } @@ -1437,18 +1453,20 @@ mod tests { name: &'static str, backend: &'static str, configured_timeout_ms: u32, - observed_timeouts: Arc>>, + predicted_timeouts: Arc>>, + request_timeouts: Arc>>, ) -> Self { Self { name, backend, configured_timeout_ms, - observed_timeouts: Some(observed_timeouts), + predicted_timeouts: Some(predicted_timeouts), + request_timeouts: Some(request_timeouts), } } - fn record(&self, timeout_ms: u32) { - if let Some(observed) = &self.observed_timeouts { + fn record(slot: &Option>>>, timeout_ms: u32) { + if let Some(observed) = slot { observed .lock() .expect("should lock observed timeouts") @@ -1468,7 +1486,7 @@ mod tests { _request: &AuctionRequest, context: &AuctionContext<'_>, ) -> Result> { - self.record(context.timeout_ms); + Self::record(&self.request_timeouts, context.timeout_ms); let req = PlatformHttpRequest::new( http::Request::builder() .method("POST") @@ -1504,7 +1522,7 @@ mod tests { } fn backend_name(&self, _services: &RuntimeServices, timeout_ms: u32) -> Option { - self.record(timeout_ms); + Self::record(&self.predicted_timeouts, timeout_ms); Some(self.backend.to_string()) } } @@ -2038,94 +2056,71 @@ mod tests { ); } - #[test] - fn quantize_transport_timeout_floors_to_quantum() { - assert_eq!( - super::quantize_transport_timeout_ms(0), - 0, - "should keep zero at zero" - ); - assert_eq!( - super::quantize_transport_timeout_ms(249), - 0, - "should floor a sub-quantum budget to zero" - ); - assert_eq!( - super::quantize_transport_timeout_ms(250), - 250, - "should keep an exact quantum multiple unchanged" - ); - assert_eq!( - super::quantize_transport_timeout_ms(999), - 750, - "should floor to the next-lower quantum multiple" - ); - assert_eq!( - super::quantize_transport_timeout_ms(2000), - 2000, - "should keep a larger exact quantum multiple unchanged" - ); + /// Test backend whose [`PlatformBackend::canonicalize_transport_timeout_ms`] + /// returns a fixed value regardless of the wall-clock budget, so the + /// orchestrator's transport-timeout wiring can be asserted without timing + /// flakiness. Records every `(remaining_ms, configured_ms)` pair it sees. + /// + /// The exact quantization arithmetic lives in the Fastly adapter (the only + /// platform that overrides `canonicalize_transport_timeout_ms`); these core + /// tests only prove the orchestrator applies whatever the platform returns + /// and applies it identically to the predicted name and the launched + /// request. + struct CanonicalTimeoutBackend { + canonical_ms: u32, + calls: Arc>>, } - #[test] - fn effective_transport_timeout_prefers_configured_constant() { - assert_eq!( - super::effective_transport_timeout_ms(2000, 1000), - 1000, - "should use the configured timeout verbatim when the budget allows" - ); - assert_eq!( - super::effective_transport_timeout_ms(2000, 100), - 100, - "should preserve a sub-quantum configured timeout — quantizing it away would permanently disable the provider" - ); - assert_eq!( - super::effective_transport_timeout_ms(999, 2000), - 750, - "should quantize the budget-bound value down to the 750ms bucket" - ); - assert_eq!( - super::effective_transport_timeout_ms(300, 2000), - 250, - "should quantize a tight budget down to one quantum" - ); - assert_eq!( - super::effective_transport_timeout_ms(200, 2000), - 200, - "should pass a sub-quantum budget through exactly instead of rounding to zero" - ); - assert_eq!( - super::effective_transport_timeout_ms(50, 100), - 50, - "should pass through when the budget is below both the quantum and the configured timeout" - ); - assert_eq!( - super::effective_transport_timeout_ms(0, 1000), - 0, - "should return zero for an exhausted budget so the launch is skipped" - ); - assert_eq!( - super::effective_transport_timeout_ms(100, 0), - 0, - "should return zero for a zero configured timeout so the launch is skipped" - ); + impl CanonicalTimeoutBackend { + fn new(canonical_ms: u32, calls: Arc>>) -> Self { + Self { + canonical_ms, + calls, + } + } + } + + impl PlatformBackend for CanonicalTimeoutBackend { + fn predict_name( + &self, + _spec: &PlatformBackendSpec, + ) -> Result> { + Ok("stub-backend".to_owned()) + } + + fn ensure(&self, _spec: &PlatformBackendSpec) -> Result> { + Ok("stub-backend".to_owned()) + } + + fn canonicalize_transport_timeout_ms(&self, remaining_ms: u32, configured_ms: u32) -> u32 { + self.calls + .lock() + .expect("should lock canonicalize calls") + .push((remaining_ms, configured_ms)); + self.canonical_ms + } } #[test] - fn sub_quantum_configured_timeout_still_launches_provider() { + fn parallel_launch_applies_canonical_timeout_to_name_and_request() { futures::executor::block_on(async { - // A provider whose configured timeout is below one quantum must - // still launch with its exact configured value: the constant is - // name-stable on its own, so only budget-derived values are - // quantized. + // The orchestrator must hand the platform-canonicalized value to + // BOTH `backend_name` (which derives the correlation key) and + // `request_bids` (via `context.timeout_ms`). Recording them + // separately and asserting exact equality catches a regression that + // predicts one bucket but registers another — which would drop the + // response into the "unknown backend" branch. let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); - let services = build_services_with_http_client(stub); + let calls = Arc::new(Mutex::new(Vec::new())); + let backend = Arc::new(CanonicalTimeoutBackend::new(750, Arc::clone(&calls))); + let services = build_services_with_backend_and_http_client(backend, stub); // SAFETY: `Box::leak` creates a `'static` reference for test use only. // The leaked allocation is bounded to the test process lifetime. let services: &'static RuntimeServices = Box::leak(Box::new(services)); - let observed = Arc::new(Mutex::new(Vec::new())); + let predicted = Arc::new(Mutex::new(Vec::new())); + let requested = Arc::new(Mutex::new(Vec::new())); let config = AuctionConfig { enabled: true, providers: vec!["bidder".to_string()], @@ -2137,8 +2132,9 @@ mod tests { orchestrator.register_provider(Arc::new(StubAuctionProvider::recording( "bidder", "bidder-backend", - 100, - Arc::clone(&observed), + 1000, + Arc::clone(&predicted), + Arc::clone(&requested), ))); let request = create_test_auction_request(); @@ -2161,49 +2157,62 @@ mod tests { .await .expect("should complete auction"); - let observed = observed.lock().expect("should lock observed timeouts"); + let predicted = predicted.lock().expect("should lock predicted"); + let requested = requested.lock().expect("should lock requested"); + assert_eq!( + *predicted, + vec![750], + "backend_name should receive the canonicalized value" + ); + assert_eq!( + *requested, + vec![750], + "request_bids should receive the same canonicalized value" + ); + assert_eq!( + *predicted, *requested, + "predicted and registered transport timeouts must be identical" + ); + + let calls = calls.lock().expect("should lock calls"); + assert_eq!(calls.len(), 1, "should canonicalize once for the launch"); + let (remaining_ms, configured_ms) = calls[0]; + assert_eq!( + configured_ms, 1000, + "should pass the provider's configured timeout as the configured bound" + ); assert!( - !observed.is_empty(), - "should launch the sub-quantum-configured provider" + remaining_ms > 0 && remaining_ms <= 2000, + "should pass the live remaining budget, got {remaining_ms}ms" ); - for timeout in observed.iter() { - assert_eq!( - *timeout, 100, - "should pass the configured 100ms timeout through unchanged" - ); - } }); } #[test] - fn parallel_path_quantizes_provider_transport_timeout() { + fn zero_canonical_timeout_skips_parallel_launch() { futures::executor::block_on(async { - // A 999ms budget must reach the provider as the 750ms quantum - // bucket — both in backend_name (which derives the Fastly backend - // name) and in context.timeout_ms (which configures the backend - // and payload deadlines) — so the backend name stays stable - // across requests with slightly different remaining budgets. + // A platform that canonicalizes to zero signals "budget exhausted"; + // the orchestrator must skip the launch. With the only provider + // skipped, no requests launch and the auction errors. let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - let services = build_services_with_http_client(stub); + let calls = Arc::new(Mutex::new(Vec::new())); + let backend = Arc::new(CanonicalTimeoutBackend::new(0, Arc::clone(&calls))); + let services = build_services_with_backend_and_http_client(backend, stub); // SAFETY: `Box::leak` creates a `'static` reference for test use only. // The leaked allocation is bounded to the test process lifetime. let services: &'static RuntimeServices = Box::leak(Box::new(services)); - let observed = Arc::new(Mutex::new(Vec::new())); let config = AuctionConfig { enabled: true, providers: vec!["bidder".to_string()], - timeout_ms: 999, + timeout_ms: 2000, mediator: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider::recording( + orchestrator.register_provider(Arc::new(StubAuctionProvider::new( "bidder", "bidder-backend", - 2000, - Arc::clone(&observed), ))); let request = create_test_auction_request(); @@ -2216,60 +2225,55 @@ mod tests { let context = AuctionContext { settings: &settings, request: &req, - timeout_ms: 999, + timeout_ms: 2000, provider_responses: None, services, }; - orchestrator - .run_auction(&request, &context) - .await - .expect("should complete auction"); - - let observed = observed.lock().expect("should lock observed timeouts"); + let result = orchestrator.run_auction(&request, &context).await; assert!( - !observed.is_empty(), - "should record provider transport timeouts" + result.is_err(), + "should error when the only provider is skipped for an exhausted budget" ); - for timeout in observed.iter() { - assert!( - *timeout % super::TRANSPORT_TIMEOUT_QUANTUM_MS == 0 - && *timeout > 0 - && *timeout <= 750, - "should floor the 999ms budget to a quantum bucket at or below 750ms, got {timeout}ms" - ); - } }); } #[test] - fn sub_quantum_budget_launches_with_exact_remaining_timeout() { + fn synchronous_mediation_applies_canonical_timeout_to_mediator() { futures::executor::block_on(async { - // A configured auction budget below one quantum must still launch - // providers with the exact remaining budget — rounding it to zero - // would hard-fail every auction for publishers with sub-250ms - // budgets. + // The mediator runs after the bidding phase and has no select-loop + // backstop; it must still receive the platform-canonicalized value + // for both prediction and request. let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); - let services = build_services_with_http_client(stub); + stub.push_response(200, b"{}".to_vec()); // bidder send_async + stub.push_response(200, b"{}".to_vec()); // mediator send_async + let calls = Arc::new(Mutex::new(Vec::new())); + let backend = Arc::new(CanonicalTimeoutBackend::new(500, Arc::clone(&calls))); + let services = build_services_with_backend_and_http_client(backend, stub); // SAFETY: `Box::leak` creates a `'static` reference for test use only. // The leaked allocation is bounded to the test process lifetime. let services: &'static RuntimeServices = Box::leak(Box::new(services)); - let observed = Arc::new(Mutex::new(Vec::new())); + let predicted = Arc::new(Mutex::new(Vec::new())); + let requested = Arc::new(Mutex::new(Vec::new())); let config = AuctionConfig { enabled: true, providers: vec!["bidder".to_string()], - timeout_ms: 200, - mediator: None, + mediator: Some("mediator".to_string()), + timeout_ms: 2000, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider::recording( + orchestrator.register_provider(Arc::new(StubAuctionProvider::new( "bidder", "bidder-backend", + ))); + orchestrator.register_provider(Arc::new(StubAuctionProvider::recording( + "mediator", + "mediator-backend", 2000, - Arc::clone(&observed), + Arc::clone(&predicted), + Arc::clone(&requested), ))); let request = create_test_auction_request(); @@ -2282,67 +2286,76 @@ mod tests { let context = AuctionContext { settings: &settings, request: &req, - timeout_ms: 200, + timeout_ms: 2000, provider_responses: None, services, }; - let result = orchestrator + orchestrator .run_auction(&request, &context) .await - .expect("should complete auction with a sub-quantum budget"); + .expect("should complete mediated auction"); - assert_eq!( - result.provider_responses.len(), - 1, - "should launch the provider despite the sub-quantum budget" - ); - let observed = observed.lock().expect("should lock observed timeouts"); + let predicted = predicted.lock().expect("should lock predicted"); + let requested = requested.lock().expect("should lock requested"); + // The orchestrator hands the mediator its budget through + // `context.timeout_ms` and calls `request_bids` directly; it does not + // call the mediator's `backend_name` (the mediator self-registers its + // backend), so only the request side is observed here. assert!( - !observed.is_empty(), - "should record provider transport timeouts" + predicted.is_empty(), + "orchestrator should not separately predict a backend name for the mediator" + ); + assert_eq!( + *requested, + vec![500], + "mediator request should use the canonical value" ); - for timeout in observed.iter() { - assert!( - *timeout > 0 && *timeout <= 200, - "should pass the exact sub-quantum remaining budget through, got {timeout}ms" - ); - } }); } #[test] - fn synchronous_mediation_quantizes_mediator_timeout() { + fn dispatched_collect_applies_canonical_timeout_to_both_paths() { futures::executor::block_on(async { - // The mediator has no select-loop deadline backstop, so its - // transport timeout must be quantized by rounding down: a - // quantum-aligned value no larger than the remaining budget. + // Same wiring invariant on the split dispatch/collect path used by + // publisher page rendering: the dispatched bidder and the collected + // mediator both receive the canonicalized value for prediction and + // request. let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); // bidder send_async stub.push_response(200, b"{}".to_vec()); // mediator send_async - let services = build_services_with_http_client(stub); + let calls = Arc::new(Mutex::new(Vec::new())); + let backend = Arc::new(CanonicalTimeoutBackend::new(500, Arc::clone(&calls))); + let services = build_services_with_backend_and_http_client(backend, stub); // SAFETY: `Box::leak` creates a `'static` reference for test use only. // The leaked allocation is bounded to the test process lifetime. let services: &'static RuntimeServices = Box::leak(Box::new(services)); - let observed = Arc::new(Mutex::new(Vec::new())); + let bidder_predicted = Arc::new(Mutex::new(Vec::new())); + let bidder_requested = Arc::new(Mutex::new(Vec::new())); + let mediator_predicted = Arc::new(Mutex::new(Vec::new())); + let mediator_requested = Arc::new(Mutex::new(Vec::new())); let config = AuctionConfig { enabled: true, providers: vec!["bidder".to_string()], mediator: Some("mediator".to_string()), - timeout_ms: 999, + timeout_ms: 2000, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider::new( + orchestrator.register_provider(Arc::new(StubAuctionProvider::recording( "bidder", "bidder-backend", + 2000, + Arc::clone(&bidder_predicted), + Arc::clone(&bidder_requested), ))); orchestrator.register_provider(Arc::new(StubAuctionProvider::recording( "mediator", "mediator-backend", 2000, - Arc::clone(&observed), + Arc::clone(&mediator_predicted), + Arc::clone(&mediator_requested), ))); let request = create_test_auction_request(); @@ -2355,65 +2368,168 @@ mod tests { let context = AuctionContext { settings: &settings, request: &req, - timeout_ms: 999, + timeout_ms: 2000, provider_responses: None, services, }; + let dispatched = match orchestrator.dispatch_auction(&request, &context).await { + DispatchAuctionOutcome::Dispatched(dispatched) => dispatched, + _ => panic!("should dispatch the bidder request"), + }; orchestrator + .collect_dispatched_auction(dispatched, services, &context) + .await; + + let bidder_predicted = bidder_predicted + .lock() + .expect("should lock bidder predicted"); + let bidder_requested = bidder_requested + .lock() + .expect("should lock bidder requested"); + assert_eq!( + *bidder_predicted, + vec![500], + "dispatched bidder name should use canonical value" + ); + assert_eq!( + *bidder_requested, + vec![500], + "dispatched bidder request should use canonical value" + ); + assert_eq!( + *bidder_predicted, *bidder_requested, + "dispatched bidder predicted and registered timeouts must be identical" + ); + + let mediator_predicted = mediator_predicted + .lock() + .expect("should lock mediator predicted"); + let mediator_requested = mediator_requested + .lock() + .expect("should lock mediator requested"); + // As on the synchronous path, the orchestrator calls the mediator's + // `request_bids` directly without predicting a backend name for it. + assert!( + mediator_predicted.is_empty(), + "orchestrator should not separately predict a backend name for the mediator" + ); + assert_eq!( + *mediator_requested, + vec![500], + "mediator request should use the canonical value" + ); + }); + } + + #[test] + fn parallel_duplicate_backend_name_fails_second_provider_attributably() { + futures::executor::block_on(async { + // Two providers that canonicalize to the SAME backend name (e.g. two + // auction providers behind one gateway origin). The correlation map + // keys on backend name, so the second must not silently overwrite + // the first — it must fail attributably so no bid is misparsed or + // lost. + let stub = Arc::new(StubHttpClient::new()); + stub.push_response(200, b"{}".to_vec()); // provider-a send_async + stub.push_response(200, b"{}".to_vec()); // provider-b send_async (dropped after guard) + let services = build_services_with_http_client(stub); + // SAFETY: `Box::leak` creates a `'static` reference for test use only. + // The leaked allocation is bounded to the test process lifetime. + let services: &'static RuntimeServices = Box::leak(Box::new(services)); + + let config = AuctionConfig { + enabled: true, + providers: vec!["provider-a".to_string(), "provider-b".to_string()], + timeout_ms: 2000, + mediator: None, + ..Default::default() + }; + let mut orchestrator = AuctionOrchestrator::new(config); + orchestrator.register_provider(Arc::new(StubAuctionProvider::new( + "provider-a", + "shared-backend", + ))); + orchestrator.register_provider(Arc::new(StubAuctionProvider::new( + "provider-b", + "shared-backend", + ))); + + let request = create_test_auction_request(); + let settings = create_test_settings(); + let req = http::Request::builder() + .method(http::Method::GET) + .uri("https://example.com/test") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let context = AuctionContext { + settings: &settings, + request: &req, + timeout_ms: 2000, + provider_responses: None, + services, + }; + + let result = orchestrator .run_auction(&request, &context) .await - .expect("should complete mediated auction"); + .expect("should complete auction despite the name collision"); - let observed = observed.lock().expect("should lock observed timeouts"); - assert!(!observed.is_empty(), "should run the mediator"); - for timeout in observed.iter() { - assert!( - *timeout % super::TRANSPORT_TIMEOUT_QUANTUM_MS == 0, - "mediator timeout {timeout}ms should be quantum-aligned" - ); - assert!( - *timeout > 0 && *timeout <= 750, - "mediator timeout {timeout}ms should be positive and floored below the 999ms budget" - ); - } + assert_eq!( + result.provider_responses.len(), + 2, + "should account for both providers" + ); + let provider_a = result + .provider_responses + .iter() + .find(|r| r.provider == "provider-a") + .expect("should have provider-a response"); + let provider_b = result + .provider_responses + .iter() + .find(|r| r.provider == "provider-b") + .expect("should have provider-b response"); + assert_eq!( + provider_a.status, + BidStatus::Success, + "the first provider on the shared name should launch and succeed" + ); + assert_eq!( + provider_b.status, + BidStatus::Error, + "the second provider on the shared name should fail attributably, not be dropped" + ); }); } #[test] - fn dispatched_collect_quantizes_mediator_timeout() { + fn dispatched_duplicate_backend_name_fails_second_provider_attributably() { futures::executor::block_on(async { - // Same invariant as the synchronous path, on the split - // dispatch/collect path used by publisher page rendering. + // Same collision defense on the dispatch/collect path. let stub = Arc::new(StubHttpClient::new()); - stub.push_response(200, b"{}".to_vec()); // bidder send_async - stub.push_response(200, b"{}".to_vec()); // mediator send_async + stub.push_response(200, b"{}".to_vec()); // provider-a send_async + stub.push_response(200, b"{}".to_vec()); // provider-b send_async (dropped after guard) let services = build_services_with_http_client(stub); // SAFETY: `Box::leak` creates a `'static` reference for test use only. // The leaked allocation is bounded to the test process lifetime. let services: &'static RuntimeServices = Box::leak(Box::new(services)); - let observed_bidder = Arc::new(Mutex::new(Vec::new())); - let observed_mediator = Arc::new(Mutex::new(Vec::new())); let config = AuctionConfig { enabled: true, - providers: vec!["bidder".to_string()], - mediator: Some("mediator".to_string()), - timeout_ms: 999, + providers: vec!["provider-a".to_string(), "provider-b".to_string()], + timeout_ms: 2000, + mediator: None, ..Default::default() }; let mut orchestrator = AuctionOrchestrator::new(config); - orchestrator.register_provider(Arc::new(StubAuctionProvider::recording( - "bidder", - "bidder-backend", - 2000, - Arc::clone(&observed_bidder), + orchestrator.register_provider(Arc::new(StubAuctionProvider::new( + "provider-a", + "shared-backend", ))); - orchestrator.register_provider(Arc::new(StubAuctionProvider::recording( - "mediator", - "mediator-backend", - 2000, - Arc::clone(&observed_mediator), + orchestrator.register_provider(Arc::new(StubAuctionProvider::new( + "provider-b", + "shared-backend", ))); let request = create_test_auction_request(); @@ -2426,47 +2542,29 @@ mod tests { let context = AuctionContext { settings: &settings, request: &req, - timeout_ms: 999, + timeout_ms: 2000, provider_responses: None, services, }; let dispatched = match orchestrator.dispatch_auction(&request, &context).await { DispatchAuctionOutcome::Dispatched(dispatched) => dispatched, - _ => panic!("should dispatch the bidder request"), + _ => panic!("should dispatch the first provider despite the name collision"), }; - orchestrator + let result = orchestrator .collect_dispatched_auction(dispatched, services, &context) .await; - let observed_bidder = observed_bidder.lock().expect("should lock bidder timeouts"); - assert!( - !observed_bidder.is_empty(), - "should record dispatched bidder timeouts" + let provider_b = result + .provider_responses + .iter() + .find(|r| r.provider == "provider-b") + .expect("should have provider-b response"); + assert_eq!( + provider_b.status, + BidStatus::Error, + "the second provider on the shared name should fail attributably, not be dropped" ); - for timeout in observed_bidder.iter() { - assert!( - *timeout % super::TRANSPORT_TIMEOUT_QUANTUM_MS == 0 - && *timeout > 0 - && *timeout <= 750, - "dispatched bidder timeout should floor 999ms to a quantum bucket at or below 750ms, got {timeout}ms" - ); - } - - let observed_mediator = observed_mediator - .lock() - .expect("should lock mediator timeouts"); - assert!(!observed_mediator.is_empty(), "should run the mediator"); - for timeout in observed_mediator.iter() { - assert!( - *timeout % super::TRANSPORT_TIMEOUT_QUANTUM_MS == 0, - "mediator timeout {timeout}ms should be quantum-aligned" - ); - assert!( - *timeout > 0 && *timeout <= 750, - "mediator timeout {timeout}ms should be positive and floored below the 999ms budget" - ); - } }); } diff --git a/crates/trusted-server-core/src/ec/pull_sync.rs b/crates/trusted-server-core/src/ec/pull_sync.rs index fa096d59d..833898b50 100644 --- a/crates/trusted-server-core/src/ec/pull_sync.rs +++ b/crates/trusted-server-core/src/ec/pull_sync.rs @@ -174,6 +174,7 @@ pub fn dispatch_pull_sync( certificate_check: settings.proxy.certificate_check, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + discriminator: None, }) { Ok(name) => name, Err(err) => { diff --git a/crates/trusted-server-core/src/integrations/datadome/protection.rs b/crates/trusted-server-core/src/integrations/datadome/protection.rs index 717ad46e8..eedcf7cd2 100644 --- a/crates/trusted-server-core/src/integrations/datadome/protection.rs +++ b/crates/trusted-server-core/src/integrations/datadome/protection.rs @@ -160,6 +160,7 @@ impl DataDomeIntegration { certificate_check: true, first_byte_timeout: Duration::from_millis(u64::from(self.config.timeout_ms)), between_bytes_timeout: Duration::from_millis(u64::from(self.config.timeout_ms)), + discriminator: None, }; services.backend().ensure(&spec).change_context(Self::error( diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 2052215f2..75b4693ff 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -153,6 +153,10 @@ fn integration_backend_spec( certificate_check, first_byte_timeout, between_bytes_timeout: first_byte_timeout, + // Distinguish this integration's backend from any other provider that + // targets the same origin, so auction response correlation by backend + // name cannot cross providers. + discriminator: Some(integration.to_string()), }) } diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index ee7201fb8..d6ffee23b 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -649,6 +649,33 @@ pub(crate) fn noop_services_with_client_ip(ip: IpAddr) -> RuntimeServices { .build() } +/// Build a [`RuntimeServices`] with a caller-supplied [`PlatformBackend`] and +/// HTTP client. +/// +/// Lets auction tests inject a backend whose +/// [`PlatformBackend::canonicalize_transport_timeout_ms`] returns a controlled +/// value, so the orchestrator's transport-timeout wiring can be asserted +/// deterministically without depending on wall-clock timing. +pub(crate) fn build_services_with_backend_and_http_client( + backend: Arc, + http_client: Arc, +) -> RuntimeServices { + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(backend) + .http_client(http_client) + .geo(Arc::new(NoopGeo)) + .client_info(ClientInfo { + client_ip: None, + tls_protocol: None, + tls_cipher: None, + ..ClientInfo::default() + }) + .build() +} + /// Build a [`RuntimeServices`] with a custom secret store, [`StubBackend`], and HTTP client. pub(crate) fn build_services_with_secret_and_http_client( secret_store: impl PlatformSecretStore + 'static, @@ -856,6 +883,7 @@ mod tests { certificate_check: true, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + discriminator: None, }; let name = stub.ensure(&spec).expect("should return a backend name"); assert_eq!(name, "stub-backend", "should return fixed name"); diff --git a/crates/trusted-server-core/src/platform/traits.rs b/crates/trusted-server-core/src/platform/traits.rs index ecc886c9d..c6af0a307 100644 --- a/crates/trusted-server-core/src/platform/traits.rs +++ b/crates/trusted-server-core/src/platform/traits.rs @@ -110,6 +110,28 @@ pub trait PlatformBackend: Send + Sync { /// Returns [`PlatformError::Backend`] when the backend cannot be /// registered on the platform. fn ensure(&self, spec: &PlatformBackendSpec) -> Result>; + + /// Canonicalize a per-provider transport timeout for backend-name stability. + /// + /// `remaining_ms` is the wall-clock budget left in the auction and + /// `configured_ms` is the provider's own configured timeout. The returned + /// value is used both to derive the dynamic backend name and as the + /// provider's request deadline, so it must be identical for prediction and + /// registration of the same launch. + /// + /// Adapters that embed the transport timeout in the dynamic backend name + /// (Fastly) override this to round budget-derived values to a coarse + /// ladder, so per-request wall-clock jitter neither defeats cross-request + /// connection pooling nor accumulates registrations toward the per-service + /// dynamic backend limit. + /// + /// The default returns the exact budget-bound value + /// (`remaining_ms.min(configured_ms)`): adapters that neither register nor + /// enforce a backend-name transport timeout gain nothing from rounding and + /// must not shorten bidder deadlines for no benefit. + fn canonicalize_transport_timeout_ms(&self, remaining_ms: u32, configured_ms: u32) -> u32 { + remaining_ms.min(configured_ms) + } } /// Synchronous, object-safe geo lookup. diff --git a/crates/trusted-server-core/src/platform/types.rs b/crates/trusted-server-core/src/platform/types.rs index a81c24105..a39a26430 100644 --- a/crates/trusted-server-core/src/platform/types.rs +++ b/crates/trusted-server-core/src/platform/types.rs @@ -143,6 +143,16 @@ pub struct PlatformBackendSpec { pub first_byte_timeout: Duration, /// Maximum time to wait between response body bytes. pub between_bytes_timeout: Duration, + /// Optional stable discriminator folded into the backend name. + /// + /// Two callers can target the same origin (scheme, host, port, TLS) with + /// the same transport timeout yet need distinct dynamic backends — for + /// example two auction providers behind one gateway host. Because the + /// auction orchestrator correlates responses back to providers by backend + /// name, a shared name would let one provider's response be parsed as + /// another's. Setting this to a per-provider/integration identifier keeps + /// their names distinct while remaining stable across requests. + pub discriminator: Option, } /// Cloneable container of platform services for a single request. diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 19c10a80d..d1a5cdc57 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -1099,6 +1099,7 @@ pub async fn handle_asset_proxy_request( certificate_check: settings.proxy.certificate_check, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + discriminator: None, }) .change_context(TrustedServerError::Proxy { message: "asset backend registration failed".to_string(), @@ -1289,6 +1290,7 @@ async fn proxy_with_redirects( certificate_check: settings.proxy.certificate_check, first_byte_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, between_bytes_timeout: DEFAULT_FIRST_BYTE_TIMEOUT, + discriminator: None, }) .change_context(TrustedServerError::Proxy { message: "backend registration failed".to_string(), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 6b0ea3a5d..7a2b9b437 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1368,6 +1368,7 @@ pub async fn handle_publisher_request( certificate_check: settings.proxy.certificate_check, first_byte_timeout: DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT, between_bytes_timeout: DEFAULT_PUBLISHER_FIRST_BYTE_TIMEOUT, + discriminator: None, }) .change_context(TrustedServerError::Proxy { message: "backend registration failed".to_string(), From 62a2dd054adf18f13e2157ce9a3df4e18101bf80 Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 10 Jul 2026 14:56:06 -0500 Subject: [PATCH 021/198] Add Prebid error diagnostics Prebid non-2xx responses were reduced to bare provider errors, making intermittent failures difficult to diagnose. Surface safe HTTP metadata and bounded debug details while correlating server logs with the auction ID. --- .../src/integrations/prebid.rs | 551 ++++++++++++++---- docs/guide/integrations/prebid.md | 28 +- 2 files changed, 450 insertions(+), 129 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 22c972f5c..2e7655382 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -62,20 +62,134 @@ const ZONE_KEY: &str = "zone"; /// Default currency for `OpenRTB` bid floors and responses. const DEFAULT_CURRENCY: &str = "USD"; -#[cfg(test)] +const PREBID_ERROR_TYPE_UPSTREAM_HTTP: &str = "upstream_http"; +const PREBID_PUBLIC_ERROR_MESSAGE_CHARS: usize = 500; const PREBID_ERROR_BODY_PREVIEW_CHARS: usize = 1000; - -#[cfg(test)] const PREBID_ERROR_BODY_PREVIEW_BYTES: usize = PREBID_ERROR_BODY_PREVIEW_CHARS * 4; +const PREBID_ERROR_JSON_MAX_DEPTH: usize = 6; +const PREBID_ERROR_JSON_KEYS: [&str; 6] = + ["message", "error", "errors", "detail", "title", "reason"]; + +#[derive(Debug, Eq, PartialEq)] +struct BoundedPrebidErrorText { + text: String, + truncated: bool, +} -#[cfg(test)] -fn prebid_body_preview(body: &[u8]) -> String { +fn bounded_prebid_error_text(value: &str, max_chars: usize) -> Option { + let mut text = String::new(); + let mut char_count = 0; + let mut pending_space = false; + let mut truncated = false; + + for character in value.chars() { + if character.is_whitespace() || character.is_control() { + pending_space = !text.is_empty(); + continue; + } + + if pending_space { + if char_count == max_chars { + truncated = true; + break; + } + text.push(' '); + char_count += 1; + pending_space = false; + } + + if char_count == max_chars { + truncated = true; + break; + } + text.push(character); + char_count += 1; + } + + (!text.is_empty()).then_some(BoundedPrebidErrorText { text, truncated }) +} + +fn prebid_body_preview(body: &[u8]) -> Option { let bounded_body = &body[..body.len().min(PREBID_ERROR_BODY_PREVIEW_BYTES)]; + let mut preview = bounded_prebid_error_text( + &String::from_utf8_lossy(bounded_body), + PREBID_ERROR_BODY_PREVIEW_CHARS, + )?; + preview.truncated |= body.len() > bounded_body.len(); + Some(preview) +} - String::from_utf8_lossy(bounded_body) - .chars() - .take(PREBID_ERROR_BODY_PREVIEW_CHARS) - .collect() +fn nested_prebid_json_error_message( + value: &Json, + depth: usize, + allow_direct_string: bool, +) -> Option<&str> { + if depth > PREBID_ERROR_JSON_MAX_DEPTH { + return None; + } + + match value { + Json::String(message) if allow_direct_string => { + (!message.trim().is_empty()).then_some(message.as_str()) + } + Json::Array(values) => values.iter().find_map(|value| { + nested_prebid_json_error_message(value, depth + 1, allow_direct_string) + }), + Json::Object(values) => PREBID_ERROR_JSON_KEYS + .iter() + .find_map(|key| { + values + .get(*key) + .and_then(|value| nested_prebid_json_error_message(value, depth + 1, true)) + }) + .or_else(|| { + values + .values() + .find_map(|value| nested_prebid_json_error_message(value, depth + 1, false)) + }), + _ => None, + } +} + +fn prebid_json_error_message(value: &Json) -> Option<&str> { + let Json::Object(values) = value else { + return None; + }; + + PREBID_ERROR_JSON_KEYS.iter().find_map(|key| { + values + .get(*key) + .and_then(|value| nested_prebid_json_error_message(value, 0, true)) + }) +} + +fn is_plain_text_content_type(content_type: Option<&str>) -> bool { + content_type.is_some_and(|value| { + value + .split(';') + .next() + .is_some_and(|mime| mime.trim().eq_ignore_ascii_case("text/plain")) + }) +} + +fn extract_prebid_error_message( + body: &[u8], + content_type: Option<&str>, +) -> Option { + let candidate = match serde_json::from_slice::(body) { + Ok(value) => prebid_json_error_message(&value)?.to_owned(), + Err(_) if is_plain_text_content_type(content_type) => { + std::str::from_utf8(body).ok()?.to_owned() + } + Err(_) => return None, + }; + + // Do not expose an HTML error page even if an intermediary labels it as text/plain. + if candidate.trim_start().starts_with('<') { + return None; + } + + bounded_prebid_error_text(&candidate, PREBID_PUBLIC_ERROR_MESSAGE_CHARS) } /// CCPA/US-privacy string sent when the `Sec-GPC` header signals opt-out. @@ -1845,6 +1959,113 @@ impl PrebidAuctionProvider { } } + async fn parse_response_inner( + &self, + response: PlatformResponse, + response_time_ms: u64, + auction_id: Option<&str>, + ) -> Result> { + let response = response.response; + let status = response.status(); + let content_type = response + .headers() + .get(header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_owned); + + // Parse response — collect_response_bounded caps memory from misbehaving providers. + let body_bytes = collect_response_bounded( + response.into_body(), + UPSTREAM_RTB_MAX_RESPONSE_BYTES, + "prebid", + ) + .await + .change_context(TrustedServerError::Prebid { + message: "Failed to read Prebid response body".to_string(), + })?; + + if !status.is_success() { + let auction_id = auction_id.unwrap_or(""); + log::warn!("Prebid auction {auction_id:?} returned non-success status: {status}"); + + if self.config.debug { + match prebid_body_preview(&body_bytes) { + Some(preview) => { + let truncation = if preview.truncated { + " (truncated)" + } else { + "" + }; + log::warn!( + "Prebid auction {auction_id:?} error response body preview{truncation}: {}", + preview.text + ); + } + None => log::warn!( + "Prebid auction {auction_id:?} returned an empty error response body" + ), + } + } + + let status_code = status.as_u16(); + let mut auction_response = + AuctionResponse::error(PREBID_INTEGRATION_ID, response_time_ms) + .with_metadata( + "error_type", + serde_json::json!(PREBID_ERROR_TYPE_UPSTREAM_HTTP), + ) + .with_metadata("http_status", serde_json::json!(status_code)) + .with_metadata( + "message", + serde_json::json!(format!("Prebid Server returned HTTP {status_code}")), + ); + + if self.config.debug { + if let Some(message) = + extract_prebid_error_message(&body_bytes, content_type.as_deref()) + { + auction_response.metadata.insert( + "upstream_message".to_string(), + serde_json::json!(message.text), + ); + auction_response.metadata.insert( + "upstream_message_truncated".to_string(), + serde_json::json!(message.truncated), + ); + } + } + + return Ok(auction_response); + } + + let response_json: Json = + serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Prebid { + message: "Failed to parse Prebid response".to_string(), + })?; + + // Log the full response body when debug is enabled to surface + // ext.debug.httpcalls, resolvedrequest, bidstatus, errors, etc. + if self.config.debug && log::log_enabled!(log::Level::Trace) { + match serde_json::to_string_pretty(&response_json) { + Ok(json) => log::trace!("Prebid OpenRTB response:\n{json}"), + Err(e) => { + log::warn!("Prebid: failed to serialize response for logging: {e}"); + } + } + } + + let mut auction_response = self.parse_openrtb_response(&response_json, response_time_ms); + self.enrich_response_metadata(&response_json, &mut auction_response); + + log::info!( + "Prebid returned {} bids in {}ms", + auction_response.bids.len(), + response_time_ms + ); + + Ok(auction_response) + } + fn should_suppress_bid_notifications(&self, bidder: &str) -> bool { self.config.suppress_nurl || self @@ -2108,68 +2329,19 @@ impl AuctionProvider for PrebidAuctionProvider { response: PlatformResponse, response_time_ms: u64, ) -> Result> { - let response = response.response; - let status = response.status(); - - // Parse response — collect_response_bounded caps memory from misbehaving providers. - let body_bytes = collect_response_bounded( - response.into_body(), - UPSTREAM_RTB_MAX_RESPONSE_BYTES, - "prebid", - ) - .await - .change_context(TrustedServerError::Prebid { - message: "Failed to read Prebid response body".to_string(), - })?; - - if !status.is_success() { - let body_preview = String::from_utf8_lossy(&body_bytes); - // SECURITY: the PBS response body is upstream-controlled and may leak - // internal detail (hostnames, stack traces, auth hints). Per the - // invariant documented in `auction/orchestrator.rs`, it MUST NOT reach - // the public `/auction` response, which happens if it lands in - // `AuctionResponse.metadata` (cloned verbatim into - // `ext.orchestrator.provider_details[].metadata`). Log the snippet - // server-side and surface only the numeric HTTP status — enough for an - // operator to tell an error from a no-bid without publishing the body. - log::warn!( - "Prebid returned non-success status {status}: {}", - &body_preview[..body_preview.floor_char_boundary(512)] - ); - return Ok(AuctionResponse::error("prebid", response_time_ms) - .with_metadata( - "error_type", - serde_json::json!(crate::auction::orchestrator::ERROR_TYPE_HTTP_STATUS), - ) - .with_metadata("status", serde_json::json!(status.as_u16()))); - } - - let response_json: Json = - serde_json::from_slice(&body_bytes).change_context(TrustedServerError::Prebid { - message: "Failed to parse Prebid response".to_string(), - })?; - - // Log the full response body when debug is enabled to surface - // ext.debug.httpcalls, resolvedrequest, bidstatus, errors, etc. - if self.config.debug && log::log_enabled!(log::Level::Trace) { - match serde_json::to_string_pretty(&response_json) { - Ok(json) => log::trace!("Prebid OpenRTB response:\n{json}"), - Err(e) => { - log::warn!("Prebid: failed to serialize response for logging: {e}"); - } - } - } - - let mut auction_response = self.parse_openrtb_response(&response_json, response_time_ms); - self.enrich_response_metadata(&response_json, &mut auction_response); - - log::info!( - "Prebid returned {} bids in {}ms", - auction_response.bids.len(), - response_time_ms - ); + self.parse_response_inner(response, response_time_ms, None) + .await + } - Ok(auction_response) + async fn parse_response_with_context( + &self, + response: PlatformResponse, + response_time_ms: u64, + request: &AuctionRequest, + _context: &AuctionContext<'_>, + ) -> Result> { + self.parse_response_inner(response, response_time_ms, Some(request.id.as_str())) + .await } fn supports_media_type(&self, media_type: &MediaType) -> bool { @@ -2243,8 +2415,7 @@ mod tests { use super::*; use crate::auction::test_support::create_test_auction_context as shared_test_auction_context; use crate::auction::types::{ - AdFormat, AdSlot, AuctionContext, AuctionRequest, BidStatus, DeviceInfo, PublisherInfo, - UserInfo, + AdFormat, AdSlot, AuctionContext, AuctionRequest, DeviceInfo, PublisherInfo, UserInfo, }; use crate::consent::{ConsentContext, ConsentSource}; @@ -2352,50 +2523,6 @@ mod tests { ); } - #[test] - fn parse_response_attaches_status_metadata_without_leaking_body_on_http_error() { - let provider = PrebidAuctionProvider::new(base_config()); - let response = PlatformResponse::new( - edgezero_core::http::response_builder() - .status(403) - .body(EdgeBody::from( - br#"{"error":"upstream-secret-detail"}"#.to_vec(), - )) - .expect("should build test response"), - ); - - let result = futures::executor::block_on(provider.parse_response(response, 643)) - .expect("should return Ok(error response) for non-success status"); - - assert_eq!( - result.status, - BidStatus::Error, - "non-success HTTP status should map to an error response" - ); - assert_eq!( - result.metadata["error_type"], - json!("http_status"), - "should tag the error path so telemetry buckets it as an http status error" - ); - assert_eq!( - result.metadata["status"], - json!(403), - "should surface the upstream HTTP status code" - ); - // SECURITY: the upstream response body must never reach the public - // /auction response via AuctionResponse.metadata. - assert!( - !result.metadata.contains_key("body"), - "upstream response body must not be surfaced on the response metadata" - ); - assert!( - !result.metadata.values().any(|v| v - .as_str() - .is_some_and(|s| s.contains("upstream-secret-detail"))), - "no metadata value may contain the upstream body" - ); - } - fn test_sri(algorithm: &str, digest: &[u8]) -> String { format!("{algorithm}-{}", TEST_BASE64_STANDARD.encode(digest)) } @@ -2430,6 +2557,23 @@ mod tests { .expect("should parse response body as utf-8") } + fn prebid_platform_response( + status: StatusCode, + content_type: Option<&str>, + body: impl Into>, + ) -> PlatformResponse { + let mut builder = http::Response::builder().status(status); + if let Some(content_type) = content_type { + builder = builder.header(header::CONTENT_TYPE, content_type); + } + + PlatformResponse::new( + builder + .body(EdgeBody::from(body.into())) + .expect("should build Prebid platform response"), + ) + } + fn create_test_auction_request() -> AuctionRequest { AuctionRequest { id: "auction-123".to_string(), @@ -4835,27 +4979,42 @@ external_bundle_sri = "sha384-AAAA" ); } + #[test] + fn bounded_prebid_error_text_normalizes_control_characters_and_whitespace() { + let message = bounded_prebid_error_text("\n invalid\trequest\0 payload \r\n", 100) + .expect("should extract bounded text"); + + assert_eq!( + message.text, "invalid request payload", + "should make upstream text safe for one-line responses and logs" + ); + assert!(!message.truncated, "should retain the complete message"); + } + #[test] fn prebid_body_preview_truncates_to_character_limit() { let body = "x".repeat(PREBID_ERROR_BODY_PREVIEW_CHARS + 100); - let preview = prebid_body_preview(body.as_bytes()); + let preview = prebid_body_preview(body.as_bytes()).expect("should build body preview"); assert_eq!( - preview.chars().count(), + preview.text.chars().count(), PREBID_ERROR_BODY_PREVIEW_CHARS, "should cap the upstream body preview" ); + assert!(preview.truncated, "should report body preview truncation"); } #[test] fn prebid_body_preview_handles_non_utf8_lossily() { - let preview = prebid_body_preview(&[b'o', b'k', 0xff, b'!']); + let preview = + prebid_body_preview(&[b'o', b'k', 0xff, b'!']).expect("should build body preview"); assert_eq!( - preview, "ok\u{fffd}!", + preview.text, "ok\u{fffd}!", "should replace invalid UTF-8 bytes without panicking" ); + assert!(!preview.truncated, "should retain the complete preview"); } #[test] @@ -4863,17 +5022,18 @@ external_bundle_sri = "sha384-AAAA" let mut body = vec![b'x'; PREBID_ERROR_BODY_PREVIEW_BYTES]; body.extend_from_slice(&[0xff, b't', b'a', b'i', b'l']); - let preview = prebid_body_preview(&body); + let preview = prebid_body_preview(&body).expect("should build body preview"); assert_eq!( - preview.chars().count(), + preview.text.chars().count(), PREBID_ERROR_BODY_PREVIEW_CHARS, - "should keep the public preview capped" + "should keep the log preview capped" ); assert!( - !preview.contains('\u{fffd}') && !preview.contains("tail"), + !preview.text.contains('\u{fffd}') && !preview.text.contains("tail"), "should not process bytes beyond the bounded preview slice" ); + assert!(preview.truncated, "should report bounded-slice truncation"); } #[test] @@ -4882,17 +5042,152 @@ external_bundle_sri = "sha384-AAAA" body.extend_from_slice("\u{2603}".as_bytes()); body.extend_from_slice(b"tail"); - let preview = prebid_body_preview(&body); + let preview = prebid_body_preview(&body).expect("should build body preview"); assert_eq!( - preview.chars().count(), + preview.text.chars().count(), PREBID_ERROR_BODY_PREVIEW_CHARS, - "should keep the public preview capped" + "should keep the log preview capped" ); assert!( - !preview.contains("tail"), + !preview.text.contains("tail"), "should not include bytes beyond the bounded preview slice" ); + assert!(preview.truncated, "should report partial-body truncation"); + } + + #[test] + fn extract_prebid_error_message_reads_nested_json_message() { + let body = br#"{ + "errors": { + "exampleBidder": [{"code": 1, "message": " invalid\nrequest "}] + } + }"#; + + let message = extract_prebid_error_message(body, Some("application/json")) + .expect("should extract nested JSON error message"); + + assert_eq!(message.text, "invalid request"); + assert!(!message.truncated, "should retain the complete message"); + } + + #[test] + fn extract_prebid_error_message_reads_plain_text() { + let message = extract_prebid_error_message( + b" request rejected\r\nby Prebid Server ", + Some("Text/Plain; charset=utf-8"), + ) + .expect("should extract plain-text error message"); + + assert_eq!(message.text, "request rejected by Prebid Server"); + assert!(!message.truncated, "should retain the complete message"); + } + + #[test] + fn extract_prebid_error_message_rejects_html_and_unknown_json_fields() { + assert!( + extract_prebid_error_message( + b"internal proxy error", + Some("text/plain"), + ) + .is_none(), + "should not expose HTML error pages" + ); + + for body in [ + br#"{"resolvedrequest":{"account":"internal"}}"#.as_slice(), + br#"{"errors":{"resolvedrequest":{"account":"internal"}}}"#.as_slice(), + br#""internal""#.as_slice(), + br#"["internal"]"#.as_slice(), + ] { + assert!( + extract_prebid_error_message(body, Some("application/json")).is_none(), + "should only expose strings associated with allowlisted JSON error fields" + ); + } + } + + #[test] + fn extract_prebid_error_message_truncates_public_message() { + let body = serde_json::to_vec(&json!({ + "message": "x".repeat(PREBID_PUBLIC_ERROR_MESSAGE_CHARS + 100), + })) + .expect("should serialize test error response"); + + let message = extract_prebid_error_message(&body, Some("application/json")) + .expect("should extract JSON error message"); + + assert_eq!( + message.text.chars().count(), + PREBID_PUBLIC_ERROR_MESSAGE_CHARS, + "should cap the browser-visible upstream message" + ); + assert!(message.truncated, "should report public message truncation"); + } + + #[test] + fn non_success_prebid_response_always_includes_safe_http_metadata() { + let provider = PrebidAuctionProvider::new(base_config()); + let response = prebid_platform_response( + StatusCode::BAD_REQUEST, + Some("application/json"), + br#"{"message":"request details should remain hidden"}"#.to_vec(), + ); + + let auction_response = futures::executor::block_on(provider.parse_response(response, 42)) + .expect("should convert upstream HTTP failure to auction response"); + + assert_eq!( + auction_response.status, + crate::auction::types::BidStatus::Error + ); + assert_eq!( + auction_response.metadata["error_type"], + json!(PREBID_ERROR_TYPE_UPSTREAM_HTTP) + ); + assert_eq!(auction_response.metadata["http_status"], json!(400)); + assert_eq!( + auction_response.metadata["message"], + json!("Prebid Server returned HTTP 400") + ); + assert!( + !auction_response.metadata.contains_key("upstream_message"), + "should hide upstream text when Prebid debug is disabled" + ); + } + + #[test] + fn debug_non_success_prebid_response_includes_bounded_upstream_message() { + let mut config = base_config(); + config.debug = true; + let provider = PrebidAuctionProvider::new(config); + let response = prebid_platform_response( + StatusCode::UNPROCESSABLE_ENTITY, + Some("application/json; charset=utf-8"), + br#"{"error":{"message":"imp[0] has no valid bidders"}}"#.to_vec(), + ); + let settings = make_settings(); + let http_request = build_test_request(); + let context = create_test_auction_context(&settings, &http_request); + let auction_request = create_test_auction_request(); + + let auction_response = futures::executor::block_on(provider.parse_response_with_context( + response, + 66, + &auction_request, + &context, + )) + .expect("should convert upstream HTTP failure to debug auction response"); + + assert_eq!(auction_response.metadata["http_status"], json!(422)); + assert_eq!( + auction_response.metadata["upstream_message"], + json!("imp[0] has no valid bidders") + ); + assert_eq!( + auction_response.metadata["upstream_message_truncated"], + json!(false) + ); } fn make_auction_request(slots: Vec) -> AuctionRequest { diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index c1e85a553..b8340cfb5 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -132,8 +132,34 @@ The Prebid provider extracts metadata from the Prebid Server response and attach | `debug` | `ext.debug` | Prebid Server debug payload (httpcalls, resolvedrequest) | | `bidstatus` | `ext.prebid.bidstatus` | Per-bid status from every invited bidder | +### Upstream HTTP errors + +When Prebid Server returns a non-2xx status, the provider detail always includes a safe error classification, HTTP status, and generic message: + +```json +{ + "error_type": "upstream_http", + "http_status": 400, + "message": "Prebid Server returned HTTP 400" +} +``` + +With `debug = true`, Trusted Server also extracts the first error message from allowlisted JSON fields (`message`, `error`, `errors`, `detail`, `title`, or `reason`) or a plain-text response. The message is normalized to one line and limited to 500 characters: + +```json +{ + "error_type": "upstream_http", + "http_status": 400, + "message": "Prebid Server returned HTTP 400", + "upstream_message": "Invalid request: imp[0] has no valid bidders", + "upstream_message_truncated": false +} +``` + +HTML error pages and unrecognized JSON payloads are not exposed. Debug mode also writes a bounded error-body preview to `tslog`, correlated with the auction ID. + ::: warning -Enabling `debug` increases response sizes and adds overhead. Use it in development or when diagnosing auction issues — not in production. +Enabling `debug` increases response sizes and adds overhead. It can also expose bounded upstream diagnostics to `/auction` callers and logs. Use it temporarily when diagnosing auction issues, not as a permanent production setting. ::: ### Test mode vs. debug From ef619539659d9126c11d6f0b22892af91c9b1163 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Tue, 14 Jul 2026 23:20:24 +0530 Subject: [PATCH 022/198] Suppress fabricated empty Prebid bidder params A configured bidder with no inline params and no matching override expanded to `"bidder": {}`, which PBS rejects. After applying overrides, drop fabricated empty bidders, preserve an explicitly supplied empty object so genuine misconfiguration stays visible, and fall back to the stored-request path when no eligible bidders remain. --- .../src/integrations/prebid.rs | 179 ++++++++++++++++-- 1 file changed, 163 insertions(+), 16 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 32a1c2a85..68d718a16 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::Duration; @@ -1424,10 +1424,22 @@ impl PrebidAuctionProvider { // Only pass through keys that are known PBS bidders — skip provider-specific // keys like "aps" which belong to their own separate auction provider. let mut bidder: HashMap = HashMap::new(); + // Bidders the publisher explicitly supplied — including an + // explicit empty `{}`. A configured bidder that exists only + // because `expand_trusted_server_bidders` fabricated an empty + // params object is NOT explicit and must not ship as + // `"bidder": {}` (which PBS rejects). + let mut explicit_bidders: HashSet = HashSet::new(); for (name, params) in &slot.bidders { if name == TRUSTED_SERVER_BIDDER { + if let Some(per_bidder) = + params.get(BIDDER_PARAMS_KEY).and_then(Json::as_object) + { + explicit_bidders.extend(per_bidder.keys().cloned()); + } bidder.extend(expand_trusted_server_bidders(&self.config.bidders, params)); } else if self.config.bidders.iter().any(|b| b == name) { + explicit_bidders.insert(name.clone()); bidder.insert(name.clone(), params.clone()); } else if name != "aps" { // `aps` is intentionally handled by its own provider. Any @@ -1443,16 +1455,32 @@ impl PrebidAuctionProvider { } } - // When no inline PBS bidder params exist (e.g. creative-opportunity slots - // whose PBS params live in stored requests), tell PBS to resolve bidder - // config from the stored request keyed by this slot ID. + // Apply canonical and compatibility-derived rules in normalized + // order. An override rule can populate a bidder that arrived with + // empty params, promoting a fabricated empty into a valid bidder. + for (name, params) in &mut bidder { + self.bid_param_override_engine + .apply(BidParamOverrideFacts { bidder: name, zone }, params); + } + + // Drop bidders that are still an empty object after overrides and + // were not explicitly supplied. Shipping `"bidder": {}` makes PBS + // reject the imp; an explicit empty object is preserved so genuine + // publisher misconfiguration stays visible. + bidder.retain(|name, params| { + let is_empty_object = params.as_object().is_some_and(serde_json::Map::is_empty); + !is_empty_object || explicit_bidders.contains(name) + }); + + // When no eligible PBS bidder params remain (e.g. creative-opportunity + // slots whose PBS params live in stored requests, or a slot whose + // configured bidders all resolved to fabricated empties), tell PBS to + // resolve bidder config from the stored request keyed by this slot ID. // - // This cannot fire for the client /auction path: the JS adapter - // injects a `trustedServer` entry into every ad unit, so `bidder` - // is only empty for server-side creative-opportunity slots with - // no inline provider params (or when `config.bidders` is empty, - // where PBS previously received an empty bidder map and returned - // no bids — a stored-request miss is the same no-bid outcome). + // This cannot fire for a client /auction slot that carries real + // inline params: the JS adapter injects a `trustedServer` entry, and + // any bidder with params survives the drop above. It falls back only + // when nothing eligible remains — the same no-bid outcome as before. let storedrequest = if bidder.is_empty() { Some(ImpStoredRequest { id: slot.id.clone(), @@ -1461,12 +1489,6 @@ impl PrebidAuctionProvider { None }; - // Apply canonical and compatibility-derived rules in normalized order. - for (name, params) in &mut bidder { - self.bid_param_override_engine - .apply(BidParamOverrideFacts { bidder: name, zone }, params); - } - Some(Imp { id: Some(slot.id.clone()), banner: Some(Banner { @@ -5950,6 +5972,131 @@ set = { placementId = "explicit_header" } ); } + #[test] + fn to_openrtb_drops_fabricated_empty_bidder_params() { + // config.bidders lists three, but the slot supplies inline params only + // for kargo. Without a matching override, triplelift and criteo would + // expand to empty `{}` objects — invalid bidder entries PBS rejects. + // They must be dropped; the valid kargo bidder must still ship. + let config = parse_prebid_toml( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example" +bidders = ["kargo", "triplelift", "criteo"] +"#, + ); + + let slot = make_ts_slot( + "ad-header-0", + &json!({ "kargo": { "placementId": "kn1" } }), + None, + ); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(config, &request); + let params = bidder_params(&ortb); + + assert_eq!( + params["kargo"]["placementId"], "kn1", + "should keep the valid inline bidder" + ); + assert!( + !params.contains_key("triplelift"), + "should drop a fabricated empty bidder with no inline params or override" + ); + assert!( + !params.contains_key("criteo"), + "should drop a fabricated empty bidder with no inline params or override" + ); + } + + #[test] + fn to_openrtb_preserves_an_explicitly_empty_bidder() { + // A publisher-supplied empty `{}` is a real (if misconfigured) signal and + // must survive so the misconfiguration stays visible — unlike a fabricated + // empty, which is dropped. + let config = parse_prebid_toml( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example" +bidders = ["kargo"] +"#, + ); + + let slot = make_ts_slot("ad-header-0", &json!({ "kargo": {} }), None); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(config, &request); + let params = bidder_params(&ortb); + + assert_eq!( + params["kargo"], + json!({}), + "should preserve an explicitly supplied empty bidder object" + ); + } + + #[test] + fn to_openrtb_keeps_a_fabricated_bidder_that_an_override_populates() { + // criteo has no inline params (fabricated empty), but an override rule + // fills it — so it is valid and must ship, not be dropped. + let config = parse_prebid_toml( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example" +bidders = ["criteo"] + +[integrations.prebid.bid_param_overrides.criteo] +networkId = 99999 +"#, + ); + + let slot = make_ts_slot("ad-header-0", &json!({}), None); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(config, &request); + let params = bidder_params(&ortb); + + assert_eq!( + params["criteo"]["networkId"], 99999, + "override should populate the fabricated empty bidder, keeping it" + ); + } + + #[test] + fn to_openrtb_falls_back_to_stored_request_when_all_bidders_are_fabricated_empty() { + // config.bidders present, but the slot supplies no inline params and no + // override matches — every configured bidder resolves to a fabricated + // empty and is dropped, leaving PBS to resolve via the stored request. + let config = parse_prebid_toml( + r#" +[integrations.prebid] +enabled = true +server_url = "https://prebid.example" +bidders = ["kargo", "triplelift"] +"#, + ); + + let slot = make_ts_slot("ad-header-0", &json!({}), None); + let request = make_auction_request(vec![slot]); + + let ortb = call_to_openrtb(config, &request); + let ext = ortb.imp[0].ext.as_ref().expect("should have imp ext"); + let prebid = ext.get("prebid").expect("should have prebid in ext"); + + assert!( + prebid.get("bidder").is_none(), + "should drop all fabricated empty bidders" + ); + assert_eq!( + prebid["storedrequest"]["id"], "ad-header-0", + "should fall back to stored request when no eligible bidders remain" + ); + } + #[test] fn to_openrtb_skips_aps_key_from_slot_bidders_in_pbs_request() { let slot = make_slot( From 0acac4b207f26697084c8ca56368615136625911 Mon Sep 17 00:00:00 2001 From: Christian Date: Tue, 14 Jul 2026 14:07:32 -0500 Subject: [PATCH 023/198] Preserve Prebid ad units across GPT refreshes --- .../lib/src/integrations/prebid/index.ts | 239 +++++++- .../test/integrations/prebid/index.test.ts | 509 ++++++++++++++++++ 2 files changed, 731 insertions(+), 17 deletions(-) diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index be839d8fc..65932d5ba 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -228,6 +228,17 @@ type TrustedServerAdUnit = { mediaTypes?: { banner?: TrustedServerBanner }; bids?: TrustedServerBid[]; }; +type ClientSideBidSnapshot = { bidder: string; params: Record }; +type PublisherAdUnitSnapshot = { + bidderParams: Record>; + clientSideBids: ClientSideBidSnapshot[]; + zone?: string; +}; +type PublisherDeliveryContext = { remainingCodes: Set }; + +let publisherAdUnitSnapshots = new Map(); +let syntheticRefreshAdUnits = new WeakSet(); +const activePublisherDeliveryContexts: PublisherDeliveryContext[] = []; type TrustedServerBidRequest = { adUnitCode?: string; code?: string; @@ -363,6 +374,17 @@ function firstTargetingValue(values: string[] | undefined): string | undefined { * code in order and return the first matching ad unit, so container-backed slots * still recover the publisher's configured params and bidders. */ +function findRefreshSnapshot( + candidateCodes: Array +): PublisherAdUnitSnapshot | undefined { + for (const code of candidateCodes) { + if (!code) continue; + const snapshot = publisherAdUnitSnapshots.get(code); + if (snapshot) return snapshot; + } + return undefined; +} + function findRefreshAdUnit( candidateCodes: Array ): TrustedServerAdUnit | undefined { @@ -375,6 +397,89 @@ function findRefreshAdUnit( return undefined; } +function copyParamValue(value: unknown, seen = new WeakMap()): unknown { + if (Array.isArray(value)) { + const existing = seen.get(value); + if (existing) return existing; + const copy: unknown[] = []; + seen.set(value, copy); + value.forEach((entry) => copy.push(copyParamValue(entry, seen))); + return copy; + } + + if (value && typeof value === 'object') { + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) return value; + + const existing = seen.get(value); + if (existing) return existing; + const copy = Object.create(prototype) as Record; + seen.set(value, copy); + for (const [key, entry] of Object.entries(value)) { + Object.defineProperty(copy, key, { + value: copyParamValue(entry, seen), + enumerable: true, + configurable: true, + writable: true, + }); + } + return copy; + } + + return value; +} + +function copyParams(params: Record | undefined): Record { + return copyParamValue(params ?? {}) as Record; +} + +function foldedBidderParams( + bid: TrustedServerBid | undefined +): Record> { + const folded = (bid?.params?.[BIDDER_PARAMS_KEY] ?? {}) as Record< + string, + Record + >; + return Object.fromEntries( + Object.entries(folded).map(([bidder, params]) => [bidder, copyParams(params)]) + ); +} + +function capturePublisherAdUnitSnapshot( + unit: TrustedServerAdUnit, + clientSideBidders: Set +): PublisherAdUnitSnapshot | undefined { + if (typeof unit.code !== 'string' || unit.code.length === 0) return undefined; + + const rawBidderParams: Record> = {}; + const clientSideBids: ClientSideBidSnapshot[] = []; + let existingTsBid: TrustedServerBid | undefined; + + const bids = Array.isArray(unit.bids) ? unit.bids : []; + for (const bid of bids) { + if (!bid?.bidder) continue; + if (bid.bidder === ADAPTER_CODE) { + existingTsBid ??= bid; + continue; + } + if (clientSideBidders.has(bid.bidder)) { + clientSideBids.push({ bidder: bid.bidder, params: copyParams(bid.params) }); + continue; + } + rawBidderParams[bid.bidder] = copyParams(bid.params); + } + + const bidderParams = + Object.keys(rawBidderParams).length > 0 ? rawBidderParams : foldedBidderParams(existingTsBid); + const zone = unit.mediaTypes?.banner?.name; + + return { + bidderParams, + clientSideBids, + ...(zone ? { zone } : {}), + }; +} + /** * Collect the configured client-side bidder entries for a refreshing slot. * @@ -389,6 +494,14 @@ function findRefreshAdUnit( function clientSideBidsForRefresh( candidateCodes: Array ): Array<{ bidder: string; params: Record }> { + const snapshot = findRefreshSnapshot(candidateCodes); + if (snapshot) { + return snapshot.clientSideBids.map((bid) => ({ + bidder: bid.bidder, + params: copyParams(bid.params), + })); + } + const clientSideBidders = new Set(getInjectedConfig()?.clientSideBidders ?? []); if (clientSideBidders.size === 0) return []; @@ -398,7 +511,7 @@ function clientSideBidsForRefresh( const bids: Array<{ bidder: string; params: Record }> = []; for (const bid of match.bids) { if (bid?.bidder && clientSideBidders.has(bid.bidder)) { - bids.push({ bidder: bid.bidder, params: bid.params ?? {} }); + bids.push({ bidder: bid.bidder, params: copyParams(bid.params) }); } } return bids; @@ -420,6 +533,13 @@ function clientSideBidsForRefresh( function serverSideBidderParamsForRefresh( candidateCodes: Array ): Record> { + const snapshot = findRefreshSnapshot(candidateCodes); + if (snapshot) { + return Object.fromEntries( + Object.entries(snapshot.bidderParams).map(([bidder, params]) => [bidder, copyParams(params)]) + ); + } + const match = findRefreshAdUnit(candidateCodes); if (!match?.bids) return {}; @@ -456,6 +576,50 @@ function clearRefreshTargeting(slot: RefreshGptSlot): void { } } +function removePublisherDeliveryContext(context: PublisherDeliveryContext): void { + const index = activePublisherDeliveryContexts.lastIndexOf(context); + if (index >= 0) activePublisherDeliveryContexts.splice(index, 1); +} + +function consumeBarePublisherDeliveryContext(): boolean { + for (let index = activePublisherDeliveryContexts.length - 1; index >= 0; index -= 1) { + const context = activePublisherDeliveryContexts[index]; + if (context.remainingCodes.size === 0) continue; + context.remainingCodes.clear(); + return true; + } + return false; +} + +function consumeExplicitPublisherDeliveryContext(targetSlots: RefreshGptSlot[]): boolean { + if (targetSlots.length === 0) return false; + + for (let index = activePublisherDeliveryContexts.length - 1; index >= 0; index -= 1) { + const context = activePublisherDeliveryContexts[index]; + const coveredCodes: string[] = []; + let allCovered = true; + + for (const slot of targetSlots) { + const injectedSlot = findInjectedSlotForRefresh(slot); + const candidates = [refreshSlotElementId(slot), injectedSlot?.div_id]; + const coveredCode = candidates.find( + (code): code is string => !!code && context.remainingCodes.has(code) + ); + if (!coveredCode) { + allCovered = false; + break; + } + coveredCodes.push(coveredCode); + } + + if (!allCovered) continue; + coveredCodes.forEach((code) => context.remainingCodes.delete(code)); + return true; + } + + return false; +} + function collectAuctionEids(): AuctionEid[] | undefined { if (typeof pbjs.getUserIdsAsEids !== 'function') { return undefined; @@ -492,6 +656,10 @@ function collectAuctionEids(): AuctionEid[] | undefined { * 2. `config` argument — explicit overrides from the publisher's JS */ export function installPrebidNpm(config?: Partial): typeof pbjs { + publisherAdUnitSnapshots = new Map(); + syntheticRefreshAdUnits = new WeakSet(); + activePublisherDeliveryContexts.length = 0; + const injected = getInjectedConfig(); const merged: PrebidNpmConfig = { endpoint: config?.endpoint, @@ -563,9 +731,20 @@ export function installPrebidNpm(config?: Partial): typeof pbjs const opts = requestObj || {}; // eslint-disable-next-line @typescript-eslint/no-explicit-any const adUnits = ((opts as any).adUnits || pbjs.adUnits || []) as TrustedServerAdUnit[]; + const isSyntheticRefresh = + adUnits.length > 0 && adUnits.every((unit) => syntheticRefreshAdUnits.has(unit)); + const publisherAdUnitCodes = new Set(); // Ensure every ad unit has a trustedServer bid entry for (const unit of adUnits) { + if (!syntheticRefreshAdUnits.has(unit)) { + const snapshot = capturePublisherAdUnitSnapshot(unit, clientSideBidders); + if (snapshot && unit.code) { + publisherAdUnitSnapshots.set(unit.code, snapshot); + publisherAdUnitCodes.add(unit.code); + } + } + if (!Array.isArray(unit.bids)) { unit.bids = []; } @@ -649,8 +828,22 @@ export function installPrebidNpm(config?: Partial): typeof pbjs const originalBidsBack = opts.bidsBackHandler; opts.bidsBackHandler = function (...args: unknown[]) { syncPrebidEidsCookie(); - if (typeof originalBidsBack === 'function') { - originalBidsBack.apply(this, args); + if (typeof originalBidsBack !== 'function') return; + if (isSyntheticRefresh || publisherAdUnitCodes.size === 0) { + originalBidsBack.apply(this, args as Parameters); + return; + } + + const context: PublisherDeliveryContext = { + remainingCodes: new Set(publisherAdUnitCodes), + }; + // Delivery attribution is intentionally synchronous and ends as soon as + // the publisher's original callback returns. + activePublisherDeliveryContexts.push(context); + try { + originalBidsBack.apply(this, args as Parameters); + } finally { + removePublisherDeliveryContext(context); } }; @@ -734,6 +927,14 @@ export function installRefreshHandler(timeoutMs = 1500): void { const originalRefresh = pubads.refresh.bind(pubads); pubads.refresh = function (slots?: unknown[], opts?: unknown) { + // For bare refresh() calls (no slots arg), get all registered slots from GPT + // so we can auction the same concrete slot list and avoid stale targeting. + const targetSlots = ( + slots ?? + (pubads as { getSlots?: () => unknown[] }).getSlots?.() ?? + [] + ).filter((slot): slot is RefreshGptSlot => typeof slot === 'object' && slot !== null); + // One-shot bypass for adInit()'s internal refresh: that refresh delivers // freshly applied server-side targeting to GAM and must not be turned // into a client-side auction (which would clear the TS targeting). @@ -743,13 +944,14 @@ export function installRefreshHandler(timeoutMs = 1500): void { return originalRefresh(slots, opts); } - // For bare refresh() calls (no slots arg), get all registered slots from GPT - // so we can auction the same concrete slot list and avoid stale targeting. - const targetSlots = ( - slots ?? - (pubads as { getSlots?: () => unknown[] }).getSlots?.() ?? - [] - ).filter((slot): slot is RefreshGptSlot => typeof slot === 'object' && slot !== null); + const isExplicitSlotList = slots !== undefined; + const hasOnlyValidExplicitSlots = !isExplicitSlotList || targetSlots.length === slots.length; + const isPublisherDeliveryRefresh = isExplicitSlotList + ? hasOnlyValidExplicitSlots && consumeExplicitPublisherDeliveryContext(targetSlots) + : consumeBarePublisherDeliveryContext(); + if (isPublisherDeliveryRefresh) { + return originalRefresh(slots, opts); + } if (!targetSlots.length) { return originalRefresh(slots, opts); @@ -759,8 +961,16 @@ export function installRefreshHandler(timeoutMs = 1500): void { const adUnits = targetSlots.map((slot) => { const injectedSlot = findInjectedSlotForRefresh(slot); + const code = refreshSlotElementId(slot) ?? 'refresh-slot'; + // A TS-owned slot may be defined on `${div_id}-container`, so the GPT + // element id used as the synthetic refresh code can differ from the + // inner `div_id` the publisher keyed their ad unit by. Recover from both. + const candidateCodes = [code, injectedSlot?.div_id]; + const snapshot = findRefreshSnapshot(candidateCodes); const zone = - injectedSlot?.targeting?.[ZONE_KEY] ?? firstTargetingValue(slot.getTargeting?.(ZONE_KEY)); + injectedSlot?.targeting?.[ZONE_KEY] ?? + firstTargetingValue(slot.getTargeting?.(ZONE_KEY)) ?? + snapshot?.zone; const banner: TrustedServerBanner = { sizes: bannerSizesFromInjectedSlot(injectedSlot) ?? @@ -768,12 +978,6 @@ export function installRefreshHandler(timeoutMs = 1500): void { DEFAULT_REFRESH_SIZES, ...(zone ? { name: zone } : {}), }; - - const code = refreshSlotElementId(slot) ?? 'refresh-slot'; - // A TS-owned slot may be defined on `${div_id}-container`, so the GPT - // element id used as the synthetic refresh code can differ from the - // inner `div_id` the publisher keyed their ad unit by. Recover from both. - const candidateCodes = [code, injectedSlot?.div_id]; const tsParams: Record = zone ? { [ZONE_KEY]: zone } : {}; // Carry the publisher's inline server-side (PBS) bidder params captured // on the initial ad unit so refresh/scroll auctions don't drop them. @@ -796,6 +1000,7 @@ export function installRefreshHandler(timeoutMs = 1500): void { // unrelated GPT slots whose targeting this wrapper only cleared for // `targetSlots` — leaving their next request dependent on stale state. const refreshAdUnitCodes = adUnits.map((unit) => unit.code); + adUnits.forEach((unit) => syntheticRefreshAdUnits.add(unit)); pbjs.requestBids({ adUnits, bidsBackHandler: () => { diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 9ad7945c4..6435d9708 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -617,6 +617,15 @@ describe('prebid/installPrebidNpm', () => { expect(adUnits[0].bids[0].bidder).toBe('trustedServer'); }); + it('normalizes a truthy non-array bids value without throwing', () => { + const pbjs = installPrebidNpm(); + const adUnits = [{ code: 'example-malformed-slot', bids: { malformed: true } }] as any[]; + + expect(() => pbjs.requestBids({ adUnits } as any)).not.toThrow(); + + expect(adUnits[0].bids).toEqual([{ bidder: 'trustedServer', params: { bidderParams: {} } }]); + }); + it('includes zone from mediaTypes.banner.name in trustedServer params', () => { const pbjs = installPrebidNpm(); @@ -1352,6 +1361,506 @@ describe('prebid/installRefreshHandler', () => { }); }); +describe('prebid publisher snapshots and delivery refreshes', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockRequestBids.mockReset(); + mockPbjs.requestBids = mockRequestBids; + mockPbjs.adUnits = []; + mockGetUserIdsAsEids.mockReset(); + mockGetUserIdsAsEids.mockReturnValue([]); + mockGetBidAdapter.mockReturnValue({}); + delete (mockPbjs as any).setTargetingForGPTAsync; + delete (window as any).__tsjs_prebid; + (window as any).tsjs = undefined; + delete (window as any).googletag; + }); + + afterEach(() => { + delete (window as any).__tsjs_prebid; + (window as any).tsjs = undefined; + delete (window as any).googletag; + }); + + function installGpt(slots: any[]) { + const originalRefresh = vi.fn(); + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => slots), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + installRefreshHandler(640); + return { originalRefresh, pubads }; + } + + function refreshAdUnitFromLastRequest(): any { + const lastCall = mockRequestBids.mock.calls[mockRequestBids.mock.calls.length - 1]; + return lastCall?.[0]?.adUnits?.[0]; + } + + it('recovers inline params, ordered client bids, and zone when pbjs.adUnits is empty', () => { + (window as any).__tsjs_prebid = { clientSideBidders: ['exampleBrowser'] }; + const runtimeInstance = 'example-runtime-instance'; + const code = `example-slot-${runtimeInstance}`; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [{ getWidth: () => 320, getHeight: () => 100 }], + clearTargeting: vi.fn(), + }; + const { pubads } = installGpt([slot]); + const pbjs = installPrebidNpm(); + const firstParams = { placement: 'first' }; + const effectiveParams = { placement: 'effective' }; + + pbjs.requestBids({ + adUnits: [ + { + code, + mediaTypes: { banner: { name: 'example-zone', sizes: [[320, 100]] } }, + bids: [ + { bidder: 'exampleServer', params: firstParams }, + { bidder: 'exampleBrowser', params: { placement: 'browser-one' } }, + { bidder: 'exampleServer', params: effectiveParams }, + { bidder: 'exampleBrowser', params: { placement: 'browser-two' } }, + ], + }, + ], + } as any); + effectiveParams.placement = 'changed-after-auction'; + + pubads.refresh([slot]); + + expect(mockPbjs.adUnits).toEqual([]); + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(refreshAdUnitFromLastRequest()).toEqual({ + code, + mediaTypes: { banner: { name: 'example-zone', sizes: [[320, 100]] } }, + bids: [ + { + bidder: 'trustedServer', + params: { + bidderParams: { exampleServer: { placement: 'effective' } }, + zone: 'example-zone', + }, + }, + { bidder: 'exampleBrowser', params: { placement: 'browser-one' } }, + { bidder: 'exampleBrowser', params: { placement: 'browser-two' } }, + ], + }); + }); + + it('isolates nested bidder-param objects and arrays from later publisher mutation', () => { + (window as any).__tsjs_prebid = { clientSideBidders: ['exampleBrowser'] }; + const code = 'example-nested-params-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { pubads } = installGpt([slot]); + const pbjs = installPrebidNpm(); + const serverParams = { + placement: { + rules: [{ label: 'original-rule' }], + sizes: [300, 250], + }, + }; + const browserParams = { + groups: [{ values: ['original-value'] }], + }; + + pbjs.requestBids({ + adUnits: [ + { + code, + bids: [ + { bidder: 'exampleServer', params: serverParams }, + { bidder: 'exampleBrowser', params: browserParams }, + ], + }, + ], + } as any); + serverParams.placement.rules[0].label = 'changed-rule'; + serverParams.placement.sizes.push(999); + browserParams.groups[0].values[0] = 'changed-value'; + + pubads.refresh([slot]); + + const expectedBids = [ + { + bidder: 'trustedServer', + params: { + bidderParams: { + exampleServer: { + placement: { + rules: [{ label: 'original-rule' }], + sizes: [300, 250], + }, + }, + }, + }, + }, + { + bidder: 'exampleBrowser', + params: { groups: [{ values: ['original-value'] }] }, + }, + ]; + const firstRefreshBids = refreshAdUnitFromLastRequest().bids; + expect(firstRefreshBids).toEqual(expectedBids); + + firstRefreshBids[0].params.bidderParams.exampleServer.placement.rules[0].label = + 'changed-refresh-rule'; + firstRefreshBids[0].params.bidderParams.exampleServer.placement.sizes.push(777); + firstRefreshBids[1].params.groups[0].values[0] = 'changed-refresh-value'; + pubads.refresh([slot]); + + expect(refreshAdUnitFromLastRequest().bids).toEqual(expectedBids); + }); + + it('keeps snapshots across repeated synthetic refreshes and overwrites newer publisher config', () => { + const code = 'example-dynamic-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { pubads } = installGpt([slot]); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { + code, + mediaTypes: { banner: { name: 'example-zone-one', sizes: [[300, 250]] } }, + bids: [{ bidder: 'exampleServer', params: { placement: 'one' } }], + }, + ], + } as any); + pubads.refresh([slot]); + expect(refreshAdUnitFromLastRequest().bids[0].params).toEqual({ + bidderParams: { exampleServer: { placement: 'one' } }, + zone: 'example-zone-one', + }); + + pubads.refresh([slot]); + expect(refreshAdUnitFromLastRequest().bids[0].params).toEqual({ + bidderParams: { exampleServer: { placement: 'one' } }, + zone: 'example-zone-one', + }); + + pbjs.requestBids({ + adUnits: [ + { + code, + mediaTypes: { banner: { name: 'example-zone-two', sizes: [[300, 250]] } }, + bids: [{ bidder: 'exampleServer', params: { placement: 'two' } }], + }, + ], + } as any); + pubads.refresh([slot]); + + expect(refreshAdUnitFromLastRequest().bids[0].params).toEqual({ + bidderParams: { exampleServer: { placement: 'two' } }, + zone: 'example-zone-two', + }); + }); + + it('does not cross-contaminate dynamic-code snapshots and retains the global fallback', () => { + const slotOne = { + getSlotElementId: () => 'example-code-one', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const slotTwo = { + getSlotElementId: () => 'example-code-two', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const globalSlot = { + getSlotElementId: () => 'example-global-code', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { pubads } = installGpt([slotOne, slotTwo, globalSlot]); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { + code: 'example-code-one', + bids: [{ bidder: 'exampleServer', params: { placement: 'one' } }], + }, + { + code: 'example-code-two', + bids: [{ bidder: 'exampleServer', params: { placement: 'two' } }], + }, + ], + } as any); + mockPbjs.adUnits = [ + { + code: 'example-global-code', + bids: [{ bidder: 'exampleFallback', params: { placement: 'global' } }], + }, + ]; + + pubads.refresh([slotOne]); + expect(refreshAdUnitFromLastRequest().bids[0].params.bidderParams).toEqual({ + exampleServer: { placement: 'one' }, + }); + pubads.refresh([slotTwo]); + expect(refreshAdUnitFromLastRequest().bids[0].params.bidderParams).toEqual({ + exampleServer: { placement: 'two' }, + }); + pubads.refresh([globalSlot]); + expect(refreshAdUnitFromLastRequest().bids[0].params.bidderParams).toEqual({ + exampleFallback: { placement: 'global' }, + }); + }); + + it('bypasses explicit covered subset delivery refreshes without clearing targeting', () => { + const slotOne = { + getSlotElementId: () => 'example-covered-one', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const slotTwo = { + getSlotElementId: () => 'example-covered-two-container', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + (window as any).tsjs = { + adSlots: [{ div_id: 'example-covered-two', formats: [[300, 250]], targeting: {} }], + }; + const { originalRefresh, pubads } = installGpt([slotOne, slotTwo]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { code: 'example-covered-one', bids: [{ bidder: 'exampleServer', params: {} }] }, + { code: 'example-covered-two', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => { + pubads.refresh([slotOne]); + pubads.refresh([slotTwo]); + }, + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(slotOne.clearTargeting).not.toHaveBeenCalled(); + expect(slotTwo.clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenNthCalledWith(1, [slotOne], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [slotTwo], undefined); + }); + + it('bypasses a bare delivery refresh even when GPT includes a GAM-only extra slot', () => { + const coveredSlot = { + getSlotElementId: () => 'example-covered', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const gamOnlySlot = { + getSlotElementId: () => 'example-gam-only-interstitial', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([coveredSlot, gamOnlySlot]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code: 'example-covered', bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh(), + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(coveredSlot.clearTargeting).not.toHaveBeenCalled(); + expect(gamOnlySlot.clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith(undefined, undefined); + }); + + it('keeps explicit unrelated and mixed delivery lists on the synthetic path', () => { + const coveredSlot = { + getSlotElementId: () => 'example-covered', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const unrelatedSlot = { + getSlotElementId: () => 'example-unrelated', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([coveredSlot, unrelatedSlot]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code: 'example-covered', bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => { + pubads.refresh([unrelatedSlot]); + pubads.refresh([coveredSlot, unrelatedSlot]); + }, + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(3); + expect(mockRequestBids.mock.calls[1][0].adUnits.map((unit: any) => unit.code)).toEqual([ + 'example-unrelated', + ]); + expect(mockRequestBids.mock.calls[2][0].adUnits.map((unit: any) => unit.code)).toEqual([ + 'example-covered', + 'example-unrelated', + ]); + expect(coveredSlot.clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(unrelatedSlot.clearTargeting).toHaveBeenCalledWith('ts_initial'); + expect(unrelatedSlot.clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(originalRefresh).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenNthCalledWith(1, [unrelatedSlot], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [coveredSlot, unrelatedSlot], undefined); + }); + + it('treats a microtask refresh after publisher delivery as an independent auction', async () => { + const slot = { + getSlotElementId: () => 'example-deferred-refresh', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + let deferredRefresh: Promise | undefined; + + pbjs.requestBids({ + adUnits: [ + { code: 'example-deferred-refresh', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => { + deferredRefresh = Promise.resolve().then(() => pubads.refresh([slot])); + }, + } as any); + await deferredRefresh; + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(slot.clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); + + it('keeps nested publisher delivery contexts isolated during reentrant auctions', () => { + const outerSlot = { + getSlotElementId: () => 'example-outer-delivery', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const innerSlot = { + getSlotElementId: () => 'example-inner-delivery', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([outerSlot, innerSlot]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { code: 'example-outer-delivery', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => { + pbjs.requestBids({ + adUnits: [ + { code: 'example-inner-delivery', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => pubads.refresh([innerSlot]), + } as any); + pubads.refresh([outerSlot]); + }, + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(innerSlot.clearTargeting).not.toHaveBeenCalled(); + expect(outerSlot.clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenNthCalledWith(1, [innerSlot], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [outerSlot], undefined); + }); + + it('cleans delivery context after a publisher callback throws', () => { + const slot = { + getSlotElementId: () => 'example-throwing-callback', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + expect(() => + pbjs.requestBids({ + adUnits: [ + { + code: 'example-throwing-callback', + bids: [{ bidder: 'exampleServer', params: {} }], + }, + ], + bidsBackHandler: () => { + throw new Error('example callback failure'); + }, + } as any) + ).toThrow('example callback failure'); + + pubads.refresh([slot]); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(slot.clearTargeting).toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledTimes(1); + }); + + it('completes an internal synthetic refresh once without recursion', () => { + const slot = { + getSlotElementId: () => 'example-independent-refresh', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + installPrebidNpm(); + + pubads.refresh([slot]); + + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); +}); + describe('prebid/client-side bidders', () => { beforeEach(() => { vi.clearAllMocks(); From c59a064e6ef27a822b1986df65135b64d63ef8d5 Mon Sep 17 00:00:00 2001 From: Christian Date: Tue, 14 Jul 2026 16:21:19 -0500 Subject: [PATCH 024/198] Handle deferred Prebid delivery refreshes --- .../lib/src/integrations/prebid/index.ts | 101 ++++++++-- .../test/integrations/prebid/index.test.ts | 188 +++++++++++++++++- 2 files changed, 259 insertions(+), 30 deletions(-) diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 65932d5ba..e01cdd879 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -45,6 +45,7 @@ const TS_REFRESH_TARGETING_KEYS = [ 'hb_cache_host', 'hb_cache_path', ] as const; +const PUBLISHER_DELIVERY_CONTEXT_TIMEOUT_MS = 1000; /** Configuration options for the Prebid integration. */ export interface PrebidNpmConfig { @@ -234,7 +235,12 @@ type PublisherAdUnitSnapshot = { clientSideBids: ClientSideBidSnapshot[]; zone?: string; }; -type PublisherDeliveryContext = { remainingCodes: Set }; +type PublisherDeliveryContext = { + remainingCodes: Set; + retainForTargetedRefresh: boolean; + cleanupTimer?: ReturnType; +}; +type SetTargetingForGptAsync = (...args: unknown[]) => unknown; let publisherAdUnitSnapshots = new Map(); let syntheticRefreshAdUnits = new WeakSet(); @@ -577,15 +583,32 @@ function clearRefreshTargeting(slot: RefreshGptSlot): void { } function removePublisherDeliveryContext(context: PublisherDeliveryContext): void { + if (context.cleanupTimer !== undefined) { + clearTimeout(context.cleanupTimer); + context.cleanupTimer = undefined; + } const index = activePublisherDeliveryContexts.lastIndexOf(context); if (index >= 0) activePublisherDeliveryContexts.splice(index, 1); } +function targetingCoversPublisherDeliveryContext( + adUnitCodes: unknown, + context: PublisherDeliveryContext +): boolean { + if (adUnitCodes === undefined) return context.remainingCodes.size > 0; + const codes = typeof adUnitCodes === 'string' ? [adUnitCodes] : adUnitCodes; + return ( + Array.isArray(codes) && + codes.some((code) => typeof code === 'string' && context.remainingCodes.has(code)) + ); +} + function consumeBarePublisherDeliveryContext(): boolean { for (let index = activePublisherDeliveryContexts.length - 1; index >= 0; index -= 1) { const context = activePublisherDeliveryContexts[index]; if (context.remainingCodes.size === 0) continue; context.remainingCodes.clear(); + removePublisherDeliveryContext(context); return true; } return false; @@ -594,30 +617,35 @@ function consumeBarePublisherDeliveryContext(): boolean { function consumeExplicitPublisherDeliveryContext(targetSlots: RefreshGptSlot[]): boolean { if (targetSlots.length === 0) return false; - for (let index = activePublisherDeliveryContexts.length - 1; index >= 0; index -= 1) { - const context = activePublisherDeliveryContexts[index]; - const coveredCodes: string[] = []; - let allCovered = true; - - for (const slot of targetSlots) { - const injectedSlot = findInjectedSlotForRefresh(slot); - const candidates = [refreshSlotElementId(slot), injectedSlot?.div_id]; + // Publishers may include GAM-only slots in the same explicit refresh that + // delivers a completed Prebid auction. Attribute the call to delivery when + // any slot is covered, while consuming only the covered codes so an + // unrelated-only refresh still follows the synthetic auction path. + const matches = new Map>(); + for (const slot of targetSlots) { + const injectedSlot = findInjectedSlotForRefresh(slot); + const candidates = [refreshSlotElementId(slot), injectedSlot?.div_id]; + + for (let index = activePublisherDeliveryContexts.length - 1; index >= 0; index -= 1) { + const context = activePublisherDeliveryContexts[index]; const coveredCode = candidates.find( (code): code is string => !!code && context.remainingCodes.has(code) ); - if (!coveredCode) { - allCovered = false; - break; - } - coveredCodes.push(coveredCode); + if (!coveredCode) continue; + + const contextMatches = matches.get(context) ?? new Set(); + contextMatches.add(coveredCode); + matches.set(context, contextMatches); + break; } + } - if (!allCovered) continue; + if (matches.size === 0) return false; + for (const [context, coveredCodes] of matches) { coveredCodes.forEach((code) => context.remainingCodes.delete(code)); - return true; + if (context.remainingCodes.size === 0) removePublisherDeliveryContext(context); } - - return false; + return true; } function collectAuctionEids(): AuctionEid[] | undefined { @@ -658,7 +686,7 @@ function collectAuctionEids(): AuctionEid[] | undefined { export function installPrebidNpm(config?: Partial): typeof pbjs { publisherAdUnitSnapshots = new Map(); syntheticRefreshAdUnits = new WeakSet(); - activePublisherDeliveryContexts.length = 0; + [...activePublisherDeliveryContexts].forEach(removePublisherDeliveryContext); const injected = getInjectedConfig(); const merged: PrebidNpmConfig = { @@ -836,14 +864,43 @@ export function installPrebidNpm(config?: Partial): typeof pbjs const context: PublisherDeliveryContext = { remainingCodes: new Set(publisherAdUnitCodes), + retainForTargetedRefresh: false, + }; + const targetingPbjs = pbjs as unknown as { + setTargetingForGPTAsync?: SetTargetingForGptAsync; }; - // Delivery attribution is intentionally synchronous and ends as soon as - // the publisher's original callback returns. + const originalSetTargeting = targetingPbjs.setTargetingForGPTAsync; + let targetingWrapper: SetTargetingForGptAsync | undefined; + if (typeof originalSetTargeting === 'function') { + targetingWrapper = (...targetingArgs: unknown[]) => { + const result = originalSetTargeting.apply(targetingPbjs, targetingArgs); + if (targetingCoversPublisherDeliveryContext(targetingArgs[0], context)) { + context.retainForTargetedRefresh = true; + } + return result; + }; + targetingPbjs.setTargetingForGPTAsync = targetingWrapper; + } + activePublisherDeliveryContexts.push(context); try { originalBidsBack.apply(this, args as Parameters); } finally { - removePublisherDeliveryContext(context); + if (targetingWrapper && targetingPbjs.setTargetingForGPTAsync === targetingWrapper) { + targetingPbjs.setTargetingForGPTAsync = originalSetTargeting; + } + if (context.retainForTargetedRefresh && context.remainingCodes.size > 0) { + // Some publisher wrappers set targeting in bidsBackHandler, return, + // and schedule the matching GPT refresh shortly afterward. Retain + // this one-shot context only after that targeting signal, with a + // bounded expiry so a later independent refresh remains independent. + context.cleanupTimer = setTimeout( + () => removePublisherDeliveryContext(context), + PUBLISHER_DELIVERY_CONTEXT_TIMEOUT_MS + ); + } else { + removePublisherDeliveryContext(context); + } } }; diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 6435d9708..bdb336fcd 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -1694,7 +1694,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(originalRefresh).toHaveBeenCalledWith(undefined, undefined); }); - it('keeps explicit unrelated and mixed delivery lists on the synthetic path', () => { + it('keeps explicit unrelated lists synthetic and bypasses mixed delivery lists', () => { const coveredSlot = { getSlotElementId: () => 'example-covered', getTargeting: () => [], @@ -1721,15 +1721,11 @@ describe('prebid publisher snapshots and delivery refreshes', () => { }, } as any); - expect(mockRequestBids).toHaveBeenCalledTimes(3); + expect(mockRequestBids).toHaveBeenCalledTimes(2); expect(mockRequestBids.mock.calls[1][0].adUnits.map((unit: any) => unit.code)).toEqual([ 'example-unrelated', ]); - expect(mockRequestBids.mock.calls[2][0].adUnits.map((unit: any) => unit.code)).toEqual([ - 'example-covered', - 'example-unrelated', - ]); - expect(coveredSlot.clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(coveredSlot.clearTargeting).not.toHaveBeenCalled(); expect(unrelatedSlot.clearTargeting).toHaveBeenCalledWith('ts_initial'); expect(unrelatedSlot.clearTargeting).toHaveBeenCalledWith('hb_cache_path'); expect(originalRefresh).toHaveBeenCalledTimes(2); @@ -1737,7 +1733,183 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(originalRefresh).toHaveBeenNthCalledWith(2, [coveredSlot, unrelatedSlot], undefined); }); - it('treats a microtask refresh after publisher delivery as an independent auction', async () => { + it('bypasses an explicit delivery refresh with four covered slots and a GAM-only extra', () => { + const coveredSlots = Array.from({ length: 4 }, (_, index) => ({ + getSlotElementId: () => `example-covered-${index}`, + getTargeting: () => [], + clearTargeting: vi.fn(), + })); + const gamOnlySlot = { + getSlotElementId: () => 'example-gam-only-interstitial', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const refreshSlots = [...coveredSlots, gamOnlySlot]; + const { originalRefresh, pubads } = installGpt(refreshSlots); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: coveredSlots.map((_, index) => ({ + code: `example-covered-${index}`, + bids: [{ bidder: 'exampleServer', params: { placement: index } }], + })), + bidsBackHandler: () => pubads.refresh(refreshSlots), + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(1); + refreshSlots.forEach((slot) => expect(slot.clearTargeting).not.toHaveBeenCalled()); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith(refreshSlots, undefined); + }); + + it('bypasses a targeted delivery refresh shortly after the publisher callback returns', () => { + vi.useFakeTimers(); + try { + const coveredSlots = Array.from({ length: 4 }, (_, index) => ({ + getSlotElementId: () => `example-targeted-${index}`, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + })); + const gamOnlySlot = { + getSlotElementId: () => 'example-targeted-interstitial', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const refreshSlots = [...coveredSlots, gamOnlySlot]; + const { originalRefresh, pubads } = installGpt(refreshSlots); + const setTargetingForGPTAsync = vi.fn(); + (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; + let refreshAfterCallback: (() => void) | undefined; + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + const pendingRefresh = refreshAfterCallback; + refreshAfterCallback = undefined; + if (pendingRefresh) setTimeout(pendingRefresh, 750); + }); + const pbjs = installPrebidNpm(); + const coveredCodes = coveredSlots.map((slot) => slot.getSlotElementId()); + + pbjs.requestBids({ + adUnits: coveredCodes.map((code, index) => ({ + code, + bids: [{ bidder: 'exampleServer', params: { placement: index } }], + })), + bidsBackHandler: () => { + (pbjs as any).setTargetingForGPTAsync([gamOnlySlot.getSlotElementId(), ...coveredCodes]); + refreshAfterCallback = () => pubads.refresh(refreshSlots); + }, + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(setTargetingForGPTAsync).toHaveBeenCalledWith([ + gamOnlySlot.getSlotElementId(), + ...coveredCodes, + ]); + expect((mockPbjs as any).setTargetingForGPTAsync).toBe(setTargetingForGPTAsync); + + vi.advanceTimersByTime(750); + + refreshSlots.forEach((slot) => expect(slot.clearTargeting).not.toHaveBeenCalled()); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith(refreshSlots, undefined); + + vi.runOnlyPendingTimers(); + pubads.refresh([coveredSlots[0]]); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(coveredSlots[0].clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(originalRefresh).toHaveBeenCalledTimes(2); + } finally { + vi.runOnlyPendingTimers(); + vi.useRealTimers(); + delete (mockPbjs as any).setTargetingForGPTAsync; + } + }); + + it('expires a targeted delivery context before a later event-loop task', () => { + vi.useFakeTimers(); + try { + const slot = { + getSlotElementId: () => 'example-expiring-delivery', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + (mockPbjs as any).setTargetingForGPTAsync = vi.fn(); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { code: 'example-expiring-delivery', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => (pbjs as any).setTargetingForGPTAsync(['example-expiring-delivery']), + } as any); + vi.runOnlyPendingTimers(); + pubads.refresh([slot]); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(slot.clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + } finally { + vi.runOnlyPendingTimers(); + vi.useRealTimers(); + delete (mockPbjs as any).setTargetingForGPTAsync; + } + }); + + it('bypasses a mixed explicit delivery list spanning nested contexts', () => { + const outerSlot = { + getSlotElementId: () => 'example-outer-delivery', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const innerSlot = { + getSlotElementId: () => 'example-inner-delivery', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const gamOnlySlot = { + getSlotElementId: () => 'example-gam-only-interstitial', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const refreshSlots = [innerSlot, outerSlot, gamOnlySlot]; + const { originalRefresh, pubads } = installGpt(refreshSlots); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { code: 'example-outer-delivery', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => { + pbjs.requestBids({ + adUnits: [ + { code: 'example-inner-delivery', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => pubads.refresh(refreshSlots), + } as any); + }, + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + refreshSlots.forEach((slot) => expect(slot.clearTargeting).not.toHaveBeenCalled()); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith(refreshSlots, undefined); + }); + + it('treats a microtask refresh without a targeting signal as an independent auction', async () => { const slot = { getSlotElementId: () => 'example-deferred-refresh', getTargeting: () => [], From 310e62f307a8320639cee58938142de1afe3a78e Mon Sep 17 00:00:00 2001 From: Christian Date: Tue, 14 Jul 2026 16:57:56 -0500 Subject: [PATCH 025/198] Use HTTP status error type for Prebid failures --- crates/trusted-server-core/src/integrations/prebid.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 2e7655382..9366512eb 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -18,6 +18,7 @@ use serde_json::Value as Json; use url::{Url, Url as ParsedUrl}; use validator::{Validate, ValidationError}; +use crate::auction::orchestrator::ERROR_TYPE_HTTP_STATUS; use crate::auction::provider::AuctionProvider; use crate::auction::types::{ AuctionContext, AuctionRequest, AuctionResponse, Bid as AuctionBid, MediaType, @@ -62,7 +63,6 @@ const ZONE_KEY: &str = "zone"; /// Default currency for `OpenRTB` bid floors and responses. const DEFAULT_CURRENCY: &str = "USD"; -const PREBID_ERROR_TYPE_UPSTREAM_HTTP: &str = "upstream_http"; const PREBID_PUBLIC_ERROR_MESSAGE_CHARS: usize = 500; const PREBID_ERROR_BODY_PREVIEW_CHARS: usize = 1000; const PREBID_ERROR_BODY_PREVIEW_BYTES: usize = PREBID_ERROR_BODY_PREVIEW_CHARS * 4; @@ -2010,10 +2010,7 @@ impl PrebidAuctionProvider { let status_code = status.as_u16(); let mut auction_response = AuctionResponse::error(PREBID_INTEGRATION_ID, response_time_ms) - .with_metadata( - "error_type", - serde_json::json!(PREBID_ERROR_TYPE_UPSTREAM_HTTP), - ) + .with_metadata("error_type", serde_json::json!(ERROR_TYPE_HTTP_STATUS)) .with_metadata("http_status", serde_json::json!(status_code)) .with_metadata( "message", @@ -5143,7 +5140,7 @@ external_bundle_sri = "sha384-AAAA" ); assert_eq!( auction_response.metadata["error_type"], - json!(PREBID_ERROR_TYPE_UPSTREAM_HTTP) + json!(ERROR_TYPE_HTTP_STATUS) ); assert_eq!(auction_response.metadata["http_status"], json!(400)); assert_eq!( From 6b4e82ed594d77fdc686107298823d83b2e3c19f Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 15 Jul 2026 12:04:27 -0500 Subject: [PATCH 026/198] Make auction creative rewriting optional Allow operators to retain sanitizer-accepted external URLs in POST /auction adm while preserving mandatory server-side sanitization and the existing default behavior. --- CHANGELOG.md | 1 + .../src/auction/endpoints.rs | 7 +- .../src/auction/formats.rs | 141 +++++++++++++++++- .../src/auction/orchestrator.rs | 1 + .../src/auction_config_types.rs | 22 +++ .../trusted-server-core/src/config_payload.rs | 24 +++ crates/trusted-server-core/src/proxy.rs | 45 ++++++ crates/trusted-server-core/src/settings.rs | 35 +++++ docs/guide/auction-orchestration.md | 69 ++++++--- docs/guide/configuration.md | 26 +++- docs/guide/creative-processing.md | 54 +++++-- trusted-server.example.toml | 4 + 12 files changed, 382 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bc49c80b..fddc8009d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added the default-true `[auction].rewrite_creatives` option. Setting it to `false` preserves mandatory `/auction` creative sanitization while skipping first-party resource/click URL rewriting and creative TSJS injection. - Added Osano consent mirror integration docs and public enablement guidance. - Implemented basic authentication for configurable endpoint paths (#73) - Added integrations guide with example `testlight` integration diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 825129b1a..e7d4f0de4 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -76,9 +76,10 @@ const MAX_AUCTION_BODY_SIZE: usize = 256 * 1024; /// ## Response /// /// Returns an `OpenRTB 2.x` response. Creative HTML is inlined in each bid's -/// `adm` field after sanitisation and first-party URL rewriting. Response -/// headers include `X-TS-EC` (the caller's Edge Cookie ID) and -/// `X-TS-EC-Fresh` (a freshly generated ID for cookie renewal). +/// `adm` field after mandatory server-side sanitization. First-party resource +/// and click URL rewriting plus creative TSJS injection are enabled by default; +/// setting [`auction.rewrite_creatives`][`crate::auction_config_types::AuctionConfig::rewrite_creatives`] +/// to `false` skips only that rewrite pass. /// /// ## Scroll, refresh, and SPA navigation /// diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index 441828a18..71f9a290c 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -217,7 +217,8 @@ pub fn convert_tsjs_to_auction_request( /// Convert `OrchestrationResult` to `OpenRTB` response format. /// -/// Returns rewritten creative HTML directly in the `adm` field for inline delivery. +/// Always sanitizes creative HTML in the `adm` field and optionally rewrites it +/// according to the auction configuration. /// /// # Errors /// @@ -250,21 +251,34 @@ pub fn convert_to_openrtb_response( let width = to_openrtb_i32(bid.width, "width", &bid_context); let height = to_openrtb_i32(bid.height, "height", &bid_context); - // Process creative HTML if present - — sanitize dangerous markup first, then rewrite URLs. + // Process creative HTML if present — always sanitize dangerous markup first. let creative_html = if let Some(ref raw_creative) = bid.creative { let sanitized = creative::sanitize_creative_html(raw_creative); - let rewritten = creative::rewrite_creative_html(settings, &sanitized); + let sanitized_len = sanitized.len(); + let rewrite_creatives = settings.auction.rewrite_creatives; + let processed = if rewrite_creatives { + creative::rewrite_creative_html(settings, &sanitized) + } else { + sanitized + }; + let rewrite_mode = if rewrite_creatives { + "enabled" + } else { + "disabled" + }; log::debug!( - "Processed creative for auction {} slot {} ({} → {} → {} bytes)", + "Processed creative for auction {} slot {} bidder {} (rewrite {}, raw {} bytes, sanitized {} bytes, output {} bytes)", auction_request.id, slot_id, + bid.bidder, + rewrite_mode, raw_creative.len(), - sanitized.len(), - rewritten.len() + sanitized_len, + processed.len() ); - rewritten + processed } else { // No creative provided (e.g., from mediation layer that returns iframe URLs) log::warn!( @@ -445,6 +459,15 @@ mod tests { } } + fn make_complete_creative_bid() -> Bid { + let mut bid = make_bid("div-gpt-top", "appnexus", Some(2.75)); + bid.creative = Some( + r#""# + .to_string(), + ); + bid + } + fn make_result(bid: Bid) -> OrchestrationResult { OrchestrationResult { provider_responses: vec![AuctionResponse { @@ -466,6 +489,13 @@ mod tests { .expect("should parse JSON response") } + fn response_adm(response: Response) -> String { + response_json(response)["seatbid"][0]["bid"][0]["adm"] + .as_str() + .expect("should serialize adm as a string") + .to_string() + } + fn make_banner_body(config: Option) -> AdRequest { AdRequest { ad_units: vec![AdUnit { @@ -932,6 +962,103 @@ mod tests { ); } + #[test] + fn convert_to_openrtb_response_rewrites_sanitized_creative_by_default() { + let settings = make_settings(); + let auction_request = make_auction_request(); + let result = make_result(make_complete_creative_bid()); + + let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) + .expect("should convert creative with rewriting enabled"); + let adm = response_adm(response); + + assert!( + adm.matches("/first-party/proxy?tsurl=").count() >= 2, + "should rewrite image and inline CSS URLs through the proxy: {adm}" + ); + assert!( + adm.contains("/first-party/click?tsurl="), + "should rewrite click URLs: {adm}" + ); + assert!( + adm.contains("data-tsclick"), + "should add the click guard attribute: {adm}" + ); + assert!( + adm.contains("tsjs-unified.min.js"), + "should inject the unified creative runtime: {adm}" + ); + assert!( + !adm.contains(r#"src="https://cdn.example.com/ad.png""#), + "should not retain the image URL as a direct attribute: {adm}" + ); + assert!( + !adm.contains(r#"href="https://advertiser.example.com/landing""#), + "should not retain the click URL as a direct attribute: {adm}" + ); + assert!( + !adm.contains("url(https://styles.example.com/bg.png)"), + "should not retain the CSS URL as a direct value: {adm}" + ); + assert!( + !adm.contains("auction-script-marker"), + "should remove malicious script content before rewriting: {adm}" + ); + assert!( + !adm.contains("auction-handler-marker") && !adm.contains("onerror"), + "should remove event handlers before rewriting: {adm}" + ); + } + + #[test] + fn convert_to_openrtb_response_can_skip_rewriting_but_not_sanitization() { + let mut settings = make_settings(); + settings.auction.rewrite_creatives = false; + let auction_request = make_auction_request(); + let result = make_result(make_complete_creative_bid()); + + let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) + .expect("should convert creative with rewriting disabled"); + let adm = response_adm(response); + + assert!( + adm.contains(r#"src="https://cdn.example.com/ad.png""#), + "should retain the sanitizer-accepted image URL: {adm}" + ); + assert!( + adm.contains(r#"href="https://advertiser.example.com/landing""#), + "should retain the sanitizer-accepted click URL: {adm}" + ); + assert!( + adm.contains("url(https://styles.example.com/bg.png)"), + "should retain the sanitizer-accepted CSS URL: {adm}" + ); + assert!( + !adm.contains("/first-party/proxy"), + "should not rewrite resource URLs: {adm}" + ); + assert!( + !adm.contains("/first-party/click"), + "should not rewrite click URLs: {adm}" + ); + assert!( + !adm.contains("data-tsclick"), + "should not add the click guard attribute: {adm}" + ); + assert!( + !adm.contains("tsjs-unified.min.js"), + "should not inject the unified creative runtime: {adm}" + ); + assert!( + !adm.contains("auction-script-marker"), + "should still remove malicious script content: {adm}" + ); + assert!( + !adm.contains("auction-handler-marker") && !adm.contains("onerror"), + "should still remove event handlers: {adm}" + ); + } + #[test] fn convert_to_openrtb_response_serializes_missing_creative_as_empty_adm() { let settings = make_settings(); diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 4ef73e581..fb9255825 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -1818,6 +1818,7 @@ mod tests { futures::executor::block_on(async { let config = AuctionConfig { enabled: true, + rewrite_creatives: true, providers: vec![], mediator: None, timeout_ms: 2000, diff --git a/crates/trusted-server-core/src/auction_config_types.rs b/crates/trusted-server-core/src/auction_config_types.rs index 3bd747f64..f1d1a5cf0 100644 --- a/crates/trusted-server-core/src/auction_config_types.rs +++ b/crates/trusted-server-core/src/auction_config_types.rs @@ -11,6 +11,10 @@ pub struct AuctionConfig { #[serde(default)] pub enabled: bool, + /// Rewrite sanitized winning-bid creative HTML to first-party endpoints. + #[serde(default = "default_rewrite_creatives")] + pub rewrite_creatives: bool, + /// Provider names that participate in bidding /// Simply list the provider names (e.g., ["prebid", "aps"]) #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] @@ -41,6 +45,7 @@ impl Default for AuctionConfig { fn default() -> Self { Self { enabled: false, + rewrite_creatives: default_rewrite_creatives(), providers: Vec::new(), mediator: None, timeout_ms: default_timeout(), @@ -54,6 +59,10 @@ fn default_timeout() -> u32 { 2000 } +fn default_rewrite_creatives() -> bool { + true +} + fn default_creative_store() -> String { "creative_store".to_owned() } @@ -79,3 +88,16 @@ impl AuctionConfig { self.mediator.is_some() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rewrite_creatives_defaults_to_true() { + assert!( + AuctionConfig::default().rewrite_creatives, + "should enable creative rewriting by default" + ); + } +} diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index dd0b35337..58c185381 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -78,6 +78,30 @@ mod tests { ); } + #[test] + fn legacy_blob_without_rewrite_creatives_preserves_rewriting() { + let mut data = + serde_json::to_value(test_settings()).expect("should serialize settings to JSON"); + let auction = data + .get_mut("auction") + .and_then(serde_json::Value::as_object_mut) + .expect("should serialize auction settings as an object"); + assert!( + auction.remove("rewrite_creatives").is_some(), + "should remove the newly serialized setting" + ); + let envelope = BlobEnvelope::new(data, "2026-01-01T00:00:00Z".to_string()); + let envelope_json = serde_json::to_string(&envelope).expect("should serialize envelope"); + + let reconstructed = + settings_from_config_blob(&envelope_json).expect("should reconstruct legacy settings"); + + assert!( + reconstructed.auction.rewrite_creatives, + "should enable creative rewriting for legacy blobs" + ); + } + #[test] fn strings_that_look_like_json_scalars_round_trip_as_strings() { let mut original = test_settings(); diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 9e03f4dbd..9bc71fb22 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -2882,6 +2882,51 @@ mod tests { assert_eq!(ct, "text/css; charset=utf-8"); } + #[test] + fn auction_rewrite_setting_does_not_change_proxied_html_or_css_rewriting() { + let mut settings = create_test_settings(); + settings.auction.rewrite_creatives = false; + let req = build_http_request(Method::GET, "https://edge.example/first-party/proxy"); + + let html = r#""#; + let mut html_response = build_http_response(StatusCode::OK, EdgeBody::from(html)); + html_response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/html; charset=utf-8"), + ); + let html_output = finalize( + &settings, + &req, + "https://cdn.example/creative.html", + html_response, + ) + .expect("should finalize proxied HTML"); + let html_body = response_body_string(html_output); + + let css = "body{background:url(https://cdn.example/bg.png)}"; + let mut css_response = build_http_response(StatusCode::OK, EdgeBody::from(css)); + css_response + .headers_mut() + .insert(header::CONTENT_TYPE, HeaderValue::from_static("text/css")); + let css_output = finalize( + &settings, + &req, + "https://cdn.example/creative.css", + css_response, + ) + .expect("should finalize proxied CSS"); + let css_body = response_body_string(css_output); + + assert!( + html_body.contains("/first-party/proxy?tsurl="), + "should keep rewriting proxied HTML when auction rewriting is disabled: {html_body}" + ); + assert!( + css_body.contains("/first-party/proxy?tsurl="), + "should keep rewriting proxied CSS when auction rewriting is disabled: {css_body}" + ); + } + #[test] fn html_response_rewrite_preserves_non_standard_port() { // Verify that HTML rewriting preserves non-standard ports in sub-resource URLs. diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 03cc535c8..b177a23e7 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -4334,6 +4334,41 @@ origin_host_header_overide = "www.example.com""#, assert!(!rewrite.is_excluded("")); } + #[test] + fn test_auction_rewrite_creatives_defaults_to_true_when_omitted() { + let toml_str = crate_test_settings_str() + + r#" + [auction] + enabled = true + providers = [] + "#; + + let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); + + assert!( + settings.auction.rewrite_creatives, + "should preserve creative rewriting when the setting is omitted" + ); + } + + #[test] + fn test_auction_rewrite_creatives_accepts_explicit_false() { + let toml_str = crate_test_settings_str() + + r#" + [auction] + enabled = true + providers = [] + rewrite_creatives = false + "#; + + let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); + + assert!( + !settings.auction.rewrite_creatives, + "should disable creative rewriting when explicitly configured" + ); + } + #[test] fn test_auction_allowed_context_keys_defaults_to_empty() { let settings = create_test_settings(); diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index d75958812..c6c82dac3 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -12,7 +12,7 @@ Key capabilities: - **Strategy-based winner selection** — Automatic strategy detection based on configuration - **Mediator support** — Optional external mediator for decoding encoded prices (e.g., APS) and applying unified floor pricing - **Provider abstraction** — Pluggable provider interface for adding new demand sources -- **Creative rewriting** — Winning creatives automatically rewritten with first-party proxy URLs +- **Creative rewriting** — Winning creatives are sanitized and rewritten with first-party proxy URLs by default ## System Flow (Prebid + APS) @@ -147,7 +147,7 @@ sequenceDiagram Note over Client,Mock: Response Assembly activate TS activate Client - Orch->>Orch: Transform to OpenRTB response
Generate iframe creatives
Rewrite creative URLs
Add orchestrator metadata + Orch->>Orch: Transform to OpenRTB response
Sanitize creative HTML
Optionally rewrite creative URLs
Add orchestrator metadata Orch-->>TS: OpenRTB BidResponse Note right of Orch: { "id": "auction-response",
"seatbid": [{ "seat": "amazon-aps",
"bid": [{ "price": 2.50,
"adm": "', + }, + }, + }; + + try { + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + const ts = (window as TestWindow).tsjs!; + ts.adInit!(); + capturedListener!({ isEmpty: false, slot: mockSlot }); + + const oldRecord = ts.renders?.['atf_sidebar_ad']; + expect(oldRecord?.injected).toBe(false); + expect(runDeferredPlacement).toBeDefined(); + + // A newer route/adInit starts before the animation-frame retry runs while + // retaining the same publisher-owned slot element. The old callback must + // not mutate that shared element before its trace guard runs. + ts.adInit!(); + const reusedSlot = document.getElementById('div-atf-sidebar')!; + runDeferredPlacement!(0); + + expect(oldRecord?.injected).toBe(false); + expect(reusedSlot.querySelector('iframe')).toBeNull(); + expect(reusedSlot.getAttribute('data-ts-injected')).toBe('false'); + } finally { + vi.unstubAllGlobals(); + } + }); + + it('confirms a successful deferred ADM placement on the original record', async () => { + let capturedListener: ((e: SlotRenderEvent) => void) | undefined; + let runDeferredPlacement: FrameRequestCallback | undefined; + vi.stubGlobal( + 'requestAnimationFrame', + vi.fn((callback: FrameRequestCallback) => { + runDeferredPlacement = callback; + return 1; + }) + ); + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([mockSlot]), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, fn: (e: SlotRenderEvent) => void) => { + if (event === 'slotRenderEnded') capturedListener = fn; + }), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: { + atf_sidebar_ad: { + hb_adid: 'deferred-ad', + adm: '', + }, + }, + }; + + try { + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + const ts = (window as TestWindow).tsjs!; + ts.adInit!(); + capturedListener!({ isEmpty: false, slot: mockSlot }); + const record = ts.renders?.atf_sidebar_ad; + const originalBookkeeping = { + seq: record?.seq, + count: record?.count, + at: record?.at, + historyLength: ts.renderLog?.length, + }; + + runDeferredPlacement!(0); + + expect(ts.renders?.atf_sidebar_ad).toBe(record); + expect(record?.injected).toBe(true); + expect({ + seq: record?.seq, + count: record?.count, + at: record?.at, + historyLength: ts.renderLog?.length, + }).toEqual(originalBookkeeping); + } finally { + vi.unstubAllGlobals(); + } + }); + it('does not attribute a later GAM refresh to the finished server-side auction', async () => { let capturedListener: ((e: SlotRenderEvent) => void) | undefined; @@ -1117,7 +1350,9 @@ describe('installTsRenderBridge', () => { expect(fetchStub).toHaveBeenCalledWith( 'https://openads.example.com/cache?uuid=test-cache-uuid', - { mode: 'cors' } + // Carries an abort signal so a navigation can cancel a render belonging + // to the route it is leaving. + { mode: 'cors', signal: expect.any(AbortSignal) } ); expect(stopSpy).toHaveBeenCalled(); expect(portMessages).toHaveLength(1); @@ -1157,6 +1392,80 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); + it('keeps GAM and bridge signals for both arrival orders on one record per impression', async () => { + const source = createTrustedSlotIframe(); + let slotRenderListener: ((event: SlotRenderEvent) => void) | undefined; + const gptSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + clearTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-header'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([gptSlot]), + refresh: vi.fn(), + addEventListener: vi.fn((event: string, listener: (event: SlotRenderEvent) => void) => { + if (event === 'slotRenderEnded') slotRenderListener = listener; + }), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(gptSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + const ts = (window as TestWindow).tsjs!; + ts.bids.homepage_header = { + hb_adid: 'debug-first', + hb_bidder: 'mocktioneer', + }; + const bridgeListener = await captureBridgeListener(); + ts.adInit!(); + + const sendBridgeRequest = (adId: string): void => { + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId }), + ports: [{ postMessage: vi.fn() }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + }; + + // GAM first, bridge second. + slotRenderListener!({ isEmpty: false, slot: gptSlot }); + const firstRecord = ts.renders?.homepage_header; + ts.bids.homepage_header = { + ...ts.bids.homepage_header, + adm: '

First creative
', + }; + sendBridgeRequest('debug-first'); + expect(ts.renders?.homepage_header).toBe(firstRecord); + expect(firstRecord).toEqual( + expect.objectContaining({ count: 1, injected: true, servedFrom: 'debug-adm' }) + ); + expect(ts.renderLog).toHaveLength(1); + + // Bridge first, GAM second for the next impression. + ts.bids.homepage_header = { + hb_adid: 'debug-second', + hb_bidder: 'mocktioneer', + adm: '', + }; + ts.adInit!(); + sendBridgeRequest('debug-second'); + const secondRecord = ts.renders?.homepage_header; + slotRenderListener!({ isEmpty: false, slot: gptSlot }); + expect(ts.renders?.homepage_header).toBe(secondRecord); + expect(secondRecord).toEqual( + expect.objectContaining({ count: 2, injected: true, gamEmpty: false }) + ); + expect(ts.renderLog).toHaveLength(2); + }); + it('fetches PBS Cache once when two same-adId messages race before the fetch resolves', async () => { // Concurrent render double-fire guard: two 'Prebid Request' messages for the // same adId can arrive before the first cache fetch settles. The in-flight @@ -1210,6 +1519,47 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); + it('drops a PBS Cache result when the live bid changed before fetch completion', async () => { + let resolveFetch: (value: Response) => void = () => {}; + fetchStub.mockReturnValue( + new Promise((resolve) => { + resolveFetch = resolve; + }) + ); + const ts = (window as TestWindow).tsjs!; + ts.bids.homepage_header = { + ...ts.bids.homepage_header, + hb_auction_id: 'auction-1', + hb_bid_id: 'bid-1', + }; + const bridgeListener = await captureBridgeListener(); + const portMessages: string[] = []; + const source = createTrustedSlotIframe(); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [{ postMessage: (message: string) => portMessages.push(message) }], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + + // Same bridge ad ID and auction ID, but a different bid object/trace ID. + // Comparing only hb_adid + hb_auction_id would incorrectly accept the old + // creative and stamp it with the captured route's data. + ts.bids.homepage_header = { + ...ts.bids.homepage_header, + hb_bid_id: 'bid-2', + hb_adm_hash: 'new-creative-hash', + }; + resolveFetch({ ok: true, text: () => Promise.resolve('
Old creative
') } as Response); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(portMessages).toHaveLength(0); + expect(ts.renders?.homepage_header).toBeUndefined(); + }); + it('responds with adm without fetching PBS Cache when debug adm is available', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); const debugAdm = '
Debug Creative
'; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 9a08defcb..58291329b 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -51,6 +51,7 @@ describe('installSpaAuctionHook', () => { originalReplaceState({}, '', '/'); // Drop any ad containers inserted by a test so DOM state does not leak. document.body.innerHTML = ''; + delete (window as TestWindow).googletag; // Remove this test's popstate listener(s) so they do not fire in later tests. popstateHandlers.forEach((handler) => window.removeEventListener('popstate', handler)); popstateHandlers = []; @@ -304,6 +305,59 @@ describe('installSpaAuctionHook', () => { expect(adInit).toHaveBeenCalledTimes(1); }); + it('stops orphan recovery before a fast route DOM swap can replay old bids', async () => { + document.body.innerHTML = '
'; + const definedDivs: string[] = []; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([]), + refresh: vi.fn(), + addEventListener: vi.fn(), + }; + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn((_path: string, _sizes: unknown, divId: string) => { + definedDivs.push(divId); + return { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + clearTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue(divId), + getTargeting: vi.fn().mockReturnValue([]), + }; + }), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + display: vi.fn(), + destroySlots: vi.fn(), + }; + // Keep page-bids slower than the orphan observer's 250 ms debounce. + fetchStub.mockReturnValue(new Promise(() => {})); + + await importGptModule(); + const ts = (window as TestWindow).tsjs!; + ts.adSlots = [ + { + id: 'ad-header-0', + gam_unit_path: '/123/header', + div_id: 'ad-header-0', + formats: [[728, 90]], + targeting: {}, + }, + ]; + ts.bids = { 'ad-header-0': { hb_adid: 'old-route-ad' } }; + ts.adInit!(); + expect(definedDivs).toEqual(['ad-header-0-_R_old_']); + + history.pushState({}, '', '/new-route'); + document.body.innerHTML = '
'; + await new Promise((resolve) => setTimeout(resolve, 350)); + + // The pending old-route watcher was disconnected synchronously when + // navigation began, so it never rebound or re-requested the old auction. + expect(definedDivs).toEqual(['ad-header-0-_R_old_']); + }); + it('leaves slots and bids untouched on a non-OK response', async () => { fetchStub.mockResolvedValue({ ok: false, status: 500 }); const { installSpaAuctionHook } = await importGptModule(); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index c6d47eb46..4b5fffe52 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -8,6 +8,7 @@ const { mockRegisterBidAdapter, mockGetUserIdsAsEids, mockGetConfig, + mockOnEvent, mockPbjs, mockGetBidAdapter, mockAdapterManager, @@ -21,6 +22,7 @@ const { () => [] as Array<{ source: string; uids?: Array<{ id: string; atype?: number }> }> ); const mockGetConfig = vi.fn(); + const mockOnEvent = vi.fn(); const mockPbjs = { setConfig: mockSetConfig, processQueue: mockProcessQueue, @@ -28,6 +30,7 @@ const { registerBidAdapter: mockRegisterBidAdapter, getUserIdsAsEids: mockGetUserIdsAsEids, getConfig: mockGetConfig, + onEvent: mockOnEvent, adUnits: [] as any[], }; const mockAdapterManager = { @@ -40,6 +43,7 @@ const { mockRegisterBidAdapter, mockGetUserIdsAsEids, mockGetConfig, + mockOnEvent, mockPbjs, mockGetBidAdapter, mockAdapterManager, @@ -69,7 +73,8 @@ import { auctionBidsToPrebidBids, installPrebidNpm, installRefreshHandler, - recordPrebidBidWon, + installPrebidRenderTrace, + recordPrebidAdRender, } from '../../../src/integrations/prebid/index'; import type { AuctionBid } from '../../../src/core/auction'; import type { TsjsApi } from '../../../src/core/types'; @@ -219,6 +224,7 @@ describe('prebid/auctionBidsToPrebidBids', () => { creativeId: 'KM-CREA-1', adomain: ['kargo.com'], auctionId: 'ts-auction-xyz', + bidId: 'bid-abc-1', admHash: 'a1b2c3d4e5f60718', }, ]; @@ -227,10 +233,13 @@ describe('prebid/auctionBidsToPrebidBids', () => { expect(result[0].meta.tsAuctionId).toBe('ts-auction-xyz'); expect(result[0].meta.tsAdmHash).toBe('a1b2c3d4e5f60718'); + // The bid's own OpenRTB id, distinct from the advertiser creative id. + expect(result[0].meta.tsBidId).toBe('bid-abc-1'); + expect(result[0].creativeId).toBe('KM-CREA-1'); }); }); -describe('prebid/recordPrebidBidWon', () => { +describe('prebid/recordPrebidAdRender', () => { beforeEach(() => { delete (window as { tsjs?: TsjsApi }).tsjs; document.body.innerHTML = ''; @@ -242,12 +251,19 @@ describe('prebid/recordPrebidBidWon', () => { it('records an auction-path render for a server-side bid', () => { document.body.innerHTML = '
'; - const record = recordPrebidBidWon({ - adUnitCode: 'ad-header-0-_R_x_', - bidderCode: 'kargo', - creativeId: 'KM-CREA-1', - meta: { tsAuctionId: '265dcedd-aa0a', tsAdmHash: 'f68044ca9f68c88c' }, - }); + const record = recordPrebidAdRender( + { + adUnitCode: 'ad-header-0-_R_x_', + bidderCode: 'kargo', + creativeId: 'KM-CREA-1', + meta: { + tsAuctionId: '265dcedd-aa0a', + tsBidId: 'bid-abc-1', + tsAdmHash: 'f68044ca9f68c88c', + }, + }, + 'succeeded' + ); expect(record).toBeDefined(); expect(record).toEqual( @@ -257,6 +273,7 @@ describe('prebid/recordPrebidBidWon', () => { rendered: true, injected: true, auctionId: '265dcedd-aa0a', + bidId: 'bid-abc-1', admHash: 'f68044ca9f68c88c', bidder: 'kargo', creativeId: 'KM-CREA-1', @@ -266,21 +283,111 @@ describe('prebid/recordPrebidBidWon', () => { ); // Written into the shared registry the panel reads. expect((window as { tsjs?: TsjsApi }).tsjs?.renders?.['ad-header-0-_R_x_']).toBeDefined(); + // The bid id must reach the DOM as its own attribute, never folded into + // data-ts-ad-id. + const el = document.getElementById('ad-header-0-_R_x_')!; + expect(el.getAttribute('data-ts-bid-id')).toBe('bid-abc-1'); + }); + + it('records a failed render as unconfirmed, not as a green render', () => { + document.body.innerHTML = '
'; + const record = recordPrebidAdRender( + { + adUnitCode: 'ad-header-0', + bidderCode: 'kargo', + meta: { tsAuctionId: '265dcedd-aa0a' }, + }, + 'failed' + ); + + expect(record).toEqual( + expect.objectContaining({ rendered: false, injected: false, visible: false }) + ); }); it('skips a bid without the server-side trace tuple (client-side bidder)', () => { - const record = recordPrebidBidWon({ - adUnitCode: 'ad-header-0', - bidderCode: 'appnexus', - meta: { advertiserDomains: ['x.com'] }, - }); + const record = recordPrebidAdRender( + { + adUnitCode: 'ad-header-0', + bidderCode: 'appnexus', + meta: { advertiserDomains: ['x.com'] }, + }, + 'succeeded' + ); expect(record).toBeUndefined(); expect((window as { tsjs?: TsjsApi }).tsjs?.renders).toBeUndefined(); }); it('skips a bid with no adUnitCode', () => { - expect(recordPrebidBidWon({ meta: { tsAuctionId: 'x' } })).toBeUndefined(); - expect(recordPrebidBidWon(undefined)).toBeUndefined(); + expect(recordPrebidAdRender({ meta: { tsAuctionId: 'x' } }, 'succeeded')).toBeUndefined(); + expect(recordPrebidAdRender(undefined, 'succeeded')).toBeUndefined(); + }); +}); + +describe('prebid/installPrebidRenderTrace', () => { + beforeEach(() => { + delete (window as { tsjs?: TsjsApi }).tsjs; + document.body.innerHTML = ''; + mockOnEvent.mockReset(); + delete (mockPbjs as { __tsRenderTraceInstalled?: boolean }).__tsRenderTraceInstalled; + }); + afterEach(() => { + delete (window as { tsjs?: TsjsApi }).tsjs; + document.body.innerHTML = ''; + }); + + it('confirms renders from adRenderSucceeded, never from bidWon', () => { + document.body.innerHTML = '
'; + installPrebidRenderTrace(); + + const events = mockOnEvent.mock.calls.map(([name]) => name); + expect(events).toEqual(['adRenderSucceeded', 'adRenderFailed']); + // bidWon fires when a bid is marked the winner — before the renderer runs, + // and so before the render can fail. Confirming on it would show a green + // render for a creative that never reached the page. + expect(events).not.toContain('bidWon'); + + const handlers = Object.fromEntries(mockOnEvent.mock.calls) as Record< + string, + (event: unknown) => void + >; + handlers['adRenderSucceeded']({ + bid: { + adUnitCode: 'ad-header-0', + bidderCode: 'kargo', + meta: { tsAuctionId: 'auction-success', tsBidId: 'bid-success' }, + }, + }); + expect((window as { tsjs?: TsjsApi }).tsjs?.renders?.['ad-header-0']).toEqual( + expect.objectContaining({ rendered: true, injected: true, bidId: 'bid-success' }) + ); + }); + + it('does not produce a confirmed record when the render fails after the win', () => { + document.body.innerHTML = '
'; + installPrebidRenderTrace(); + + const handlers = Object.fromEntries(mockOnEvent.mock.calls) as Record< + string, + (event: unknown) => void + >; + const bid = { + adUnitCode: 'ad-header-0', + bidderCode: 'kargo', + meta: { tsAuctionId: '265dcedd-aa0a' }, + }; + + // Prebid marks the bid as won, then its renderer fails asynchronously. + handlers['adRenderFailed']({ reason: 'exception', message: 'boom', bid }); + + const record = (window as { tsjs?: TsjsApi }).tsjs?.renders?.['ad-header-0']; + expect(record).toEqual( + expect.objectContaining({ rendered: false, injected: false, visible: false }) + ); + // No green badge and no confirmed-render attributes on the slot. + const el = document.getElementById('ad-header-0')!; + expect(el.getAttribute('data-ts-rendered')).toBe('false'); + expect(el.getAttribute('data-ts-injected')).toBe('false'); }); }); From be02574d83c09c44862d90a29f1e8497c6888562 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 22 Jul 2026 20:43:59 +0530 Subject: [PATCH 084/198] Document SSAT root document 304 prevention --- ...sat-root-document-304-prevention-design.md | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md diff --git a/docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md b/docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md new file mode 100644 index 000000000..d790d2a83 --- /dev/null +++ b/docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md @@ -0,0 +1,146 @@ +# SSAT Root Document 304 Prevention Design + +## Problem + +An auction-eligible publisher navigation can currently return `304 Not Modified` +on reload. Trusted Server starts the server-side auction before fetching the +publisher document, but a 304 has no HTML body. The HTML processor therefore +cannot inject the current request's slot state or auction result, and the browser +reuses a previously synthesized document. + +The behavior has two independent causes: + +- Trusted Server forwards browser validators (`If-None-Match` and + `If-Modified-Since`) to the publisher origin. +- The Fastly adapter sends publisher requests through its read-through cache. + +Successful synthesized HTML is also returned with `private, max-age=0` while +retaining the publisher's `ETag` and `Last-Modified`. That explicitly permits +browser storage and revalidation even though those validators describe the +unmodified origin representation, not the personalized document returned by +Trusted Server. + +## Scope + +This change applies only when the existing `should_run_ad_stack` decision is +true. That decision already limits the path to GET document navigations that are +not prefetches or bots and that have matched ad slots, permitted consent, and an +enabled auction. + +The change does not alter: + +- HEAD requests; +- bots or prefetches; +- requests without matching slots or auction consent; +- publisher requests when the auction is disabled; +- static Trusted Server assets and their intentional conditional responses; +- the `/page-bids` client-side auction endpoint; or +- auction identifiers. + +## Design + +### Publisher request + +Immediately before the publisher-origin fetch, an auction-eligible request will +remove `If-None-Match` and `If-Modified-Since`. This forces the publisher origin +to return a complete representation instead of validating a browser-cached +copy. + +The corresponding `PlatformHttpRequest` will carry an explicit, default-false +cache-bypass option. The Fastly adapter will translate that option to +`fastly::Request::set_pass(true)` before both synchronous and asynchronous sends. +All other `PlatformHttpRequest` call sites retain their current behavior because +the option defaults to false. Adapters without an intermediary read-through +cache require no runtime change. + +This bypass is deliberately scoped to the publisher fetch for an eligible SSAT +navigation. It must not be set on assets, image optimization, SSP fan-out, or +integration calls. + +### Publisher response + +When the eligible publisher response is HTML, Trusted Server will: + +- set `Cache-Control: private, no-store`; +- remove `ETag` and `Last-Modified`; +- continue removing `Surrogate-Control` and + `Fastly-Surrogate-Control`. + +`no-store` is an intentional correctness choice. It prevents the browser from +retaining a synthesized document that could later be resurrected through +revalidation. This trades away some browser back/forward-cache eligibility and +increases publisher-origin traffic, but it guarantees that an eligible +navigation receives a fresh body for SSAT injection. + +### Unexpected origin 304 + +An eligible publisher request will already be unconditional and will bypass the +Fastly cache. If the publisher nevertheless returns 304, Trusted Server will not +forward it to the browser. It will abandon the in-flight auction using a distinct +reason and return a synthetic `502 Bad Gateway` response with +`Cache-Control: private, no-store` and no validators or surrogate cache headers. + +The implementation will not retry. The first request is already unconditional +and cache-bypassed, so repeating it is unlikely to produce a body and would add +origin traffic and latency. Returning an explicit non-cacheable error is safer +than allowing the browser to reuse stale personalized HTML. + +## Data Flow + +1. Trusted Server evaluates the existing SSAT eligibility gates. +2. If eligible, it dispatches the server-side auction as it does today. +3. Before the publisher fetch, it removes browser conditional headers and marks + the platform request as cache-bypassed. +4. Fastly sends the request directly to the configured publisher backend. +5. A complete HTML response enters the existing buffering/HTML injection path. +6. Trusted Server injects current slot and auction data and removes all storage + and validation metadata before responding. +7. If the origin unexpectedly returns 304, Trusted Server abandons the auction + and returns the non-cacheable 502 instead. + +## Error Handling and Observability + +Existing publisher transport-error handling remains unchanged. The unexpected +304 case will reuse the existing abandoned-auction event mechanism with a +specific reason such as `unexpected_origin_304`, allowing it to be distinguished +from transport failures and ordinary bodiless responses. + +Noneligible publisher requests retain their existing 304 behavior. This avoids a +global semantic change to the proxy and keeps normal conditional caching intact +outside personalized SSAT documents. + +## Testing + +Tests will prove the behavior at the relevant boundaries: + +- `PlatformHttpRequest` defaults to ordinary cache behavior and its builder + enables bypass explicitly. +- The Fastly adapter applies pass mode only when requested. +- An eligible publisher request removes both conditional headers and requests a + platform cache bypass. +- A noneligible request preserves its conditional headers and ordinary cache + behavior. +- Eligible HTML receives `private, no-store` and has origin and surrogate + validators removed. +- An eligible origin 304, with or without `Content-Type`, is never returned as a + client 304 and produces one abandoned-auction observation. +- Existing HEAD, prefetch, bot, noneligible 304, asset, and `/page-bids` tests + remain unchanged. + +Targeted tests will be written before implementation. Final verification will +use the repository's target-specific test, formatting, and lint commands rather +than a bare workspace build or test. + +## Risks + +- Every eligible SSAT navigation reaches the publisher origin and transfers a + full document, increasing origin load and potentially TTFB. +- `no-store` can reduce back/forward-cache effectiveness, depending on browser + behavior. +- A publisher that incorrectly emits 304 for an unconditional request will now + expose a visible 502 instead of stale content. The distinct telemetry reason + makes this condition diagnosable. + +These costs are accepted because the requested invariant is that every eligible +SSAT navigation receives a complete document into which the current auction can +be injected. From 19c3a25268772816591760a1dd9c71cbd258dad9 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 22 Jul 2026 20:47:06 +0530 Subject: [PATCH 085/198] Clarify unexpected origin 304 telemetry --- .../2026-07-22-ssat-root-document-304-prevention-design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md b/docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md index d790d2a83..38cb1a8a6 100644 --- a/docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md +++ b/docs/superpowers/specs/2026-07-22-ssat-root-document-304-prevention-design.md @@ -102,7 +102,7 @@ than allowing the browser to reuse stale personalized HTML. Existing publisher transport-error handling remains unchanged. The unexpected 304 case will reuse the existing abandoned-auction event mechanism with a -specific reason such as `unexpected_origin_304`, allowing it to be distinguished +specific reason `unexpected_origin_304`, allowing it to be distinguished from transport failures and ordinary bodiless responses. Noneligible publisher requests retain their existing 304 behavior. This avoids a From 1eb09cba99d0ef38a3ccdeca5e0c53dabe8d6256 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 22 Jul 2026 20:58:27 +0530 Subject: [PATCH 086/198] Plan SSAT root document 304 prevention --- ...07-22-ssat-root-document-304-prevention.md | 612 ++++++++++++++++++ 1 file changed, 612 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-22-ssat-root-document-304-prevention.md diff --git a/docs/superpowers/plans/2026-07-22-ssat-root-document-304-prevention.md b/docs/superpowers/plans/2026-07-22-ssat-root-document-304-prevention.md new file mode 100644 index 000000000..9f87ca9d4 --- /dev/null +++ b/docs/superpowers/plans/2026-07-22-ssat-root-document-304-prevention.md @@ -0,0 +1,612 @@ +# SSAT Root Document 304 Prevention Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Guarantee that every auction-eligible SSAT navigation receives a complete, non-stored publisher document instead of a browser- or Fastly-generated 304. + +**Architecture:** Add a default-off cache-bypass capability to the platform HTTP request and map it to Fastly pass mode. The publisher path enables it only for the existing `should_run_ad_stack` gate, strips browser validators before the origin fetch, removes response validators while setting `private, no-store`, and converts an unexpected eligible-origin 304 into a non-cacheable 502 with abandoned-auction telemetry. + +**Tech Stack:** Rust 2024, `edgezero_core` HTTP types, Fastly Rust SDK 0.12.1, async traits, Viceroy tests, `error-stack`. + +--- + +## File Map + +| File | Responsibility | +| --- | --- | +| `crates/trusted-server-core/src/platform/http.rs` | Define the platform-neutral, default-off cache-bypass request option. | +| `crates/trusted-server-core/src/platform/test_support.rs` | Record cache-bypass options in the shared stub HTTP client for publisher tests. | +| `crates/trusted-server-adapter-fastly/src/platform.rs` | Translate the platform option to Fastly `Request::set_pass(true)` in both send paths. | +| `crates/trusted-server-core/src/publisher.rs` | Apply the eligibility gate, strip validators, set the synthesized response policy, fail closed on unexpected 304, and test the complete behavior. | + +No configuration schema, JavaScript, `/page-bids`, auction-ID, asset, or integration files change. + +### Task 1: Add Platform Cache-Bypass Metadata and Test Recording + +**Files:** + +- Modify: `crates/trusted-server-core/src/platform/http.rs` +- Modify: `crates/trusted-server-core/src/platform/test_support.rs` + +- [ ] **Step 1: Write failing constructor and builder tests** + +Add these tests to `platform::http::tests`: + +```rust +#[test] +fn platform_http_request_cache_bypass_defaults_to_false() { + let request = edgezero_core::http::request_builder() + .uri("https://example.com/") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let request = PlatformHttpRequest::new(request, "origin"); + + assert!( + !request.bypass_cache, + "ordinary platform requests should retain normal cache behavior" + ); +} + +#[test] +fn platform_http_request_cache_bypass_builder_enables_bypass() { + let request = edgezero_core::http::request_builder() + .uri("https://example.com/") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let request = PlatformHttpRequest::new(request, "origin").with_cache_bypass(); + + assert!( + request.bypass_cache, + "cache-bypass builder should enable platform cache bypass" + ); +} +``` + +- [ ] **Step 2: Run the tests and verify they fail** + +Run: + +```bash +cargo test-fastly platform_http_request_cache_bypass -- --nocapture +``` + +Expected: compilation fails because `bypass_cache` and `with_cache_bypass` do not exist. + +- [ ] **Step 3: Add the minimal platform request option** + +Add a documented public field and builder to `PlatformHttpRequest`: + +```rust +/// Whether the platform's intermediary response cache must be bypassed. +/// +/// Adapters without an intermediary outbound cache may treat this as already +/// satisfied. The option defaults to `false` so existing call sites preserve +/// their current cache behavior. +pub bypass_cache: bool, +``` + +Initialize it to `false` in `new`, then add: + +```rust +/// Bypass the platform's intermediary response cache for this request. +#[must_use] +pub fn with_cache_bypass(mut self) -> Self { + self.bypass_cache = true; + self +} +``` + +- [ ] **Step 4: Extend the shared HTTP stub** + +Add `cache_bypass_flags: Mutex>` to `StubHttpClient`, initialize it, +record `request.bypass_cache` in both `send` and `send_async`, and expose: + +```rust +pub fn recorded_cache_bypass_flags(&self) -> Vec { + self.cache_bypass_flags + .lock() + .expect("should lock cache bypass flags") + .clone() +} +``` + +Record the flag before consuming `request.request`. + +- [ ] **Step 5: Run targeted platform tests** + +Run: + +```bash +cargo test-fastly platform_http_request_cache_bypass -- --nocapture +cargo test-fastly platform::test_support -- --nocapture +``` + +Expected: both commands pass. + +- [ ] **Step 6: Commit** + +```bash +git add crates/trusted-server-core/src/platform/http.rs crates/trusted-server-core/src/platform/test_support.rs +git commit -m "Add platform HTTP cache bypass option" +``` + +### Task 2: Map Cache Bypass to Fastly Pass Mode + +**Files:** + +- Modify: `crates/trusted-server-adapter-fastly/src/platform.rs` + +- [ ] **Step 1: Write failing Fastly cache-override tests** + +Introduce a private helper named `apply_fastly_cache_bypass` and add tests that +construct a `fastly::Request`, invoke the helper, and inspect the SDK's derived +debug representation: + +```rust +#[test] +fn apply_fastly_cache_bypass_sets_pass_when_enabled() { + let mut request = fastly::Request::get("https://example.com/"); + + apply_fastly_cache_bypass(&mut request, true); + + assert!( + format!("{request:?}").contains("cache_override: Pass"), + "enabled bypass should select Fastly pass mode" + ); +} + +#[test] +fn apply_fastly_cache_bypass_preserves_default_when_disabled() { + let mut request = fastly::Request::get("https://example.com/"); + + apply_fastly_cache_bypass(&mut request, false); + + assert!( + format!("{request:?}").contains("cache_override: None"), + "disabled bypass should preserve Fastly read-through caching" + ); +} +``` + +- [ ] **Step 2: Run the tests and verify they fail** + +Run: + +```bash +cargo test-fastly apply_fastly_cache_bypass -- --nocapture +``` + +Expected: compilation fails because the helper does not exist. + +- [ ] **Step 3: Implement and use the Fastly mapping** + +Add: + +```rust +fn apply_fastly_cache_bypass(request: &mut fastly::Request, bypass_cache: bool) { + if bypass_cache { + request.set_pass(true); + } +} +``` + +In `FastlyPlatformHttpClient::send`, copy `request.bypass_cache` before moving +the inner request, make the converted Fastly request mutable, and invoke the +helper before `.send()`. + +Do the same in `send_async` before `.send_async()`. Preserve the existing Image +Optimizer and streaming-response rejection behavior. + +- [ ] **Step 4: Run targeted Fastly adapter tests** + +Run: + +```bash +cargo test-fastly apply_fastly_cache_bypass -- --nocapture +cargo test-fastly fastly_platform_http_client -- --nocapture +``` + +Expected: all matching tests pass. + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-adapter-fastly/src/platform.rs +git commit -m "Bypass Fastly cache for marked HTTP requests" +``` + +### Task 3: Protect Eligible Publisher Requests and Successful HTML Responses + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Add focused eligible- and ineligible-request test helpers** + +Add a `ssat_cache_policy_tests` module beside the existing handler-level test +modules. Reuse `StubHttpClient`, `StubBackend`, no-op services, a +non-regulated `EcContext`, a slot matching `/article`, and an enabled +orchestrator. A launch-failing provider is sufficient for these policy tests: +eligibility depends on the configured gate, not the dispatch outcome. + +The eligible request must be: + +```rust +HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .header(header::IF_NONE_MATCH, "\"origin-tag\"") + .header(header::IF_MODIFIED_SINCE, "Wed, 21 Oct 2015 07:28:00 GMT") + .body(EdgeBody::empty()) + .expect("should build eligible navigation") +``` + +Queue an origin 200 with `Content-Type: text/html`, `Cache-Control: public, +max-age=300`, `ETag`, `Last-Modified`, `Surrogate-Control`, and +`Fastly-Surrogate-Control`. + +- [ ] **Step 2: Write the failing eligible-request test** + +Drive `handle_publisher_request` and assert: + +```rust +assert_eq!(stub.recorded_cache_bypass_flags(), vec![true]); +let origin_headers = stub + .recorded_request_headers() + .into_iter() + .last() + .expect("should record publisher request headers"); +assert!(!origin_headers.iter().any(|(name, _)| name == "if-none-match")); +assert!(!origin_headers.iter().any(|(name, _)| name == "if-modified-since")); +``` + +Extract the response headers from the returned `PublisherResponse` and assert: + +```rust +assert_eq!( + response.headers().get(header::CACHE_CONTROL), + Some(&HeaderValue::from_static("private, no-store")) +); +for name in [ + header::ETAG.as_str(), + header::LAST_MODIFIED.as_str(), + "surrogate-control", + "fastly-surrogate-control", +] { + assert!(response.headers().get(name).is_none(), "{name} should be removed"); +} +``` + +- [ ] **Step 3: Write the failing noneligible-request test** + +Use the existing `run_publisher_proxy` helper with no slots and the same +conditional headers. Queue a normal response and assert: + +```rust +assert_eq!(stub.recorded_cache_bypass_flags(), vec![false]); +assert!(origin_headers.iter().any(|(name, _)| name == "if-none-match")); +assert!(origin_headers.iter().any(|(name, _)| name == "if-modified-since")); +``` + +Also assert the origin's cache policy and validators remain unchanged. This is +the regression guard for HEAD, bots, prefetches, no-slot pages, and every other +request that fails the existing gate. + +- [ ] **Step 4: Run the publisher policy tests and verify they fail** + +Run: + +```bash +cargo test-fastly ssat_cache_policy_tests -- --nocapture +``` + +Expected: the eligible assertions fail because validators are forwarded, +bypass is false, the response uses `private, max-age=0`, and validators remain. + +- [ ] **Step 5: Implement request protection** + +Immediately after auction dispatch and before URI/Host rewriting, add: + +```rust +if should_run_ad_stack { + req.headers_mut().remove(header::IF_NONE_MATCH); + req.headers_mut().remove(header::IF_MODIFIED_SINCE); +} +``` + +Build the publisher request once, conditionally apply the builder, and send it: + +```rust +let platform_request = PlatformHttpRequest::new(req, backend_name); +let platform_request = if should_run_ad_stack { + platform_request.with_cache_bypass() +} else { + platform_request +}; +``` + +- [ ] **Step 6: Implement successful HTML response protection** + +Within the existing `should_run_ad_stack && is_html_content_type(...)` branch: + +```rust +response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), +); +response.headers_mut().remove(header::ETAG); +response.headers_mut().remove(header::LAST_MODIFIED); +response.headers_mut().remove("surrogate-control"); +response.headers_mut().remove("fastly-surrogate-control"); +``` + +Update the adjacent rationale: synthesized, per-navigation auction state must +not be stored or validated as though it were the origin representation. + +- [ ] **Step 7: Run targeted publisher tests** + +Run: + +```bash +cargo test-fastly ssat_cache_policy_tests -- --nocapture +cargo test-fastly publisher_request_uses_platform_http_client_with_http_types -- --nocapture +cargo test-fastly response_carries_body_preserves_bodiless_metadata -- --nocapture +``` + +Expected: all commands pass. + +- [ ] **Step 8: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs +git commit -m "Prevent SSAT publisher document revalidation" +``` + +### Task 4: Fail Closed on an Unexpected Eligible-Origin 304 + +**Files:** + +- Modify: `crates/trusted-server-core/src/publisher.rs` + +- [ ] **Step 1: Add a dispatching test provider** + +Within `ssat_cache_policy_tests`, add a provider whose `request_bids` sends one +request through `context.services.http_client().send_async(...)`. Give it a +stable backend name and make `parse_response` panic because an unexpected 304 +must abandon, not collect, the pending request. + +Queue responses in this order because `StubHttpClient` consumes the provider +response during `send_async` before the publisher response during `send`: + +```rust +stub.push_response(200, b"unused provider response".to_vec()); +stub.push_response_with_headers( + 304, + Vec::new(), + vec![("etag", "\"origin-tag\"")], +); +``` + +- [ ] **Step 2: Write the failing unexpected-304 test** + +Drive an eligible navigation with the dispatching provider and recording +telemetry sink. Assert the returned variant is `PublisherResponse::Buffered` +with: + +```rust +assert_eq!(response.status(), StatusCode::BAD_GATEWAY); +assert_eq!( + response.headers().get(header::CACHE_CONTROL), + Some(&HeaderValue::from_static("private, no-store")) +); +assert!(response.headers().get(header::ETAG).is_none()); +assert!(response.headers().get(header::LAST_MODIFIED).is_none()); +assert!(response.headers().get("surrogate-control").is_none()); +assert!(response.headers().get("fastly-surrogate-control").is_none()); +``` + +Flatten telemetry rows and assert exactly one summary row has +`terminal_status == Some("abandoned")` and +`terminal_reason == Some("unexpected_origin_304")`. Assert no provider parse or +auction collection occurred. + +Cover both a typical 304 without `Content-Type` and a 304 carrying +`Content-Type: text/html` using a small table/helper so response classification +cannot affect the guard. + +- [ ] **Step 3: Verify the test fails** + +Run: + +```bash +cargo test-fastly unexpected_origin_304 -- --nocapture +``` + +Expected: the handler returns 304 and no `unexpected_origin_304` telemetry. + +- [ ] **Step 4: Add a noneligible-304 regression test** + +Use `run_publisher_proxy` with no slots, queue a 304 carrying `ETag`, +`Last-Modified`, and origin cache headers, and assert: + +```rust +let response = match run_publisher_proxy(&settings, &services, request).await { + PublisherResponse::Buffered(response) => response, + _ => panic!("noneligible 304 should remain a buffered response"), +}; +assert_eq!(response.status(), StatusCode::NOT_MODIFIED); +assert_eq!(response.headers().get(header::ETAG), Some(&origin_etag)); +assert_eq!( + response.headers().get(header::LAST_MODIFIED), + Some(&origin_last_modified) +); +``` + +Also assert the request used `bypass_cache == false` and preserved its incoming +conditional headers. This proves the 304-to-502 guard is eligibility-scoped +rather than global. + +- [ ] **Step 5: Implement the fail-closed guard before content classification** + +Immediately after receiving/logging the publisher response and before reading +its content type: + +```rust +if should_run_ad_stack && response.status() == StatusCode::NOT_MODIFIED { + if let Some(dispatched) = dispatched_auction.take() { + emit_abandoned_auction( + services, + auction_observation.take(), + dispatched, + "unexpected_origin_304", + ) + .await; + } + + let response = Response::builder() + .status(StatusCode::BAD_GATEWAY) + .header(header::CACHE_CONTROL, "private, no-store") + .header(header::CONTENT_TYPE, "text/plain; charset=utf-8") + .body(EdgeBody::from("Publisher origin returned an invalid conditional response")) + .change_context(TrustedServerError::Proxy { + message: "failed to build unexpected origin 304 response".to_string(), + })?; + return Ok(PublisherResponse::Buffered(response)); +} +``` + +Because the response is built from a fresh builder, it contains no origin +validators or surrogate cache headers and still goes through the adapter's +normal finalization after the publisher handler returns. + +- [ ] **Step 6: Run the unexpected-304 and generic bodiless tests** + +Run: + +```bash +cargo test-fastly ssat_cache_policy_tests -- --nocapture +cargo test-fastly response_carries_body_preserves_bodiless_metadata -- --nocapture +cargo test-fastly serve_static -- --nocapture +``` + +Expected: eligible publisher 304 tests return 502; generic publisher/static +conditional semantics remain passing. + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs +git commit -m "Reject unexpected SSAT origin 304 responses" +``` + +### Task 5: Full Verification and Scope Audit + +**Files:** + +- Verify only; modify production files only if a verification failure exposes a defect in the approved scope. + +- [ ] **Step 1: Format and inspect the diff** + +Run: + +```bash +cargo fmt --all +git diff --check origin/main...HEAD +git diff --stat origin/main...HEAD +git status --short +``` + +Expected: formatting succeeds; no whitespace errors; only the spec, plan, two +core platform files, publisher, and Fastly platform adapter are changed. Local +`fastly.toml` remains untouched. + +- [ ] **Step 2: Run all target-specific test suites** + +Run: + +```bash +cargo test-fastly +cargo test-axum +cargo test-cloudflare +cargo test-spin +./scripts/test-cli.sh +cargo test --manifest-path crates/trusted-server-integration-tests/Cargo.toml --test parity +``` + +Expected: all tests pass. + +- [ ] **Step 3: Run JavaScript and documentation gates** + +Run from `crates/trusted-server-js/lib`: + +```bash +npx vitest run +npm run format +``` + +Then run from `docs`: + +```bash +npm run format +``` + +Expected: JavaScript tests pass and both format commands complete without +errors. Inspect `git status --short` afterward; formatting must not introduce +unrelated content changes. + +- [ ] **Step 4: Run formatting and target-specific lints/checks** + +Run: + +```bash +cargo fmt --all -- --check +cargo clippy-fastly +cargo clippy-axum +cargo clippy-cloudflare +cargo clippy-cloudflare-wasm +cargo clippy-spin-native +cargo clippy-spin-wasm +cargo check-fastly +cargo check-axum +cargo check-cloudflare +cargo check-spin +``` + +Expected: all checks pass with no warnings promoted to errors. + +- [ ] **Step 5: Audit constructors and behavior boundaries** + +Run: + +```bash +rg -n "with_cache_bypass|bypass_cache|set_pass" crates +rg -n "If-None-Match|If-Modified-Since|private, no-store|unexpected_origin_304" crates/trusted-server-core/src/publisher.rs +git diff origin/main...HEAD -- fastly.toml crates/trusted-server-js +``` + +Expected: + +- `with_cache_bypass` is used only by the eligible publisher fetch. +- Fastly honors the option in both send paths. +- all other request constructors default to false; +- `fastly.toml` and JavaScript have no branch diff. + +- [ ] **Step 6: Commit any formatting-only changes if needed** + +```bash +git add crates/trusted-server-core/src/platform/http.rs \ + crates/trusted-server-core/src/platform/test_support.rs \ + crates/trusted-server-core/src/publisher.rs \ + crates/trusted-server-adapter-fastly/src/platform.rs +git commit -m "Format SSAT 304 prevention changes" +``` + +Skip this commit when `cargo fmt --all` produces no new diff. + +- [ ] **Step 7: Request final code review** + +Run the repository's code-review workflow against `origin/main...HEAD`. Resolve +only correctness, security, test, or approved-scope findings, then repeat the +affected verification commands before reporting completion. From 3360e6a4756e816530838aa6943dadcb9b992412 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 22 Jul 2026 10:38:52 -0500 Subject: [PATCH 087/198] Handle oversized HEAD response metadata HEAD Content-Length describes the corresponding GET representation, not a body that will be buffered. Applying the buffered-response limit to that metadata prevents valid S3 Image Optimizer preflights from reaching their streamed GET request. Resolves: #950 --- .../src/platform.rs | 68 +++++++++++++++++-- 1 file changed, 64 insertions(+), 4 deletions(-) diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index b1bab232b..dc61d2b34 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -360,11 +360,14 @@ fn fastly_response_to_platform( mut resp: fastly::Response, backend_name: impl Into, stream_response: bool, + response_body_expected: bool, ) -> Result> { // Pre-flight: reject oversized responses before copying bytes into WASM heap. // Content-Length is advisory but covers most origin responses; chunked // responses without it fall through to the post-materialization check below. - if !stream_response + // HEAD responses report the corresponding GET size but contain no body. + if response_body_expected + && !stream_response && let Some(claimed_len) = resp .get_header("content-length") .and_then(|v| v.to_str().ok()) @@ -382,7 +385,9 @@ fn fastly_response_to_platform( for (name, value) in resp.get_headers() { builder = builder.header(name.as_str(), value.as_bytes()); } - let body = if stream_response { + let body = if !response_body_expected { + edgezero_core::body::Body::empty() + } else if stream_response { fastly_body_to_edge_stream(resp.take_body()) } else { let body_bytes = resp.take_body_bytes(); @@ -431,6 +436,7 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { let backend_name = request.backend_name.clone(); let image_optimizer = request.image_optimizer; let stream_response = request.stream_response; + let response_body_expected = request.request.method() != edgezero_core::http::Method::HEAD; let mut fastly_req = edge_request_to_fastly(request.request)?; if let Some(options) = image_optimizer { apply_fastly_image_optimizer(&mut fastly_req, options)?; @@ -438,7 +444,12 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { let fastly_resp = fastly_req .send(&backend_name) .change_context(PlatformError::HttpClient)?; - fastly_response_to_platform(fastly_resp, backend_name, stream_response) + fastly_response_to_platform( + fastly_resp, + backend_name, + stream_response, + response_body_expected, + ) } async fn send_async( @@ -501,7 +512,7 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { .attach("select: response has no backend name; correlation impossible")); }; ( - fastly_response_to_platform(fastly_resp, backend_name, false), + fastly_response_to_platform(fastly_resp, backend_name, false, true), None, ) } @@ -736,6 +747,55 @@ mod tests { // --- FastlyPlatformHttpClient ------------------------------------------- + #[test] + fn fastly_response_to_platform_allows_oversized_head_content_length() { + let mut fastly_response = fastly::Response::from_status(200); + fastly_response.set_header( + fastly::http::header::CONTENT_LENGTH, + (MAX_PLATFORM_RESPONSE_BODY_BYTES + 1).to_string(), + ); + + let platform_response = + fastly_response_to_platform(fastly_response, "origin", false, false) + .expect("should allow HEAD metadata for an oversized object"); + + assert_eq!( + platform_response + .response + .headers() + .get(edgezero_core::http::header::CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()), + Some("10485761"), + "should preserve the origin Content-Length" + ); + assert!( + platform_response + .response + .into_body() + .into_bytes() + .unwrap_or_default() + .is_empty(), + "should return an empty HEAD response body" + ); + } + + #[test] + fn fastly_response_to_platform_rejects_oversized_buffered_get_content_length() { + let mut fastly_response = fastly::Response::from_status(200); + fastly_response.set_header( + fastly::http::header::CONTENT_LENGTH, + (MAX_PLATFORM_RESPONSE_BODY_BYTES + 1).to_string(), + ); + + let error = fastly_response_to_platform(fastly_response, "origin", false, true) + .expect_err("should reject oversized buffered GET metadata"); + + assert!( + format!("{error:?}").contains("exceeds 10485760-byte response body limit"), + "should retain the buffered response size limit: {error:?}" + ); + } + #[test] fn fastly_platform_http_client_send_returns_error_for_unregistered_backend() { let client = FastlyPlatformHttpClient; From 8bdbd61e7d3cf0e6150824cbca1b690d106d1685 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 22 Jul 2026 21:08:57 +0530 Subject: [PATCH 088/198] Add platform HTTP cache bypass option --- .../trusted-server-core/src/platform/http.rs | 48 +++++++++++++++++++ .../src/platform/test_support.rs | 33 ++++++++++++- 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/crates/trusted-server-core/src/platform/http.rs b/crates/trusted-server-core/src/platform/http.rs index da99487a3..2e1ec51a4 100644 --- a/crates/trusted-server-core/src/platform/http.rs +++ b/crates/trusted-server-core/src/platform/http.rs @@ -23,6 +23,12 @@ pub struct PlatformHttpRequest { /// Adapters that cannot attach this metadata to their send path should /// return an error rather than silently dropping transformations. pub image_optimizer: Option, + /// Whether the platform's intermediary response cache must be bypassed. + /// + /// Adapters without an intermediary outbound cache may treat this as already + /// satisfied. The option defaults to `false` so existing call sites preserve + /// their current cache behavior. + pub bypass_cache: bool, /// Whether the response body should stay streaming in the platform response. /// /// Adapters that cannot preserve streaming response bodies should return an @@ -38,6 +44,7 @@ impl PlatformHttpRequest { request, backend_name: backend_name.into(), image_optimizer: None, + bypass_cache: false, stream_response: false, } } @@ -53,6 +60,13 @@ impl PlatformHttpRequest { self } + /// Bypass the platform's intermediary response cache for this request. + #[must_use] + pub fn with_cache_bypass(mut self) -> Self { + self.bypass_cache = true; + self + } + /// Preserve the upstream response body as a stream when the adapter supports it. /// /// Asset routes use this to avoid materializing large image/static responses @@ -306,8 +320,42 @@ pub trait PlatformHttpClient: Send + Sync { #[cfg(test)] mod tests { + use edgezero_core::body::Body; + use edgezero_core::http::request_builder; + use super::*; + #[test] + fn platform_http_request_cache_bypass_defaults_to_false() { + let request = PlatformHttpRequest::new( + request_builder() + .body(Body::empty()) + .expect("should build request"), + "stub-backend", + ); + + assert!( + !request.bypass_cache, + "should preserve existing cache behavior by default" + ); + } + + #[test] + fn platform_http_request_cache_bypass_builder_enables_bypass() { + let request = PlatformHttpRequest::new( + request_builder() + .body(Body::empty()) + .expect("should build request"), + "stub-backend", + ) + .with_cache_bypass(); + + assert!( + request.bypass_cache, + "should enable intermediary cache bypass" + ); + } + // --------------------------------------------------------------------------- // Error-correlation interim scope (before EdgeZero #213) // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-core/src/platform/test_support.rs b/crates/trusted-server-core/src/platform/test_support.rs index 4235b4ed7..9de2de134 100644 --- a/crates/trusted-server-core/src/platform/test_support.rs +++ b/crates/trusted-server-core/src/platform/test_support.rs @@ -225,6 +225,7 @@ pub(crate) struct StubHttpClient { // platforms whose send_async executes eagerly (e.g. Cloudflare Workers). concurrent_fanout: std::sync::atomic::AtomicBool, image_optimizer_options: Mutex>>, + cache_bypass_flags: Mutex>, stream_response_flags: Mutex>, request_methods: Mutex>, request_uris: Mutex>, @@ -247,6 +248,7 @@ impl StubHttpClient { select_errors: Mutex::new(VecDeque::new()), concurrent_fanout: std::sync::atomic::AtomicBool::new(true), image_optimizer_options: Mutex::new(Vec::new()), + cache_bypass_flags: Mutex::new(Vec::new()), stream_response_flags: Mutex::new(Vec::new()), request_methods: Mutex::new(Vec::new()), request_uris: Mutex::new(Vec::new()), @@ -319,6 +321,14 @@ impl StubHttpClient { .clone() } + /// Return cache-bypass flags captured per `send` or `send_async` call, in order. + pub(crate) fn recorded_cache_bypass_flags(&self) -> Vec { + self.cache_bypass_flags + .lock() + .expect("should lock cache bypass flags") + .clone() + } + /// Return streaming-response flags captured per `send` call, in order. pub fn recorded_stream_response_flags(&self) -> Vec { self.stream_response_flags @@ -376,6 +386,10 @@ impl PlatformHttpClient for StubHttpClient { .lock() .expect("should lock image optimizer options") .push(request.image_optimizer.clone()); + self.cache_bypass_flags + .lock() + .expect("should lock cache bypass flags") + .push(request.bypass_cache); self.stream_response_flags .lock() .expect("should lock stream response flags") @@ -456,6 +470,10 @@ impl PlatformHttpClient for StubHttpClient { .lock() .expect("should lock calls") .push(backend_name.clone()); + self.cache_bypass_flags + .lock() + .expect("should lock cache bypass flags") + .push(request.bypass_cache); let headers: Vec<(String, String)> = request .request @@ -715,6 +733,11 @@ mod tests { vec!["stub-backend"], "should record the backend name" ); + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "should record the default cache-bypass flag" + ); } #[test] @@ -760,8 +783,9 @@ mod tests { let pending_a = futures::executor::block_on(stub.send_async(make_req("backend-a"))) .expect("should start request a"); - let pending_b = futures::executor::block_on(stub.send_async(make_req("backend-b"))) - .expect("should start request b"); + let pending_b = + futures::executor::block_on(stub.send_async(make_req("backend-b").with_cache_bypass())) + .expect("should start request b"); assert_eq!( pending_a.backend_name(), @@ -800,6 +824,11 @@ mod tests { vec!["backend-a", "backend-b"], "should record both send_async calls in order" ); + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false, true], + "should record both send_async cache-bypass flags in order" + ); } #[test] From 15f4565a06a1c3c2506f2358eecfa086809d52a0 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 22 Jul 2026 21:19:08 +0530 Subject: [PATCH 089/198] Bypass Fastly cache for marked HTTP requests --- .../src/platform.rs | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-adapter-fastly/src/platform.rs b/crates/trusted-server-adapter-fastly/src/platform.rs index b1bab232b..78ac11db4 100644 --- a/crates/trusted-server-adapter-fastly/src/platform.rs +++ b/crates/trusted-server-adapter-fastly/src/platform.rs @@ -408,6 +408,12 @@ fn fastly_response_to_platform( // FastlyPlatformHttpClient // --------------------------------------------------------------------------- +fn apply_fastly_cache_bypass(request: &mut fastly::Request, bypass_cache: bool) { + if bypass_cache { + request.set_pass(true); + } +} + /// Fastly implementation of [`PlatformHttpClient`]. /// /// - [`send`](PlatformHttpClient::send) converts the platform request to a @@ -431,10 +437,12 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { let backend_name = request.backend_name.clone(); let image_optimizer = request.image_optimizer; let stream_response = request.stream_response; + let bypass_cache = request.bypass_cache; let mut fastly_req = edge_request_to_fastly(request.request)?; if let Some(options) = image_optimizer { apply_fastly_image_optimizer(&mut fastly_req, options)?; } + apply_fastly_cache_bypass(&mut fastly_req, bypass_cache); let fastly_resp = fastly_req .send(&backend_name) .change_context(PlatformError::HttpClient)?; @@ -454,7 +462,9 @@ impl PlatformHttpClient for FastlyPlatformHttpClient { return Err(Report::new(PlatformError::HttpClient) .attach("streaming responses are not supported with Fastly send_async")); } - let fastly_req = edge_request_to_fastly(request.request)?; + let bypass_cache = request.bypass_cache; + let mut fastly_req = edge_request_to_fastly(request.request)?; + apply_fastly_cache_bypass(&mut fastly_req, bypass_cache); let pending = fastly_req .send_async(&backend_name) .change_context(PlatformError::HttpClient)?; @@ -736,6 +746,26 @@ mod tests { // --- FastlyPlatformHttpClient ------------------------------------------- + #[test] + fn apply_fastly_cache_bypass_sets_pass_when_enabled() { + let mut request = fastly::Request::get("https://example.com/"); + apply_fastly_cache_bypass(&mut request, true); + assert!( + format!("{request:?}").contains("cache_override: Pass"), + "enabled bypass should select Fastly pass mode" + ); + } + + #[test] + fn apply_fastly_cache_bypass_preserves_default_when_disabled() { + let mut request = fastly::Request::get("https://example.com/"); + apply_fastly_cache_bypass(&mut request, false); + assert!( + format!("{request:?}").contains("cache_override: None"), + "disabled bypass should preserve Fastly read-through caching" + ); + } + #[test] fn fastly_platform_http_client_send_returns_error_for_unregistered_backend() { let client = FastlyPlatformHttpClient; From 92f4169a432207db29a6c081033fc016300f0c42 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 22 Jul 2026 21:38:42 +0530 Subject: [PATCH 090/198] Prevent SSAT publisher document revalidation --- crates/trusted-server-core/src/publisher.rs | 251 +++++++++++++++++++- 1 file changed, 241 insertions(+), 10 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 34909efe7..bd016ced0 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1738,6 +1738,11 @@ pub async fn handle_publisher_request( } ); + if should_run_ad_stack { + req.headers_mut().remove(header::IF_NONE_MATCH); + req.headers_mut().remove(header::IF_MODIFIED_SINCE); + } + // Only advertise encodings the rewrite pipeline can decode and re-encode. restrict_accept_encoding(&mut req); // Strip the internal `fastly-ssl` scheme signal before forwarding to the @@ -1756,11 +1761,11 @@ pub async fn handle_publisher_request( // SSP requests are already racing through the platform HTTP client, so // origin TTFB tracks origin latency rather than the auction timeout. - let mut response = match services - .http_client() - .send(PlatformHttpRequest::new(req, backend_name)) - .await - { + let mut publisher_request = PlatformHttpRequest::new(req, backend_name); + if should_run_ad_stack { + publisher_request = publisher_request.with_cache_bypass(); + } + let mut response = match services.http_client().send(publisher_request).await { Ok(platform_response) => platform_response.response, Err(err) => { if let Some(dispatched) = dispatched_auction.take() { @@ -1793,10 +1798,9 @@ pub async fn handle_publisher_request( None }; - // §4.7: HTML carrying inline per-user bid data must never be shared-cached. - // `private, max-age=0` is deliberate (not `no-store`): it keeps the page - // BFCache-eligible while restricting reuse to the same user's browser with - // revalidation; `Surrogate-Control` removal handles the Fastly shared cache. + // §4.7: HTML with synthesized per-navigation auction state must not be + // stored or validated as an origin representation. Strip both browser and + // surrogate validators/cache directives before returning it. // // Gate on `should_run_ad_stack` rather than content-type alone: when no slot // matched, the feature is disabled, or this is not an ad-eligible navigation, @@ -1813,8 +1817,10 @@ pub async fn handle_publisher_request( if should_run_ad_stack && is_html_content_type(origin_content_type) { response.headers_mut().insert( header::CACHE_CONTROL, - HeaderValue::from_static("private, max-age=0"), + HeaderValue::from_static("private, no-store"), ); + response.headers_mut().remove(header::ETAG); + response.headers_mut().remove(header::LAST_MODIFIED); response.headers_mut().remove("surrogate-control"); response.headers_mut().remove("fastly-surrogate-control"); } @@ -3032,6 +3038,231 @@ mod tests { .expect("should proxy publisher request") } + mod ssat_cache_policy_tests { + use super::*; + use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; + use crate::test_support::tests::crate_test_settings_str; + + const ORIGIN_ETAG: &str = "\"origin-tag\""; + const ORIGIN_LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT"; + + fn settings_with_enabled_auction_and_creative_opportunities() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\n\n\ + [creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml) + .expect("should parse settings with auction and creative opportunities enabled") + } + + fn article_slot() -> CreativeOpportunitySlot { + CreativeOpportunitySlot { + id: "article-slot".to_string(), + gam_unit_path: None, + div_id: None, + page_patterns: vec!["/article".to_string()], + formats: vec![CreativeOpportunityFormat { + width: 300, + height: 250, + media_type: MediaType::Banner, + }], + floor_price: None, + targeting: Default::default(), + providers: Default::default(), + compiled_patterns: Vec::new(), + } + } + + fn conditional_navigation_request() -> Request { + HttpRequest::builder() + .method(Method::GET) + .uri("https://ts.example.com/article") + .header(header::HOST, "ts.example.com") + .header("sec-fetch-dest", "document") + .header(header::IF_NONE_MATCH, ORIGIN_ETAG) + .header(header::IF_MODIFIED_SINCE, ORIGIN_LAST_MODIFIED) + .body(EdgeBody::empty()) + .expect("should build conditional navigation request") + } + + fn queue_cacheable_html_response(stub: &StubHttpClient) { + stub.push_response_with_headers( + 200, + b"origin".to_vec(), + vec![ + ("content-type", "text/html; charset=utf-8"), + ("cache-control", "public, max-age=300"), + ("etag", ORIGIN_ETAG), + ("last-modified", ORIGIN_LAST_MODIFIED), + ("surrogate-control", "max-age=300"), + ("fastly-surrogate-control", "max-age=300"), + ], + ); + } + + async fn run_with_slots( + settings: &Settings, + services: &RuntimeServices, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> PublisherResponse { + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let consent = crate::consent::ConsentContext { + jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, + ..Default::default() + }; + let mut ec_context = EcContext::new_for_test(None, consent); + + handle_publisher_request( + settings, + services, + None, + &mut ec_context, + AuctionDispatch { + orchestrator: &orchestrator, + slots, + registry: None, + }, + req, + ) + .await + .expect("should proxy publisher request") + } + + fn response_head(response: PublisherResponse) -> http::response::Parts { + match response { + PublisherResponse::Buffered(response) + | PublisherResponse::Stream { response, .. } + | PublisherResponse::PassThrough { response, .. } => response.into_parts().0, + } + } + + fn recorded_header<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } + + #[tokio::test] + async fn eligible_navigation_bypasses_cache_and_returns_non_storable_html() { + // Arrange + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let stub = Arc::new(StubHttpClient::new()); + queue_cacheable_html_response(&stub); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let slots = [article_slot()]; + let req = conditional_navigation_request(); + + // Act + let response = run_with_slots(&settings, &services, &slots, req).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![true], + "eligible publisher navigation should bypass the platform cache" + ); + let recorded_requests = stub.recorded_request_headers(); + let outbound_headers = recorded_requests + .first() + .expect("should record the outbound publisher request"); + assert_eq!( + recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), + None, + "eligible publisher request should not forward If-None-Match" + ); + assert_eq!( + recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), + None, + "eligible publisher request should not forward If-Modified-Since" + ); + assert_eq!( + response_head + .headers + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, no-store"), + "eligible HTML response should be private and non-storable" + ); + for header_name in [ + header::ETAG, + header::LAST_MODIFIED, + header::HeaderName::from_static("surrogate-control"), + header::HeaderName::from_static("fastly-surrogate-control"), + ] { + assert!( + !response_head.headers.contains_key(&header_name), + "eligible HTML response should remove {header_name}" + ); + } + } + + #[tokio::test] + async fn navigation_without_matched_slots_preserves_origin_cache_policy() { + // Arrange + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let stub = Arc::new(StubHttpClient::new()); + queue_cacheable_html_response(&stub); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + let req = conditional_navigation_request(); + + // Act + let response = run_with_slots(&settings, &services, &[], req).await; + let response_head = response_head(response); + + // Assert + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "publisher navigation without matched slots should use the default cache mode" + ); + let recorded_requests = stub.recorded_request_headers(); + let outbound_headers = recorded_requests + .first() + .expect("should record the outbound publisher request"); + assert_eq!( + recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), + Some(ORIGIN_ETAG), + "publisher request without matched slots should preserve If-None-Match" + ); + assert_eq!( + recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), + Some(ORIGIN_LAST_MODIFIED), + "publisher request without matched slots should preserve If-Modified-Since" + ); + + for (header_name, expected) in [ + (header::CACHE_CONTROL, "public, max-age=300"), + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ] { + assert_eq!( + response_head + .headers + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "publisher response without matched slots should preserve {header_name}" + ); + } + } + } + #[tokio::test] async fn publisher_request_uses_platform_http_client_with_http_types() { let settings = create_test_settings(); From fc5611a9b1235c7b614bd60b093b63b0e098e0df Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 22 Jul 2026 21:55:25 +0530 Subject: [PATCH 091/198] Reject unexpected SSAT origin 304 responses --- crates/trusted-server-core/src/publisher.rs | 321 +++++++++++++++++++- 1 file changed, 320 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index bd016ced0..71a362118 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1789,6 +1789,30 @@ pub async fn handle_publisher_request( response.headers().len() ); + if should_run_ad_stack && response.status() == StatusCode::NOT_MODIFIED { + if let Some(dispatched) = dispatched_auction.take() { + emit_abandoned_auction( + services, + auction_observation.take(), + dispatched, + "unexpected_origin_304", + ) + .await; + } + + let response = Response::builder() + .status(StatusCode::BAD_GATEWAY) + .header(header::CACHE_CONTROL, "private, no-store") + .header(header::CONTENT_TYPE, "text/plain; charset=utf-8") + .body(EdgeBody::from( + "Publisher origin returned an invalid conditional response", + )) + .change_context(TrustedServerError::Proxy { + message: "failed to build unexpected origin 304 response".to_string(), + })?; + return Ok(PublisherResponse::Buffered(response)); + } + let ad_slots_script = if should_run_ad_stack { settings .creative_opportunities @@ -3040,11 +3064,91 @@ mod tests { mod ssat_cache_policy_tests { use super::*; + use crate::auction::provider::AuctionProvider; + use crate::auction::telemetry::{AuctionEventBatch, AuctionTelemetrySink}; use crate::creative_opportunities::{CreativeOpportunityFormat, CreativeOpportunitySlot}; + use crate::platform::test_support::{ + NoopConfigStore, NoopGeo, NoopSecretStore, StubBackend, + }; + use crate::platform::{ClientInfo, PlatformPendingRequest, PlatformResponse}; use crate::test_support::tests::crate_test_settings_str; const ORIGIN_ETAG: &str = "\"origin-tag\""; const ORIGIN_LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT"; + const UNEXPECTED_304_PROVIDER: &str = "example_navigation_bidder"; + const UNEXPECTED_304_BACKEND: &str = "example-navigation-bidder-backend"; + + struct DispatchingTestProvider; + + #[async_trait::async_trait(?Send)] + impl AuctionProvider for DispatchingTestProvider { + fn provider_name(&self) -> &'static str { + UNEXPECTED_304_PROVIDER + } + + async fn request_bids( + &self, + _request: &AuctionRequest, + context: &AuctionContext<'_>, + ) -> Result> { + let request = PlatformHttpRequest::new( + HttpRequest::builder() + .method(Method::POST) + .uri("https://bidder.example.com/navigation-bids") + .body(EdgeBody::empty()) + .expect("should build test provider request"), + UNEXPECTED_304_BACKEND, + ); + context + .services + .http_client() + .send_async(request) + .await + .change_context(TrustedServerError::Auction { + message: "test provider launch failed".to_string(), + }) + } + + async fn parse_response( + &self, + _response: PlatformResponse, + _response_time_ms: u64, + ) -> Result> { + panic!("parse_response must not run for an unexpected origin 304"); + } + + fn timeout_ms(&self) -> u32 { + 100 + } + + fn backend_name( + &self, + _services: &RuntimeServices, + _timeout_ms: u32, + ) -> Option { + Some(UNEXPECTED_304_BACKEND.to_string()) + } + } + + #[derive(Default)] + struct RecordingTelemetrySink { + batches: Mutex>, + } + + #[async_trait::async_trait(?Send)] + impl AuctionTelemetrySink for RecordingTelemetrySink { + async fn emit_auction_events( + &self, + _services: &RuntimeServices, + batch: AuctionEventBatch, + ) -> Result<(), Report> { + self.batches + .lock() + .expect("should lock telemetry batches") + .push(batch); + Ok(()) + } + } fn settings_with_enabled_auction_and_creative_opportunities() -> Settings { let toml = format!( @@ -3056,6 +3160,33 @@ mod tests { .expect("should parse settings with auction and creative opportunities enabled") } + fn settings_with_dispatching_provider() -> Settings { + let toml = format!( + "{}\n[auction]\nenabled = true\nproviders = [\"{UNEXPECTED_304_PROVIDER}\"]\n\n\ + [creative_opportunities]\ngam_network_id = \"12345\"\n", + crate_test_settings_str() + ); + Settings::from_toml(&toml) + .expect("should parse settings with the dispatching test provider") + } + + fn services_with_telemetry( + http_client: Arc, + telemetry_sink: Arc, + ) -> RuntimeServices { + let telemetry_sink: Arc = telemetry_sink; + RuntimeServices::builder() + .config_store(Arc::new(NoopConfigStore)) + .secret_store(Arc::new(NoopSecretStore)) + .kv_store(Arc::new(edgezero_core::key_value_store::NoopKvStore)) + .backend(Arc::new(StubBackend)) + .http_client(http_client) + .geo(Arc::new(NoopGeo)) + .auction_telemetry_sink(telemetry_sink) + .client_info(ClientInfo::default()) + .build() + } + fn article_slot() -> CreativeOpportunitySlot { CreativeOpportunitySlot { id: "article-slot".to_string(), @@ -3108,6 +3239,16 @@ mod tests { req: Request, ) -> PublisherResponse { let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + run_with_orchestrator(settings, services, &orchestrator, slots, req).await + } + + async fn run_with_orchestrator( + settings: &Settings, + services: &RuntimeServices, + orchestrator: &AuctionOrchestrator, + slots: &[CreativeOpportunitySlot], + req: Request, + ) -> PublisherResponse { let consent = crate::consent::ConsentContext { jurisdiction: crate::consent::jurisdiction::Jurisdiction::NonRegulated, ..Default::default() @@ -3120,7 +3261,7 @@ mod tests { None, &mut ec_context, AuctionDispatch { - orchestrator: &orchestrator, + orchestrator, slots, registry: None, }, @@ -3261,6 +3402,184 @@ mod tests { ); } } + + #[tokio::test] + async fn eligible_navigation_rejects_unexpected_origin_304() { + for content_type in [None, Some("text/html; charset=utf-8")] { + // Arrange + let settings = settings_with_dispatching_provider(); + let mut orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + orchestrator.register_provider(Arc::new(DispatchingTestProvider)); + let telemetry_sink = Arc::new(RecordingTelemetrySink::default()); + let stub = Arc::new(StubHttpClient::new()); + + // `send_async` consumes the first response before the publisher + // origin request consumes the second response. + stub.push_response(200, b"unused provider response".to_vec()); + let mut origin_headers = vec![ + ("cache-control", "public, max-age=300"), + ("etag", ORIGIN_ETAG), + ("last-modified", ORIGIN_LAST_MODIFIED), + ("surrogate-control", "max-age=300"), + ("fastly-surrogate-control", "max-age=300"), + ]; + if let Some(content_type) = content_type { + origin_headers.push(("content-type", content_type)); + } + stub.push_response_with_headers(304, Vec::new(), origin_headers); + let services = services_with_telemetry( + Arc::clone(&stub) as Arc, + Arc::clone(&telemetry_sink), + ); + let slots = [article_slot()]; + + // Act + let response = run_with_orchestrator( + &settings, + &services, + &orchestrator, + &slots, + conditional_navigation_request(), + ) + .await; + + // Assert + let response = match response { + PublisherResponse::Buffered(response) => response, + PublisherResponse::PassThrough { .. } | PublisherResponse::Stream { .. } => { + panic!("unexpected origin 304 should return a buffered response") + } + }; + assert_eq!( + response.status(), + StatusCode::BAD_GATEWAY, + "eligible origin 304 should fail closed with or without Content-Type" + ); + assert_eq!( + response + .headers() + .get(header::CACHE_CONTROL) + .and_then(|value| value.to_str().ok()), + Some("private, no-store"), + "eligible origin 304 should return an explicitly non-storable response" + ); + for header_name in [ + header::ETAG, + header::LAST_MODIFIED, + header::HeaderName::from_static("surrogate-control"), + header::HeaderName::from_static("fastly-surrogate-control"), + ] { + assert!( + !response.headers().contains_key(&header_name), + "eligible origin 304 should not forward {header_name}" + ); + } + + let batches = telemetry_sink + .batches + .lock() + .expect("should lock telemetry batches"); + let summary_rows: Vec<_> = batches + .iter() + .flat_map(AuctionEventBatch::rows) + .filter(|row| row.event_kind == "summary") + .collect(); + assert_eq!( + summary_rows.len(), + 1, + "unexpected origin 304 should emit exactly one summary row" + ); + assert_eq!( + summary_rows[0].terminal_status.as_deref(), + Some("abandoned"), + "unexpected origin 304 should abandon the dispatched auction" + ); + assert_eq!( + summary_rows[0].terminal_reason.as_deref(), + Some("unexpected_origin_304"), + "unexpected origin 304 should use the bounded telemetry reason" + ); + } + } + + #[tokio::test] + async fn noneligible_origin_304_preserves_conditional_response_metadata() { + // Arrange + let settings = settings_with_enabled_auction_and_creative_opportunities(); + let stub = Arc::new(StubHttpClient::new()); + stub.push_response_with_headers( + 304, + Vec::new(), + vec![ + ("cache-control", "public, max-age=300"), + ("etag", ORIGIN_ETAG), + ("last-modified", ORIGIN_LAST_MODIFIED), + ("surrogate-control", "max-age=300"), + ("fastly-surrogate-control", "max-age=300"), + ], + ); + let services = build_services_with_http_client( + Arc::clone(&stub) as Arc + ); + + // Act + let response = + run_with_slots(&settings, &services, &[], conditional_navigation_request()).await; + + // Assert + let response = match response { + PublisherResponse::Buffered(response) => response, + PublisherResponse::PassThrough { .. } | PublisherResponse::Stream { .. } => { + panic!("noneligible origin 304 should remain buffered") + } + }; + assert_eq!( + response.status(), + StatusCode::NOT_MODIFIED, + "noneligible origin 304 should preserve its status" + ); + for (header_name, expected) in [ + (header::CACHE_CONTROL, "public, max-age=300"), + (header::ETAG, ORIGIN_ETAG), + (header::LAST_MODIFIED, ORIGIN_LAST_MODIFIED), + ( + header::HeaderName::from_static("surrogate-control"), + "max-age=300", + ), + ( + header::HeaderName::from_static("fastly-surrogate-control"), + "max-age=300", + ), + ] { + assert_eq!( + response + .headers() + .get(&header_name) + .and_then(|value| value.to_str().ok()), + Some(expected), + "noneligible origin 304 should preserve {header_name}" + ); + } + assert_eq!( + stub.recorded_cache_bypass_flags(), + vec![false], + "noneligible publisher navigation should use the default cache mode" + ); + let recorded_requests = stub.recorded_request_headers(); + let outbound_headers = recorded_requests + .first() + .expect("should record the outbound publisher request"); + assert_eq!( + recorded_header(outbound_headers, header::IF_NONE_MATCH.as_str()), + Some(ORIGIN_ETAG), + "noneligible publisher request should preserve If-None-Match" + ); + assert_eq!( + recorded_header(outbound_headers, header::IF_MODIFIED_SINCE.as_str()), + Some(ORIGIN_LAST_MODIFIED), + "noneligible publisher request should preserve If-Modified-Since" + ); + } } #[tokio::test] From 79f6053ef1beb03fdf0b30733d53a38028657e4e Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Wed, 22 Jul 2026 22:12:57 +0530 Subject: [PATCH 092/198] Format SSAT 304 implementation plan --- .../2026-07-22-ssat-root-document-304-prevention.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/superpowers/plans/2026-07-22-ssat-root-document-304-prevention.md b/docs/superpowers/plans/2026-07-22-ssat-root-document-304-prevention.md index 9f87ca9d4..cd89dea13 100644 --- a/docs/superpowers/plans/2026-07-22-ssat-root-document-304-prevention.md +++ b/docs/superpowers/plans/2026-07-22-ssat-root-document-304-prevention.md @@ -12,12 +12,12 @@ ## File Map -| File | Responsibility | -| --- | --- | -| `crates/trusted-server-core/src/platform/http.rs` | Define the platform-neutral, default-off cache-bypass request option. | -| `crates/trusted-server-core/src/platform/test_support.rs` | Record cache-bypass options in the shared stub HTTP client for publisher tests. | -| `crates/trusted-server-adapter-fastly/src/platform.rs` | Translate the platform option to Fastly `Request::set_pass(true)` in both send paths. | -| `crates/trusted-server-core/src/publisher.rs` | Apply the eligibility gate, strip validators, set the synthesized response policy, fail closed on unexpected 304, and test the complete behavior. | +| File | Responsibility | +| --------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-core/src/platform/http.rs` | Define the platform-neutral, default-off cache-bypass request option. | +| `crates/trusted-server-core/src/platform/test_support.rs` | Record cache-bypass options in the shared stub HTTP client for publisher tests. | +| `crates/trusted-server-adapter-fastly/src/platform.rs` | Translate the platform option to Fastly `Request::set_pass(true)` in both send paths. | +| `crates/trusted-server-core/src/publisher.rs` | Apply the eligibility gate, strip validators, set the synthesized response policy, fail closed on unexpected 304, and test the complete behavior. | No configuration schema, JavaScript, `/page-bids`, auction-ID, asset, or integration files change. From 78bb93eb13722c3d5e0f58d90bc006604e3226a6 Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:24:15 -0700 Subject: [PATCH 093/198] Make creative sanitization opt-in and restore creative iframe origin isolation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Creative sanitization ran unconditionally on every markup bid, stripping `script`/`object`/`embed`/`form` and friends together with their inner content. For script-based creatives — the majority of programmatic display — that leaves nothing renderable, and the slot goes blank with no error: the leftover markup is usually a tracking pixel, so the ad server reports a successful render. Measured over 291 creative deliveries on a live publisher: a median 43% of bytes removed, 29 creatives reduced by more than 80%, and 20 reduced below 500 bytes. One bidder lost 100% of every creative; another lost 76% across 43 of them. Add `auction.sanitize_creatives` so sanitization can be disabled where creatives render in a foreign-origin frame (the Prebid Universal Creative inside the ad server's iframe), and make both creative controls opt-in: `sanitize_creatives` and `rewrite_creatives` now default to false, so a creative ships exactly as the bidder returned it unless a publisher asks for processing. Removing `allow-same-origin` from the creative iframe sandbox is part of the same change rather than a follow-up. Sanitization was documented as "the primary defense against malicious markup", with the sandbox as defense-in-depth — but the sandbox granted `allow-same-origin` alongside `allow-scripts`, which removes its origin isolation entirely. With sanitization now optional, that pairing would leave creative markup able to reach publisher cookies, storage, and same-origin fetches. The two sibling sandboxes (APS_RENDERER_SANDBOX, ADM_IFRAME_SANDBOX) already omit the token for exactly this reason; this brings the third in line, so the origin boundary no longer depends on an optional transform. Note the default change alters behaviour for deployments that never set `rewrite_creatives`: creative URL rewriting is now off unless enabled explicitly. Verified end to end: creatives pass through byte-for-byte (triplelift 8902 -> 8902, openx 22069 -> 22069, previously 100% and 35% losses), page renders with ads serving and no hydration errors. --- .../src/auction/formats.rs | 66 +++++++++++++++++-- .../src/auction/orchestrator.rs | 1 + .../src/auction_config_types.rs | 31 +++++++-- .../trusted-server-core/src/config_payload.rs | 6 +- crates/trusted-server-core/src/settings.rs | 10 ++- .../trusted-server-js/lib/src/core/render.ts | 14 ++-- .../lib/test/core/render.test.ts | 6 +- trusted-server.example.toml | 20 ++++-- 8 files changed, 130 insertions(+), 24 deletions(-) diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index c2da85393..2754861d1 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -288,7 +288,12 @@ pub fn convert_to_openrtb_response( // Ordinary markup remains on the mandatory sanitize/rewrite path. A // typed renderer is serialized separately and never enters the HTML sanitizer. let (adm, ext) = if let Some(ref raw_creative) = bid.creative { - let sanitized = creative::sanitize_creative_html(raw_creative); + let sanitize_creatives = settings.auction.sanitize_creatives; + let sanitized = if sanitize_creatives { + creative::sanitize_creative_html(raw_creative) + } else { + raw_creative.clone() + }; let sanitized_len = sanitized.len(); let rewrite_creatives = settings.auction.rewrite_creatives; let processed = if rewrite_creatives { @@ -296,6 +301,11 @@ pub fn convert_to_openrtb_response( } else { sanitized }; + let sanitize_mode = if sanitize_creatives { + "enabled" + } else { + "disabled" + }; let rewrite_mode = if rewrite_creatives { "enabled" } else { @@ -303,10 +313,11 @@ pub fn convert_to_openrtb_response( }; log::debug!( - "Processed creative for auction {} slot {} bidder {} (rewrite {}, raw {} bytes, sanitized {} bytes, output {} bytes)", + "Processed creative for auction {} slot {} bidder {} (sanitize {}, rewrite {}, raw {} bytes, sanitized {} bytes, output {} bytes)", auction_request.id, slot_id, bid.bidder, + sanitize_mode, rewrite_mode, raw_creative.len(), sanitized_len, @@ -1087,8 +1098,10 @@ mod tests { } #[test] - fn convert_to_openrtb_response_rewrites_sanitized_creative_by_default() { - let settings = make_settings(); + fn convert_to_openrtb_response_rewrites_sanitized_creative_when_enabled() { + let mut settings = make_settings(); + settings.auction.sanitize_creatives = true; + settings.auction.rewrite_creatives = true; let auction_request = make_auction_request(); let result = make_result(make_complete_creative_bid()); @@ -1135,9 +1148,52 @@ mod tests { } #[test] - fn convert_to_openrtb_response_can_skip_rewriting_but_not_sanitization() { + fn convert_to_openrtb_response_can_skip_sanitization_when_disabled() { + // Sanitization strips every executable element with its inner content, which + // destroys script-based creatives (the majority of programmatic display). + // Publishers whose creatives render in a foreign-origin frame — where the + // markup cannot reach the publisher origin — can opt out and deliver the + // creative exactly as the bidder returned it. + let mut settings = make_settings(); + settings.auction.sanitize_creatives = false; + settings.auction.rewrite_creatives = false; + let auction_request = make_auction_request(); + let result = make_result(make_complete_creative_bid()); + + let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) + .expect("should convert creative with sanitization disabled"); + let adm = response_adm(response); + + assert!( + adm.contains("auction-script-marker"), + "should retain script content when sanitization is disabled: {adm}" + ); + assert!( + adm.contains("auction-handler-marker"), + "should retain event handlers when sanitization is disabled: {adm}" + ); + } + + #[test] + fn sanitize_creatives_defaults_to_disabled() { + let config = crate::auction_config_types::AuctionConfig::default(); + assert!( + !config.sanitize_creatives, + "creatives are delivered as the bidder returned them unless a publisher opts in" + ); + assert!( + !config.rewrite_creatives, + "creative URL rewriting is opt-in" + ); + } + + #[test] + fn convert_to_openrtb_response_can_skip_rewriting_while_sanitizing() { + // The two controls are independent: sanitization can stay on while URL + // rewriting is off. let mut settings = make_settings(); settings.auction.rewrite_creatives = false; + settings.auction.sanitize_creatives = true; let auction_request = make_auction_request(); let result = make_result(make_complete_creative_bid()); diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 8e4ad6c25..9a17553c9 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -2122,6 +2122,7 @@ mod tests { futures::executor::block_on(async { let config = AuctionConfig { enabled: true, + sanitize_creatives: true, rewrite_creatives: true, providers: vec![], mediator: None, diff --git a/crates/trusted-server-core/src/auction_config_types.rs b/crates/trusted-server-core/src/auction_config_types.rs index f1d1a5cf0..fb9ef92a0 100644 --- a/crates/trusted-server-core/src/auction_config_types.rs +++ b/crates/trusted-server-core/src/auction_config_types.rs @@ -11,6 +11,19 @@ pub struct AuctionConfig { #[serde(default)] pub enabled: bool, + /// Strip executable markup from winning-bid creative HTML before delivery. + /// + /// Sanitization removes `script`/`object`/`embed`/`form`/etc. **with their inner + /// content**, which blanks script-based creatives — the majority of programmatic + /// display. It is the primary defence when the creative renders in a context that + /// shares the publisher's origin. + /// + /// Disable only when creatives render in a foreign-origin frame (for example the + /// Prebid Universal Creative inside the ad server's iframe), where the markup + /// cannot reach the publisher origin. Defaults to disabled. + #[serde(default = "default_sanitize_creatives")] + pub sanitize_creatives: bool, + /// Rewrite sanitized winning-bid creative HTML to first-party endpoints. #[serde(default = "default_rewrite_creatives")] pub rewrite_creatives: bool, @@ -45,6 +58,7 @@ impl Default for AuctionConfig { fn default() -> Self { Self { enabled: false, + sanitize_creatives: default_sanitize_creatives(), rewrite_creatives: default_rewrite_creatives(), providers: Vec::new(), mediator: None, @@ -59,8 +73,12 @@ fn default_timeout() -> u32 { 2000 } +fn default_sanitize_creatives() -> bool { + false +} + fn default_rewrite_creatives() -> bool { - true + false } fn default_creative_store() -> String { @@ -94,10 +112,15 @@ mod tests { use super::*; #[test] - fn rewrite_creatives_defaults_to_true() { + fn creative_processing_defaults_to_disabled() { + let config = AuctionConfig::default(); + assert!( + !config.rewrite_creatives, + "creative rewriting is opt-in: creatives ship as the bidder returned them" + ); assert!( - AuctionConfig::default().rewrite_creatives, - "should enable creative rewriting by default" + !config.sanitize_creatives, + "creative sanitization is opt-in: it strips executable markup with its content" ); } } diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index 58c185381..8842162bc 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -79,7 +79,7 @@ mod tests { } #[test] - fn legacy_blob_without_rewrite_creatives_preserves_rewriting() { + fn legacy_blob_without_rewrite_creatives_leaves_rewriting_disabled() { let mut data = serde_json::to_value(test_settings()).expect("should serialize settings to JSON"); let auction = data @@ -97,8 +97,8 @@ mod tests { settings_from_config_blob(&envelope_json).expect("should reconstruct legacy settings"); assert!( - reconstructed.auction.rewrite_creatives, - "should enable creative rewriting for legacy blobs" + !reconstructed.auction.rewrite_creatives, + "creative rewriting is opt-in: a blob without the field leaves it disabled" ); } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 4fe066997..74cc72027 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -4438,7 +4438,7 @@ adSlot = "67890" } #[test] - fn test_auction_rewrite_creatives_defaults_to_true_when_omitted() { + fn test_auction_creative_processing_defaults_to_false_when_omitted() { let toml_str = crate_test_settings_str() + r#" [auction] @@ -4449,8 +4449,12 @@ adSlot = "67890" let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); assert!( - settings.auction.rewrite_creatives, - "should preserve creative rewriting when the setting is omitted" + !settings.auction.rewrite_creatives, + "creative rewriting is opt-in when the setting is omitted" + ); + assert!( + !settings.auction.sanitize_creatives, + "creative sanitization is opt-in when the setting is omitted" ); } diff --git a/crates/trusted-server-js/lib/src/core/render.ts b/crates/trusted-server-js/lib/src/core/render.ts index ee08ef288..f00525b4e 100644 --- a/crates/trusted-server-js/lib/src/core/render.ts +++ b/crates/trusted-server-js/lib/src/core/render.ts @@ -7,15 +7,21 @@ import NORMALIZE_CSS from './styles/normalize.css?inline'; import IFRAME_TEMPLATE from './templates/iframe.html?raw'; // Sandbox permissions granted to creative iframes. +// // Ad creatives routinely contain scripts for tracking, click handling, and -// viewability measurement, so allow-scripts and allow-same-origin are required -// for creatives to render correctly. Server-side sanitization is the primary -// defense against malicious markup; the sandbox provides defense-in-depth. +// viewability measurement, so `allow-scripts` is required for them to render. +// +// `allow-same-origin` is deliberately excluded: combined with `allow-scripts` on +// srcdoc (or first-party src) content, that pair effectively removes the sandbox's +// origin isolation and would let SSP-provided markup run with the publisher +// origin's privileges — cookies, storage, and same-origin fetches. The origin +// boundary must not depend on server-side sanitization, which is optional +// (`auction.sanitize_creatives`) and cannot run at all for renderer-based bids. +// Matches APS_RENDERER_SANDBOX and ADM_IFRAME_SANDBOX, which already omit it. const CREATIVE_SANDBOX_TOKENS = [ 'allow-forms', 'allow-popups', 'allow-popups-to-escape-sandbox', - 'allow-same-origin', 'allow-scripts', 'allow-top-navigation-by-user-activation', ] as const; diff --git a/crates/trusted-server-js/lib/test/core/render.test.ts b/crates/trusted-server-js/lib/test/core/render.test.ts index a81486cf3..63a33c8a9 100644 --- a/crates/trusted-server-js/lib/test/core/render.test.ts +++ b/crates/trusted-server-js/lib/test/core/render.test.ts @@ -31,8 +31,12 @@ describe('render', () => { expect(sandbox).toContain('allow-popups'); expect(sandbox).toContain('allow-popups-to-escape-sandbox'); expect(sandbox).toContain('allow-top-navigation-by-user-activation'); - expect(sandbox).toContain('allow-same-origin'); expect(sandbox).toContain('allow-scripts'); + // `allow-scripts` + `allow-same-origin` together defeat the sandbox: creative + // markup would run with the publisher origin's privileges (cookies, storage, + // same-origin fetches). Matches APS_RENDERER_SANDBOX and ADM_IFRAME_SANDBOX, + // which already omit it. + expect(sandbox).not.toContain('allow-same-origin'); }); it('preserves dollar sequences when building the creative document', async () => { diff --git a/trusted-server.example.toml b/trusted-server.example.toml index ef3edc2af..7cd16133e 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -112,10 +112,22 @@ rewrite_script = true [auction] enabled = false -# Defaults to true. Set false to return sanitized but unre-written winning-bid adm, -# skipping proxy/click URL conversion and creative TSJS injection. -# Sanitization is always applied and cannot be disabled by this setting. -rewrite_creatives = true +# Defaults to false. Set true to rewrite winning-bid adm to first-party endpoints, +# converting proxy/click URLs and injecting the creative TSJS runtime. +# Sanitization is controlled separately by `sanitize_creatives` below. +rewrite_creatives = false +# Strip executable markup (script/object/embed/form/...) from winning-bid adm, +# removing those elements together with their inner content. +# +# Defaults to false: creatives are delivered exactly as the bidder returned them. +# Enable whenever creatives can render in a context that shares the publisher's +# origin — it is the primary defence there. +# +# Set false only when creatives render in a foreign-origin frame (for example the +# Prebid Universal Creative inside the ad server's iframe), where the markup cannot +# reach the publisher origin. Sanitization removes script-based creatives entirely, +# so leaving it enabled on a script-heavy demand stack silently blanks those slots. +sanitize_creatives = false providers = [] timeout_ms = 2000 allowed_context_keys = [] From b4f5def9160b79863e7b8ae4342c78440e54dace Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:24:15 -0700 Subject: [PATCH 094/198] Make creative sanitization opt-in and restore creative iframe origin isolation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Creative sanitization ran unconditionally on every markup bid, stripping `script`/`object`/`embed`/`form` and friends together with their inner content. For script-based creatives — the majority of programmatic display — that leaves nothing renderable, and the slot goes blank with no error: the leftover markup is usually a tracking pixel, so the ad server reports a successful render. Measured over 291 creative deliveries on a live publisher: a median 43% of bytes removed, 29 creatives reduced by more than 80%, and 20 reduced below 500 bytes. One bidder lost 100% of every creative; another lost 76% across 43 of them. Add `auction.sanitize_creatives` so sanitization can be disabled where creatives render in a foreign-origin frame (the Prebid Universal Creative inside the ad server's iframe), and make both creative controls opt-in: `sanitize_creatives` and `rewrite_creatives` now default to false, so a creative ships exactly as the bidder returned it unless a publisher asks for processing. Removing `allow-same-origin` from the creative iframe sandbox is part of the same change rather than a follow-up. Sanitization was documented as "the primary defense against malicious markup", with the sandbox as defense-in-depth — but the sandbox granted `allow-same-origin` alongside `allow-scripts`, which removes its origin isolation entirely. With sanitization now optional, that pairing would leave creative markup able to reach publisher cookies, storage, and same-origin fetches. The two sibling sandboxes (APS_RENDERER_SANDBOX, ADM_IFRAME_SANDBOX) already omit the token for exactly this reason; this brings the third in line, so the origin boundary no longer depends on an optional transform. Note the default change alters behaviour for deployments that never set `rewrite_creatives`: creative URL rewriting is now off unless enabled explicitly. Verified end to end: creatives pass through byte-for-byte (triplelift 8902 -> 8902, openx 22069 -> 22069, previously 100% and 35% losses), page renders with ads serving and no hydration errors. --- .../src/auction/formats.rs | 69 +++++++++++++++-- .../src/auction/orchestrator.rs | 1 + .../src/auction_config_types.rs | 74 ++++++++++++++++--- .../trusted-server-core/src/config_payload.rs | 8 +- crates/trusted-server-core/src/settings.rs | 10 ++- .../trusted-server-js/lib/src/core/render.ts | 14 +++- .../lib/test/core/render.test.ts | 6 +- trusted-server.example.toml | 22 ++++-- 8 files changed, 172 insertions(+), 32 deletions(-) diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index 71f9a290c..b23331552 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -251,9 +251,15 @@ pub fn convert_to_openrtb_response( let width = to_openrtb_i32(bid.width, "width", &bid_context); let height = to_openrtb_i32(bid.height, "height", &bid_context); - // Process creative HTML if present — always sanitize dangerous markup first. + // Process creative HTML if present. Sanitization is opt-in: when disabled + // the creative ships exactly as the bidder returned it. let creative_html = if let Some(ref raw_creative) = bid.creative { - let sanitized = creative::sanitize_creative_html(raw_creative); + let sanitize_creatives = settings.auction.sanitize_creatives; + let sanitized = if sanitize_creatives { + creative::sanitize_creative_html(raw_creative) + } else { + raw_creative.clone() + }; let sanitized_len = sanitized.len(); let rewrite_creatives = settings.auction.rewrite_creatives; let processed = if rewrite_creatives { @@ -261,6 +267,11 @@ pub fn convert_to_openrtb_response( } else { sanitized }; + let sanitize_mode = if sanitize_creatives { + "enabled" + } else { + "disabled" + }; let rewrite_mode = if rewrite_creatives { "enabled" } else { @@ -268,10 +279,11 @@ pub fn convert_to_openrtb_response( }; log::debug!( - "Processed creative for auction {} slot {} bidder {} (rewrite {}, raw {} bytes, sanitized {} bytes, output {} bytes)", + "Processed creative for auction {} slot {} bidder {} (sanitize {}, rewrite {}, raw {} bytes, sanitized {} bytes, output {} bytes)", auction_request.id, slot_id, bid.bidder, + sanitize_mode, rewrite_mode, raw_creative.len(), sanitized_len, @@ -963,8 +975,10 @@ mod tests { } #[test] - fn convert_to_openrtb_response_rewrites_sanitized_creative_by_default() { - let settings = make_settings(); + fn convert_to_openrtb_response_rewrites_sanitized_creative_when_enabled() { + let mut settings = make_settings(); + settings.auction.sanitize_creatives = true; + settings.auction.rewrite_creatives = true; let auction_request = make_auction_request(); let result = make_result(make_complete_creative_bid()); @@ -1011,9 +1025,52 @@ mod tests { } #[test] - fn convert_to_openrtb_response_can_skip_rewriting_but_not_sanitization() { + fn convert_to_openrtb_response_can_skip_sanitization_when_disabled() { + // Sanitization strips every executable element with its inner content, which + // destroys script-based creatives (the majority of programmatic display). + // Publishers whose creatives render in a foreign-origin frame — where the + // markup cannot reach the publisher origin — can opt out and deliver the + // creative exactly as the bidder returned it. + let mut settings = make_settings(); + settings.auction.sanitize_creatives = false; + settings.auction.rewrite_creatives = false; + let auction_request = make_auction_request(); + let result = make_result(make_complete_creative_bid()); + + let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) + .expect("should convert creative with sanitization disabled"); + let adm = response_adm(response); + + assert!( + adm.contains("auction-script-marker"), + "should retain script content when sanitization is disabled: {adm}" + ); + assert!( + adm.contains("auction-handler-marker"), + "should retain event handlers when sanitization is disabled: {adm}" + ); + } + + #[test] + fn sanitize_creatives_defaults_to_disabled() { + let config = crate::auction_config_types::AuctionConfig::default(); + assert!( + !config.sanitize_creatives, + "creatives are delivered as the bidder returned them unless a publisher opts in" + ); + assert!( + !config.rewrite_creatives, + "creative URL rewriting is opt-in" + ); + } + + #[test] + fn convert_to_openrtb_response_can_skip_rewriting_while_sanitizing() { + // The two controls are independent: sanitization can stay on while URL + // rewriting is off. let mut settings = make_settings(); settings.auction.rewrite_creatives = false; + settings.auction.sanitize_creatives = true; let auction_request = make_auction_request(); let result = make_result(make_complete_creative_bid()); diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index b018518d5..68cc27289 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -1823,6 +1823,7 @@ mod tests { futures::executor::block_on(async { let config = AuctionConfig { enabled: true, + sanitize_creatives: true, rewrite_creatives: true, providers: vec![], mediator: None, diff --git a/crates/trusted-server-core/src/auction_config_types.rs b/crates/trusted-server-core/src/auction_config_types.rs index eb93adbd1..f05c4df22 100644 --- a/crates/trusted-server-core/src/auction_config_types.rs +++ b/crates/trusted-server-core/src/auction_config_types.rs @@ -11,6 +11,22 @@ pub struct AuctionConfig { #[serde(default)] pub enabled: bool, + /// Strip executable markup from winning-bid creative HTML before delivery. + /// + /// Sanitization removes `script`/`object`/`embed`/`form`/etc. **with their inner + /// content**, which blanks script-based creatives — the majority of programmatic + /// display. It is the primary defence when the creative renders in a context that + /// shares the publisher's origin. + /// + /// Disable only when creatives render in a foreign-origin frame (for example the + /// Prebid Universal Creative inside the ad server's iframe), where the markup + /// cannot reach the publisher origin. Defaults to disabled. + #[serde( + default = "default_sanitize_creatives", + skip_serializing_if = "is_default_sanitize_creatives" + )] + pub sanitize_creatives: bool, + /// Rewrite sanitized winning-bid creative HTML to first-party endpoints. #[serde( default = "default_rewrite_creatives", @@ -48,6 +64,7 @@ impl Default for AuctionConfig { fn default() -> Self { Self { enabled: false, + sanitize_creatives: default_sanitize_creatives(), rewrite_creatives: default_rewrite_creatives(), providers: Vec::new(), mediator: None, @@ -62,14 +79,22 @@ fn default_timeout() -> u32 { 2000 } +fn default_sanitize_creatives() -> bool { + false +} + fn default_rewrite_creatives() -> bool { - true + false } fn is_default_rewrite_creatives(value: &bool) -> bool { *value == default_rewrite_creatives() } +fn is_default_sanitize_creatives(value: &bool) -> bool { + *value == default_sanitize_creatives() +} + fn default_creative_store() -> String { "creative_store".to_owned() } @@ -101,13 +126,17 @@ mod tests { use super::*; #[test] - fn rewrite_creatives_defaults_to_true() { + fn creative_processing_defaults_to_disabled() { let config: AuctionConfig = serde_json::from_value(serde_json::json!({})).expect("should deserialize defaults"); assert!( - config.rewrite_creatives, - "should enable creative rewriting by default" + !config.rewrite_creatives, + "creative rewriting is opt-in: creatives ship as the bidder returned them" + ); + assert!( + !config.sanitize_creatives, + "creative sanitization is opt-in: it strips executable markup with its content" ); } @@ -123,17 +152,44 @@ mod tests { } #[test] - fn disabled_rewrite_creatives_is_serialized() { + fn enabled_rewrite_creatives_is_serialized() { let config = AuctionConfig { - rewrite_creatives: false, + rewrite_creatives: true, ..AuctionConfig::default() }; - let serialized = serde_json::to_value(config).expect("should serialize disabled rewriting"); + let serialized = serde_json::to_value(config).expect("should serialize enabled rewriting"); assert_eq!( serialized.get("rewrite_creatives"), - Some(&serde_json::Value::Bool(false)), - "should preserve an explicit rewrite opt-out" + Some(&serde_json::Value::Bool(true)), + "should preserve an explicit rewrite opt-in" + ); + } + + #[test] + fn default_sanitize_creatives_is_not_serialized() { + let serialized = + serde_json::to_value(AuctionConfig::default()).expect("should serialize defaults"); + + assert!( + serialized.get("sanitize_creatives").is_none(), + "should omit the default sanitize setting" + ); + } + + #[test] + fn enabled_sanitize_creatives_is_serialized() { + let config = AuctionConfig { + sanitize_creatives: true, + ..AuctionConfig::default() + }; + let serialized = + serde_json::to_value(config).expect("should serialize enabled sanitization"); + + assert_eq!( + serialized.get("sanitize_creatives"), + Some(&serde_json::Value::Bool(true)), + "should preserve an explicit sanitize opt-in" ); } } diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index f7ae531ea..bf32b0102 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -97,8 +97,8 @@ mod tests { } #[test] - fn legacy_blob_without_rewrite_creatives_preserves_rewriting() { - let data = + fn legacy_blob_without_rewrite_creatives_leaves_rewriting_disabled() { + let mut data = serde_json::to_value(test_settings()).expect("should serialize settings to JSON"); let auction = data .get("auction") @@ -115,8 +115,8 @@ mod tests { settings_from_config_blob(&envelope_json).expect("should reconstruct legacy settings"); assert!( - reconstructed.auction.rewrite_creatives, - "should enable creative rewriting for legacy blobs" + !reconstructed.auction.rewrite_creatives, + "creative rewriting is opt-in: a blob without the field leaves it disabled" ); } diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index bdba093b7..ef5130d3d 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -4380,7 +4380,7 @@ origin_host_header_overide = "www.example.com""#, } #[test] - fn test_auction_rewrite_creatives_defaults_to_true_when_omitted() { + fn test_auction_creative_processing_defaults_to_false_when_omitted() { let toml_str = crate_test_settings_str() + r#" [auction] @@ -4391,8 +4391,12 @@ origin_host_header_overide = "www.example.com""#, let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); assert!( - settings.auction.rewrite_creatives, - "should preserve creative rewriting when the setting is omitted" + !settings.auction.rewrite_creatives, + "creative rewriting is opt-in when the setting is omitted" + ); + assert!( + !settings.auction.sanitize_creatives, + "creative sanitization is opt-in when the setting is omitted" ); } diff --git a/crates/trusted-server-js/lib/src/core/render.ts b/crates/trusted-server-js/lib/src/core/render.ts index ee08ef288..f00525b4e 100644 --- a/crates/trusted-server-js/lib/src/core/render.ts +++ b/crates/trusted-server-js/lib/src/core/render.ts @@ -7,15 +7,21 @@ import NORMALIZE_CSS from './styles/normalize.css?inline'; import IFRAME_TEMPLATE from './templates/iframe.html?raw'; // Sandbox permissions granted to creative iframes. +// // Ad creatives routinely contain scripts for tracking, click handling, and -// viewability measurement, so allow-scripts and allow-same-origin are required -// for creatives to render correctly. Server-side sanitization is the primary -// defense against malicious markup; the sandbox provides defense-in-depth. +// viewability measurement, so `allow-scripts` is required for them to render. +// +// `allow-same-origin` is deliberately excluded: combined with `allow-scripts` on +// srcdoc (or first-party src) content, that pair effectively removes the sandbox's +// origin isolation and would let SSP-provided markup run with the publisher +// origin's privileges — cookies, storage, and same-origin fetches. The origin +// boundary must not depend on server-side sanitization, which is optional +// (`auction.sanitize_creatives`) and cannot run at all for renderer-based bids. +// Matches APS_RENDERER_SANDBOX and ADM_IFRAME_SANDBOX, which already omit it. const CREATIVE_SANDBOX_TOKENS = [ 'allow-forms', 'allow-popups', 'allow-popups-to-escape-sandbox', - 'allow-same-origin', 'allow-scripts', 'allow-top-navigation-by-user-activation', ] as const; diff --git a/crates/trusted-server-js/lib/test/core/render.test.ts b/crates/trusted-server-js/lib/test/core/render.test.ts index a81486cf3..63a33c8a9 100644 --- a/crates/trusted-server-js/lib/test/core/render.test.ts +++ b/crates/trusted-server-js/lib/test/core/render.test.ts @@ -31,8 +31,12 @@ describe('render', () => { expect(sandbox).toContain('allow-popups'); expect(sandbox).toContain('allow-popups-to-escape-sandbox'); expect(sandbox).toContain('allow-top-navigation-by-user-activation'); - expect(sandbox).toContain('allow-same-origin'); expect(sandbox).toContain('allow-scripts'); + // `allow-scripts` + `allow-same-origin` together defeat the sandbox: creative + // markup would run with the publisher origin's privileges (cookies, storage, + // same-origin fetches). Matches APS_RENDERER_SANDBOX and ADM_IFRAME_SANDBOX, + // which already omit it. + expect(sandbox).not.toContain('allow-same-origin'); }); it('preserves dollar sequences when building the creative document', async () => { diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 99fabf3f2..d21a56ac7 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -112,11 +112,23 @@ rewrite_script = true [auction] enabled = false -# Defaults to true. Keep this leaf present when using the EdgeZero v0.0.4 -# environment override. Set false to return sanitized but unre-written winning-bid -# adm, skipping proxy/click URL conversion and creative TSJS injection. -# Sanitization is always applied. Restore and push true before an older-binary rollback. -rewrite_creatives = true +# Defaults to false. Keep this leaf present when using the EdgeZero v0.0.4 +# environment override. Set true to rewrite winning-bid adm to first-party +# endpoints, converting proxy/click URLs and injecting the creative TSJS runtime. +# Sanitization is controlled separately by `sanitize_creatives` below. +rewrite_creatives = false +# Strip executable markup (script/object/embed/form/...) from winning-bid adm, +# removing those elements together with their inner content. +# +# Defaults to false: creatives are delivered exactly as the bidder returned them. +# Enable whenever creatives can render in a context that shares the publisher's +# origin — it is the primary defence there. +# +# Leave disabled when creatives render in a foreign-origin frame (for example the +# Prebid Universal Creative inside the ad server's iframe), where the markup cannot +# reach the publisher origin. Sanitization removes script-based creatives entirely, +# so enabling it on a script-heavy demand stack silently blanks those slots. +sanitize_creatives = false providers = [] timeout_ms = 2000 allowed_context_keys = [] From f59fd85a5830edcaab437603799702c34bb1c83b Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 23 Jul 2026 13:24:31 +0530 Subject: [PATCH 095/198] Add gam_unit_path template parser --- .../src/creative_opportunities.rs | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 9ce741f3f..15bf95e69 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -14,6 +14,72 @@ use crate::auction::types::{AdFormat, AdSlot, MediaType}; use crate::price_bucket::PriceGranularity; use crate::settings::vec_from_seq_or_map; +/// A single parsed segment of a [`gam_unit_path`](CreativeOpportunitySlot::gam_unit_path) template. +#[derive(Debug, Clone)] +pub(crate) enum UnitTemplatePart { + /// Verbatim text between placeholders. + Literal(String), + /// `{network_id}` — replaced with the GAM network id. + NetworkId, + /// `{section}` — replaced with the request-derived section. + Section, + /// `{slot_id}` — replaced with the slot id. + SlotId, +} + +/// Parses a `gam_unit_path` template into an ordered list of parts. +/// +/// Supported placeholders: `{network_id}`, `{section}`, `{slot_id}`. A template +/// with no placeholders is a single [`UnitTemplatePart::Literal`] and renders +/// verbatim. +/// +/// # Errors +/// +/// Returns an error string for an empty template, an unmatched or nested `{`, +/// a stray `}`, or an unknown placeholder name. +fn parse_unit_template(raw: &str) -> Result, String> { + if raw.is_empty() { + return Err("gam_unit_path template must not be empty".to_string()); + } + let mut parts = Vec::new(); + let mut literal = String::new(); + let mut chars = raw.chars(); + while let Some(c) = chars.next() { + match c { + '{' => { + if !literal.is_empty() { + parts.push(UnitTemplatePart::Literal(std::mem::take(&mut literal))); + } + let mut name = String::new(); + loop { + match chars.next() { + Some('}') => break, + Some('{') => return Err(format!("nested '{{' in template `{raw}`")), + Some(ch) => name.push(ch), + None => return Err(format!("unmatched '{{' in template `{raw}`")), + } + } + match name.as_str() { + "network_id" => parts.push(UnitTemplatePart::NetworkId), + "section" => parts.push(UnitTemplatePart::Section), + "slot_id" => parts.push(UnitTemplatePart::SlotId), + other => { + return Err(format!( + "unknown placeholder `{{{other}}}` in template `{raw}`" + )); + } + } + } + '}' => return Err(format!("stray '}}' in template `{raw}`")), + other => literal.push(other), + } + } + if !literal.is_empty() { + parts.push(UnitTemplatePart::Literal(literal)); + } + Ok(parts) +} + /// Top-level configuration for the creative opportunities system. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -554,6 +620,46 @@ mod tests { assert_eq!(slot.resolved_div_id(), "atf"); } + #[test] + fn parse_unit_template_accepts_known_placeholders() { + let parts = parse_unit_template("/{network_id}/autoblog/{section}") + .expect("should parse valid template"); + assert_eq!(parts.len(), 4, "should split into literal+ph+literal+ph"); + } + + #[test] + fn parse_unit_template_accepts_static_path() { + let parts = parse_unit_template("/88059007/autoblog/homepage") + .expect("should parse a static path as a single literal"); + assert!( + matches!(parts.as_slice(), [UnitTemplatePart::Literal(s)] if s == "/88059007/autoblog/homepage"), + "should be one literal part" + ); + } + + #[test] + fn parse_unit_template_rejects_unknown_placeholder() { + let err = parse_unit_template("/{network_id}/{oops}") + .expect_err("should reject unknown placeholder"); + assert!(err.contains("oops"), "error should name the bad placeholder"); + } + + #[test] + fn parse_unit_template_rejects_unmatched_brace() { + parse_unit_template("/{network_id}/{section").expect_err("should reject unmatched '{'"); + parse_unit_template("/a}b").expect_err("should reject stray '}'"); + } + + #[test] + fn parse_unit_template_rejects_nested_brace() { + parse_unit_template("/{net{work}_id}").expect_err("should reject nested '{'"); + } + + #[test] + fn parse_unit_template_rejects_empty() { + parse_unit_template("").expect_err("should reject empty template"); + } + #[test] fn validate_runtime_rejects_empty_div_id_override() { // An empty/whitespace div_id would resolve every slot to the first From 9a71f556920215067c6f52cca12044945844a389 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 23 Jul 2026 13:31:04 +0530 Subject: [PATCH 096/198] Add request-path section derivation --- .../src/creative_opportunities.rs | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 15bf95e69..0f5dc9acc 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -80,6 +80,39 @@ fn parse_unit_template(raw: &str) -> Result, String> { Ok(parts) } +/// Collapses each run of characters outside `[A-Za-z0-9_-]` to a single `_`. +/// +/// Returns a non-empty string for any non-empty input. +fn sanitize_section(segment: &str) -> String { + let mut out = String::with_capacity(segment.len()); + let mut in_bad_run = false; + for ch in segment.chars() { + if ch.is_ascii_alphanumeric() || ch == '_' || ch == '-' { + out.push(ch); + in_bad_run = false; + } else if !in_bad_run { + out.push('_'); + in_bad_run = true; + } + } + out +} + +/// Derives the `{section}` value from a request path. +/// +/// Uses the first non-empty path segment, sanitized to `[A-Za-z0-9_-]`. Falls +/// back to `section_root` when the path has no segment (`/`, repeated slashes). +/// +/// The path is used **raw** (not percent-decoded) so this stays consistent with +/// how [`page_patterns`](CreativeOpportunitySlot::page_patterns) glob-match the +/// same path — e.g. `/new%20s` yields `new_20s`, never the decoded `new_s`. +pub(crate) fn derive_section(path: &str, section_root: &str) -> String { + match path.split('/').find(|segment| !segment.is_empty()) { + Some(segment) => sanitize_section(segment), + None => section_root.to_string(), + } +} + /// Top-level configuration for the creative opportunities system. #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(deny_unknown_fields)] @@ -660,6 +693,35 @@ mod tests { parse_unit_template("").expect_err("should reject empty template"); } + #[test] + fn derive_section_uses_first_segment() { + assert_eq!(derive_section("/news", "home"), "news"); + assert_eq!(derive_section("/news/gm-cadillac", "home"), "news"); + assert_eq!(derive_section("/car-research/x", "home"), "car-research"); + } + + #[test] + fn derive_section_uses_root_when_no_segment() { + assert_eq!(derive_section("/", "homepage"), "homepage"); + assert_eq!(derive_section("///", "homepage"), "homepage"); + } + + #[test] + fn derive_section_sanitizes_unsafe_runs_to_single_underscore() { + // Not decoded: in "new%20s" only '%' is disallowed ('2' and '0' are + // alphanumeric), so it collapses to a single '_' -> "new_20s". This is + // exactly the no-decode contract: had we decoded, %20 would be a space + // and yield "new_s"; we do NOT decode. + assert_eq!(derive_section("/new%20s", "home"), "new_20s"); + // A run of disallowed chars collapses to one '_'. + assert_eq!(derive_section("/a..b", "home"), "a_b"); + } + + #[test] + fn derive_section_is_non_empty_for_all_disallowed_segment() { + assert_eq!(derive_section("/%%%/x", "home"), "_"); + } + #[test] fn validate_runtime_rejects_empty_div_id_override() { // An empty/whitespace div_id would resolve every slot to the first From 75758730b7acfbd1e4f8779b43a90982538e8e9f Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 23 Jul 2026 15:02:58 +0530 Subject: [PATCH 097/198] Add section_root, unit-template compile/render, and startup validation --- .../src/creative_opportunities.rs | 223 ++++++++++++++++-- crates/trusted-server-core/src/publisher.rs | 4 + 2 files changed, 208 insertions(+), 19 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 0f5dc9acc..59ce6d46b 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -139,6 +139,14 @@ pub struct CreativeOpportunitiesConfig { /// Price granularity for header-bidding price bucketing. Defaults to `Dense`. #[serde(default)] pub price_granularity: PriceGranularity, + /// Value substituted for `{section}` when the request path has no first + /// segment (e.g. `/`). + /// + /// Required when any slot's [`gam_unit_path`](CreativeOpportunitySlot::gam_unit_path) + /// template contains `{section}`. No default — a home-section name is + /// publisher-specific, so the URL→section convention stays in config, not core. + #[serde(default)] + pub section_root: Option, /// Slot templates. Empty vec = feature disabled (no auction fired, no globals injected). #[serde(default, deserialize_with = "vec_from_seq_or_map")] pub slot: Vec, @@ -152,15 +160,48 @@ impl CreativeOpportunitiesConfig { } } + /// Parse every slot's [`gam_unit_path`](CreativeOpportunitySlot::gam_unit_path) + /// template. Call once after deserialization, before [`validate_runtime`](Self::validate_runtime). + /// + /// # Errors + /// + /// Returns an error string when any slot's template is malformed. + pub fn compile_unit_templates(&mut self) -> Result<(), String> { + for slot in &mut self.slot { + slot.compile_unit_template()?; + } + Ok(()) + } + /// Validate all slot definitions after runtime preparation. /// /// # Errors /// /// Returns an error string when a slot has an invalid identifier, page - /// pattern set, format list, dimensions, or resolved GAM unit path. + /// pattern set, format list, or dimensions, or when a slot's `gam_unit_path` + /// template uses `{section}` without a valid [`section_root`](Self::section_root). pub fn validate_runtime(&self) -> Result<(), String> { for slot in &self.slot { - slot.validate_runtime(&self.gam_network_id)?; + slot.validate_runtime()?; + } + + if self + .slot + .iter() + .any(CreativeOpportunitySlot::template_uses_section) + { + match self.section_root.as_deref() { + Some(root) + if !root.is_empty() + && root + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') => {} + _ => { + return Err("section_root is required and must match [A-Za-z0-9_-]+ \ + when a gam_unit_path template uses {section}" + .to_string()); + } + } } Ok(()) @@ -205,6 +246,14 @@ pub struct CreativeOpportunitySlot { /// crate can construct slots via struct-literal syntax with an empty cache. #[serde(skip, default)] pub(crate) compiled_patterns: Vec, + /// Pre-parsed [`gam_unit_path`](Self::gam_unit_path) template, populated by + /// [`compile_unit_template`](Self::compile_unit_template) at startup. + /// + /// `None` when the slot has no explicit `gam_unit_path` (renders the default + /// `//`). `pub(crate)` so cross-module test helpers can build + /// slots via struct-literal syntax with an empty cache. + #[serde(skip, default)] + pub(crate) compiled_unit: Option>, } impl CreativeOpportunitySlot { @@ -214,7 +263,7 @@ impl CreativeOpportunitySlot { /// /// Returns an error string when required slot fields are empty, invalid, /// or semantically unusable at runtime. - pub fn validate_runtime(&self, gam_network_id: &str) -> Result<(), String> { + pub fn validate_runtime(&self) -> Result<(), String> { validate_slot_id(&self.id)?; if self.page_patterns.is_empty() { @@ -269,15 +318,14 @@ impl CreativeOpportunitySlot { )); } - if self - .resolved_gam_unit_path(gam_network_id) - .trim() - .is_empty() + // A present-but-blank `gam_unit_path` renders to an empty/whitespace + // unit path. An empty string also fails template parsing at startup; + // this keeps the slot-level check self-contained (tests call + // `validate_runtime` without compiling templates first). + if let Some(raw) = &self.gam_unit_path + && raw.trim().is_empty() { - return Err(format!( - "slot `{}` resolved GAM unit path must not be empty", - self.id - )); + return Err(format!("slot `{}` gam_unit_path must not be empty", self.id)); } Ok(()) @@ -364,6 +412,52 @@ impl CreativeOpportunitySlot { .unwrap_or_else(|| format!("/{}/{}", gam_network_id, self.id)) } + /// Parses [`gam_unit_path`](Self::gam_unit_path) into + /// [`compiled_unit`](Self::compiled_unit). Call once at startup via + /// [`CreativeOpportunitiesConfig::compile_unit_templates`]. + /// + /// # Errors + /// + /// Returns an error string (prefixed with the slot id) when the template is + /// malformed. See [`parse_unit_template`]. + pub fn compile_unit_template(&mut self) -> Result<(), String> { + self.compiled_unit = match &self.gam_unit_path { + Some(raw) => { + Some(parse_unit_template(raw).map_err(|e| format!("slot `{}`: {e}", self.id))?) + } + None => None, + }; + Ok(()) + } + + /// Renders the resolved GAM unit path for a given network id and section. + /// + /// Substitutes `{network_id}`, `{section}`, and `{slot_id}` in the parsed + /// template. Falls back to `//` when the slot has no template. + #[must_use] + pub fn render_gam_unit_path(&self, gam_network_id: &str, section: &str) -> String { + match &self.compiled_unit { + Some(parts) => parts + .iter() + .map(|part| match part { + UnitTemplatePart::Literal(s) => s.as_str(), + UnitTemplatePart::NetworkId => gam_network_id, + UnitTemplatePart::Section => section, + UnitTemplatePart::SlotId => self.id.as_str(), + }) + .collect(), + None => format!("/{}/{}", gam_network_id, self.id), + } + } + + /// Returns `true` if this slot's compiled template contains `{section}`. + #[must_use] + pub(crate) fn template_uses_section(&self) -> bool { + self.compiled_unit + .as_ref() + .is_some_and(|parts| parts.iter().any(|p| matches!(p, UnitTemplatePart::Section))) + } + /// Returns the div element ID for this slot. /// /// Returns the [`div_id`](Self::div_id) override when set, otherwise returns [`id`](Self::id). @@ -554,6 +648,7 @@ mod tests { targeting: Default::default(), providers: Default::default(), compiled_patterns: Vec::new(), + compiled_unit: None, } } @@ -722,6 +817,96 @@ mod tests { assert_eq!(derive_section("/%%%/x", "home"), "_"); } + fn make_config_with_section_template(section_root: Option<&str>) -> CreativeOpportunitiesConfig { + let mut slot = make_slot("ad-header-0", vec!["/news/*"]); + slot.gam_unit_path = Some("/{network_id}/autoblog/{section}".to_string()); + CreativeOpportunitiesConfig { + gam_network_id: "88059007".to_string(), + auction_timeout_ms: None, + price_granularity: PriceGranularity::default(), + section_root: section_root.map(str::to_string), + slot: vec![slot], + } + } + + #[test] + fn render_gam_unit_path_substitutes_placeholders() { + let mut slot = make_slot("ad-header-0", vec!["/news/*"]); + slot.gam_unit_path = Some("/{network_id}/autoblog/{section}".to_string()); + slot.compile_unit_template().expect("should compile template"); + assert_eq!( + slot.render_gam_unit_path("88059007", "news"), + "/88059007/autoblog/news" + ); + } + + #[test] + fn render_gam_unit_path_defaults_when_no_template() { + let mut slot = make_slot("sidebar", vec!["/*"]); + slot.gam_unit_path = None; + slot.compile_unit_template().expect("should compile (no template)"); + assert_eq!(slot.render_gam_unit_path("99999", "ignored"), "/99999/sidebar"); + } + + #[test] + fn render_gam_unit_path_uses_static_template_verbatim() { + let mut slot = make_slot("atf", vec!["/"]); + slot.gam_unit_path = Some("/99999/example/homepage".to_string()); + slot.compile_unit_template() + .expect("should compile static template"); + assert_eq!( + slot.render_gam_unit_path("99999", "news"), + "/99999/example/homepage" + ); + } + + #[test] + fn validate_runtime_requires_section_root_when_template_uses_section() { + let mut config = make_config_with_section_template(None); + config.compile_slots(); + config + .compile_unit_templates() + .expect("templates should compile"); + let err = config + .validate_runtime() + .expect_err("should require section_root"); + assert!(err.contains("section_root"), "error should mention section_root"); + } + + #[test] + fn validate_runtime_rejects_invalid_section_root() { + let mut config = make_config_with_section_template(Some("has space")); + config.compile_slots(); + config + .compile_unit_templates() + .expect("templates should compile"); + config + .validate_runtime() + .expect_err("should reject non [A-Za-z0-9_-] root"); + } + + #[test] + fn validate_runtime_accepts_section_template_with_valid_root() { + let mut config = make_config_with_section_template(Some("homepage")); + config.compile_slots(); + config + .compile_unit_templates() + .expect("templates should compile"); + config + .validate_runtime() + .expect("should accept valid section_root"); + } + + #[test] + fn compile_unit_templates_surfaces_parse_error() { + let mut config = make_config_with_section_template(Some("home")); + config.slot[0].gam_unit_path = Some("/{bad}".to_string()); + config.compile_slots(); + config + .compile_unit_templates() + .expect_err("should surface unknown-placeholder error"); + } + #[test] fn validate_runtime_rejects_empty_div_id_override() { // An empty/whitespace div_id would resolve every slot to the first @@ -731,19 +916,19 @@ mod tests { slot.div_id = Some(String::new()); assert!( - slot.validate_runtime("1234").is_err(), + slot.validate_runtime().is_err(), "empty div_id override should fail validation" ); slot.div_id = Some(" ".to_string()); assert!( - slot.validate_runtime("1234").is_err(), + slot.validate_runtime().is_err(), "whitespace-only div_id override should fail validation" ); slot.div_id = Some("div-ad-x".to_string()); assert!( - slot.validate_runtime("1234").is_ok(), + slot.validate_runtime().is_ok(), "a concrete div_id override should pass validation" ); } @@ -755,31 +940,31 @@ mod tests { slot.floor_price = Some(-0.01); assert!( - slot.validate_runtime("1234").is_err(), + slot.validate_runtime().is_err(), "negative floor_price should fail validation" ); slot.floor_price = Some(f64::NAN); assert!( - slot.validate_runtime("1234").is_err(), + slot.validate_runtime().is_err(), "NaN floor_price should fail validation" ); slot.floor_price = Some(f64::INFINITY); assert!( - slot.validate_runtime("1234").is_err(), + slot.validate_runtime().is_err(), "infinite floor_price should fail validation" ); slot.floor_price = Some(0.0); assert!( - slot.validate_runtime("1234").is_ok(), + slot.validate_runtime().is_ok(), "zero floor_price should pass validation" ); slot.floor_price = None; assert!( - slot.validate_runtime("1234").is_ok(), + slot.validate_runtime().is_ok(), "absent floor_price should pass validation" ); } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 34909efe7..7edd35cf2 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -4274,6 +4274,7 @@ mod tests { gam_network_id: "21765378893".to_string(), auction_timeout_ms: Some(500), price_granularity: PriceGranularity::Dense, + section_root: None, slot: Vec::new(), } } @@ -4295,6 +4296,7 @@ mod tests { .collect(), providers: Default::default(), compiled_patterns: Vec::new(), + compiled_unit: None, } } @@ -4942,6 +4944,7 @@ mod tests { targeting: Default::default(), providers: Default::default(), compiled_patterns: Vec::new(), + compiled_unit: None, }] } @@ -5444,6 +5447,7 @@ mod tests { targeting: Default::default(), providers: Default::default(), compiled_patterns: Vec::new(), + compiled_unit: None, }] } From 860ed0b263750be22f62fe2cdd13235726b00267 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 23 Jul 2026 15:11:46 +0530 Subject: [PATCH 098/198] Render gam_unit_path template per request across initial and SPA paths --- crates/trusted-server-core/src/publisher.rs | 43 +++++++++++++++++---- crates/trusted-server-core/src/settings.rs | 9 ++++- 2 files changed, 44 insertions(+), 8 deletions(-) diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 7edd35cf2..90bdc6c1d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1788,7 +1788,7 @@ pub async fn handle_publisher_request( settings .creative_opportunities .as_ref() - .map(|co_config| build_ad_slots_script(&matched_slots, co_config)) + .map(|co_config| build_ad_slots_script(&matched_slots, co_config, &request_path)) } else { None }; @@ -2201,11 +2201,18 @@ pub(crate) fn build_empty_bids_script() -> String { /// definition and the two paths cannot silently diverge. Property names match /// what the client-side TSJS bundle expects: `gam_unit_path`, `div_id`, /// `formats`, and `targeting`. -fn build_slot_json( +pub(crate) fn build_slot_json( slot: &crate::creative_opportunities::CreativeOpportunitySlot, co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, + request_path: &str, ) -> serde_json::Value { - let gam_path = slot.resolved_gam_unit_path(&co_config.gam_network_id); + // `{section}` derives from the same raw path `page_patterns` matched + // against; `section_root` covers the no-segment case (`/`). + let section = crate::creative_opportunities::derive_section( + request_path, + co_config.section_root.as_deref().unwrap_or_default(), + ); + let gam_path = slot.render_gam_unit_path(&co_config.gam_network_id, §ion); let div_id = slot.resolved_div_id(); let formats: Vec = slot .formats @@ -2233,10 +2240,11 @@ fn build_slot_json( pub(crate) fn build_ad_slots_script( matched_slots: &[crate::creative_opportunities::CreativeOpportunitySlot], co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, + request_path: &str, ) -> String { let slots: Vec = matched_slots .iter() - .map(|slot| build_slot_json(slot, co_config)) + .map(|slot| build_slot_json(slot, co_config, request_path)) .collect(); let json = serde_json::to_string(&slots) .expect("serde_json::to_string of Vec should be infallible"); @@ -2562,7 +2570,7 @@ pub async fn handle_page_bids( let slots_json: Vec = if ad_stack_enabled { matched_slots .iter() - .map(|slot| build_slot_json(slot, co_config)) + .map(|slot| build_slot_json(slot, co_config, &path_param)) .collect() } else { Vec::new() @@ -4331,7 +4339,7 @@ mod tests { fn ad_slots_script_contains_slot_data() { let slots = vec![make_slot()]; let config = make_config(); - let script = build_ad_slots_script(&slots, &config); + let script = build_ad_slots_script(&slots, &config, "/"); assert!( script.contains("window.tsjs=window.tsjs||{}"), "should initialise tsjs namespace" @@ -4352,7 +4360,7 @@ mod tests { fn ad_slots_script_is_xss_safe() { let slots = vec![make_slot()]; let config = make_config(); - let script = build_ad_slots_script(&slots, &config); + let script = build_ad_slots_script(&slots, &config, "/"); let inner = script .trim_start_matches(""); @@ -4360,6 +4368,27 @@ mod tests { assert!(!inner.contains('>'), "no unescaped > in script content"); } + #[test] + fn build_slot_json_renders_section_from_request_path() { + let mut config = make_config(); + config.section_root = Some("homepage".to_string()); + let mut slot = make_slot(); + slot.gam_unit_path = Some("/{network_id}/autoblog/{section}".to_string()); + slot.compile_unit_template().expect("template should compile"); + + let news = crate::publisher::build_slot_json(&slot, &config, "/news/gm-cadillac"); + assert_eq!( + news["gam_unit_path"], "/21765378893/autoblog/news", + "section should derive from the first path segment" + ); + + let home = crate::publisher::build_slot_json(&slot, &config, "/"); + assert_eq!( + home["gam_unit_path"], "/21765378893/autoblog/homepage", + "root path should use section_root" + ); + } + #[test] fn bid_map_includes_nurl_and_burl() { let mut winning_bids = HashMap::new(); diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index c49e99686..4514d12bd 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -2077,6 +2077,13 @@ impl Settings { if let Some(co) = &mut self.creative_opportunities { co.compile_slots(); + // Parse `gam_unit_path` templates once here (mirrors the compiled + // glob cache) so request-time rendering is substitution-only. + co.compile_unit_templates().map_err(|err| { + Report::new(TrustedServerError::Configuration { + message: format!("Invalid creative opportunity gam_unit_path template: {err}"), + }) + })?; // Slots flow into injected HTML/JS, provider payloads, and GPT // calls. Env/private config can bypass static review, so validate // the full runtime shape on every load path. @@ -5602,7 +5609,7 @@ gam_unit_path = "" page_patterns = ["/"] formats = [{ width = 300, height = 250 }] "#, - "resolved GAM unit path must not be empty", + "gam_unit_path template must not be empty", ); } From 21a35239b82ebe251ba2840075667f52b7fd3c20 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 23 Jul 2026 15:13:39 +0530 Subject: [PATCH 099/198] Add spec and plan for per-section gam_unit_path --- .../2026-07-23-per-section-gam-unit-path.md | 746 ++++++++++++++++++ ...-07-23-per-section-gam-unit-path-design.md | 208 +++++ 2 files changed, 954 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-23-per-section-gam-unit-path.md create mode 100644 docs/superpowers/specs/2026-07-23-per-section-gam-unit-path-design.md diff --git a/docs/superpowers/plans/2026-07-23-per-section-gam-unit-path.md b/docs/superpowers/plans/2026-07-23-per-section-gam-unit-path.md new file mode 100644 index 000000000..375795a6b --- /dev/null +++ b/docs/superpowers/plans/2026-07-23-per-section-gam-unit-path.md @@ -0,0 +1,746 @@ +# Per-Section `gam_unit_path` Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `creative_opportunities.slot.gam_unit_path` a template with a +`{section}` placeholder derived from the request path, so one slot rule serves +all site sections instead of one rule per (slot × section). + +**Architecture:** Parse each slot's `gam_unit_path` into a cached template at +startup (alongside the existing compiled-glob cache); reject malformed templates +and a `{section}` template missing its `section_root`. At request time derive +`{section}` from the raw path (sanitized) and render the template inside +`build_slot_json`, which gains a `request_path` argument. Server-only — the +client keeps receiving a resolved `gam_unit_path` string, so no JS change. + +**Tech Stack:** Rust 2024, `trusted-server-core`. Tests via `cargo test_details` +(native host, `aarch64-apple-darwin`) for iteration and `cargo test-fastly` +(core + fastly on `wasm32-wasip1` via Viceroy) for the CI gate. + +**Spec:** `docs/superpowers/specs/2026-07-23-per-section-gam-unit-path-design.md` + +**Issue:** https://github.com/IABTechLab/trusted-server/issues/954 + +--- + +## File Structure + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` + - new: `UnitTemplatePart` enum, `parse_unit_template`, `sanitize_section`, + `derive_section` + - new on `CreativeOpportunitySlot`: `compiled_unit` field, + `compile_unit_template`, `render_gam_unit_path`, `template_uses_section` + - new on `CreativeOpportunitiesConfig`: `section_root` field, + `compile_unit_templates`; extend `validate_runtime` + - unit tests in the existing `#[cfg(test)] mod tests` +- Modify: `crates/trusted-server-core/src/publisher.rs` + - `build_slot_json` gains `request_path: &str`; renders via `render_gam_unit_path` + - `build_ad_slots_script` gains `request_path: &str`; threads it through + - `handle_page_bids` passes its normalized `path` to `build_slot_json` +- Modify: `crates/trusted-server-core/src/settings.rs` + - `prepare_runtime` calls `compile_unit_templates` and surfaces parse errors +- Modify: `docs/guide/configuration.md` (add creative_opportunities section) +- Modify: `trusted-server.example.toml` and the live autoblog config + +Notes on lifecycle: `page_patterns` inheritance is **out of scope** (sibling +issue). Templates are parsed at startup and cached with `#[serde(skip)]`, +mirroring the existing `compiled_patterns` field. + +--- + +## Task 1: Template parser + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Test: same file, `#[cfg(test)] mod tests` + +- [ ] **Step 1: Write the failing tests** + +Add to `mod tests`: + +```rust +#[test] +fn parse_unit_template_accepts_known_placeholders() { + let parts = parse_unit_template("/{network_id}/autoblog/{section}") + .expect("should parse valid template"); + assert_eq!(parts.len(), 4, "should split into literal+ph+literal+ph"); +} + +#[test] +fn parse_unit_template_accepts_static_path() { + let parts = parse_unit_template("/88059007/autoblog/homepage") + .expect("should parse a static path as a single literal"); + assert!( + matches!(parts.as_slice(), [UnitTemplatePart::Literal(s)] if s == "/88059007/autoblog/homepage"), + "should be one literal part" + ); +} + +#[test] +fn parse_unit_template_rejects_unknown_placeholder() { + let err = parse_unit_template("/{network_id}/{oops}").expect_err("should reject unknown placeholder"); + assert!(err.contains("oops"), "error should name the bad placeholder"); +} + +#[test] +fn parse_unit_template_rejects_unmatched_brace() { + parse_unit_template("/{network_id}/{section").expect_err("should reject unmatched '{'"); + parse_unit_template("/a}b").expect_err("should reject stray '}'"); +} + +#[test] +fn parse_unit_template_rejects_nested_brace() { + parse_unit_template("/{net{work}_id}").expect_err("should reject nested '{'"); +} + +#[test] +fn parse_unit_template_rejects_empty() { + parse_unit_template("").expect_err("should reject empty template"); +} +``` + +- [ ] **Step 2: Run tests, verify they fail** + +Run: `cargo test_details -p trusted-server-core creative_opportunities::tests::parse_unit_template` +Expected: FAIL — `cannot find function parse_unit_template` / `UnitTemplatePart`. + +- [ ] **Step 3: Implement the enum + parser** + +Add near the top of the module body (after imports): + +```rust +/// A single parsed segment of a `gam_unit_path` template. +#[derive(Debug, Clone)] +pub(crate) enum UnitTemplatePart { + /// Verbatim text between placeholders. + Literal(String), + /// `{network_id}` — replaced with the GAM network id. + NetworkId, + /// `{section}` — replaced with the request-derived section. + Section, + /// `{slot_id}` — replaced with the slot id. + SlotId, +} + +/// Parses a `gam_unit_path` template into an ordered list of parts. +/// +/// # Errors +/// +/// Returns an error string for an empty template, an unmatched or nested `{`, +/// a stray `}`, or an unknown placeholder name. +fn parse_unit_template(raw: &str) -> Result, String> { + if raw.is_empty() { + return Err("gam_unit_path template must not be empty".to_string()); + } + let mut parts = Vec::new(); + let mut literal = String::new(); + let mut chars = raw.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '{' => { + if !literal.is_empty() { + parts.push(UnitTemplatePart::Literal(std::mem::take(&mut literal))); + } + let mut name = String::new(); + loop { + match chars.next() { + Some('}') => break, + Some('{') => { + return Err(format!("nested '{{' in template `{raw}`")); + } + Some(ch) => name.push(ch), + None => return Err(format!("unmatched '{{' in template `{raw}`")), + } + } + match name.as_str() { + "network_id" => parts.push(UnitTemplatePart::NetworkId), + "section" => parts.push(UnitTemplatePart::Section), + "slot_id" => parts.push(UnitTemplatePart::SlotId), + other => { + return Err(format!( + "unknown placeholder `{{{other}}}` in template `{raw}`" + )); + } + } + } + '}' => return Err(format!("stray '}}' in template `{raw}`")), + other => literal.push(other), + } + } + if !literal.is_empty() { + parts.push(UnitTemplatePart::Literal(literal)); + } + Ok(parts) +} +``` + +- [ ] **Step 4: Run tests, verify they pass** + +Run: `cargo test_details -p trusted-server-core creative_opportunities::tests::parse_unit_template` +Expected: PASS (6 tests). + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/creative_opportunities.rs +git commit -m "Add gam_unit_path template parser" +``` + +--- + +## Task 2: Section derivation + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Test: same file + +- [ ] **Step 1: Write the failing tests** + +```rust +#[test] +fn derive_section_uses_first_segment() { + assert_eq!(derive_section("/news", "home"), "news"); + assert_eq!(derive_section("/news/gm-cadillac", "home"), "news"); + assert_eq!(derive_section("/car-research/x", "home"), "car-research"); +} + +#[test] +fn derive_section_uses_root_when_no_segment() { + assert_eq!(derive_section("/", "homepage"), "homepage"); + assert_eq!(derive_section("///", "homepage"), "homepage"); +} + +#[test] +fn derive_section_sanitizes_unsafe_runs_to_single_underscore() { + // Not decoded: in "new%20s" only '%' is disallowed ('2' and '0' are + // alphanumeric), so it collapses to a single '_' -> "new_20s". This is + // exactly the no-decode contract: had we decoded, %20 would be a space and + // yield "new_s"; we do NOT decode. + assert_eq!(derive_section("/new%20s", "home"), "new_20s"); + // A run of disallowed chars collapses to one '_'. + assert_eq!(derive_section("/a..b", "home"), "a_b"); +} + +#[test] +fn derive_section_is_non_empty_for_all_disallowed_segment() { + assert_eq!(derive_section("/%%%/x", "home"), "_"); +} +``` + +- [ ] **Step 2: Run tests, verify they fail** + +Run: `cargo test_details -p trusted-server-core creative_opportunities::tests::derive_section` +Expected: FAIL — `cannot find function derive_section`. + +- [ ] **Step 3: Implement the two functions** + +```rust +/// Collapses each run of characters outside `[A-Za-z0-9_-]` to a single `_`. +/// +/// Returns a non-empty string for any non-empty input. +fn sanitize_section(segment: &str) -> String { + let mut out = String::with_capacity(segment.len()); + let mut in_bad_run = false; + for ch in segment.chars() { + if ch.is_ascii_alphanumeric() || ch == '_' || ch == '-' { + out.push(ch); + in_bad_run = false; + } else if !in_bad_run { + out.push('_'); + in_bad_run = true; + } + } + out +} + +/// Derives the `{section}` value from a request path. +/// +/// Uses the first non-empty path segment, sanitized to `[A-Za-z0-9_-]`. Falls +/// back to `section_root` when the path has no segment (`/`, repeated slashes). +/// The path is used **raw** (not percent-decoded) so this stays consistent with +/// how `page_patterns` glob-match the same path. +pub(crate) fn derive_section(path: &str, section_root: &str) -> String { + match path.split('/').find(|segment| !segment.is_empty()) { + Some(segment) => sanitize_section(segment), + None => section_root.to_string(), + } +} +``` + +- [ ] **Step 4: Run tests, verify they pass** + +Run: `cargo test_details -p trusted-server-core creative_opportunities::tests::derive_section` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/creative_opportunities.rs +git commit -m "Add request-path section derivation" +``` + +--- + +## Task 3: Config field, template compile + render, startup validation + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` +- Test: same file + +- [ ] **Step 1: Write the failing tests** + +```rust +// NOTE: the existing helper signature is `make_slot(id: &str, patterns: Vec<&str>)` +// (see creative_opportunities.rs:443) — pass `vec![...]`, not `&[...]`. +#[test] +fn render_gam_unit_path_substitutes_placeholders() { + let mut slot = make_slot("ad-header-0", vec!["/news/*"]); + slot.gam_unit_path = Some("/{network_id}/autoblog/{section}".to_string()); + slot.compile_unit_template().expect("should compile template"); + assert_eq!( + slot.render_gam_unit_path("88059007", "news"), + "/88059007/autoblog/news" + ); +} + +#[test] +fn render_gam_unit_path_defaults_when_no_template() { + let mut slot = make_slot("sidebar", vec!["/*"]); + slot.gam_unit_path = None; + slot.compile_unit_template().expect("should compile (no template)"); + assert_eq!(slot.render_gam_unit_path("99999", "ignored"), "/99999/sidebar"); +} + +#[test] +fn render_gam_unit_path_uses_static_template_verbatim() { + let mut slot = make_slot("atf", vec!["/"]); + slot.gam_unit_path = Some("/99999/example/homepage".to_string()); + slot.compile_unit_template().expect("should compile static template"); + assert_eq!(slot.render_gam_unit_path("99999", "news"), "/99999/example/homepage"); +} + +#[test] +fn validate_runtime_requires_section_root_when_template_uses_section() { + let mut config = make_config_with_section_template(None); // section_root = None + config.compile_slots(); + config.compile_unit_templates().expect("templates compile"); + let err = config.validate_runtime().expect_err("should require section_root"); + assert!(err.contains("section_root"), "error should mention section_root"); +} + +#[test] +fn validate_runtime_rejects_invalid_section_root() { + let mut config = make_config_with_section_template(Some("has space")); + config.compile_slots(); + config.compile_unit_templates().expect("templates compile"); + config.validate_runtime().expect_err("should reject non [A-Za-z0-9_-] root"); +} + +#[test] +fn validate_runtime_accepts_section_template_with_valid_root() { + let mut config = make_config_with_section_template(Some("homepage")); + config.compile_slots(); + config.compile_unit_templates().expect("templates compile"); + config.validate_runtime().expect("should accept valid section_root"); +} + +#[test] +fn compile_unit_templates_surfaces_parse_error() { + let mut config = make_config_with_section_template(Some("home")); + config.slot[0].gam_unit_path = Some("/{bad}".to_string()); + config.compile_slots(); + config.compile_unit_templates().expect_err("should surface unknown-placeholder error"); +} +``` + +Add test helpers to `mod tests` if not present (adapt to the existing helper +style in this module): + +```rust +fn make_config_with_section_template(section_root: Option<&str>) -> CreativeOpportunitiesConfig { + let mut slot = make_slot("ad-header-0", vec!["/news/*"]); + slot.gam_unit_path = Some("/{network_id}/autoblog/{section}".to_string()); + CreativeOpportunitiesConfig { + gam_network_id: "88059007".to_string(), + auction_timeout_ms: None, + price_granularity: PriceGranularity::default(), + section_root: section_root.map(str::to_string), + slot: vec![slot], + } +} +``` + +The `make_slot(id: &str, patterns: Vec<&str>)` helper **already exists** at +`creative_opportunities.rs:443` and constructs a `CreativeOpportunitySlot` via +struct-literal syntax. Because the struct uses `#[serde(deny_unknown_fields)]` +and the helper names every field explicitly, adding `compiled_unit` to the +struct makes this helper fail to compile until updated — see Step 3's helper-fix +sub-step. + +- [ ] **Step 2: Run tests, verify they fail** + +Run: `cargo test_details -p trusted-server-core creative_opportunities::tests` +Expected: FAIL — missing `section_root`, `compiled_unit`, `compile_unit_template`, +`render_gam_unit_path`, `compile_unit_templates`. + +- [ ] **Step 3: Add the field, cache, methods, and validation** + +On `CreativeOpportunitiesConfig` (add field): + +```rust +/// Value substituted for `{section}` when the request path has no first +/// segment (e.g. `/`). Required when any slot's `gam_unit_path` template +/// contains `{section}`. No default — a home-section name is publisher-specific. +#[serde(default)] +pub section_root: Option, +``` + +On `CreativeOpportunitySlot` (add cached template, parallel to `compiled_patterns`): + +```rust +/// Pre-parsed [`gam_unit_path`](Self::gam_unit_path) template, populated by +/// [`compile_unit_template`](Self::compile_unit_template) at startup. `None` +/// when the slot has no explicit `gam_unit_path` (uses the default path). +#[serde(skip, default)] +pub(crate) compiled_unit: Option>, +``` + +Slot methods: + +```rust +/// Parses [`gam_unit_path`](Self::gam_unit_path) into [`compiled_unit`](Self::compiled_unit). +/// +/// # Errors +/// +/// Returns an error string when the template is malformed (see +/// [`parse_unit_template`]). +pub fn compile_unit_template(&mut self) -> Result<(), String> { + self.compiled_unit = match &self.gam_unit_path { + Some(raw) => Some(parse_unit_template(raw).map_err(|e| format!("slot `{}`: {e}", self.id))?), + None => None, + }; + Ok(()) +} + +/// Renders the resolved GAM unit path for a given network id and section. +/// +/// Uses the parsed template when present, otherwise the default +/// `//`. +#[must_use] +pub fn render_gam_unit_path(&self, gam_network_id: &str, section: &str) -> String { + match &self.compiled_unit { + Some(parts) => parts + .iter() + .map(|part| match part { + UnitTemplatePart::Literal(s) => s.as_str(), + UnitTemplatePart::NetworkId => gam_network_id, + UnitTemplatePart::Section => section, + UnitTemplatePart::SlotId => self.id.as_str(), + }) + .collect(), + None => format!("/{}/{}", gam_network_id, self.id), + } +} + +/// Returns `true` if this slot's compiled template contains `{section}`. +#[must_use] +pub(crate) fn template_uses_section(&self) -> bool { + self.compiled_unit + .as_ref() + .is_some_and(|parts| parts.iter().any(|p| matches!(p, UnitTemplatePart::Section))) +} +``` + +On `CreativeOpportunitiesConfig` (compile all templates + extend validation): + +```rust +/// Parse every slot's `gam_unit_path` template. Call once after deserialization. +/// +/// # Errors +/// +/// Returns an error string when any slot's template is malformed. +pub fn compile_unit_templates(&mut self) -> Result<(), String> { + for slot in &mut self.slot { + slot.compile_unit_template()?; + } + Ok(()) +} +``` + +In `validate_runtime`, after the existing per-slot loop, add: + +```rust +if self.slot.iter().any(CreativeOpportunitySlot::template_uses_section) { + match self.section_root.as_deref() { + Some(root) + if !root.is_empty() + && root.chars().all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') => {} + _ => { + return Err( + "section_root is required and must match [A-Za-z0-9_-]+ when a \ + gam_unit_path template uses {section}" + .to_string(), + ); + } + } +} +``` + +Remove the old path-render emptiness check in `validate_runtime` +(the block calling `resolved_gam_unit_path(...).trim().is_empty()`); malformed or +empty templates are now caught at parse time by `compile_unit_templates`, and a +rendered result is non-empty by construction. + +**Update the existing test helper (required — adding `compiled_unit` breaks it):** +Add `compiled_unit: None` to the `CreativeOpportunitySlot` struct-literal in +`make_slot` at `crates/trusted-server-core/src/creative_opportunities.rs:443`. +The struct uses `#[serde(deny_unknown_fields)]` and the helper names every field, +so a missing field is a compile error, not a `#[serde(default)]` fill-in. + +- [ ] **Step 4: Run tests, verify they pass** + +Run: `cargo test_details -p trusted-server-core creative_opportunities::tests` +Expected: PASS (Task 1–3 tests). + +- [ ] **Step 5: Commit** + +```bash +git add crates/trusted-server-core/src/creative_opportunities.rs +git commit -m "Add section_root, unit-template compile/render, and startup validation" +``` + +--- + +## Task 4: Render at request time (thread the path through publisher.rs) + +**Files:** + +- Modify: `crates/trusted-server-core/src/creative_opportunities.rs` (remove now-unused `resolved_gam_unit_path`, or keep if other callers remain — grep first) +- Modify: `crates/trusted-server-core/src/publisher.rs` +- Modify: `crates/trusted-server-core/src/settings.rs` +- Test: `crates/trusted-server-core/src/publisher.rs` `#[cfg(test)] mod tests` + +- [ ] **Step 0: Update publisher.rs struct-literal test helpers (required — new fields break them)** + +Adding `section_root` to `CreativeOpportunitiesConfig` and `compiled_unit` to +`CreativeOpportunitySlot` breaks every hand-built literal in `publisher.rs` +tests. Add the new fields to each: + +- `crates/trusted-server-core/src/publisher.rs:4272` — `make_config()`: add `section_root: None`. +- `crates/trusted-server-core/src/publisher.rs:4282` — `make_slot()`: add `compiled_unit: None`. +- `crates/trusted-server-core/src/publisher.rs:4931` — `article_slot()`: add `compiled_unit: None`. +- `crates/trusted-server-core/src/publisher.rs:5433` — `article_slot()` (second module): add `compiled_unit: None`. + +Run: `cargo test_details -p trusted-server-core publisher:: --no-run` +Expected: compiles (no `missing field` errors) before writing the new test. + +- [ ] **Step 1: Write the failing test (equivalence + per-section)** + +In `publisher.rs` tests, add (adapt to the existing test helpers/config builders +in that module): + +```rust +#[test] +fn build_slot_json_renders_section_from_request_path() { + let config = creative_opportunities_config_with_template(); // gam_unit_path = "/{network_id}/autoblog/{section}", section_root = "homepage" + let slot = &config.slot[0]; + + let news = build_slot_json(slot, &config, "/news/gm-cadillac"); + assert_eq!(news["gam_unit_path"], "/88059007/autoblog/news"); + + let home = build_slot_json(slot, &config, "/"); + assert_eq!(home["gam_unit_path"], "/88059007/autoblog/homepage"); +} +``` + +- [ ] **Step 2: Run test, verify it fails** + +Run: `cargo test_details -p trusted-server-core publisher::tests::build_slot_json_renders_section` +Expected: FAIL — `build_slot_json` takes 2 args / wrong unit value. + +- [ ] **Step 3: Thread `request_path` and render** + +In `build_slot_json` (`crates/trusted-server-core/src/publisher.rs` ~2204): + +```rust +fn build_slot_json( + slot: &crate::creative_opportunities::CreativeOpportunitySlot, + co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, + request_path: &str, +) -> serde_json::Value { + let section = crate::creative_opportunities::derive_section( + request_path, + co_config.section_root.as_deref().unwrap_or_default(), + ); + let gam_path = slot.render_gam_unit_path(&co_config.gam_network_id, §ion); + // ...rest unchanged (div_id, formats, targeting, json!)... +} +``` + +In `build_ad_slots_script` (~2233) add `request_path: &str` and pass it: + +```rust +pub(crate) fn build_ad_slots_script( + matched_slots: &[crate::creative_opportunities::CreativeOpportunitySlot], + co_config: &crate::creative_opportunities::CreativeOpportunitiesConfig, + request_path: &str, +) -> String { + let slots: Vec = matched_slots + .iter() + .map(|slot| build_slot_json(slot, co_config, request_path)) + .collect(); + // ...unchanged... +} +``` + +At the initial-render caller (~publisher.rs:1791) pass `&request_path`: + +```rust +.map(|co_config| build_ad_slots_script(&matched_slots, co_config, &request_path)) +``` + +In `handle_page_bids` (~2562) pass the already-normalized path +(`path_param` / the value from `normalize_page_bids_path`) to `build_slot_json`: + +```rust +.map(|slot| build_slot_json(slot, co_config, &path_param)) +``` + +Update any existing `build_ad_slots_script(...)` / `build_slot_json(...)` test +call sites in `publisher.rs` to pass a path argument (e.g. `"/"`). + +- [ ] **Step 4: Update `settings.rs::prepare_runtime`** + +In `crates/trusted-server-core/src/settings.rs` (~2078), compile templates and +surface parse errors: + +```rust +if let Some(co) = &mut self.creative_opportunities { + co.compile_slots(); + co.compile_unit_templates().map_err(|err| { + Report::new(TrustedServerError::Configuration { + message: format!("Invalid creative opportunity gam_unit_path template: {err}"), + }) + })?; + co.validate_runtime().map_err(|err| { + Report::new(TrustedServerError::Configuration { + message: format!("Invalid creative opportunity slot config: {err}"), + }) + })?; +} +``` + +- [ ] **Step 5: Run tests, verify they pass** + +Run: `cargo test_details -p trusted-server-core publisher::tests` +Expected: PASS. + +- [ ] **Step 6: Fix the existing empty-`gam_unit_path` settings test if needed** + +`settings.rs::settings_rejects_creative_opportunity_slot_with_empty_gam_unit_path` +now fails at template-parse (empty template) rather than the render check. Verify +it still asserts rejection; update the expected error substring if it pins a +message. + +Run: `cargo test_details -p trusted-server-core settings::` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add crates/trusted-server-core/src/publisher.rs crates/trusted-server-core/src/creative_opportunities.rs crates/trusted-server-core/src/settings.rs +git commit -m "Render gam_unit_path template per request across initial and SPA paths" +``` + +--- + +## Task 5: Docs + config + +**Files:** + +- Modify: `docs/guide/configuration.md` +- Modify: `trusted-server.example.toml` +- Modify: the live autoblog `trusted-server.toml` (operator-owned, gitignored — update locally, do not commit) + +- [ ] **Step 1: Add a creative_opportunities section to configuration.md** + +Cover: the placeholder set (`{network_id}`, `{section}`, `{slot_id}`); section +derivation (first path segment, sanitized to `[A-Za-z0-9_-]`, raw/undecoded); +`section_root` requirement and validation; behavior on an unmatched route (no +slot, template never rendered); back-compat (static path used verbatim; no +`gam_unit_path` → `//`). Use fictional values +(`example.com`, network `99999`) per the repo's docs rule. + +- [ ] **Step 2: Update `trusted-server.example.toml`** + +Show one templated slot with `section_root` and a `{section}` `gam_unit_path`, +using fictional values. + +- [ ] **Step 3: Docs format check** + +Run: `cd docs && npm run format` +Expected: no diff / formatting clean. + +- [ ] **Step 4: Commit** + +```bash +git add docs/guide/configuration.md trusted-server.example.toml +git commit -m "Document per-section gam_unit_path templating" +``` + +--- + +## Task 6: Full verification (CI gate) + +- [ ] **Step 1: Format** + +Run: `cargo fmt --all -- --check` +Expected: clean. + +- [ ] **Step 2: Core + Fastly tests under Viceroy (full module, not filtered)** + +Run: `cargo test-fastly` +Expected: PASS. (Runs the full creative_opportunities + publisher test modules on +`wasm32-wasip1`; a format-changing edit can hide later failures when filtered, so +run the whole suite here.) + +- [ ] **Step 3: Other adapters (no behavior change expected, guard against signature breaks)** + +Run: `cargo test-axum && cargo test-cloudflare && cargo test-spin` +Expected: PASS. + +- [ ] **Step 4: Clippy across adapter targets** + +Run: `cargo clippy-fastly && cargo clippy-axum && cargo clippy-cloudflare && cargo clippy-cloudflare-wasm && cargo clippy-spin-native && cargo clippy-spin-wasm` +Expected: no warnings. + +- [ ] **Step 5: JS unaffected (sanity)** + +Run: `cd crates/trusted-server-js/lib && npx vitest run` +Expected: PASS (no JS change; confirms wire shape unbroken). + +- [ ] **Step 6: Final commit if any fixups** + +```bash +git add -A +git commit -m "Fix clippy/fmt for per-section gam_unit_path" +``` + +--- + +## Acceptance criteria mapping + +- **N slots × M sections without N×M rules** — Task 1–4 (one templated slot rule + serves all sections). +- **Resolution tested (`/`, single/multi-segment, no-match, encoded)** — Task 2 + tests + Task 4 equivalence + the unmatched-route case (no slot matched → no + `build_slot_json` call; covered by existing `match_slots` empty tests). +- **Existing static configs unchanged** — Task 3 `render_gam_unit_path` verbatim + - default tests. +- **Startup catches empty/unknown/malformed template + missing/invalid + `section_root`** — Task 1 + Task 3 validation tests. +- **`{section}` sanitized, raw path** — Task 2 tests. +- **Documented** — Task 5. diff --git a/docs/superpowers/specs/2026-07-23-per-section-gam-unit-path-design.md b/docs/superpowers/specs/2026-07-23-per-section-gam-unit-path-design.md new file mode 100644 index 000000000..39e25d42f --- /dev/null +++ b/docs/superpowers/specs/2026-07-23-per-section-gam-unit-path-design.md @@ -0,0 +1,208 @@ +# Per-Section `gam_unit_path` Design + +**Date:** 2026-07-23 + +**Status:** Proposed + +**Issue:** [IABTechLab/trusted-server#954](https://github.com/IABTechLab/trusted-server/issues/954) + +## Summary + +`creative_opportunities.slot.gam_unit_path` is a static string, so a publisher +whose GAM ad unit varies by site section cannot express that in one rule. The +only way to model it today is one slot rule per (slot × section), which +multiplies out fast: 3 slots across 10 sections needs 30 near-identical rules. + +This design makes `gam_unit_path` a **template** with a small, fixed placeholder +set — `{network_id}`, `{section}`, `{slot_id}` — where `{section}` is derived +from the request path at render time. One slot rule then covers all sections. + +The derivation policy that matters (`{section}` for the site root) lives in +config via a required `section_root`, not in core — honoring the issue's +constraint that the URL→section convention is publisher-specific. + +Scope is deliberately narrow: **only** `gam_unit_path` templating. Sharing +`page_patterns`/`gam_unit_path` defaults across slots is a related but distinct +duplication problem, tracked as a sibling issue, not built here. + +## Goals + +1. A publisher with N slots across M sections expresses per-section ad units + without N×M rules. +2. `{section}` is derived from the request path with a config-supplied value for + the site root; no URL convention is hardcoded in core. +3. Existing static `gam_unit_path` configs keep working, byte-for-byte + unchanged. +4. Startup rejects unresolvable configuration: unknown placeholders, malformed + templates, and a `{section}` template missing its `section_root`. +5. Resolution is covered by tests including `/`, single- and multi-segment + paths, unsafe/encoded segments, and paths matching no slot. +6. Documented in `docs/guide/configuration.md`, which currently has no + creative_opportunities section. + +## Non-goals + +Documented here so onboarding publishers know the boundary. Each is an additive +extension that does **not** change the config shape below. + +1. **Locale offset** — deriving `{section}` from a segment other than the first + (e.g. `/en/news` → `news`). `{section}` is the first path segment. A + `section_segment` index knob can be added later. +2. **Full-path mirror** — `{section}` spanning multiple segments (`/a/b` → + `a/b`). Real GAM trees bucket by section, not per-article, so this is rare; + use a static per-slot `gam_unit_path` for the exception. +3. **Named per-section overrides** — mapping an irregular section to a renamed + unit (`/reviews` → `editorial/reviews-v2`). Set that one slot's + `gam_unit_path` explicitly, or add named overrides later. +4. **Host- or query-derived sections** — path-only. Out of scope entirely. +5. **Slot-defaults inheritance** — sharing `page_patterns`/`gam_unit_path` at + the `[creative_opportunities]` level. Separate issue; has a startup-lifecycle + concern this design intentionally avoids. + +## Background: how `gam_unit_path` is used + +- `gam_unit_path` is **client-side only**. The resolved string reaches + `googletag.defineSlot(path, sizes, div)` in + `crates/trusted-server-js/lib/src/integrations/gpt/index.ts`. It is **not** in + the OpenRTB bid request — `CreativeOpportunitySlot::to_ad_slot` never emits it. + Therefore this change is **server-only; no JS wire change**. The client keeps + receiving a resolved `gam_unit_path` string. +- Today's resolver is literal-or-default and path-independent + (`crates/trusted-server-core/src/creative_opportunities.rs`): + + ```rust + pub fn resolved_gam_unit_path(&self, gam_network_id: &str) -> String { + self.gam_unit_path + .clone() + .unwrap_or_else(|| format!("/{}/{}", gam_network_id, self.id)) + } + ``` + +- The value is emitted in `build_slot_json` + (`crates/trusted-server-core/src/publisher.rs`), shared by two paths: + - initial render via `build_ad_slots_script` (called where `request_path` is + in scope); + - SPA navigation via `handle_page_bids` (has the normalized `path` param). + + Neither currently passes the path into `build_slot_json`. + +## Design + +### Config shape + +```toml +[creative_opportunities] +gam_network_id = "88059007" +auction_timeout_ms = 2000 +price_granularity = "dense" +section_root = "homepage" # required when a template uses {section} + +[[creative_opportunities.slot]] +id = "ad-header-0" +gam_unit_path = "/{network_id}/autoblog/{section}" +page_patterns = ["/", "/news/*", "/reviews/*", "/deals/*"] +formats = [{ width = 970, height = 90 }, { width = 728, height = 90 }] +[creative_opportunities.slot.providers.prebid] +bidders = {} +``` + +### Placeholders + +| placeholder | resolves to | +| -------------- | ----------------------------------------------------- | +| `{network_id}` | `gam_network_id` | +| `{slot_id}` | slot `id` | +| `{section}` | first path segment; `section_root` when path has none | + +### Resolution model + +```text +startup (prepare_runtime, once): + for each slot: + parse slot.gam_unit_path (if Some) into a template: + reject unknown placeholder, unmatched/nested brace, empty template + cache the parsed template (serde-skipped, like compiled_patterns) + if any slot's template contains {section}: + require section_root present AND matching ^[A-Za-z0-9_-]+$ + +request (per matched slot, path known): + if slot has a parsed template: + section = first non-empty segment of the RAW path, + runs of [^A-Za-z0-9_-] replaced with a single '_'; + section_root when the path has no segment ("/", repeated slashes) + render template + else: + "/{network_id}/{slot_id}" # existing default (back-compat) +``` + +### Section derivation rules (deterministic) + +- Extract the **first non-empty** path segment. +- Replace each run of disallowed characters (`[^A-Za-z0-9_-]`) with a single + `_`. Guarantees a non-empty result for any non-empty segment. Because the path + is **not** decoded, `new%20s` → `new_20s` (only `%` is disallowed; `2` and `0` + are alphanumeric) — never silently `news`, and never the decoded `new_s`. +- Use `section_root` **only** when there is no segment (`/`, repeated slashes). +- Derive from the **raw, undecoded** path — the same string `page_patterns` + glob-match against — so matching and derivation never disagree. Percent-encoded + segments are **not** decoded. +- `section_root` validated at startup: non-empty, entirely `[A-Za-z0-9_-]`. + +### Back-compat + +- No template placeholders in a slot's `gam_unit_path` → used verbatim. +- No `gam_unit_path` set on a slot → `/{network_id}/{slot_id}` (unchanged). +- A config with no `{section}` anywhere never requires `section_root`. + +### Validation moves from render to parse + +`validate_runtime` currently calls `resolved_gam_unit_path` and rejects an empty +result. That check becomes path-dependent under templating, so it is replaced by +**startup template validation**: the template parses, all placeholders are +known, and `section_root` is present when `{section}` is used. The rendered +result is non-empty by construction (literals plus non-empty substitutions, or +the `/{network_id}/{slot_id}` default), so no per-request emptiness check is +needed. + +## Alternatives considered + +- **Named sections** (`[section.NAME]` blocks carrying patterns + unit): more + general (expresses irregular units) but forces enumerating every section, and + centralizes patterns — a bigger change that overlaps the deferred + slot-defaults concern. Rejected as the base; the `unit`-override variant is a + possible future extension. +- **Explicit `unit_by_pattern` map per slot** (issue option 2): fully + data-driven but repeats the section→unit table inside every slot, so adding a + section still edits all N slots. Rejected. +- **Hardcoded first-segment derivation** (issue option 3, literal): smallest, + but bakes one site's URL convention into core, which the issue forbids. The + chosen design keeps the one publisher-specific knob (`section_root`) in config. + +## Risks + +- **Client-influenced path.** `{section}` is derived from a request path the + client controls (especially the SPA `path` param). Mitigated by: sanitizing to + `[A-Za-z0-9_-]`; deriving only for paths that already matched a slot's + `page_patterns`; and the fact that `gam_unit_path` is not in the bid request, + so a crafted section only affects the caller's own `defineSlot`. +- **Two render paths drift.** Initial-render and SPA must produce identical + units for the same path. Covered by an equivalence test. + +## Acceptance criteria + +- [ ] N slots × M sections without N×M rules. +- [ ] Resolution tested: `/`, single-segment, multi-segment, no-match, encoded + segment. +- [ ] Existing static `gam_unit_path` configs unchanged. +- [ ] `validate()` (startup) catches empty/unknown/malformed template and a + `{section}` template with missing/invalid `section_root`. +- [ ] `{section}` sanitized to `[A-Za-z0-9_-]`, derived from the raw path. +- [ ] Documented in `docs/guide/configuration.md`, including unmatched-route + behavior and the no-decode rule; example and live autoblog configs updated. + +## Sibling issue (not built here) + +"creative_opportunities: support shared slot defaults for `page_patterns` and +`gam_unit_path`." Inheritance of `page_patterns` must materialize onto each slot +at startup **before** `compile_slots()` (because `match_slots` never sees the +top-level config), which is the lifecycle subtlety this scoped design avoids. From 070fd944b6cb08a667d885fc1277adb4aaa33ca3 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 23 Jul 2026 15:13:39 +0530 Subject: [PATCH 100/198] Document per-section gam_unit_path templating --- docs/guide/configuration.md | 75 +++++++++++++++++++++++++++++++++++++ trusted-server.example.toml | 22 +++++++++++ 2 files changed, 97 insertions(+) diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index b975590c6..f5b778a15 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -1229,6 +1229,81 @@ TRUSTED_SERVER__AUCTION__TIMEOUT_MS=2000 TRUSTED_SERVER__AUCTION__CREATIVE_STORE=creative_store ``` +## Creative Opportunities Configuration + +### `[creative_opportunities]` + +Defines the ad slots the trusted server offers on a page: which pages each slot +appears on (`page_patterns`), its supported sizes (`formats`), and the GAM ad +unit it maps to (`gam_unit_path`). + +```toml +[creative_opportunities] +gam_network_id = "123456789" +price_granularity = "dense" + +# Shared placeholder value for the site root ("/") — see {section} below. +section_root = "home" + +[[creative_opportunities.slot]] +id = "ad-header" +gam_unit_path = "/{network_id}/example/{section}" +page_patterns = ["/", "/news/*", "/reviews/*"] +formats = [{ width = 728, height = 90 }] +``` + +### `gam_unit_path` templating + +`gam_unit_path` is a template. A publisher whose ad unit varies by site section +expresses that in **one** slot rule instead of one rule per (slot × section). + +Supported placeholders: + +| Placeholder | Resolves to | +| -------------- | -------------------------------------------------------------- | +| `{network_id}` | `gam_network_id` | +| `{slot_id}` | the slot's `id` | +| `{section}` | first path segment of the request (see derivation rules below) | + +A template with **no** placeholders is used verbatim. A slot with **no** +`gam_unit_path` falls back to `//`. Both preserve the +pre-templating behavior, so existing static configs are unchanged. + +### `{section}` derivation + +`{section}` is derived from the request path at request time: + +- It is the **first non-empty path segment**. `/news/article-123` → `news`. +- It is sanitized: each run of characters outside `[A-Za-z0-9_-]` becomes a + single `_`. +- The path is used **raw — it is not percent-decoded**. So `/new%20s` → + `new_20s` (only `%` is disallowed; `2` and `0` are kept), never the decoded + `new_s`. This keeps `{section}` consistent with how `page_patterns` match the + same raw path. +- When the path has no segment (`/`, or repeated slashes), `{section}` is + `section_root`. + +`section_root` is **required** whenever any slot's template uses `{section}`, +and must match `[A-Za-z0-9_-]+`. There is no default: the home-section name is +publisher-specific, so the URL→section convention lives in config, not core. +Startup fails if `{section}` is used without a valid `section_root`. + +Example resolution for `gam_unit_path = "/{network_id}/example/{section}"` with +`gam_network_id = "123456789"` and `section_root = "home"`: + +| Request path | `gam_unit_path` | +| --------------- | ---------------------------- | +| `/` | `/123456789/example/home` | +| `/news` | `/123456789/example/news` | +| `/news/article` | `/123456789/example/news` | +| `/reviews/x` | `/123456789/example/reviews` | + +An **unmatched route** — a path matched by no slot's `page_patterns` — produces +no slot at all, so no template is rendered for it. + +Startup validation rejects a malformed template: an unknown placeholder (e.g. +`{oops}`), an unmatched or nested `{`, a stray `}`, or an empty `gam_unit_path`. + ## Fastly Runtime Config Store After the EdgeZero cutover, the Fastly adapter always dispatches through the diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 26d95d681..e2f8994f6 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -157,7 +157,29 @@ gam_network_id = "123456789" auction_timeout_ms = 500 # override via TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__AUCTION_TIMEOUT_MS price_granularity = "dense" +# `gam_unit_path` may be a template. Supported placeholders: +# {network_id} -> gam_network_id +# {slot_id} -> the slot's id +# {section} -> first path segment of the request, sanitized to +# [A-Za-z0-9_-]; `section_root` below is used for "/". +# A template with no placeholders (or an absent gam_unit_path) keeps the old +# behavior: verbatim path, or the default `//`. +# +# `section_root` is REQUIRED when any slot's template uses {section}. There is no +# default — the home-section name is publisher-specific. Must be [A-Za-z0-9_-]+. +section_root = "home" + # No slot templates are enabled in the checked-in default config. Add # `[[creative_opportunities.slot]]` entries via private config or override the # entire array via: # TRUSTED_SERVER__CREATIVE_OPPORTUNITIES__SLOT='[{"id":"...","gam_unit_path":"...",...}]' +# +# Example templated slot (one rule serves every section): +# [[creative_opportunities.slot]] +# id = "ad-header" +# gam_unit_path = "/{network_id}/example/{section}" +# page_patterns = ["/", "/news/*", "/reviews/*"] +# formats = [{ width = 728, height = 90 }] +# "/" -> /123456789/example/home +# "/news/x" -> /123456789/example/news +# "/reviews/y" -> /123456789/example/reviews From 20f5f8e3a6b4ac4d4e7711f1f5c0d7ee08bea3a2 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Thu, 23 Jul 2026 15:23:17 +0530 Subject: [PATCH 101/198] Use fictional example values in tests and docs --- .../src/creative_opportunities.rs | 50 ++++++++++++------- crates/trusted-server-core/src/publisher.rs | 17 ++++--- .../2026-07-23-per-section-gam-unit-path.md | 32 ++++++------ ...-07-23-per-section-gam-unit-path-design.md | 6 +-- 4 files changed, 61 insertions(+), 44 deletions(-) diff --git a/crates/trusted-server-core/src/creative_opportunities.rs b/crates/trusted-server-core/src/creative_opportunities.rs index 59ce6d46b..cc1ac67a3 100644 --- a/crates/trusted-server-core/src/creative_opportunities.rs +++ b/crates/trusted-server-core/src/creative_opportunities.rs @@ -325,7 +325,10 @@ impl CreativeOpportunitySlot { if let Some(raw) = &self.gam_unit_path && raw.trim().is_empty() { - return Err(format!("slot `{}` gam_unit_path must not be empty", self.id)); + return Err(format!( + "slot `{}` gam_unit_path must not be empty", + self.id + )); } Ok(()) @@ -750,17 +753,17 @@ mod tests { #[test] fn parse_unit_template_accepts_known_placeholders() { - let parts = parse_unit_template("/{network_id}/autoblog/{section}") + let parts = parse_unit_template("/{network_id}/example/{section}") .expect("should parse valid template"); assert_eq!(parts.len(), 4, "should split into literal+ph+literal+ph"); } #[test] fn parse_unit_template_accepts_static_path() { - let parts = parse_unit_template("/88059007/autoblog/homepage") + let parts = parse_unit_template("/99999/example/homepage") .expect("should parse a static path as a single literal"); assert!( - matches!(parts.as_slice(), [UnitTemplatePart::Literal(s)] if s == "/88059007/autoblog/homepage"), + matches!(parts.as_slice(), [UnitTemplatePart::Literal(s)] if s == "/99999/example/homepage"), "should be one literal part" ); } @@ -769,7 +772,10 @@ mod tests { fn parse_unit_template_rejects_unknown_placeholder() { let err = parse_unit_template("/{network_id}/{oops}") .expect_err("should reject unknown placeholder"); - assert!(err.contains("oops"), "error should name the bad placeholder"); + assert!( + err.contains("oops"), + "error should name the bad placeholder" + ); } #[test] @@ -791,8 +797,8 @@ mod tests { #[test] fn derive_section_uses_first_segment() { assert_eq!(derive_section("/news", "home"), "news"); - assert_eq!(derive_section("/news/gm-cadillac", "home"), "news"); - assert_eq!(derive_section("/car-research/x", "home"), "car-research"); + assert_eq!(derive_section("/news/article-123", "home"), "news"); + assert_eq!(derive_section("/my-section/x", "home"), "my-section"); } #[test] @@ -817,11 +823,13 @@ mod tests { assert_eq!(derive_section("/%%%/x", "home"), "_"); } - fn make_config_with_section_template(section_root: Option<&str>) -> CreativeOpportunitiesConfig { + fn make_config_with_section_template( + section_root: Option<&str>, + ) -> CreativeOpportunitiesConfig { let mut slot = make_slot("ad-header-0", vec!["/news/*"]); - slot.gam_unit_path = Some("/{network_id}/autoblog/{section}".to_string()); + slot.gam_unit_path = Some("/{network_id}/example/{section}".to_string()); CreativeOpportunitiesConfig { - gam_network_id: "88059007".to_string(), + gam_network_id: "99999".to_string(), auction_timeout_ms: None, price_granularity: PriceGranularity::default(), section_root: section_root.map(str::to_string), @@ -832,11 +840,12 @@ mod tests { #[test] fn render_gam_unit_path_substitutes_placeholders() { let mut slot = make_slot("ad-header-0", vec!["/news/*"]); - slot.gam_unit_path = Some("/{network_id}/autoblog/{section}".to_string()); - slot.compile_unit_template().expect("should compile template"); + slot.gam_unit_path = Some("/{network_id}/example/{section}".to_string()); + slot.compile_unit_template() + .expect("should compile template"); assert_eq!( - slot.render_gam_unit_path("88059007", "news"), - "/88059007/autoblog/news" + slot.render_gam_unit_path("99999", "news"), + "/99999/example/news" ); } @@ -844,8 +853,12 @@ mod tests { fn render_gam_unit_path_defaults_when_no_template() { let mut slot = make_slot("sidebar", vec!["/*"]); slot.gam_unit_path = None; - slot.compile_unit_template().expect("should compile (no template)"); - assert_eq!(slot.render_gam_unit_path("99999", "ignored"), "/99999/sidebar"); + slot.compile_unit_template() + .expect("should compile (no template)"); + assert_eq!( + slot.render_gam_unit_path("99999", "ignored"), + "/99999/sidebar" + ); } #[test] @@ -870,7 +883,10 @@ mod tests { let err = config .validate_runtime() .expect_err("should require section_root"); - assert!(err.contains("section_root"), "error should mention section_root"); + assert!( + err.contains("section_root"), + "error should mention section_root" + ); } #[test] diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 90bdc6c1d..e8de8582c 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -4304,7 +4304,7 @@ mod tests { .collect(), providers: Default::default(), compiled_patterns: Vec::new(), - compiled_unit: None, + compiled_unit: None, } } @@ -4373,18 +4373,19 @@ mod tests { let mut config = make_config(); config.section_root = Some("homepage".to_string()); let mut slot = make_slot(); - slot.gam_unit_path = Some("/{network_id}/autoblog/{section}".to_string()); - slot.compile_unit_template().expect("template should compile"); + slot.gam_unit_path = Some("/{network_id}/example/{section}".to_string()); + slot.compile_unit_template() + .expect("template should compile"); - let news = crate::publisher::build_slot_json(&slot, &config, "/news/gm-cadillac"); + let news = crate::publisher::build_slot_json(&slot, &config, "/news/article-123"); assert_eq!( - news["gam_unit_path"], "/21765378893/autoblog/news", + news["gam_unit_path"], "/21765378893/example/news", "section should derive from the first path segment" ); let home = crate::publisher::build_slot_json(&slot, &config, "/"); assert_eq!( - home["gam_unit_path"], "/21765378893/autoblog/homepage", + home["gam_unit_path"], "/21765378893/example/homepage", "root path should use section_root" ); } @@ -4973,7 +4974,7 @@ mod tests { targeting: Default::default(), providers: Default::default(), compiled_patterns: Vec::new(), - compiled_unit: None, + compiled_unit: None, }] } @@ -5476,7 +5477,7 @@ mod tests { targeting: Default::default(), providers: Default::default(), compiled_patterns: Vec::new(), - compiled_unit: None, + compiled_unit: None, }] } diff --git a/docs/superpowers/plans/2026-07-23-per-section-gam-unit-path.md b/docs/superpowers/plans/2026-07-23-per-section-gam-unit-path.md index 375795a6b..cbd4a2ed1 100644 --- a/docs/superpowers/plans/2026-07-23-per-section-gam-unit-path.md +++ b/docs/superpowers/plans/2026-07-23-per-section-gam-unit-path.md @@ -40,7 +40,7 @@ client keeps receiving a resolved `gam_unit_path` string, so no JS change. - Modify: `crates/trusted-server-core/src/settings.rs` - `prepare_runtime` calls `compile_unit_templates` and surfaces parse errors - Modify: `docs/guide/configuration.md` (add creative_opportunities section) -- Modify: `trusted-server.example.toml` and the live autoblog config +- Modify: `trusted-server.example.toml` and the live example config Notes on lifecycle: `page_patterns` inheritance is **out of scope** (sibling issue). Templates are parsed at startup and cached with `#[serde(skip)]`, @@ -62,17 +62,17 @@ Add to `mod tests`: ```rust #[test] fn parse_unit_template_accepts_known_placeholders() { - let parts = parse_unit_template("/{network_id}/autoblog/{section}") + let parts = parse_unit_template("/{network_id}/example/{section}") .expect("should parse valid template"); assert_eq!(parts.len(), 4, "should split into literal+ph+literal+ph"); } #[test] fn parse_unit_template_accepts_static_path() { - let parts = parse_unit_template("/88059007/autoblog/homepage") + let parts = parse_unit_template("/99999/example/homepage") .expect("should parse a static path as a single literal"); assert!( - matches!(parts.as_slice(), [UnitTemplatePart::Literal(s)] if s == "/88059007/autoblog/homepage"), + matches!(parts.as_slice(), [UnitTemplatePart::Literal(s)] if s == "/99999/example/homepage"), "should be one literal part" ); } @@ -202,8 +202,8 @@ git commit -m "Add gam_unit_path template parser" #[test] fn derive_section_uses_first_segment() { assert_eq!(derive_section("/news", "home"), "news"); - assert_eq!(derive_section("/news/gm-cadillac", "home"), "news"); - assert_eq!(derive_section("/car-research/x", "home"), "car-research"); + assert_eq!(derive_section("/news/article-123", "home"), "news"); + assert_eq!(derive_section("/my-section/x", "home"), "my-section"); } #[test] @@ -298,11 +298,11 @@ git commit -m "Add request-path section derivation" #[test] fn render_gam_unit_path_substitutes_placeholders() { let mut slot = make_slot("ad-header-0", vec!["/news/*"]); - slot.gam_unit_path = Some("/{network_id}/autoblog/{section}".to_string()); + slot.gam_unit_path = Some("/{network_id}/example/{section}".to_string()); slot.compile_unit_template().expect("should compile template"); assert_eq!( - slot.render_gam_unit_path("88059007", "news"), - "/88059007/autoblog/news" + slot.render_gam_unit_path("99999", "news"), + "/99999/example/news" ); } @@ -362,9 +362,9 @@ style in this module): ```rust fn make_config_with_section_template(section_root: Option<&str>) -> CreativeOpportunitiesConfig { let mut slot = make_slot("ad-header-0", vec!["/news/*"]); - slot.gam_unit_path = Some("/{network_id}/autoblog/{section}".to_string()); + slot.gam_unit_path = Some("/{network_id}/example/{section}".to_string()); CreativeOpportunitiesConfig { - gam_network_id: "88059007".to_string(), + gam_network_id: "99999".to_string(), auction_timeout_ms: None, price_granularity: PriceGranularity::default(), section_root: section_root.map(str::to_string), @@ -545,14 +545,14 @@ in that module): ```rust #[test] fn build_slot_json_renders_section_from_request_path() { - let config = creative_opportunities_config_with_template(); // gam_unit_path = "/{network_id}/autoblog/{section}", section_root = "homepage" + let config = creative_opportunities_config_with_template(); // gam_unit_path = "/{network_id}/example/{section}", section_root = "homepage" let slot = &config.slot[0]; - let news = build_slot_json(slot, &config, "/news/gm-cadillac"); - assert_eq!(news["gam_unit_path"], "/88059007/autoblog/news"); + let news = build_slot_json(slot, &config, "/news/article-123"); + assert_eq!(news["gam_unit_path"], "/99999/example/news"); let home = build_slot_json(slot, &config, "/"); - assert_eq!(home["gam_unit_path"], "/88059007/autoblog/homepage"); + assert_eq!(home["gam_unit_path"], "/99999/example/homepage"); } ``` @@ -663,7 +663,7 @@ git commit -m "Render gam_unit_path template per request across initial and SPA - Modify: `docs/guide/configuration.md` - Modify: `trusted-server.example.toml` -- Modify: the live autoblog `trusted-server.toml` (operator-owned, gitignored — update locally, do not commit) +- Modify: the live example `trusted-server.toml` (operator-owned, gitignored — update locally, do not commit) - [ ] **Step 1: Add a creative_opportunities section to configuration.md** diff --git a/docs/superpowers/specs/2026-07-23-per-section-gam-unit-path-design.md b/docs/superpowers/specs/2026-07-23-per-section-gam-unit-path-design.md index 39e25d42f..b18c1bcb0 100644 --- a/docs/superpowers/specs/2026-07-23-per-section-gam-unit-path-design.md +++ b/docs/superpowers/specs/2026-07-23-per-section-gam-unit-path-design.md @@ -92,14 +92,14 @@ extension that does **not** change the config shape below. ```toml [creative_opportunities] -gam_network_id = "88059007" +gam_network_id = "99999" auction_timeout_ms = 2000 price_granularity = "dense" section_root = "homepage" # required when a template uses {section} [[creative_opportunities.slot]] id = "ad-header-0" -gam_unit_path = "/{network_id}/autoblog/{section}" +gam_unit_path = "/{network_id}/example/{section}" page_patterns = ["/", "/news/*", "/reviews/*", "/deals/*"] formats = [{ width = 970, height = 90 }, { width = 728, height = 90 }] [creative_opportunities.slot.providers.prebid] @@ -198,7 +198,7 @@ needed. `{section}` template with missing/invalid `section_root`. - [ ] `{section}` sanitized to `[A-Za-z0-9_-]`, derived from the raw path. - [ ] Documented in `docs/guide/configuration.md`, including unmatched-route - behavior and the no-decode rule; example and live autoblog configs updated. + behavior and the no-decode rule; example and live example configs updated. ## Sibling issue (not built here) From ca131e0c28a55d01304bd3cb993cc2a3fd53815e Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 23 Jul 2026 11:56:34 -0500 Subject: [PATCH 102/198] Add privacy-safe auction-to-creative tracing Introduce query-activated diagnostic sessions, trace identities and telemetry, bounded GPT and Prebid render ownership, creative acknowledgements, and the browser timeline overlay. Add deterministic integration fixtures and coverage across edge adapters and browser render paths. --- .../setup-integration-test-env/action.yml | 29 + .github/workflows/integration-tests.yml | 23 +- crates/trusted-server-adapter-axum/src/app.rs | 3 +- .../src/middleware.rs | 46 + .../src/app.rs | 3 +- .../src/middleware.rs | 45 + .../trusted-server-adapter-fastly/src/app.rs | 3 +- .../trusted-server-adapter-fastly/src/main.rs | 2 + .../src/middleware.rs | 44 + .../src/tinybird.rs | 1 + crates/trusted-server-adapter-spin/src/app.rs | 9 +- .../src/middleware.rs | 38 +- .../benches/html_processor_bench.rs | 1 + .../src/auction/endpoints.rs | 42 +- .../src/auction/formats.rs | 204 +++- .../src/auction/orchestrator.rs | 493 ++++++--- .../src/auction/telemetry.rs | 222 ++-- .../src/auction/test_support.rs | 10 +- .../trusted-server-core/src/auction/types.rs | 147 +++ crates/trusted-server-core/src/config.rs | 11 +- crates/trusted-server-core/src/constants.rs | 4 + .../trusted-server-core/src/html_processor.rs | 28 +- .../src/integrations/ad_trace.rs | 620 +++++++++++ .../src/integrations/gpt_bootstrap.js | 54 + .../src/integrations/mod.rs | 5 + .../src/integrations/prebid.rs | 23 +- crates/trusted-server-core/src/openrtb.rs | 42 + crates/trusted-server-core/src/publisher.rs | 394 +++++-- .../src/response_privacy.rs | 12 +- .../browser/global-setup.ts | 13 +- .../browser/helpers/infra.ts | 1 + .../browser/helpers/state.ts | 2 +- .../browser/package.json | 1 + .../browser/playwright.config.ts | 8 +- .../tests/ad-trace/auction-trace.spec.ts | 439 ++++++++ .../tests/shared/ad-trace-gate.spec.ts | 20 + .../trusted-server.ad-trace.integration.toml | 67 ++ .../fixtures/frameworks/ad-trace/Dockerfile | 15 + .../frameworks/ad-trace/public/index.php | 201 ++++ .../frameworks/ad-trace/public/router.php | 50 + .../tests/parity.rs | 71 ++ .../lib/src/core/ad_trace.ts | 505 +++++++++ .../trusted-server-js/lib/src/core/auction.ts | 144 ++- .../lib/src/core/global.d.ts | 2 + .../trusted-server-js/lib/src/core/request.ts | 196 +++- .../trusted-server-js/lib/src/core/types.ts | 202 +++- .../lib/src/integrations/ad_trace/index.ts | 98 ++ .../lib/src/integrations/ad_trace/overlay.ts | 231 ++++ .../lib/src/integrations/gpt/index.ts | 983 +++++++++++++++++- .../lib/src/integrations/prebid/index.ts | 175 +++- .../lib/test/core/ad_trace.test.ts | 332 ++++++ .../lib/test/core/auction.test.ts | 153 ++- .../lib/test/core/request.test.ts | 142 ++- .../test/integrations/ad_trace/index.test.ts | 41 + .../integrations/ad_trace/overlay.test.ts | 110 ++ .../lib/test/integrations/gpt/ad_init.test.ts | 211 +++- .../test/integrations/gpt/ad_trace.test.ts | 327 ++++++ .../lib/test/integrations/gpt/index.test.ts | 31 +- .../test/integrations/prebid/index.test.ts | 83 ++ docs/guide/configuration.md | 17 + .../generate-integration-viceroy-configs.sh | 7 + scripts/integration-tests-browser.sh | 30 +- .../datasources/auction_events_raw.datasource | 1 + tinybird/fixtures/auction_events_raw.ndjson | 2 +- trusted-server.example.toml | 5 + 65 files changed, 6904 insertions(+), 570 deletions(-) create mode 100644 crates/trusted-server-core/src/integrations/ad_trace.rs create mode 100644 crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts create mode 100644 crates/trusted-server-integration-tests/browser/tests/shared/ad-trace-gate.spec.ts create mode 100644 crates/trusted-server-integration-tests/fixtures/configs/trusted-server.ad-trace.integration.toml create mode 100644 crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/Dockerfile create mode 100644 crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/index.php create mode 100644 crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/router.php create mode 100644 crates/trusted-server-js/lib/src/core/ad_trace.ts create mode 100644 crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts create mode 100644 crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts create mode 100644 crates/trusted-server-js/lib/test/core/ad_trace.test.ts create mode 100644 crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts create mode 100644 crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts create mode 100644 crates/trusted-server-js/lib/test/integrations/gpt/ad_trace.test.ts diff --git a/.github/actions/setup-integration-test-env/action.yml b/.github/actions/setup-integration-test-env/action.yml index 841d8d5bd..12c41502a 100644 --- a/.github/actions/setup-integration-test-env/action.yml +++ b/.github/actions/setup-integration-test-env/action.yml @@ -93,6 +93,26 @@ runs: TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: "false" run: cargo build -p trusted-server-adapter-axum + - name: Set up Node.js for browser fixtures + if: ${{ inputs.build-test-images == 'true' }} + uses: actions/setup-node@v4 + with: + node-version: ${{ steps.node-version.outputs.node-version }} + cache: npm + cache-dependency-path: crates/trusted-server-js/lib/package-lock.json + + - name: Build external Prebid fixture bundle + if: ${{ inputs.build-test-images == 'true' }} + shell: bash + run: | + rm -rf "$GITHUB_WORKSPACE/target/integration-test-artifacts/prebid" + mkdir -p "$GITHUB_WORKSPACE/target/integration-test-artifacts/prebid" + npm ci --prefix crates/trusted-server-js/lib + npm run --prefix crates/trusted-server-js/lib build:prebid-external -- \ + --adapters=rubicon \ + --user-id-modules=sharedIdSystem \ + --out "$GITHUB_WORKSPACE/target/integration-test-artifacts/prebid" + - name: Build WordPress test container if: ${{ inputs.build-test-images == 'true' }} shell: bash @@ -109,6 +129,15 @@ runs: -t test-nextjs:latest \ crates/trusted-server-integration-tests/fixtures/frameworks/nextjs/ + - name: Build ad-trace test container + if: ${{ inputs.build-test-images == 'true' }} + shell: bash + run: | + docker build \ + -f crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/Dockerfile \ + -t test-ad-trace:latest \ + . + - name: Add wasm32-unknown-unknown target for Cloudflare build if: ${{ inputs.build-cloudflare == 'true' }} shell: bash diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index ec85b96ac..da2c8c262 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -46,7 +46,7 @@ jobs: cp -r crates/trusted-server-adapter-cloudflare/build/. "$CF_BUILD_ARTIFACT_PATH/" docker save \ --output "$DOCKER_ARTIFACT_PATH" \ - test-wordpress:latest test-nextjs:latest + test-ad-trace:latest test-wordpress:latest test-nextjs:latest - name: Upload integration test artifacts uses: actions/upload-artifact@v4 @@ -228,10 +228,29 @@ jobs: path: crates/trusted-server-integration-tests/browser/playwright-report-wordpress/ retention-days: 7 + - name: Run browser tests (ad trace contract) + if: always() + working-directory: crates/trusted-server-integration-tests/browser + env: + WASM_BINARY_PATH: ${{ env.WASM_ARTIFACT_PATH }} + INTEGRATION_ORIGIN_PORT: ${{ env.ORIGIN_PORT }} + VICEROY_CONFIG_PATH: ${{ env.ARTIFACTS_DIR }}/configs/viceroy-ad-trace.toml + TEST_FRAMEWORK: ad-trace + PLAYWRIGHT_HTML_REPORT: playwright-report-ad-trace + run: npx playwright test tests/ad-trace/auction-trace.spec.ts + + - name: Upload Playwright report (ad trace contract) + uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-report-ad-trace + path: crates/trusted-server-integration-tests/browser/playwright-report-ad-trace/ + retention-days: 7 + - name: Upload Playwright traces and screenshots uses: actions/upload-artifact@v4 if: failure() with: name: playwright-traces - path: crates/trusted-server-integration-tests/browser/test-results/ + path: crates/trusted-server-integration-tests/browser/test-results-*/ retention-days: 7 diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 2f4329574..fe18e3604 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -32,7 +32,7 @@ use trusted_server_core::settings_data::{ use trusted_server_core::platform::RuntimeServices; -use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; +use crate::middleware::{AdTracePrepareMiddleware, AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{AxumPlatformConfigStore, build_runtime_services}; // --------------------------------------------------------------------------- @@ -540,6 +540,7 @@ fn build_router(state: &Arc) -> RouterService { let mut router = RouterService::builder() .middleware(FinalizeResponseMiddleware::new(Arc::clone(&state.settings))) + .middleware(AdTracePrepareMiddleware::new(Arc::clone(&state.settings))) .middleware(AuthMiddleware::new(Arc::clone(&state.settings))); router = router.route("/health", Method::GET, |_ctx: RequestContext| async { diff --git a/crates/trusted-server-adapter-axum/src/middleware.rs b/crates/trusted-server-adapter-axum/src/middleware.rs index 45cbedc2c..f3ea0d198 100644 --- a/crates/trusted-server-adapter-axum/src/middleware.rs +++ b/crates/trusted-server-adapter-axum/src/middleware.rs @@ -5,6 +5,7 @@ use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; +use edgezero_core::response::IntoResponse; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; use trusted_server_core::settings::Settings; @@ -38,6 +39,51 @@ impl Middleware for FinalizeResponseMiddleware { async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { let mut response = next.run(ctx).await?; apply_finalize_headers(&self.settings, &mut response); + trusted_server_core::integrations::ad_trace::finalize_response(&mut response); + Ok(response) + } +} + +// --------------------------------------------------------------------------- +// AdTracePrepareMiddleware +// --------------------------------------------------------------------------- + +/// Prepares and sanitizes the request before auth, routing, or downstream use. +pub struct AdTracePrepareMiddleware { + settings: Arc, +} + +impl AdTracePrepareMiddleware { + #[must_use] + pub fn new(settings: Arc) -> Self { + Self { settings } + } +} + +#[async_trait(?Send)] +impl Middleware for AdTracePrepareMiddleware { + async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { + let decision = match trusted_server_core::integrations::ad_trace::prepare_request( + &self.settings, + ctx.request_mut(), + ) { + Ok(decision) => decision, + Err(report) => { + log::error!("ad trace request preparation failed: {report:?}"); + return Ok(crate::app::http_error(&report)); + } + }; + let mut response = match next.run(ctx).await { + Ok(response) => response, + Err(error) => { + log::error!("request handler failed after ad trace preparation: {error:?}"); + error.into_response()? + } + }; + trusted_server_core::integrations::ad_trace::attach_response_decision( + &decision, + &mut response, + ); Ok(response) } } diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index c931360f6..798e2a2e0 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -29,7 +29,7 @@ use trusted_server_core::request_signing::{ }; use trusted_server_core::settings::Settings; -use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; +use crate::middleware::{AdTracePrepareMiddleware, AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::build_runtime_services; // --------------------------------------------------------------------------- @@ -432,6 +432,7 @@ fn build_router(state: &Arc) -> RouterService { let mut router = RouterService::builder() .middleware(FinalizeResponseMiddleware::new(Arc::clone(&state.settings))) + .middleware(AdTracePrepareMiddleware::new(Arc::clone(&state.settings))) .middleware(AuthMiddleware::new(Arc::clone(&state.settings))) .get( "/.well-known/trusted-server.json", diff --git a/crates/trusted-server-adapter-cloudflare/src/middleware.rs b/crates/trusted-server-adapter-cloudflare/src/middleware.rs index 5b605bcff..de15ac62b 100644 --- a/crates/trusted-server-adapter-cloudflare/src/middleware.rs +++ b/crates/trusted-server-adapter-cloudflare/src/middleware.rs @@ -5,6 +5,7 @@ use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; +use edgezero_core::response::IntoResponse; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; use trusted_server_core::settings::Settings; @@ -46,6 +47,50 @@ impl Middleware for FinalizeResponseMiddleware { let mut response = next.run(ctx).await?; apply_finalize_headers(&self.settings, geo_available, &mut response); + trusted_server_core::integrations::ad_trace::finalize_response(&mut response); + Ok(response) + } +} + +// --------------------------------------------------------------------------- +// AdTracePrepareMiddleware +// --------------------------------------------------------------------------- + +pub struct AdTracePrepareMiddleware { + settings: Arc, +} + +impl AdTracePrepareMiddleware { + #[must_use] + pub fn new(settings: Arc) -> Self { + Self { settings } + } +} + +#[async_trait(?Send)] +impl Middleware for AdTracePrepareMiddleware { + async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { + let decision = match trusted_server_core::integrations::ad_trace::prepare_request( + &self.settings, + ctx.request_mut(), + ) { + Ok(decision) => decision, + Err(report) => { + log::error!("ad trace request preparation failed: {report:?}"); + return Ok(crate::app::http_error(&report)); + } + }; + let mut response = match next.run(ctx).await { + Ok(response) => response, + Err(error) => { + log::error!("request handler failed after ad trace preparation: {error:?}"); + error.into_response()? + } + }; + trusted_server_core::integrations::ad_trace::attach_response_decision( + &decision, + &mut response, + ); Ok(response) } } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 955ff235b..841d1b731 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -129,7 +129,7 @@ use trusted_server_core::settings_data::{ }; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; -use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; +use crate::middleware::{AdTracePrepareMiddleware, AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ FastlyPlatformBackend, FastlyPlatformConfigStore, FastlyPlatformGeo, FastlyPlatformHttpClient, FastlyPlatformSecretStore, UnavailableKvStore, open_kv_store, @@ -1161,6 +1161,7 @@ impl TrustedServerApp { Arc::clone(&state.settings), Arc::new(FastlyPlatformGeo), )) + .middleware(AdTracePrepareMiddleware::new(Arc::clone(&state.settings))) .middleware(AuthMiddleware::new(Arc::clone(&state.settings))); let fallback_handler = fallback_route_handler(Arc::clone(state)); diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index ab3236c0f..8eea9f444 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -337,6 +337,8 @@ fn send_edgezero_response( // added a per-user Set-Cookie after `apply_finalize_headers` ran, so // re-apply the privacy downgrade before send. crate::middleware::enforce_set_cookie_cache_privacy(&mut response); + // Reassert console no-store after asset/EC/filter response mutations. + trusted_server_core::integrations::ad_trace::finalize_response(&mut response); let (parts, body) = response.into_parts(); diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 2c00ac2ff..f6a674301 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -85,6 +85,7 @@ impl Middleware for FinalizeResponseMiddleware { }); apply_finalize_headers(&self.settings, geo_info.as_ref(), &mut response); + trusted_server_core::integrations::ad_trace::finalize_response(&mut response); response .headers_mut() .insert(HEADER_X_TS_FINALIZED, HeaderValue::from_static("1")); @@ -93,6 +94,49 @@ impl Middleware for FinalizeResponseMiddleware { } } +// --------------------------------------------------------------------------- +// AdTracePrepareMiddleware +// --------------------------------------------------------------------------- + +/// Sanitizes and snapshots the console decision before auth and route dispatch. +pub struct AdTracePrepareMiddleware { + settings: Arc, +} + +impl AdTracePrepareMiddleware { + pub fn new(settings: Arc) -> Self { + Self { settings } + } +} + +#[async_trait(?Send)] +impl Middleware for AdTracePrepareMiddleware { + async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { + let decision = match trusted_server_core::integrations::ad_trace::prepare_request( + &self.settings, + ctx.request_mut(), + ) { + Ok(decision) => decision, + Err(report) => { + log::error!("ad trace request preparation failed: {report:?}"); + return Ok(crate::app::http_error(&report)); + } + }; + let mut response = match next.run(ctx).await { + Ok(response) => response, + Err(error) => { + log::error!("request handler failed after ad trace preparation: {error:?}"); + error.into_response()? + } + }; + trusted_server_core::integrations::ad_trace::attach_response_decision( + &decision, + &mut response, + ); + Ok(response) + } +} + // --------------------------------------------------------------------------- // AuthMiddleware // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-fastly/src/tinybird.rs b/crates/trusted-server-adapter-fastly/src/tinybird.rs index 217c167b8..6c5aeea15 100644 --- a/crates/trusted-server-adapter-fastly/src/tinybird.rs +++ b/crates/trusted-server-adapter-fastly/src/tinybird.rs @@ -418,6 +418,7 @@ mod tests { price_cpm: None, currency: None, is_win: None, + bid_trace_id: None, ad_domain: None, ad_id: None, } diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 2291fce74..dbfb5c4fd 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -705,13 +705,10 @@ fn build_router(state: &Arc) -> RouterService { let mut builder = RouterService::builder() .middleware(FinalizeResponseMiddleware::new(Arc::clone(&state.settings))) + // Normalize and sanitize outside auth so even auth short-circuits + // cannot forward reserved console inputs or skip response actions. + .middleware(NormalizeMiddleware::new(Arc::clone(&state.settings))) .middleware(AuthMiddleware::new(Arc::clone(&state.settings))) - // Innermost middleware: normalize every routed request (strip - // spoofable forwarded headers, derive the trusted Host/scheme/client-IP - // from Spin's synthetic runtime headers) so no handler can opt out of - // the de-spoofing invariant. Runs after auth so the basic-auth gate - // continues to see the original request, matching prior behaviour. - .middleware(NormalizeMiddleware::new()) // Cheap liveness probe, matching the Fastly/Axum adapters. Registered // explicitly so it is not absorbed by the publisher `/{*rest}` fallback. .get("/health", |_ctx: RequestContext| async { diff --git a/crates/trusted-server-adapter-spin/src/middleware.rs b/crates/trusted-server-adapter-spin/src/middleware.rs index 1bcede1fc..1f9178057 100644 --- a/crates/trusted-server-adapter-spin/src/middleware.rs +++ b/crates/trusted-server-adapter-spin/src/middleware.rs @@ -5,6 +5,7 @@ use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; +use edgezero_core::response::IntoResponse; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; use trusted_server_core::settings::Settings; @@ -39,6 +40,7 @@ impl Middleware for FinalizeResponseMiddleware { let mut response = next.run(ctx).await?; apply_finalize_headers(&self.settings, geo_available, &mut response); + trusted_server_core::integrations::ad_trace::finalize_response(&mut response); Ok(response) } } @@ -95,16 +97,17 @@ impl Middleware for AuthMiddleware { /// signing handler that begins deriving an issuer/audience from `RequestInfo`, /// cannot silently trust spoofable input by forgetting to opt in. /// -/// Registered after [`AuthMiddleware`] (innermost) so the basic-auth gate still -/// evaluates the original request, preserving prior behaviour. -#[derive(Default)] -pub struct NormalizeMiddleware; +/// Registered outside [`AuthMiddleware`] so de-spoofing and console sanitation +/// also apply when auth short-circuits the request. +pub struct NormalizeMiddleware { + settings: Arc, +} impl NormalizeMiddleware { /// Creates a new [`NormalizeMiddleware`]. #[must_use] - pub fn new() -> Self { - Self + pub fn new(settings: Arc) -> Self { + Self { settings } } } @@ -112,7 +115,28 @@ impl NormalizeMiddleware { impl Middleware for NormalizeMiddleware { async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { crate::app::normalize_spin_request(ctx.request_mut()); - next.run(ctx).await + let decision = match trusted_server_core::integrations::ad_trace::prepare_request( + &self.settings, + ctx.request_mut(), + ) { + Ok(decision) => decision, + Err(report) => { + log::error!("ad trace request preparation failed: {report:?}"); + return Ok(crate::app::http_error(&report)); + } + }; + let mut response = match next.run(ctx).await { + Ok(response) => response, + Err(error) => { + log::error!("request handler failed after ad trace preparation: {error:?}"); + error.into_response()? + } + }; + trusted_server_core::integrations::ad_trace::attach_response_decision( + &decision, + &mut response, + ); + Ok(response) } } diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index 6c7a397b0..24b034ec9 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -9,6 +9,7 @@ fn make_config() -> HtmlProcessorConfig { request_host: "proxy.bench.example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::default(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 1b0ced7a7..eb306dc5d 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -1,7 +1,5 @@ //! HTTP endpoint handlers for auction requests. -use std::collections::HashMap; - use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::{Request, Response, StatusCode, header}; @@ -24,12 +22,12 @@ use crate::platform::RuntimeServices; use crate::settings::Settings; use super::AuctionOrchestrator; -use super::formats::{convert_to_openrtb_response, convert_tsjs_to_auction_request}; +use super::formats::{convert_to_openrtb_response_with_trace, convert_tsjs_to_auction_request}; use super::telemetry::{ AuctionObservationContext, AuctionSource, AuctionTerminalOutcome, build_auction_events, emit_auction_events_best_effort_lazy, }; -use super::types::AuctionContext; +use super::types::{AuctionContext, AuctionPublicOutcome, AuctionTraceContext}; const MAX_CLIENT_EID_SOURCES: usize = 64; const MAX_CLIENT_UIDS_PER_SOURCE: usize = 32; @@ -159,6 +157,8 @@ pub async fn handle_auction( ); let http_req = Request::from_parts(parts, EdgeBody::empty()); + let trace_enabled = crate::integrations::ad_trace::browser_trace_enabled(&http_req); + let trace = AuctionTraceContext::new(AuctionSource::AuctionApi); // Story 5 middleware contract: auction is a read-only EC route. // It must not generate EC IDs; it only consumes pre-routed context. @@ -192,11 +192,8 @@ pub async fn handle_auction( ec_id, None, )?; - let observation = AuctionObservationContext::from_auction_request( - AuctionSource::AuctionApi, - &auction_request, - ec_context, - ); + let observation = + AuctionObservationContext::from_auction_request(&trace, &auction_request, ec_context); emit_auction_events_best_effort_lazy(services, || { build_auction_events( observation, @@ -208,18 +205,13 @@ pub async fn handle_auction( }) .await; - let empty_result = OrchestrationResult { - provider_responses: Vec::new(), - mediator_response: None, - winning_bids: HashMap::new(), - total_time_ms: 0, - metadata: HashMap::new(), - }; - return convert_to_openrtb_response( + let empty_result = OrchestrationResult::empty(trace, AuctionPublicOutcome::Skipped); + return convert_to_openrtb_response_with_trace( &empty_result, settings, &auction_request, ec_context.ec_allowed(), + trace_enabled, ); } @@ -281,6 +273,7 @@ pub async fn handle_auction( // Create auction context let context = AuctionContext { + trace: &trace, settings, request: &http_req, timeout_ms: settings.auction.timeout_ms, @@ -288,11 +281,8 @@ pub async fn handle_auction( services, }; - let observation = AuctionObservationContext::from_auction_request( - AuctionSource::AuctionApi, - &auction_request, - ec_context, - ); + let observation = + AuctionObservationContext::from_auction_request(&trace, &auction_request, ec_context); // Run the auction let result = match orchestrator.run_auction(&auction_request, &context).await { @@ -336,7 +326,13 @@ pub async fn handle_auction( ); // Convert to OpenRTB response format with inline creative HTML - convert_to_openrtb_response(&result, settings, &auction_request, ec_context.ec_allowed()) + convert_to_openrtb_response_with_trace( + &result, + settings, + &auction_request, + ec_context.ec_allowed(), + trace_enabled, + ) } /// Resolves partner EIDs from the KV identity graph for bidstream decoration. diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index 441828a18..0ed059d00 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -19,7 +19,10 @@ use crate::creative; use crate::ec::eids::encode_eids_header; use crate::error::TrustedServerError; use crate::geo::GeoInfo; -use crate::openrtb::{OpenRtbBid, OpenRtbResponse, ResponseExt, SeatBid, ToExt, to_openrtb_i32}; +use crate::openrtb::{ + AuctionTraceWire, BidTraceWire, OpenRtbBid, OpenRtbResponse, ResponseExt, SeatBid, ToExt, + TrustedServerBidExt, TrustedServerBidTraceContainer, TrustedServerResponseExt, to_openrtb_i32, +}; use crate::platform::RuntimeServices; use crate::settings::Settings; @@ -229,6 +232,21 @@ pub fn convert_to_openrtb_response( settings: &Settings, auction_request: &AuctionRequest, ec_allowed: bool, +) -> Result, Report> { + convert_to_openrtb_response_with_trace(result, settings, auction_request, ec_allowed, false) +} + +/// Convert an auction result with optional tester-gated trace extensions. +/// +/// # Errors +/// +/// Returns the same errors as [`convert_to_openrtb_response`]. +pub fn convert_to_openrtb_response_with_trace( + result: &OrchestrationResult, + settings: &Settings, + auction_request: &AuctionRequest, + ec_allowed: bool, + trace_enabled: bool, ) -> Result, Report> { // Build OpenRTB-style seatbid array let mut seatbids = Vec::with_capacity(result.winning_bids.len()); @@ -275,6 +293,24 @@ pub fn convert_to_openrtb_response( String::new() }; + let bid_ext = trace_enabled + .then(|| result.trace.winning_bids.get(slot_id)) + .flatten() + .and_then(|trace| { + TrustedServerBidExt { + trusted_server: TrustedServerBidTraceContainer { + trace: BidTraceWire { + version: 1, + bid_trace_id: trace.bid_trace_id.to_string(), + slot_id: slot_id.clone(), + provider: trace.provider.clone(), + bidder: trace.bidder.clone(), + }, + }, + } + .to_ext() + }); + let openrtb_bid = OpenRtbBid { id: Some(format!("{}-{}", bid.bidder, slot_id)), impid: Some(slot_id.to_string()), @@ -284,6 +320,7 @@ pub fn convert_to_openrtb_response( w: width, h: height, adomain: bid.adomain.clone().unwrap_or_default(), + ext: bid_ext, ..Default::default() }; @@ -319,6 +356,14 @@ pub fn convert_to_openrtb_response( time_ms: result.total_time_ms, provider_details, }, + trusted_server: trace_enabled.then(|| TrustedServerResponseExt { + trace: AuctionTraceWire { + version: 1, + auction_trace_id: result.trace.summary.auction.auction_trace_id.to_string(), + source: result.trace.summary.auction.source.as_str(), + outcome: result.trace.summary.outcome.as_str(), + }, + }), } .to_ext(), ..Default::default() @@ -416,13 +461,14 @@ mod tests { } fn make_empty_result() -> OrchestrationResult { - OrchestrationResult { - provider_responses: Vec::new(), - mediator_response: None, - winning_bids: HashMap::new(), - total_time_ms: 10, - metadata: HashMap::new(), - } + let mut result = OrchestrationResult::empty( + crate::auction::types::AuctionTraceContext::new( + crate::auction::types::AuctionSource::AuctionApi, + ), + crate::auction::types::AuctionPublicOutcome::NoBid, + ); + result.total_time_ms = 10; + result } fn make_bid(slot_id: &str, bidder: &str, price: Option) -> Bid { @@ -446,19 +492,34 @@ mod tests { } fn make_result(bid: Bid) -> OrchestrationResult { - OrchestrationResult { - provider_responses: vec![AuctionResponse { - provider: "prebid".to_string(), - bids: vec![bid.clone()], - status: BidStatus::Success, - response_time_ms: 42, - metadata: HashMap::new(), - }], - mediator_response: None, - winning_bids: HashMap::from([(bid.slot_id.clone(), bid)]), - total_time_ms: 50, + let mut result = make_empty_result(); + result.trace.summary.outcome = crate::auction::types::AuctionPublicOutcome::Completed; + result.trace.winning_bids.insert( + bid.slot_id.clone(), + crate::auction::types::WinningBidTrace { + bid_trace_id: crate::auction::types::BidTraceId::new(), + provider: "prebid".to_owned(), + bidder: bid.bidder.clone(), + }, + ); + result.winning_bid_origins.insert( + bid.slot_id.clone(), + crate::auction::types::WinningBidOrigin { + response_index: 0, + bid_index: 0, + mediated: false, + }, + ); + result.provider_responses = vec![AuctionResponse { + provider: "prebid".to_string(), + bids: vec![bid.clone()], + status: BidStatus::Success, + response_time_ms: 42, metadata: HashMap::new(), - } + }]; + result.winning_bids = HashMap::from([(bid.slot_id.clone(), bid)]); + result.total_time_ms = 50; + result } fn response_json(response: Response) -> JsonValue { @@ -974,17 +1035,69 @@ mod tests { ); } + #[test] + fn gated_response_adds_namespaced_root_and_winning_bid_trace() { + let settings = make_settings(); + let auction_request = make_auction_request(); + let result = make_result(make_bid("div-gpt-top", "appnexus", Some(2.75))); + + let response = convert_to_openrtb_response_with_trace( + &result, + &settings, + &auction_request, + false, + true, + ) + .expect("should convert traced response"); + let json = response_json(response); + + assert_eq!( + json["ext"]["trusted_server"]["trace"]["auction_trace_id"], + json!(result.trace.summary.auction.auction_trace_id.to_string()), + "should expose the shared trace identity" + ); + assert_eq!( + json["seatbid"][0]["bid"][0]["ext"]["trusted_server"]["trace"]["bid_trace_id"], + json!( + result.trace.winning_bids["div-gpt-top"] + .bid_trace_id + .to_string() + ), + "should expose only the final winner trace" + ); + assert_ne!( + json["ext"]["trusted_server"]["trace"]["auction_trace_id"], + json!(auction_request.id), + "should never expose the internal request ID as trace identity" + ); + } + + #[test] + fn ungated_response_omits_all_trace_extensions() { + let settings = make_settings(); + let auction_request = make_auction_request(); + let result = make_result(make_bid("div-gpt-top", "appnexus", Some(2.75))); + + let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) + .expect("should convert legacy response"); + let json = response_json(response); + + assert!( + json["ext"].get("trusted_server").is_none(), + "ungated root should omit trace" + ); + assert!( + json["seatbid"][0]["bid"][0].get("ext").is_none(), + "ungated bid should omit trace" + ); + } + #[test] fn convert_to_openrtb_response_allows_empty_winning_bids() { let settings = make_settings(); let auction_request = make_auction_request(); - let result = OrchestrationResult { - provider_responses: vec![], - mediator_response: None, - winning_bids: HashMap::new(), - total_time_ms: 50, - metadata: HashMap::new(), - }; + let mut result = make_empty_result(); + result.total_time_ms = 50; let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) .expect("should convert auction result without winning bids"); @@ -1009,22 +1122,27 @@ mod tests { let top_bid = make_bid("div-gpt-top", "appnexus", Some(2.75)); let mut sidebar_bid = make_bid("div-gpt-sidebar", "rubicon", Some(1.25)); sidebar_bid.creative = Some("
Sidebar
".to_string()); - let result = OrchestrationResult { - provider_responses: vec![AuctionResponse { - provider: "prebid".to_string(), - bids: vec![top_bid.clone(), sidebar_bid.clone()], - status: BidStatus::Success, - response_time_ms: 42, - metadata: HashMap::new(), - }], - mediator_response: None, - winning_bids: HashMap::from([ - (top_bid.slot_id.clone(), top_bid), - (sidebar_bid.slot_id.clone(), sidebar_bid), - ]), - total_time_ms: 50, - metadata: HashMap::new(), - }; + let mut result = make_result(top_bid.clone()); + result.provider_responses[0].bids.push(sidebar_bid.clone()); + result.trace.winning_bids.insert( + sidebar_bid.slot_id.clone(), + crate::auction::types::WinningBidTrace { + bid_trace_id: crate::auction::types::BidTraceId::new(), + provider: "prebid".to_owned(), + bidder: sidebar_bid.bidder.clone(), + }, + ); + result.winning_bid_origins.insert( + sidebar_bid.slot_id.clone(), + crate::auction::types::WinningBidOrigin { + response_index: 0, + bid_index: 1, + mediated: false, + }, + ); + result + .winning_bids + .insert(sidebar_bid.slot_id.clone(), sidebar_bid); let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) .expect("should convert multiple winning bids"); diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index bee63856f..70b829dcf 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -12,7 +12,11 @@ use crate::platform::{PlatformPendingRequest, RuntimeServices}; use super::config::AuctionConfig; use super::provider::AuctionProvider; use super::telemetry::AbandonedProviderCall; -use super::types::{AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus}; +use super::types::{ + AuctionContext, AuctionPublicOutcome, AuctionRequest, AuctionResponse, AuctionResultTrace, + AuctionTraceContext, AuctionTraceSummary, Bid, BidStatus, BidTraceId, WinningBidOrigin, + WinningBidTrace, +}; /// In-flight auction requests dispatched to SSP backends. /// @@ -22,6 +26,7 @@ use super::types::{AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStat /// race in Fastly's native layer, enabling TTFB ≈ origin latency rather than /// TTFB ≈ auction timeout. pub struct DispatchedAuction { + trace: AuctionTraceContext, pending_requests: Vec, backend_to_provider: HashMap)>, launch_responses: Vec, @@ -50,6 +55,12 @@ pub enum DispatchAuctionOutcome { } impl DispatchedAuction { + /// Return the trace context retained for split-phase collection. + #[must_use] + pub fn trace(&self) -> &AuctionTraceContext { + &self.trace + } + /// Consume the dispatch token without collecting provider responses. #[must_use] pub fn abandon( @@ -79,6 +90,7 @@ impl DispatchedAuction { impl DispatchedAuction { pub(crate) fn empty_for_test(request: AuctionRequest, timeout_ms: u32) -> Self { Self { + trace: AuctionTraceContext::new(super::types::AuctionSource::InitialNavigation), pending_requests: Vec::new(), backend_to_provider: HashMap::new(), launch_responses: Vec::new(), @@ -146,6 +158,39 @@ fn provider_transport_failed_response( .with_metadata("message", serde_json::json!("Provider request failed")) } +fn build_winning_bid_traces( + winning_bids: &HashMap, + origins: &HashMap, + provider_responses: &[AuctionResponse], + mediator_response: Option<&AuctionResponse>, + mut id_source: impl FnMut() -> BidTraceId, +) -> HashMap { + let mut traces = HashMap::with_capacity(winning_bids.len()); + for (slot_id, bid) in winning_bids { + let provider = origins + .get(slot_id) + .and_then(|origin| { + if origin.mediated { + mediator_response.map(|response| response.provider.clone()) + } else { + provider_responses + .get(origin.response_index) + .map(|response| response.provider.clone()) + } + }) + .unwrap_or_else(|| "unattributed".to_owned()); + traces.insert( + slot_id.clone(), + WinningBidTrace { + bid_trace_id: id_source(), + provider, + bidder: bid.bidder.clone(), + }, + ); + } + traces +} + fn provider_timeout_response(provider_name: &str, response_time_ms: u64) -> AuctionResponse { AuctionResponse::error(provider_name, response_time_ms) .with_metadata("error_type", serde_json::json!(ERROR_TYPE_TIMEOUT)) @@ -273,115 +318,100 @@ impl AuctionOrchestrator { let provider_responses = self.run_providers_parallel(request, context).await?; let floor_prices = self.floor_prices_by_slot(request); - let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { - let mediator = self.get_provider(mediator_name)?; - - log::info!( - "Sending {} provider responses to mediator: {}", - provider_responses.len(), - mediator.provider_name() - ); - - // Give the mediator only the remaining time from the auction - // deadline, not the full timeout — the bidding phase already - // consumed part of it. - let remaining_ms = remaining_budget_ms(mediation_start, context.timeout_ms); - - if remaining_ms == 0 { - log::warn!("Auction timeout exhausted during bidding phase; skipping mediator"); - let winning = self.select_winning_bids(&provider_responses, &floor_prices); - return Ok(OrchestrationResult { - provider_responses, - mediator_response: None, - winning_bids: winning, - total_time_ms: 0, - metadata: HashMap::new(), - }); - } - - let mediator_context = AuctionContext { - settings: context.settings, - request: context.request, - // Bound by both the remaining auction budget and the mediator's - // own configured timeout, matching the dispatched collect path. - timeout_ms: remaining_ms.min(mediator.timeout_ms()), - provider_responses: Some(&provider_responses), - services: context.services, - }; - - let start_time = Instant::now(); - let pending = mediator - .request_bids(request, &mediator_context) - .await - .change_context(TrustedServerError::Auction { - message: format!("Mediator {} failed to launch", mediator.provider_name()), - })?; + let (mediator_response, winning_bids, winning_bid_origins) = + if let Some(mediator_name) = &self.config.mediator { + let mediator = self.get_provider(mediator_name)?; - let platform_resp = mediator_context - .services - .http_client() - .wait(pending) - .await - .change_context(TrustedServerError::Auction { - message: format!("Mediator {} request failed", mediator.provider_name()), - })?; + log::info!( + "Sending {} provider responses to mediator: {}", + provider_responses.len(), + mediator.provider_name() + ); - let response_time_ms = start_time.elapsed().as_millis() as u64; - // Use the context-aware parse so mediators (e.g. adserver_mock) can - // restore nurl/burl/ad_id and PBS cache fields from the collected SSP - // responses. The dispatched collect path already does this; the - // synchronous mediation path used by POST /auction and - // /__ts/page-bids must match or mediated cache bids lose the metadata - // needed for creative rendering and win/billing beacons. - let mediator_resp = mediator - .parse_response_with_context( - platform_resp, - response_time_ms, - request, - &mediator_context, - ) - .await - .change_context(TrustedServerError::Auction { - message: format!("Mediator {} parse failed", mediator.provider_name()), - })?; + // Give the mediator only the remaining time from the auction + // deadline, not the full timeout — the bidding phase already + // consumed part of it. + let remaining_ms = remaining_budget_ms(mediation_start, context.timeout_ms); + + if remaining_ms == 0 { + log::warn!("Auction timeout exhausted during bidding phase; skipping mediator"); + let (winning_bids, winning_bid_origins) = + self.select_winning_bids(&provider_responses, &floor_prices); + return Ok(self.finalize_result( + context.trace, + provider_responses, + None, + winning_bids, + winning_bid_origins, + 0, + )); + } - // Extract winning bids from mediator response - // Filter out bids without decoded prices - mediator should have decoded all prices - let winning = mediator_resp - .bids - .iter() - .filter_map(|bid| { - if bid.price.is_none() { - log::warn!( - "Mediator '{}' returned bid for slot '{}' without decoded price - skipping. \ - Mediator should decode all prices including APS bids.", - mediator.provider_name(), - bid.slot_id - ); - None - } else { - Some((bid.slot_id.clone(), bid.clone())) - } - }) - .collect(); + let mediator_context = AuctionContext { + trace: context.trace, + settings: context.settings, + request: context.request, + // Bound by both the remaining auction budget and the mediator's + // own configured timeout, matching the dispatched collect path. + timeout_ms: remaining_ms.min(mediator.timeout_ms()), + provider_responses: Some(&provider_responses), + services: context.services, + }; - ( - Some(mediator_resp), - self.apply_floor_prices(winning, &floor_prices), - ) - } else { - // No mediator - select best bid per slot from bidder responses - let winning = self.select_winning_bids(&provider_responses, &floor_prices); - (None, winning) - }; + let start_time = Instant::now(); + let pending = mediator + .request_bids(request, &mediator_context) + .await + .change_context(TrustedServerError::Auction { + message: format!("Mediator {} failed to launch", mediator.provider_name()), + })?; + + let platform_resp = mediator_context + .services + .http_client() + .wait(pending) + .await + .change_context(TrustedServerError::Auction { + message: format!("Mediator {} request failed", mediator.provider_name()), + })?; + + let response_time_ms = start_time.elapsed().as_millis() as u64; + // Use the context-aware parse so mediators (e.g. adserver_mock) can + // restore nurl/burl/ad_id and PBS cache fields from the collected SSP + // responses. The dispatched collect path already does this; the + // synchronous mediation path used by POST /auction and + // /__ts/page-bids must match or mediated cache bids lose the metadata + // needed for creative rendering and win/billing beacons. + let mediator_resp = mediator + .parse_response_with_context( + platform_resp, + response_time_ms, + request, + &mediator_context, + ) + .await + .change_context(TrustedServerError::Auction { + message: format!("Mediator {} parse failed", mediator.provider_name()), + })?; + + let (winning_bids, winning_bid_origins) = + self.select_mediator_winning_bids(&mediator_resp, &floor_prices); + (Some(mediator_resp), winning_bids, winning_bid_origins) + } else { + // No mediator - select best bid per slot from bidder responses + let (winning_bids, winning_bid_origins) = + self.select_winning_bids(&provider_responses, &floor_prices); + (None, winning_bids, winning_bid_origins) + }; - Ok(OrchestrationResult { + Ok(self.finalize_result( + context.trace, provider_responses, mediator_response, winning_bids, - total_time_ms: 0, // Will be set by caller - metadata: HashMap::new(), - }) + winning_bid_origins, + 0, + )) } /// Run auction with only parallel bidding (no mediation). @@ -392,15 +422,17 @@ impl AuctionOrchestrator { ) -> Result> { let provider_responses = self.run_providers_parallel(request, context).await?; let floor_prices = self.floor_prices_by_slot(request); - let winning_bids = self.select_winning_bids(&provider_responses, &floor_prices); + let (winning_bids, winning_bid_origins) = + self.select_winning_bids(&provider_responses, &floor_prices); - Ok(OrchestrationResult { + Ok(self.finalize_result( + context.trace, provider_responses, - mediator_response: None, + None, winning_bids, - total_time_ms: 0, - metadata: HashMap::new(), - }) + winning_bid_origins, + 0, + )) } /// Run all providers in parallel and collect responses. @@ -495,6 +527,7 @@ impl AuctionOrchestrator { }; let provider_context = AuctionContext { + trace: context.trace, settings: context.settings, request: context.request, timeout_ms: effective_timeout, @@ -698,22 +731,23 @@ impl AuctionOrchestrator { Ok(responses) } - /// Select the best bid for each slot from all responses. + /// Select the best bid for each slot from all responses while retaining its exact origin. /// Note: Bids with None price (e.g., APS bids with encoded prices) are skipped /// when no mediator is configured, as we cannot compare them without decoding. fn select_winning_bids( &self, responses: &[AuctionResponse], floor_prices: &HashMap, - ) -> HashMap { + ) -> (HashMap, HashMap) { let mut winning_bids: HashMap = HashMap::new(); + let mut origins = HashMap::new(); - for response in responses { + for (response_index, response) in responses.iter().enumerate() { if response.status != BidStatus::Success { continue; } - for bid in &response.bids { + for (bid_index, bid) in response.bids.iter().enumerate() { // Skip bids without decoded prices (e.g., APS bids) // These require mediation layer to decode let bid_price = match bid.price { @@ -736,12 +770,91 @@ impl AuctionOrchestrator { }; if should_replace { + origins.insert( + bid.slot_id.clone(), + WinningBidOrigin { + response_index, + bid_index, + mediated: false, + }, + ); winning_bids.insert(bid.slot_id.clone(), bid.clone()); } } } - self.apply_floor_prices(winning_bids, floor_prices) + let winning_bids = self.apply_floor_prices(winning_bids, floor_prices); + origins.retain(|slot_id, _| winning_bids.contains_key(slot_id)); + (winning_bids, origins) + } + + fn select_mediator_winning_bids( + &self, + response: &AuctionResponse, + floor_prices: &HashMap, + ) -> (HashMap, HashMap) { + let mut winning_bids = HashMap::new(); + let mut origins = HashMap::new(); + for (bid_index, bid) in response.bids.iter().enumerate() { + if bid.price.is_none() { + log::warn!( + "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", + response.provider, + bid.slot_id + ); + continue; + } + origins.insert( + bid.slot_id.clone(), + WinningBidOrigin { + response_index: 0, + bid_index, + mediated: true, + }, + ); + winning_bids.insert(bid.slot_id.clone(), bid.clone()); + } + let winning_bids = self.apply_floor_prices(winning_bids, floor_prices); + origins.retain(|slot_id, _| winning_bids.contains_key(slot_id)); + (winning_bids, origins) + } + + fn finalize_result( + &self, + trace: &AuctionTraceContext, + provider_responses: Vec, + mediator_response: Option, + winning_bids: HashMap, + winning_bid_origins: HashMap, + total_time_ms: u64, + ) -> OrchestrationResult { + let outcome = if winning_bids.is_empty() { + AuctionPublicOutcome::NoBid + } else { + AuctionPublicOutcome::Completed + }; + let trace_bids = build_winning_bid_traces( + &winning_bids, + &winning_bid_origins, + &provider_responses, + mediator_response.as_ref(), + BidTraceId::new, + ); + OrchestrationResult { + trace: AuctionResultTrace { + summary: AuctionTraceSummary { + auction: trace.clone(), + outcome, + }, + winning_bids: trace_bids, + }, + winning_bid_origins, + provider_responses, + mediator_response, + winning_bids, + total_time_ms, + metadata: HashMap::new(), + } } fn apply_floor_prices( @@ -905,6 +1018,7 @@ impl AuctionOrchestrator { }; let provider_context = AuctionContext { + trace: context.trace, settings: context.settings, request: context.request, timeout_ms: effective_timeout, @@ -965,6 +1079,7 @@ impl AuctionOrchestrator { ); DispatchAuctionOutcome::Dispatched(DispatchedAuction { + trace: context.trace.clone(), pending_requests, backend_to_provider, launch_responses, @@ -992,6 +1107,7 @@ impl AuctionOrchestrator { context: &AuctionContext<'_>, ) -> OrchestrationResult { let DispatchedAuction { + trace, pending_requests, mut backend_to_provider, launch_responses, @@ -1128,7 +1244,7 @@ impl AuctionOrchestrator { } backend_to_provider.clear(); - let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { + let (mediator_response, selection) = if let Some(mediator_name) = &self.config.mediator { match self.providers.get(mediator_name.as_str()) { Some(mediator) => { // Cap the mediator at whichever is tighter: its own configured @@ -1146,14 +1262,16 @@ impl AuctionOrchestrator { mediator.provider_name(), responses.len(), ); - let winning = self.select_winning_bids(&responses, &floor_prices); - return OrchestrationResult { - provider_responses: responses, - mediator_response: None, - winning_bids: winning, - total_time_ms: auction_start.elapsed().as_millis() as u64, - metadata: HashMap::new(), - }; + let (winning_bids, winning_bid_origins) = + self.select_winning_bids(&responses, &floor_prices); + return self.finalize_result( + &trace, + responses, + None, + winning_bids, + winning_bid_origins, + auction_start.elapsed().as_millis() as u64, + ); } let mediator_timeout = remaining.min(mediator.timeout_ms()); let mediator_start = Instant::now(); @@ -1175,6 +1293,7 @@ impl AuctionOrchestrator { .body(edgezero_core::body::Body::empty()) .unwrap_or_else(|_| http::Request::new(edgezero_core::body::Body::empty())); let mediator_context = AuctionContext { + trace: &trace, settings: context.settings, request: &placeholder, timeout_ms: mediator_timeout, @@ -1206,25 +1325,11 @@ impl AuctionOrchestrator { .await { Ok(mediator_resp) => { - let winning = mediator_resp - .bids - .iter() - .filter_map(|bid| { - if bid.price.is_none() { - log::warn!( - "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", - mediator.provider_name(), - bid.slot_id - ); - None - } else { - Some((bid.slot_id.clone(), bid.clone())) - } - }) - .collect(); - let winning = - self.apply_floor_prices(winning, &floor_prices); - (Some(mediator_resp), winning) + let selection = self.select_mediator_winning_bids( + &mediator_resp, + &floor_prices, + ); + (Some(mediator_resp), selection) } Err(e) => { log::warn!( @@ -1265,13 +1370,15 @@ impl AuctionOrchestrator { (None, self.select_winning_bids(&responses, &floor_prices)) }; - OrchestrationResult { - provider_responses: responses, + let (winning_bids, winning_bid_origins) = selection; + self.finalize_result( + &trace, + responses, mediator_response, winning_bids, - total_time_ms: auction_start.elapsed().as_millis() as u64, - metadata: HashMap::new(), - } + winning_bid_origins, + auction_start.elapsed().as_millis() as u64, + ) } /// Check if orchestrator is enabled. @@ -1284,6 +1391,10 @@ impl AuctionOrchestrator { /// Result of an orchestrated auction. #[derive(Debug, Clone)] pub struct OrchestrationResult { + /// Privacy-safe tester trace for this finalized result. + pub trace: AuctionResultTrace, + /// Exact internal origin of each final winning bid. + pub(crate) winning_bid_origins: HashMap, /// All responses from providers pub provider_responses: Vec, /// Final response from mediator (if used) @@ -1297,6 +1408,32 @@ pub struct OrchestrationResult { } impl OrchestrationResult { + /// Build a no-bid result for a terminal path that already returns a response. + #[must_use] + pub fn empty(trace: AuctionTraceContext, outcome: AuctionPublicOutcome) -> Self { + Self { + trace: AuctionResultTrace { + summary: AuctionTraceSummary { + auction: trace, + outcome, + }, + winning_bids: HashMap::new(), + }, + winning_bid_origins: HashMap::new(), + provider_responses: Vec::new(), + mediator_response: None, + winning_bids: HashMap::new(), + total_time_ms: 0, + metadata: HashMap::new(), + } + } + + /// Return the exact provider/bid location for a final winning slot. + #[must_use] + pub(crate) fn winning_origin(&self, slot_id: &str) -> Option { + self.winning_bid_origins.get(slot_id).copied() + } + /// Get the winning bid for a specific slot. #[must_use] pub fn get_winning_bid(&self, slot_id: &str) -> Option<&Bid> { @@ -1331,7 +1468,7 @@ mod tests { use crate::auction::test_support::create_test_auction_context; use crate::auction::types::{ AdFormat, AdSlot, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus, - MediaType, PublisherInfo, UserInfo, + BidTraceId, MediaType, PublisherInfo, UserInfo, WinningBidOrigin, }; use crate::error::TrustedServerError; use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; @@ -1343,7 +1480,7 @@ mod tests { use std::collections::{HashMap, HashSet}; use std::sync::Arc; - use super::AuctionOrchestrator; + use super::{AuctionOrchestrator, build_winning_bid_traces}; // --------------------------------------------------------------------------- // Minimal test double for AuctionProvider @@ -1429,6 +1566,62 @@ mod tests { } } + #[test] + fn mediated_selection_retains_the_mediator_response_origin() { + let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); + let selected = mediated_bid(None); + let mediator_response = AuctionResponse::success("mediator", vec![selected], 1); + + let (_, origins) = + orchestrator.select_mediator_winning_bids(&mediator_response, &HashMap::new()); + + let origin = origins["header-banner"]; + assert!( + origin.mediated, + "mediated selection should retain mediator origin" + ); + assert_eq!( + origin.bid_index, 0, + "should retain exact mediator bid index" + ); + } + + #[test] + fn winning_trace_builder_uses_supplied_id_source_only_for_final_winners() { + let mut winner = mediated_bid(None); + winner.bidder = "example-bidder".to_owned(); + let provider_responses = vec![AuctionResponse::success( + "provider-a", + vec![winner.clone()], + 1, + )]; + let winning_bids = HashMap::from([("header-banner".to_owned(), winner)]); + let origins = HashMap::from([( + "header-banner".to_owned(), + WinningBidOrigin { + response_index: 0, + bid_index: 0, + mediated: false, + }, + )]); + let fixed = uuid::Uuid::parse_str("650e8400-e29b-41d4-a716-446655440000") + .expect("should parse fixed UUID"); + let mut calls = 0; + + let traces = + build_winning_bid_traces(&winning_bids, &origins, &provider_responses, None, || { + calls += 1; + BidTraceId::from_uuid(fixed) + }); + + assert_eq!(calls, 1, "should allocate one ID for one final winner"); + assert_eq!( + traces["header-banner"].bid_trace_id.to_string(), + fixed.to_string(), + "should use the supplied deterministic ID" + ); + } + #[async_trait::async_trait(?Send)] impl AuctionProvider for CacheRestoringMediator { fn provider_name(&self) -> &'static str { @@ -1530,6 +1723,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &req, timeout_ms: 2000, @@ -1974,6 +2168,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &req, timeout_ms: 2000, @@ -2057,6 +2252,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &req, timeout_ms: 2000, @@ -2122,6 +2318,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &req, timeout_ms: 2000, diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index d63445369..08af9ac0f 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -3,7 +3,6 @@ //! Core owns the privacy-preserving auction observation model and pure row //! builder. Platform adapters provide the concrete sink implementation. -use std::collections::HashSet; use std::time::Instant; use chrono::Utc; @@ -12,7 +11,10 @@ use serde::Serialize; use uuid::Uuid; use crate::auction::orchestrator::OrchestrationResult; -use crate::auction::types::{AuctionRequest, AuctionResponse, Bid, BidStatus, MediaType}; +pub use crate::auction::types::AuctionSource; +use crate::auction::types::{ + AuctionRequest, AuctionResponse, AuctionTraceContext, Bid, BidStatus, MediaType, +}; use crate::ec::EcContext; use crate::error::TrustedServerError; use crate::platform::RuntimeServices; @@ -20,27 +22,6 @@ use crate::platform::RuntimeServices; const MAX_PAGE_PATH_BYTES: usize = 256; const DYNAMIC_SEGMENT_REPLACEMENT: &str = ":id"; -/// Source path that initiated an auction candidate. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum AuctionSource { - /// Initial publisher navigation using server-side ad templates. - InitialNavigation, - /// SPA navigation through `GET /__ts/page-bids`. - SpaNavigation, - /// Explicit `POST /auction` API. - AuctionApi, -} - -impl AuctionSource { - fn as_str(self) -> &'static str { - match self { - Self::InitialNavigation => "initial_navigation", - Self::SpaNavigation => "spa_navigation", - Self::AuctionApi => "auction_api", - } - } -} - /// Terminal status for one auction observation. #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum AuctionTerminalStatus { @@ -123,7 +104,7 @@ impl AuctionObservationContext { /// Build an observation context from an auction request. #[must_use] pub fn from_auction_request( - auction_source: AuctionSource, + trace: &AuctionTraceContext, request: &AuctionRequest, ec_context: &EcContext, ) -> Self { @@ -135,7 +116,7 @@ impl AuctionObservationContext { .map(|url| url.path().to_owned()) .unwrap_or_else(|| "/".to_owned()); Self::from_parts( - auction_source, + trace, &request.publisher.domain, &raw_path, request.slots.len(), @@ -146,7 +127,7 @@ impl AuctionObservationContext { /// Build an observation context from publisher request parts. #[must_use] pub fn from_parts( - auction_source: AuctionSource, + trace: &AuctionTraceContext, publisher_domain: &str, raw_page_path: &str, slot_count: usize, @@ -157,8 +138,8 @@ impl AuctionObservationContext { let consent = ec_context.consent(); let slot_count = u16::try_from(slot_count).unwrap_or(u16::MAX); Self { - auction_id: Uuid::new_v4(), - auction_source, + auction_id: trace.auction_trace_id.as_uuid(), + auction_source: trace.source, publisher_domain: publisher_domain.to_owned(), page_path: normalize_page_path(raw_page_path), country: geo @@ -264,7 +245,7 @@ pub struct AuctionEventRow { pub event_ts: String, /// `summary`, `provider_call`, or `bid`. pub event_kind: String, - /// Fresh telemetry auction UUID. + /// Privacy-safe UUID shared with tester-gated trace output. pub auction_id: String, /// Source path label. pub auction_source: String, @@ -320,6 +301,8 @@ pub struct AuctionEventRow { pub currency: Option, /// Whether this is the canonical winning row for its slot. pub is_win: Option, + /// Trace UUID for the canonical winning bid only. + pub bid_trace_id: Option, /// Advertiser domain. pub ad_domain: Option, /// Creative/ad ID. @@ -359,6 +342,7 @@ impl AuctionEventRow { price_cpm: None, currency: None, is_win: None, + bid_trace_id: None, ad_domain: None, ad_id: None, } @@ -683,68 +667,84 @@ fn push_bid_rows( request: &AuctionRequest, result: &OrchestrationResult, ) { - let mut matched_wins = HashSet::new(); - - for response in &result.provider_responses { - for bid in &response.bids { - let matched_slot = result - .winning_bids - .iter() - .find(|(slot_id, winning)| { - !matched_wins.contains(*slot_id) && bid_matches_winning_bid(bid, winning) + for (response_index, response) in result.provider_responses.iter().enumerate() { + for (bid_index, bid) in response.bids.iter().enumerate() { + let winning_slot = result.winning_bids.keys().find(|slot_id| { + result.winning_origin(slot_id).is_some_and(|origin| { + !origin.mediated + && origin.response_index == response_index + && origin.bid_index == bid_index }) - .map(|(slot_id, winning)| (slot_id.clone(), winning)); - let (is_win, price) = if let Some((slot_id, winning)) = matched_slot { - matched_wins.insert(slot_id); - (1, bid.price.or(winning.price)) - } else { - (0, bid.price) - }; + }); + let trace_id = winning_slot.and_then(|slot_id| { + result + .trace + .winning_bids + .get(slot_id) + .map(|trace| trace.bid_trace_id.to_string()) + }); + let price = winning_slot + .and_then(|slot_id| result.winning_bids.get(slot_id)) + .and_then(|winning| winning.price) + .or(bid.price); rows.push(bid_row( observation, event_ts, request, &response.provider, bid, - is_win, - price, + BidRowOutcome { + is_win: u8::from(winning_slot.is_some()), + price, + bid_trace_id: trace_id, + }, )); } } if let Some(mediator_response) = &result.mediator_response { - for (slot_id, winning) in &result.winning_bids { - if matched_wins.contains(slot_id) { - continue; - } - if mediator_response - .bids - .iter() - .any(|bid| bid_matches_winning_bid(bid, winning)) - { + for (bid_index, bid) in mediator_response.bids.iter().enumerate() { + let winning_slot = result.winning_bids.keys().find(|slot_id| { + result + .winning_origin(slot_id) + .is_some_and(|origin| origin.mediated && origin.bid_index == bid_index) + }); + if let Some(slot_id) = winning_slot { + let trace_id = result + .trace + .winning_bids + .get(slot_id) + .map(|trace| trace.bid_trace_id.to_string()); rows.push(bid_row( observation, event_ts, request, &mediator_response.provider, - winning, - 1, - winning.price, + bid, + BidRowOutcome { + is_win: 1, + price: bid.price, + bid_trace_id: trace_id, + }, )); - matched_wins.insert(slot_id.clone()); } } } } +struct BidRowOutcome { + is_win: u8, + price: Option, + bid_trace_id: Option, +} + fn bid_row( observation: &AuctionObservationContext, event_ts: &str, request: &AuctionRequest, provider: &str, bid: &Bid, - is_win: u8, - price: Option, + outcome: BidRowOutcome, ) -> AuctionEventRow { let mut row = AuctionEventRow::base(observation, "bid", event_ts); row.provider = Some(provider.to_owned()); @@ -753,9 +753,10 @@ fn bid_row( row.slot_h = Some(u16::try_from(bid.height).unwrap_or(u16::MAX)); row.media_type = media_type_for_slot(request, &bid.slot_id).map(str::to_owned); row.seat = Some(bid.bidder.clone()); - row.price_cpm = price; + row.price_cpm = outcome.price; row.currency = Some(bid.currency.clone()); - row.is_win = Some(is_win); + row.is_win = Some(outcome.is_win); + row.bid_trace_id = outcome.bid_trace_id; row.ad_domain = bid .adomain .as_ref() @@ -764,16 +765,6 @@ fn bid_row( row } -fn bid_matches_winning_bid(candidate: &Bid, winning: &Bid) -> bool { - if candidate.slot_id != winning.slot_id || candidate.bidder != winning.bidder { - return false; - } - match winning.ad_id.as_deref() { - Some(winning_ad_id) => candidate.ad_id.as_deref() == Some(winning_ad_id), - None => true, - } -} - fn media_type_for_slot<'a>(request: &'a AuctionRequest, slot_id: &str) -> Option<&'a str> { request .slots @@ -948,6 +939,15 @@ mod tests { } } + fn empty_result(total_time_ms: u64) -> OrchestrationResult { + let mut result = OrchestrationResult::empty( + AuctionTraceContext::new(AuctionSource::AuctionApi), + crate::auction::types::AuctionPublicOutcome::NoBid, + ); + result.total_time_ms = total_time_ms; + result + } + fn bid(slot_id: &str, bidder: &str, ad_id: Option<&str>, price: Option) -> Bid { Bid { slot_id: slot_id.to_owned(), @@ -1024,13 +1024,27 @@ mod tests { let provider_error = AuctionResponse::error("mock", 12).with_metadata("error_type", json!("parse_response")); let winning = provider_success.bids[0].clone(); - let result = OrchestrationResult { - provider_responses: vec![provider_success, provider_no_bid, provider_error], - mediator_response: None, - winning_bids: HashMap::from([("slot-1".to_owned(), winning)]), - total_time_ms: 99, - metadata: HashMap::new(), - }; + let mut result = empty_result(99); + result.provider_responses = vec![provider_success, provider_no_bid, provider_error]; + result + .winning_bids + .insert("slot-1".to_owned(), winning.clone()); + result.winning_bid_origins.insert( + "slot-1".to_owned(), + crate::auction::types::WinningBidOrigin { + response_index: 0, + bid_index: 0, + mediated: false, + }, + ); + result.trace.winning_bids.insert( + "slot-1".to_owned(), + crate::auction::types::WinningBidTrace { + bid_trace_id: crate::auction::types::BidTraceId::new(), + provider: "prebid".to_owned(), + bidder: winning.bidder, + }, + ); let observation = AuctionObservationContext::for_test(AuctionSource::AuctionApi, "/article/1", 1); @@ -1088,13 +1102,8 @@ mod tests { let provider_http_error = AuctionResponse::error("prebid", 12) .with_metadata("error_type", json!("http_status")) .with_metadata("status", json!(403)); - let result = OrchestrationResult { - provider_responses: vec![provider_http_error], - mediator_response: None, - winning_bids: HashMap::new(), - total_time_ms: 12, - metadata: HashMap::new(), - }; + let mut result = empty_result(12); + result.provider_responses = vec![provider_http_error]; let observation = AuctionObservationContext::for_test(AuctionSource::AuctionApi, "/article/1", 1); @@ -1125,13 +1134,28 @@ mod tests { let mediator_bid = bid("slot-1", "kargo", Some("ad-1"), Some(2.0)); let mediator_response = AuctionResponse::success("adserver_mock", vec![mediator_bid.clone()], 15); - let result = OrchestrationResult { - provider_responses: vec![provider_success], - mediator_response: Some(mediator_response), - winning_bids: HashMap::from([("slot-1".to_owned(), mediator_bid)]), - total_time_ms: 80, - metadata: HashMap::new(), - }; + let mut result = empty_result(80); + result.provider_responses = vec![provider_success]; + result.mediator_response = Some(mediator_response); + result + .winning_bids + .insert("slot-1".to_owned(), mediator_bid.clone()); + result.winning_bid_origins.insert( + "slot-1".to_owned(), + crate::auction::types::WinningBidOrigin { + response_index: 0, + bid_index: 0, + mediated: false, + }, + ); + result.trace.winning_bids.insert( + "slot-1".to_owned(), + crate::auction::types::WinningBidTrace { + bid_trace_id: crate::auction::types::BidTraceId::new(), + provider: "prebid".to_owned(), + bidder: mediator_bid.bidder, + }, + ); let observation = AuctionObservationContext::for_test(AuctionSource::InitialNavigation, "/", 1); @@ -1164,13 +1188,7 @@ mod tests { #[test] fn ndjson_serialization_has_one_json_object_per_line_and_no_private_ids() { let request = test_request("ts-ec-derived-id"); - let result = OrchestrationResult { - provider_responses: Vec::new(), - mediator_response: None, - winning_bids: HashMap::new(), - total_time_ms: 1, - metadata: HashMap::new(), - }; + let result = empty_result(1); let observation = AuctionObservationContext::for_test(AuctionSource::AuctionApi, "/auction", 1); diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index e4b953e05..45d90731e 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -3,11 +3,18 @@ use std::sync::LazyLock; use edgezero_core::body::Body as EdgeBody; use http::Request; -use super::AuctionContext; +use super::{AuctionContext, AuctionSource}; +use crate::auction::types::AuctionTraceContext; use crate::platform::{RuntimeServices, test_support::noop_services}; use crate::settings::Settings; static TEST_SERVICES: LazyLock = LazyLock::new(noop_services); +static TEST_TRACE: LazyLock = + LazyLock::new(|| AuctionTraceContext::new(AuctionSource::AuctionApi)); + +pub(crate) fn test_trace() -> &'static AuctionTraceContext { + &TEST_TRACE +} pub(crate) fn create_test_auction_context<'a>( settings: &'a Settings, @@ -16,6 +23,7 @@ pub(crate) fn create_test_auction_context<'a>( ) -> AuctionContext<'a> { let services: &'static RuntimeServices = &TEST_SERVICES; AuctionContext { + trace: test_trace(), settings, request, timeout_ms, diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index 14c7713f8..a26f9b8c9 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -4,12 +4,157 @@ use edgezero_core::body::Body as EdgeBody; use http::Request; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use uuid::Uuid; use crate::auction::context::ContextValue; use crate::geo::GeoInfo; use crate::platform::RuntimeServices; use crate::settings::Settings; +/// Source path that initiated an auction candidate. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuctionSource { + /// Initial publisher navigation using server-side ad templates. + InitialNavigation, + /// SPA navigation through `GET /__ts/page-bids`. + SpaNavigation, + /// Explicit `POST /auction` API. + AuctionApi, +} + +impl AuctionSource { + /// Return the stable wire label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::InitialNavigation => "initial_navigation", + Self::SpaNavigation => "spa_navigation", + Self::AuctionApi => "auction_api", + } + } +} + +/// Privacy-safe public identity for one auction candidate. +#[derive(Debug, Clone, Copy, Eq, Hash, PartialEq, derive_more::Display)] +pub struct AuctionTraceId(Uuid); + +impl AuctionTraceId { + /// Generate a fresh random trace identity. + #[must_use] + pub fn new() -> Self { + Self(Uuid::new_v4()) + } + + /// Return the underlying UUID. + #[must_use] + pub const fn as_uuid(self) -> Uuid { + self.0 + } +} + +impl Default for AuctionTraceId { + fn default() -> Self { + Self::new() + } +} + +/// Privacy-safe public identity for one final winning bid. +#[derive(Debug, Clone, Copy, Eq, Hash, PartialEq, derive_more::Display)] +pub struct BidTraceId(Uuid); + +impl BidTraceId { + /// Generate a fresh random trace identity. + #[must_use] + pub fn new() -> Self { + Self(Uuid::new_v4()) + } + + #[cfg(test)] + pub(crate) const fn from_uuid(value: Uuid) -> Self { + Self(value) + } +} + +impl Default for BidTraceId { + fn default() -> Self { + Self::new() + } +} + +/// Trace identity and source shared throughout one auction lifecycle. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct AuctionTraceContext { + pub auction_trace_id: AuctionTraceId, + pub source: AuctionSource, +} + +impl AuctionTraceContext { + /// Generate a context for an auction candidate. + #[must_use] + pub fn new(source: AuctionSource) -> Self { + Self { + auction_trace_id: AuctionTraceId::new(), + source, + } + } +} + +/// Privacy-safe terminal state exposed to tester traffic. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuctionPublicOutcome { + Completed, + NoBid, + Skipped, + Failed, + Abandoned, +} + +impl AuctionPublicOutcome { + /// Return the stable wire label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Completed => "completed", + Self::NoBid => "no_bid", + Self::Skipped => "skipped", + Self::Failed => "failed", + Self::Abandoned => "abandoned", + } + } +} + +/// Result-independent public summary for one auction candidate. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct AuctionTraceSummary { + pub auction: AuctionTraceContext, + pub outcome: AuctionPublicOutcome, +} + +/// Public trace metadata for one final winning bid. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct WinningBidTrace { + pub bid_trace_id: BidTraceId, + pub provider: String, + pub bidder: String, +} + +/// Trace data attached to a finalized auction result. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct AuctionResultTrace { + pub summary: AuctionTraceSummary, + pub winning_bids: HashMap, +} + +/// Exact internal location of a final winning bid. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct WinningBidOrigin { + pub response_index: usize, + pub bid_index: usize, + pub mediated: bool, +} + /// Represents a unified auction request across all providers. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AuctionRequest { @@ -140,6 +285,8 @@ pub struct SiteInfo { /// [dispatch]: crate::auction::AuctionOrchestrator::dispatch_auction /// [collect]: crate::auction::AuctionOrchestrator::collect_dispatched_auction pub struct AuctionContext<'a> { + /// Trace identity owned by the auction entry point. + pub trace: &'a AuctionTraceContext, pub settings: &'a Settings, pub request: &'a Request, pub timeout_ms: u32, diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 7bbecd747..e5a59cda0 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -16,16 +16,18 @@ use validator::{Validate, ValidationError, ValidationErrors}; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; use crate::integrations::{ - adserver_mock::AdServerMockConfig, aps::ApsConfig, datadome::DataDomeConfig, - didomi::DidomiIntegrationConfig, google_tag_manager::GoogleTagManagerConfig, gpt::GptConfig, - lockr::LockrConfig, nextjs::NextJsIntegrationConfig, osano::OsanoConfig, - permutive::PermutiveConfig, prebid, sourcepoint::SourcepointConfig, testlight::TestlightConfig, + ad_trace::AdTraceConfig, adserver_mock::AdServerMockConfig, aps::ApsConfig, + datadome::DataDomeConfig, didomi::DidomiIntegrationConfig, + google_tag_manager::GoogleTagManagerConfig, gpt::GptConfig, lockr::LockrConfig, + nextjs::NextJsIntegrationConfig, osano::OsanoConfig, permutive::PermutiveConfig, prebid, + sourcepoint::SourcepointConfig, testlight::TestlightConfig, }; use crate::settings::{IntegrationConfig, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; #[cfg(test)] const DEPLOY_VALIDATED_INTEGRATION_IDS: &[&str] = &[ + "ad_trace", "prebid", "aps", "adserver_mock", @@ -136,6 +138,7 @@ fn validate_enabled_integrations( ) -> Result, Report> { let mut enabled_auction_providers = HashSet::new(); + validate_integration::(settings, "ad_trace")?; if validate_prebid(settings)? { enabled_auction_providers.insert("prebid"); } diff --git a/crates/trusted-server-core/src/constants.rs b/crates/trusted-server-core/src/constants.rs index ffcf4f034..03b5b6d24 100644 --- a/crates/trusted-server-core/src/constants.rs +++ b/crates/trusted-server-core/src/constants.rs @@ -5,6 +5,10 @@ pub const COOKIE_TS_EC: &str = "ts-ec"; /// JSON array of Extended User IDs (`[{ source, uids }]`) from identity providers. pub const COOKIE_TS_EIDS: &str = "ts-eids"; pub const COOKIE_TS_TESTER: &str = "ts-tester"; +/// Host-only browser-session cookie activated by the ad trace console query. +pub const COOKIE_TS_CONSOLE: &str = "__Host-ts-console"; +/// Reserved self-service query parameter for the ad trace console. +pub const QUERY_TS_CONSOLE: &str = "ts_console"; pub const COOKIE_SHAREDID: &str = "sharedId"; pub const HEADER_X_PUB_USER_ID: HeaderName = HeaderName::from_static("x-pub-user-id"); diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index ad69e51c5..9ef9ee0fd 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -161,6 +161,8 @@ pub struct HtmlProcessorConfig { pub request_host: String, pub request_scheme: String, pub integrations: IntegrationRegistry, + /// Request-scoped console bootstrap injected before the unified bundle. + pub head_bootstrap_script: Option, /// Pre-computed ``. /// Injected at `` open. `None` when no slots matched. pub ad_slots_script: Option, @@ -189,6 +191,7 @@ impl HtmlProcessorConfig { request_host: request_host.to_owned(), request_scheme: request_scheme.to_owned(), integrations: integrations.clone(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: settings.publisher.max_buffered_body_bytes, @@ -205,9 +208,11 @@ impl HtmlProcessorConfig { #[must_use] pub fn with_ad_state( mut self, + head_bootstrap_script: Option, ad_slots_script: Option, ad_bids_state: std::sync::Arc>>, ) -> Self { + self.head_bootstrap_script = head_bootstrap_script; self.ad_slots_script = ad_slots_script; self.ad_bids_state = ad_bids_state; self @@ -292,6 +297,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let injected_bids = Arc::new(AtomicBool::new(false)); let integration_registry = config.integrations.clone(); let script_rewriters = integration_registry.script_rewriters(); + let head_bootstrap_script = config.head_bootstrap_script.clone(); let ad_slots_script = config.ad_slots_script.clone(); let ad_bids_state = config.ad_bids_state.clone(); @@ -302,10 +308,15 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let integrations = integration_registry.clone(); let patterns = patterns.clone(); let document_state = document_state.clone(); + let head_bootstrap_script = head_bootstrap_script.clone(); let ad_slots_script = ad_slots_script.clone(); move |el| { if !injected_tsjs.get() { let mut snippet = String::new(); + // Request-scoped activation must run before every TSJS module. + if let Some(ref bootstrap) = head_bootstrap_script { + snippet.push_str(bootstrap); + } // Inject ad slots script first so it appears before tsjs bundle. if let Some(ref slots_script) = ad_slots_script { snippet.push_str(slots_script); @@ -661,6 +672,7 @@ mod tests { request_host: "test.example.com".to_owned(), request_scheme: "https".to_owned(), integrations: IntegrationRegistry::default(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, @@ -738,6 +750,8 @@ mod tests { let html = "Test"; let mut config = create_test_config(); + config.head_bootstrap_script = + Some("".to_owned()); config.integrations = IntegrationRegistry::from_rewriters_with_head_injectors( Vec::new(), Vec::new(), @@ -759,6 +773,7 @@ mod tests { let processed = String::from_utf8(output).expect("output should be valid UTF-8"); let tsjs_marker = "id=\"trustedserver-js\""; + let bootstrap_marker = "window.__tsjs_adTraceActive=true"; let head_marker = "window.__testHeadInjector=true"; assert_eq!( @@ -775,6 +790,9 @@ mod tests { let tsjs_index = processed .find(tsjs_marker) .expect("should include unified tsjs tag"); + let bootstrap_index = processed + .find(bootstrap_marker) + .expect("should include request bootstrap"); let head_index = processed .find(head_marker) .expect("should include head snippet"); @@ -783,8 +801,8 @@ mod tests { .expect("should keep existing head content"); assert!( - head_index < tsjs_index, - "should inject config before tsjs bundle so auto-init can read it" + bootstrap_index < head_index && head_index < tsjs_index, + "should inject request bootstrap and config before tsjs auto-init" ); assert!( tsjs_index < title_index, @@ -1430,6 +1448,7 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), + head_bootstrap_script: None, ad_slots_script: Some( r#""# .to_string(), @@ -1504,6 +1523,7 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), + head_bootstrap_script: None, ad_slots_script: Some( r#""#.to_string(), ), @@ -1539,6 +1559,7 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), + head_bootstrap_script: None, ad_slots_script: Some( r#""#.to_string(), ), @@ -1575,6 +1596,7 @@ mod tests { request_host: request_host.to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::default(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, @@ -1625,6 +1647,7 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), + head_bootstrap_script: None, ad_slots_script: Some( r#""#.to_string(), ), @@ -1653,6 +1676,7 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, diff --git a/crates/trusted-server-core/src/integrations/ad_trace.rs b/crates/trusted-server-core/src/integrations/ad_trace.rs new file mode 100644 index 000000000..41099d4aa --- /dev/null +++ b/crates/trusted-server-core/src/integrations/ad_trace.rs @@ -0,0 +1,620 @@ +//! Query-activated, session-scoped auction trace integration. + +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt}; +use http::{HeaderValue, Method, Request, Response, Uri, header, uri::PathAndQuery}; +use serde::Deserialize; +use validator::Validate; + +use crate::constants::{COOKIE_TS_CONSOLE, QUERY_TS_CONSOLE}; +use crate::error::TrustedServerError; +use crate::http_util::is_navigation_request; +use crate::integrations::IntegrationRegistration; +use crate::settings::{IntegrationConfig, Settings}; + +/// Stable integration identifier. +pub const AD_TRACE_INTEGRATION_ID: &str = "ad_trace"; + +const SET_CONSOLE_COOKIE: &str = "__Host-ts-console=1; Path=/; Secure; HttpOnly; SameSite=Lax"; +const CLEAR_CONSOLE_COOKIE: &str = + "__Host-ts-console=; Path=/; Secure; HttpOnly; SameSite=Lax; Max-Age=0"; + +/// Configuration for the optional browser console. +#[derive(Debug, Default, Deserialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct AdTraceConfig { + /// Enable the optional ad trace browser module and console activation. + #[serde(default)] + pub enabled: bool, +} + +impl IntegrationConfig for AdTraceConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +/// Cookie mutation attached to an eligible console-navigation response. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ConsoleCookieAction { + #[default] + None, + SetSession, + ClearSession, +} + +/// Immutable request-scoped console decision. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AdTraceRequestDecision { + enabled: bool, + browser_bootstrap: bool, + private_response: bool, + clean_browser_path_and_query: Option, + cookie_action: ConsoleCookieAction, +} + +impl AdTraceRequestDecision { + /// Whether browser-visible trace fields and targeting are enabled. + #[must_use] + pub fn enabled(&self) -> bool { + self.enabled + } + + /// Whether this response must be private and non-storeable. + #[must_use] + pub fn requires_private_no_store(&self) -> bool { + self.private_response + || self.cookie_action != ConsoleCookieAction::None + || self.clean_browser_path_and_query.is_some() + } + + /// Build the synchronous bootstrap inserted before the unified TSJS bundle. + #[must_use] + pub fn bootstrap_script(&self) -> Option { + if !self.browser_bootstrap && self.clean_browser_path_and_query.is_none() { + return None; + } + + let mut script = String::from(""); + Some(script) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum QueryDirective { + Absent, + Enable, + Disable, + Invalid, +} + +#[derive(Clone, Copy, Debug, Default)] +struct ConsoleCookieState { + occurrences: usize, + canonical: bool, +} + +#[derive(Clone, Copy, Debug, Default)] +struct AdTraceCookieApplied; + +/// Register the optional browser module. +/// +/// # Errors +/// +/// Returns a configuration error when the integration settings are invalid. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(_config) = settings.integration_config::(AD_TRACE_INTEGRATION_ID)? + else { + return Ok(None); + }; + Ok(Some( + IntegrationRegistration::builder(AD_TRACE_INTEGRATION_ID).build(), + )) +} + +/// Evaluate and sanitize the console request before routing or downstream use. +/// +/// The original query and cookie are inspected first. Every reserved query pair +/// and console cookie is then removed from the request. The immutable decision +/// is stored in request extensions for handlers to consume after sanitation. +/// +/// # Errors +/// +/// Returns an error when integration configuration or URI reconstruction fails. +pub fn prepare_request( + settings: &Settings, + request: &mut Request, +) -> Result> { + let integration_enabled = settings + .integration_config::(AD_TRACE_INTEGRATION_ID)? + .is_some(); + let (directive, clean_path, had_reserved_query) = console_query(request.uri()); + let cookie_state = console_cookie_state(request); + let eligible_navigation = is_eligible_console_navigation(request); + + sanitize_console_cookie(request); + if had_reserved_query { + replace_path_and_query(request, &clean_path)?; + } + + let mut decision = AdTraceRequestDecision::default(); + if integration_enabled && eligible_navigation && had_reserved_query { + decision.clean_browser_path_and_query = Some(clean_path); + match directive { + QueryDirective::Enable => { + decision.enabled = true; + decision.browser_bootstrap = true; + decision.cookie_action = ConsoleCookieAction::SetSession; + } + QueryDirective::Disable => { + decision.cookie_action = ConsoleCookieAction::ClearSession; + } + QueryDirective::Invalid | QueryDirective::Absent => {} + } + } else if integration_enabled + && directive == QueryDirective::Absent + && cookie_state.occurrences == 1 + && cookie_state.canonical + { + decision.enabled = true; + decision.browser_bootstrap = eligible_navigation; + } + + decision.private_response = + decision.enabled && trace_payload_request(request, eligible_navigation); + request.extensions_mut().insert(decision.clone()); + Ok(decision) +} + +/// Read the previously prepared request decision. +#[must_use] +pub fn request_decision(request: &Request) -> AdTraceRequestDecision { + request + .extensions() + .get::() + .cloned() + .unwrap_or_default() +} + +/// Return whether browser-visible trace output is active for this request. +#[must_use] +pub fn browser_trace_enabled(request: &Request) -> bool { + request_decision(request).enabled() +} + +/// Copy the prepared request decision onto a response for outer finalization. +pub fn attach_response_decision( + decision: &AdTraceRequestDecision, + response: &mut Response, +) { + response.extensions_mut().insert(decision.clone()); +} + +/// Apply the response-side session mutation and cache policy. +/// +/// Safe to call more than once. The cookie is appended once, while the +/// private/no-store policy is reasserted so later adapter cache policy cannot +/// weaken it. +pub fn finalize_response(response: &mut Response) { + let Some(decision) = response + .extensions() + .get::() + .cloned() + else { + return; + }; + + if decision.cookie_action != ConsoleCookieAction::None + && response + .extensions() + .get::() + .is_none() + { + let value = match decision.cookie_action { + ConsoleCookieAction::None => None, + ConsoleCookieAction::SetSession => Some(HeaderValue::from_static(SET_CONSOLE_COOKIE)), + ConsoleCookieAction::ClearSession => { + Some(HeaderValue::from_static(CLEAR_CONSOLE_COOKIE)) + } + }; + if let Some(value) = value { + response.headers_mut().append(header::SET_COOKIE, value); + response.extensions_mut().insert(AdTraceCookieApplied); + } + } + + if decision.requires_private_no_store() { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); + for name in crate::response_privacy::SURROGATE_CACHE_HEADERS { + response.headers_mut().remove(*name); + } + } +} + +fn trace_payload_request(request: &Request, eligible_navigation: bool) -> bool { + eligible_navigation + || request.uri().path() == "/auction" + || request.uri().path() == "/__ts/page-bids" +} + +fn is_eligible_console_navigation(request: &Request) -> bool { + request.method() == Method::GET + && is_navigation_request(request) + && !crate::publisher::is_prefetch_request(request) + && !crate::publisher::is_bot_user_agent(request) +} + +fn console_query(uri: &Uri) -> (QueryDirective, String, bool) { + let mut console_values = Vec::new(); + let mut retained = Vec::new(); + for pair in uri.query().unwrap_or_default().split('&') { + let (name, value) = pair.split_once('=').unwrap_or((pair, "")); + if name == QUERY_TS_CONSOLE { + console_values.push(value); + } else { + retained.push(pair); + } + } + + let directive = match console_values.as_slice() { + [] => QueryDirective::Absent, + ["true" | "1"] => QueryDirective::Enable, + ["false" | "0"] => QueryDirective::Disable, + _ => QueryDirective::Invalid, + }; + let mut clean = uri.path().to_owned(); + let retained_query = retained.join("&"); + if !retained_query.is_empty() { + clean.push('?'); + clean.push_str(&retained_query); + } + (directive, clean, !console_values.is_empty()) +} + +fn console_cookie_state(request: &Request) -> ConsoleCookieState { + let mut state = ConsoleCookieState::default(); + for value in request.headers().get_all(header::COOKIE) { + let Ok(value) = value.to_str() else { + continue; + }; + for cookie in value.split(';') { + let cookie = cookie.trim(); + match cookie.split_once('=') { + Some((name, value)) if name.trim() == COOKIE_TS_CONSOLE => { + state.occurrences += 1; + state.canonical |= value.trim() == "1"; + } + None if cookie == COOKIE_TS_CONSOLE => state.occurrences += 1, + _ => {} + } + } + } + state +} + +fn sanitize_console_cookie(request: &mut Request) { + let retained = request + .headers() + .get_all(header::COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(';')) + .map(str::trim) + .filter(|cookie| match cookie.split_once('=') { + Some((name, _)) => name.trim() != COOKIE_TS_CONSOLE, + None => *cookie != COOKIE_TS_CONSOLE, + }) + .filter(|cookie| !cookie.is_empty()) + .map(str::to_owned) + .collect::>(); + + request.headers_mut().remove(header::COOKIE); + if !retained.is_empty() { + let value = HeaderValue::from_str(&retained.join("; ")) + .expect("should preserve already-valid cookie header values"); + request.headers_mut().insert(header::COOKIE, value); + } +} + +fn replace_path_and_query( + request: &mut Request, + clean_path_and_query: &str, +) -> Result<(), Report> { + let mut parts = request.uri().clone().into_parts(); + parts.path_and_query = Some( + clean_path_and_query + .parse::() + .change_context(TrustedServerError::Proxy { + message: "ad trace console query produced invalid URI".to_owned(), + })?, + ); + *request.uri_mut() = Uri::from_parts(parts).change_context(TrustedServerError::Proxy { + message: "ad trace console query produced invalid URI".to_owned(), + })?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use http::{Request, Response, header}; + + use crate::test_support::tests::create_test_settings; + + use super::*; + + fn settings(enabled: bool) -> Settings { + let mut settings = create_test_settings(); + settings.integrations.insert( + AD_TRACE_INTEGRATION_ID.to_owned(), + serde_json::json!({ "enabled": enabled }), + ); + settings + } + + fn request(uri: &str, cookie: Option<&str>) -> Request { + let mut builder = Request::builder() + .method(Method::GET) + .uri(uri) + .header("sec-fetch-dest", "document"); + if let Some(cookie) = cookie { + builder = builder.header(header::COOKIE, cookie); + } + builder + .body(EdgeBody::empty()) + .expect("should build request") + } + + #[test] + fn rejects_unknown_gate_configuration() { + let mut settings = create_test_settings(); + settings.integrations.insert( + AD_TRACE_INTEGRATION_ID.to_owned(), + serde_json::json!({ "enabled": true, "enabledd": true }), + ); + + let error = settings + .integration_config::(AD_TRACE_INTEGRATION_ID) + .expect_err("should reject unknown gate field"); + assert!( + error.to_string().contains("could not be parsed"), + "should reject invalid configuration: {error}" + ); + } + + #[test] + fn query_enables_first_response_and_sanitizes_request() { + let mut req = request( + "https://publisher.example/page?x=%2F&ts_console=1&y=2", + Some("session=abc; __Host-ts-console=1"), + ); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + + assert!(decision.enabled()); + assert_eq!(decision.cookie_action, ConsoleCookieAction::SetSession); + assert_eq!( + req.uri().to_string(), + "https://publisher.example/page?x=%2F&y=2" + ); + assert_eq!( + req.headers() + .get(header::COOKIE) + .expect("should retain unrelated cookie"), + "session=abc" + ); + assert!(browser_trace_enabled(&req)); + let script = decision.bootstrap_script().expect("should bootstrap"); + assert!(script.contains("__tsjs_adTraceActive=true")); + assert!(script.contains("/page?x=%2F&y=2")); + + let mut separators = request( + "https://publisher.example/page?a=1&&ts_console=1&b=2&", + None, + ); + prepare_request(&settings(true), &mut separators).expect("should prepare"); + assert_eq!(separators.uri().query(), Some("a=1&&b=2&")); + } + + #[test] + fn exact_enable_and_disable_values_are_supported() { + for value in ["true", "1"] { + let mut req = request( + &format!("https://publisher.example/?ts_console={value}"), + None, + ); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + assert!(decision.enabled(), "{value} should enable"); + assert_eq!(decision.cookie_action, ConsoleCookieAction::SetSession); + } + for value in ["false", "0"] { + let mut req = request( + &format!("https://publisher.example/?ts_console={value}"), + Some("__Host-ts-console=1"), + ); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + assert!(!decision.enabled(), "{value} should disable"); + assert_eq!(decision.cookie_action, ConsoleCookieAction::ClearSession); + } + } + + #[test] + fn invalid_or_duplicate_query_fails_closed_without_cookie_mutation() { + for query in [ + "ts_console=True", + "ts_console=", + "ts_console=1&ts_console=true", + ] { + let mut req = request( + &format!("https://publisher.example/?{query}&keep=1"), + Some("__Host-ts-console=1"), + ); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + assert!(!decision.enabled(), "{query} should fail closed"); + assert_eq!(decision.cookie_action, ConsoleCookieAction::None); + assert_eq!(req.uri().query(), Some("keep=1")); + } + } + + #[test] + fn disabled_config_sanitizes_but_never_activates() { + let mut req = request( + "https://publisher.example/?ts_console=1&keep=1", + Some("__Host-ts-console=1; other=value; ts-tester=true"), + ); + let decision = prepare_request(&settings(false), &mut req).expect("should prepare"); + assert!(!decision.enabled()); + assert_eq!(decision.cookie_action, ConsoleCookieAction::None); + assert_eq!(decision.clean_browser_path_and_query, None); + assert_eq!(req.uri().query(), Some("keep=1")); + assert_eq!( + req.headers() + .get(header::COOKIE) + .expect("should retain unrelated cookies"), + "other=value; ts-tester=true" + ); + } + + #[test] + fn exact_session_cookie_gates_api_but_query_cannot_activate_it() { + let mut active = Request::builder() + .method(Method::POST) + .uri("https://publisher.example/auction") + .header(header::COOKIE, "__Host-ts-console=1") + .body(EdgeBody::empty()) + .expect("should build request"); + assert!( + prepare_request(&settings(true), &mut active) + .expect("should prepare") + .enabled() + ); + + let mut query_only = Request::builder() + .method(Method::POST) + .uri("https://publisher.example/auction?ts_console=1") + .body(EdgeBody::empty()) + .expect("should build request"); + let decision = prepare_request(&settings(true), &mut query_only).expect("should prepare"); + assert!(!decision.enabled()); + assert_eq!(decision.cookie_action, ConsoleCookieAction::None); + assert_eq!(query_only.uri().query(), None); + } + + #[test] + fn active_session_does_not_make_static_bundle_response_private() { + let mut req = Request::builder() + .method(Method::GET) + .uri("https://publisher.example/static/tsjs=tsjs-unified.min.js") + .header("sec-fetch-dest", "script") + .header(header::COOKIE, "__Host-ts-console=1") + .body(EdgeBody::empty()) + .expect("should build request"); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + assert!(decision.enabled()); + assert!(!decision.requires_private_no_store()); + assert_eq!(decision.bootstrap_script(), None); + } + + #[test] + fn invalid_api_query_fails_closed_even_with_session_cookie() { + let mut req = Request::builder() + .method(Method::POST) + .uri("https://publisher.example/auction?ts_console=invalid") + .header(header::COOKIE, "__Host-ts-console=1") + .body(EdgeBody::empty()) + .expect("should build request"); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + assert!(!decision.enabled()); + assert_eq!(req.uri().query(), None); + assert!(!req.headers().contains_key(header::COOKIE)); + } + + #[test] + fn duplicate_console_cookie_fails_closed_and_all_copies_are_removed() { + let mut req = request( + "https://publisher.example/", + Some("__Host-ts-console=1; a=b; __Host-ts-console=1"), + ); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + assert!(!decision.enabled()); + assert_eq!( + req.headers() + .get(header::COOKIE) + .expect("should retain unrelated cookie"), + "a=b" + ); + + let mut bare = request( + "https://publisher.example/", + Some("__Host-ts-console=1; __Host-ts-console; a=b"), + ); + let decision = prepare_request(&settings(true), &mut bare).expect("should prepare"); + assert!(!decision.enabled()); + assert_eq!( + bare.headers() + .get(header::COOKIE) + .expect("should retain unrelated cookie"), + "a=b" + ); + } + + #[test] + fn ts_tester_cookie_no_longer_activates_console() { + let mut req = request("https://publisher.example/", Some("ts-tester=true")); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + assert!(!decision.enabled()); + } + + #[test] + fn response_finalization_appends_cookie_once_and_reasserts_no_store() { + let mut req = request("https://publisher.example/?ts_console=1", None); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + let mut response = Response::builder() + .header(header::SET_COOKIE, "existing=value") + .header(header::CACHE_CONTROL, "public, max-age=60") + .header("surrogate-control", "max-age=60") + .header("cloudflare-cdn-cache-control", "public, max-age=60") + .body(EdgeBody::empty()) + .expect("should build response"); + attach_response_decision(&decision, &mut response); + + finalize_response(&mut response); + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("public, max-age=60"), + ); + finalize_response(&mut response); + + assert_eq!( + response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .count(), + 2 + ); + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "private, no-store" + ); + assert!(!response.headers().contains_key("surrogate-control")); + assert!( + !response + .headers() + .contains_key("cloudflare-cdn-cache-control") + ); + } +} diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index cc4c5c00c..884aa435a 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -42,6 +42,48 @@ pubads.__tsInitialLoadHooked = true; }); + function captureRequest(slot, trigger) { + function firstTarget(key) { + if (!slot || typeof slot.getTargeting !== "function") return undefined; + var values = slot.getTargeting(key); + return values && values.length ? String(values[0]) : undefined; + } + var divId = + slot && typeof slot.getSlotElementId === "function" + ? slot.getSlotElementId() + : ""; + var slotId = (ts.divToSlotId || {})[divId]; + var liveBid = slotId && ts.bids ? ts.bids[slotId] : undefined; + var bidSnapshot = liveBid + ? Object.freeze( + Object.assign({}, liveBid, { + trace: liveBid.trace ? Object.freeze(Object.assign({}, liveBid.trace)) : undefined, + }), + ) + : undefined; + // Freeze request-boundary attribution before display()/refresh(). If the + // optional module loads later, draining this queue never rereads mutable GPT + // targeting or the current route's bid object. + var snapshot = Object.freeze({ + slotId: slotId, + bidder: firstTarget("hb_bidder"), + adId: firstTarget("hb_adid"), + traceToken: firstTarget("ts_trace"), + bid: bidSnapshot, + }); + if (typeof ts.captureAdTraceRequest === "function") { + ts.captureAdTraceRequest(slot, trigger, snapshot); + return; + } + // The unified bundle may load after this bootstrap. Queue private request + // ownership unconditionally so trace-off traffic receives the same stale + // render and billing protection; diagnostic fields remain independently gated. + ts.pendingAdTraceRequests = ts.pendingAdTraceRequests || []; + if (ts.pendingAdTraceRequests.length < 64) { + ts.pendingAdTraceRequests.push({ slot: slot, trigger: trigger, snapshot: snapshot }); + } + } + ts.adInit = function () { var slots = ts.adSlots || []; var bids = ts.bids || {}; @@ -111,6 +153,9 @@ ].forEach(function (k) { if (b[k]) s.setTargeting(k, b[k]); }); + if (b.trace && b.trace.bidTraceId) { + s.setTargeting("ts_trace", b.trace.bidTraceId); + } // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts s.setTargeting("ts_initial", "1"); // Map both the inner div and the GPT slot's element ID (the @@ -143,6 +188,12 @@ // impression. Runs after enableServices(); on SPA navigation services are // already enabled, so this runs unconditionally for new slots. slotsToDisplay.forEach(function (divId) { + var requestSlot = newSlots.find(function (slot) { + return slot.getSlotElementId() === divId; + }); + if (requestSlot && !ts.gptInitialLoadDisabled) { + captureRequest(requestSlot, "bootstrap_display"); + } googletag.display(divId); }); // Reused publisher-owned slots always need a refresh to pick up the @@ -161,6 +212,9 @@ // bundle's adInit() in crates/trusted-server-js/lib/src/integrations/gpt/index.ts. ts.adInitRefreshInProgress = true; try { + slotsNeedingRefresh.forEach(function (slot) { + captureRequest(slot, "bootstrap_refresh"); + }); googletag.pubads().refresh(slotsNeedingRefresh); } finally { ts.adInitRefreshInProgress = false; diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index af56c3713..3431bc8e7 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -11,6 +11,7 @@ use crate::error::TrustedServerError; use crate::platform::{DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, RuntimeServices}; use crate::settings::Settings; +pub mod ad_trace; pub mod adserver_mock; pub mod aps; pub mod datadome; @@ -284,6 +285,10 @@ pub(crate) struct IntegrationBuilder { pub(crate) fn builders() -> &'static [IntegrationBuilder] { &[ + IntegrationBuilder { + id: "ad_trace", + build: ad_trace::register, + }, IntegrationBuilder { id: "prebid", build: prebid::register, diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index a1b40281d..fc2812a87 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -2664,6 +2664,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &http_req, timeout_ms: 500, @@ -2707,6 +2708,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &http_req, timeout_ms: 500, @@ -2728,6 +2730,11 @@ mod tests { request_host, auction_request.publisher.domain, "request_host should be the publisher domain, not the edge Host header" ); + assert!( + !String::from_utf8_lossy(&bodies[0]) + .contains(&context.trace.auction_trace_id.to_string()), + "internal trace UUID should never be serialized upstream" + ); } fn create_test_auction_context<'a>( @@ -5371,13 +5378,14 @@ external_bundle_sri = "sha384-AAAA" prebid_platform_response(StatusCode::BAD_REQUEST, Some("application/json"), body); let provider_response = futures::executor::block_on(provider.parse_response(response, 42)) .expect("should classify upstream HTTP error"); - let result = OrchestrationResult { - provider_responses: vec![provider_response], - mediator_response: None, - winning_bids: HashMap::new(), - total_time_ms: 42, - metadata: HashMap::new(), - }; + let mut result = OrchestrationResult::empty( + crate::auction::types::AuctionTraceContext::new( + crate::auction::types::AuctionSource::AuctionApi, + ), + crate::auction::types::AuctionPublicOutcome::NoBid, + ); + result.provider_responses = vec![provider_response]; + result.total_time_ms = 42; let response = convert_to_openrtb_response( &result, &make_settings(), @@ -5494,6 +5502,7 @@ external_bundle_sri = "sha384-AAAA" .expect("should build request"); let services = noop_services(); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &http_req, timeout_ms: 1000, diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index 65237ce0e..a06a41183 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -174,10 +174,51 @@ pub struct ImpStoredRequest { #[derive(Debug, Serialize)] pub struct ResponseExt { pub orchestrator: OrchestratorExt, + #[serde(skip_serializing_if = "Option::is_none")] + pub trusted_server: Option, } impl ToExt for ResponseExt {} +/// Namespaced Trusted Server response extensions. +#[derive(Debug, Serialize)] +pub struct TrustedServerResponseExt { + pub trace: AuctionTraceWire, +} + +/// Privacy-safe root trace extension. +#[derive(Debug, Serialize)] +pub struct AuctionTraceWire { + pub version: u8, + pub auction_trace_id: String, + pub source: &'static str, + pub outcome: &'static str, +} + +/// Namespaced Trusted Server bid extensions. +#[derive(Debug, Serialize)] +pub struct TrustedServerBidExt { + pub trusted_server: TrustedServerBidTraceContainer, +} + +impl ToExt for TrustedServerBidExt {} + +/// Container for a Trusted Server bid trace. +#[derive(Debug, Serialize)] +pub struct TrustedServerBidTraceContainer { + pub trace: BidTraceWire, +} + +/// Privacy-safe final-winning-bid trace extension. +#[derive(Debug, Serialize)] +pub struct BidTraceWire { + pub version: u8, + pub bid_trace_id: String, + pub slot_id: String, + pub provider: String, + pub bidder: String, +} + #[cfg(test)] mod tests { use super::*; @@ -211,6 +252,7 @@ mod tests { time_ms: 12, provider_details: vec![], }, + trusted_server: None, } .to_ext(); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 34909efe7..c4e64b29b 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -31,7 +31,7 @@ use crate::auction::endpoints::{ merge_auction_eids, resolve_auction_eids, resolve_client_auction_eids, }; use crate::auction::orchestrator::{ - AuctionOrchestrator, DispatchAuctionOutcome, DispatchedAuction, + AuctionOrchestrator, DispatchAuctionOutcome, DispatchedAuction, OrchestrationResult, }; use crate::auction::telemetry::{ AuctionObservationContext, AuctionSource, AuctionTerminalOutcome, build_auction_events, @@ -229,6 +229,7 @@ struct ProcessResponseParams<'a> { settings: &'a Settings, content_type: &'a str, integration_registry: &'a IntegrationRegistry, + head_bootstrap_script: Option<&'a str>, ad_slots_script: Option<&'a str>, ad_bids_state: &'a Arc>>, } @@ -273,8 +274,11 @@ fn process_response_streaming( params.request_scheme, params.settings, params.integration_registry, - params.ad_slots_script.map(str::to_string), - params.ad_bids_state.clone(), + HtmlAdState { + head_bootstrap_script: params.head_bootstrap_script.map(str::to_string), + ad_slots_script: params.ad_slots_script.map(str::to_string), + ad_bids_state: params.ad_bids_state.clone(), + }, )?; StreamingPipeline::new(config, processor).process(body_as_reader(body), output)?; } else if is_rsc_flight { @@ -312,14 +316,19 @@ fn process_response_streaming( /// `use<>` states that explicitly: without it, Rust 2024 would have the opaque /// type capture every input lifetime, forcing callers to keep the settings and /// registry alive for as long as the processor. +struct HtmlAdState { + head_bootstrap_script: Option, + ad_slots_script: Option, + ad_bids_state: Arc>>, +} + fn create_html_stream_processor( origin_host: &str, request_host: &str, request_scheme: &str, settings: &Settings, integration_registry: &IntegrationRegistry, - ad_slots_script: Option, - ad_bids_state: Arc>>, + ad_state: HtmlAdState, ) -> Result, Report> { use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; @@ -330,7 +339,11 @@ fn create_html_stream_processor( request_host, request_scheme, ) - .with_ad_state(ad_slots_script, ad_bids_state); + .with_ad_state( + ad_state.head_bootstrap_script, + ad_state.ad_slots_script, + ad_state.ad_bids_state, + ); Ok(create_html_processor(config)) } @@ -441,6 +454,7 @@ pub struct OwnedProcessResponseParams { pub(crate) request_host: String, pub(crate) request_scheme: String, pub(crate) content_type: String, + pub(crate) head_bootstrap_script: Option, pub(crate) ad_slots_script: Option, pub(crate) ad_bids_state: Arc>>, /// Observation context for the in-flight auction. @@ -453,6 +467,8 @@ pub struct OwnedProcessResponseParams { pub(crate) dispatched_auction: Option, /// Price granularity used to bucket bids when building `tsjs.bids`. pub(crate) price_granularity: PriceGranularity, + /// Whether the config and exact tester cookie permit browser trace output. + pub(crate) ad_trace_enabled: bool, } /// Buffers a [`PublisherResponse`] into a single [`Response`], collecting the @@ -620,6 +636,7 @@ pub fn stream_publisher_body( settings, content_type: ¶ms.content_type, integration_registry, + head_bootstrap_script: params.head_bootstrap_script.as_deref(), ad_slots_script: params.ad_slots_script.as_deref(), ad_bids_state: ¶ms.ad_bids_state, }; @@ -671,11 +688,12 @@ pub async fn stream_publisher_body_async( // Non-HTML: collect auction first, then stream. There is no // to hold, so delaying the entire body until collection is acceptable. let placeholder = mediator_placeholder_request(); + let trace = dispatched.trace().clone(); let result = orchestrator .collect_dispatched_auction( dispatched, services, - &make_collect_context(settings, services, &placeholder), + &make_collect_context(&trace, settings, services, &placeholder), ) .await; if let (Some(observation), Some(auction_request)) = @@ -694,10 +712,11 @@ pub async fn stream_publisher_body_async( } write_bids_to_state( - &result.winning_bids, + &result, params.price_granularity, ¶ms.ad_bids_state, settings.debug.inject_adm_for_testing, + params.ad_trace_enabled, ); return stream_publisher_body(body, output, params, settings, integration_registry); } @@ -711,8 +730,11 @@ pub async fn stream_publisher_body_async( ¶ms.request_scheme, settings, integration_registry, - params.ad_slots_script.as_deref().map(str::to_string), - params.ad_bids_state.clone(), + HtmlAdState { + head_bootstrap_script: params.head_bootstrap_script.as_deref().map(str::to_string), + ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), + ad_bids_state: params.ad_bids_state.clone(), + }, ) { Ok(processor) => processor, Err(err) => { @@ -741,6 +763,7 @@ pub async fn stream_publisher_body_async( orchestrator, services, settings, + trace_enabled: params.ad_trace_enabled, }, ) .await @@ -769,6 +792,7 @@ fn mediator_placeholder_request() -> Request { /// this argument is plumbing for the (presently unused) case where the /// orchestrator needs the caller's request shape. fn make_collect_context<'a>( + trace: &'a crate::auction::types::AuctionTraceContext, settings: &'a Settings, services: &'a RuntimeServices, placeholder: &'a Request, @@ -780,6 +804,7 @@ fn make_collect_context<'a>( callers must not forward a real client request through the collect path" ); AuctionContext { + trace, settings, request: placeholder, timeout_ms: 0, @@ -845,18 +870,27 @@ pub(crate) fn should_run_server_side_ad_stack( /// Write winning bids from an auction result into the shared `ad_bids_state` lock. pub(crate) fn write_bids_to_state( - winning_bids: &std::collections::HashMap, + result: &crate::auction::orchestrator::OrchestrationResult, price_granularity: PriceGranularity, ad_bids_state: &Arc>>, inject_adm: bool, + trace_enabled: bool, ) { log::debug!( "write_bids_to_state: {} winning bid(s): [{}]", - winning_bids.len(), - winning_bids.keys().cloned().collect::>().join(", ") + result.winning_bids.len(), + result + .winning_bids + .keys() + .cloned() + .collect::>() + .join(", ") + ); + let bid_map = build_bid_map_with_trace(result, price_granularity, inject_adm, trace_enabled); + let bids_script = build_bids_script_with_trace( + &bid_map, + trace_enabled.then(|| auction_trace_json(&result.trace.summary)), ); - let bid_map = build_bid_map(winning_bids, price_granularity, inject_adm); - let bids_script = build_bids_script(&bid_map); *ad_bids_state.lock().expect("should lock bid state") = Some(bids_script); } @@ -1065,6 +1099,7 @@ struct AuctionCollectCtx<'a> { orchestrator: &'a AuctionOrchestrator, services: &'a RuntimeServices, settings: &'a Settings, + trace_enabled: bool, } /// Run the close-body hold loop for HTML bodies, collecting the auction before @@ -1193,6 +1228,7 @@ async fn body_close_hold_loop( orchestrator, services, settings, + trace_enabled, } = ctx; let mut buffer = vec![0u8; STREAM_CHUNK_SIZE]; let mut hold = Some(BodyCloseHoldBuffer::new()); @@ -1208,11 +1244,14 @@ async fn body_close_hold_loop( collect_stream_auction( dispatched, telemetry.take(), - price_granularity, - ad_bids_state, - orchestrator, - services, - settings, + StreamAuctionFinalizeContext { + price_granularity, + ad_bids_state, + orchestrator, + services, + settings, + trace_enabled, + }, ) .await; @@ -1271,11 +1310,14 @@ async fn body_close_hold_loop( collect_stream_auction( dispatched, telemetry.take(), - price_granularity, - ad_bids_state, - orchestrator, - services, - settings, + StreamAuctionFinalizeContext { + price_granularity, + ad_bids_state, + orchestrator, + services, + settings, + trace_enabled, + }, ) .await; @@ -1351,18 +1393,32 @@ async fn emit_abandoned_auction( .await; } +struct StreamAuctionFinalizeContext<'a> { + price_granularity: PriceGranularity, + ad_bids_state: &'a Arc>>, + orchestrator: &'a AuctionOrchestrator, + services: &'a RuntimeServices, + settings: &'a Settings, + trace_enabled: bool, +} + async fn collect_stream_auction( dispatched: DispatchedAuction, telemetry: AuctionTelemetryCarry, - price_granularity: PriceGranularity, - ad_bids_state: &Arc>>, - orchestrator: &AuctionOrchestrator, - services: &RuntimeServices, - settings: &Settings, + context: StreamAuctionFinalizeContext<'_>, ) { + let StreamAuctionFinalizeContext { + price_granularity, + ad_bids_state, + orchestrator, + services, + settings, + trace_enabled, + } = context; log::info!("body_close_hold_loop: collecting dispatched auction before held body tail"); let placeholder = mediator_placeholder_request(); - let collect_ctx = make_collect_context(settings, services, &placeholder); + let trace = dispatched.trace().clone(); + let collect_ctx = make_collect_context(&trace, settings, services, &placeholder); let result = orchestrator .collect_dispatched_auction(dispatched, services, &collect_ctx) .await; @@ -1385,10 +1441,11 @@ async fn collect_stream_auction( result.winning_bids.len() ); write_bids_to_state( - &result.winning_bids, + &result, price_granularity, ad_bids_state, settings.debug.inject_adm_for_testing, + trace_enabled, ); if settings.debug.auction_html_comment { @@ -1493,6 +1550,9 @@ pub async fn handle_publisher_request( ); let consent_context = ec_context.consent().clone(); + let ad_trace_decision = crate::integrations::ad_trace::request_decision(&req); + let ad_trace_enabled = ad_trace_decision.enabled(); + let ad_trace_bootstrap = ad_trace_decision.bootstrap_script(); let ec_id = ec_context.ec_value().filter(|_| ec_allowed); let cookie_jar = handle_request_cookies(&req)?; let geo = ec_context.geo_info().cloned(); @@ -1609,13 +1669,15 @@ pub async fn handle_publisher_request( let mut dispatched_auction = if matched_slots.is_empty() { None } else { + let trace = + crate::auction::types::AuctionTraceContext::new(AuctionSource::InitialNavigation); // Telemetry attribution must use the same publisher identity as the // outbound bid request. On the navigation path `request_host` is the // trusted-server edge host, so using it here would attribute navigation // rows to the edge/staging domain while `/auction` rows (built from // `AuctionRequest::publisher.domain`) use the configured domain. let observation = AuctionObservationContext::from_parts( - AuctionSource::InitialNavigation, + &trace, &settings.publisher.domain, &request_path, matched_slots.len(), @@ -1651,6 +1713,7 @@ pub async fn handle_publisher_request( }, ); let auction_context = AuctionContext { + trace: &trace, settings, request: &req, timeout_ms: auction_timeout_ms, @@ -1672,6 +1735,19 @@ pub async fn handle_publisher_request( provider_responses, elapsed_ms, } => { + if ad_trace_enabled { + let terminal = OrchestrationResult::empty( + trace.clone(), + crate::auction::types::AuctionPublicOutcome::Failed, + ); + write_bids_to_state( + &terminal, + price_granularity, + &ad_bids_state, + settings.debug.inject_adm_for_testing, + true, + ); + } emit_auction_events_best_effort_lazy(services, || { build_auction_events( observation, @@ -1687,6 +1763,19 @@ pub async fn handle_publisher_request( None } DispatchAuctionOutcome::NotStarted => { + if ad_trace_enabled { + let terminal = OrchestrationResult::empty( + trace.clone(), + crate::auction::types::AuctionPublicOutcome::Failed, + ); + write_bids_to_state( + &terminal, + price_granularity, + &ad_bids_state, + settings.debug.inject_adm_for_testing, + true, + ); + } let elapsed_ms = observation.elapsed_ms(); emit_auction_events_best_effort_lazy(services, || { build_auction_events( @@ -1921,12 +2010,14 @@ pub async fn handle_publisher_request( request_host: request_host.to_string(), request_scheme: request_scheme.to_string(), content_type, + head_bootstrap_script: ad_trace_bootstrap.clone(), ad_slots_script: ad_slots_script.clone(), ad_bids_state: ad_bids_state.clone(), auction_observation, auction_request: auction_request_for_telemetry, dispatched_auction, price_granularity, + ad_trace_enabled, }), }) } @@ -2172,18 +2263,78 @@ pub(crate) fn build_bid_map( .collect() } +fn auction_trace_json(summary: &crate::auction::types::AuctionTraceSummary) -> serde_json::Value { + serde_json::json!({ + "version": 1, + "auctionTraceId": summary.auction.auction_trace_id.to_string(), + "source": summary.auction.source.as_str(), + "outcome": summary.outcome.as_str(), + }) +} + +fn apply_bid_traces( + bid_map: &mut serde_json::Map, + result_trace: &crate::auction::types::AuctionResultTrace, +) { + for (slot_id, trace) in &result_trace.winning_bids { + if let Some(serde_json::Value::Object(bid)) = bid_map.get_mut(slot_id) { + bid.insert( + "trace".to_owned(), + serde_json::json!({ + "version": 1, + "auctionTraceId": result_trace.summary.auction.auction_trace_id.to_string(), + "bidTraceId": trace.bid_trace_id.to_string(), + "source": result_trace.summary.auction.source.as_str(), + "slotId": slot_id, + "provider": trace.provider, + "bidder": trace.bidder, + }), + ); + } + } +} + +fn build_bid_map_with_trace( + result: &crate::auction::orchestrator::OrchestrationResult, + granularity: crate::price_bucket::PriceGranularity, + include_adm: bool, + trace_enabled: bool, +) -> serde_json::Map { + let mut bid_map = build_bid_map(&result.winning_bids, granularity, include_adm); + if !trace_enabled { + return bid_map; + } + apply_bid_traces(&mut bid_map, &result.trace); + bid_map +} + /// Build the `tsjs.bids` `` sequences inside the string. pub(crate) fn build_bids_script(bid_map: &serde_json::Map) -> String { + build_bids_script_with_trace(bid_map, None) +} + +fn build_bids_script_with_trace( + bid_map: &serde_json::Map, + auction_trace: Option, +) -> String { let json = serde_json::to_string(bid_map) .expect("serde_json::to_string of Map should be infallible"); let escaped = html_escape_for_script(&json); - format!( - "", - escaped - ) + if let Some(trace) = auction_trace { + let trace_json = serde_json::to_string(&trace) + .expect("serde_json::to_string of trace should be infallible"); + let escaped_trace = html_escape_for_script(&trace_json); + format!( + "" + ) + } else { + format!( + "" + ) + } } /// Build the empty-bids `"# .to_string(), @@ -4015,6 +4195,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let mut output = Vec::new(); @@ -4058,12 +4239,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let bogus_body = EdgeBody::from(b"not gzip".to_vec()); @@ -4165,12 +4348,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -4221,12 +4406,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let mut output = Vec::new(); @@ -4258,7 +4445,7 @@ mod tests { mod creative_opportunities_tests { use super::super::{ MatchedSlotsContext, build_ad_slots_script, build_auction_request, build_bid_map, - build_bids_script, html_escape_for_script, + build_bids_script, build_bids_script_with_trace, html_escape_for_script, }; use crate::auction::types::{Bid, MediaType}; use crate::consent::ConsentContext; @@ -4688,6 +4875,30 @@ mod tests { assert!(!inner.contains('>'), "no unescaped > in bids script"); } + #[test] + fn traced_bids_script_assigns_summary_and_bids_before_ad_init() { + let mut map = serde_json::Map::new(); + map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); + let trace = serde_json::json!({ + "version": 1, + "auctionTraceId": "550e8400-e29b-41d4-a716-446655440000", + "source": "initial_navigation", + "outcome": "completed", + }); + + let script = build_bids_script_with_trace(&map, Some(trace)); + + let trace_pos = script + .find(".auctionTrace=JSON.parse") + .expect("should assign trace"); + let bids_pos = script.find(".bids=JSON.parse").expect("should assign bids"); + let init_pos = script.find("adInit").expect("should invoke adInit"); + assert!( + trace_pos < bids_pos && bids_pos < init_pos, + "should atomically assign trace and bids before adInit" + ); + } + #[test] fn bids_script_calls_ad_init_without_retry_timer() { let mut map = serde_json::Map::new(); @@ -4982,8 +5193,10 @@ mod tests { orchestrator: &AuctionOrchestrator, slots: &[CreativeOpportunitySlot], ec_context: &EcContext, - req: Request, + mut req: Request, ) -> Response { + crate::integrations::ad_trace::prepare_request(settings, &mut req) + .expect("should prepare ad trace request"); let services = noop_services(); handle_page_bids( settings, @@ -5167,11 +5380,17 @@ mod tests { #[tokio::test] async fn url_not_matching_any_pattern_returns_empty_response() { - // Slots exist but request path does not match — no auction, no injection. - let settings = settings_with_co(); + // Slots exist but request path does not match — no auction, no injection, + // and no unjoinable trace identity even when the tester gate is open. + let mut settings = settings_with_co(); + settings + .integrations + .insert_config("ad_trace", &serde_json::json!({ "enabled": true })) + .expect("should configure ad trace"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let slots = article_slot(); // slot matches /20** only - let req = make_page_bids_request("/about"); // does not match + let mut req = make_page_bids_request("/about"); // does not match + set_test_header(&mut req, "cookie", "__Host-ts-console=1"); let body = run_page_bids(&settings, &orchestrator, &slots, req).await; @@ -5191,6 +5410,45 @@ mod tests { 0, "non-matching URL should produce zero bids" ); + assert!( + body.get("auctionTrace").is_none(), + "non-matching URL should not expose an identity without telemetry" + ); + } + + #[tokio::test] + async fn page_bids_trace_requires_config_and_console_session() { + let mut settings = settings_with_co_auction_disabled(); + settings + .integrations + .insert_config("ad_trace", &serde_json::json!({ "enabled": true })) + .expect("should configure ad trace"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + + let without_cookie = make_page_bids_request("/2024/01/my-article/"); + let without_cookie_body = + run_page_bids_consent_allowed(&settings, &orchestrator, &slots, without_cookie) + .await; + assert!( + without_cookie_body.get("auctionTrace").is_none(), + "config alone should not expose trace" + ); + + let mut gated = make_page_bids_request("/2024/01/my-article/"); + set_test_header(&mut gated, "cookie", "__Host-ts-console=1"); + let gated_body = + run_page_bids_consent_allowed(&settings, &orchestrator, &slots, gated).await; + assert_eq!( + gated_body["auctionTrace"]["source"], + serde_json::json!("spa_navigation"), + "both gates should expose generic SPA trace" + ); + assert_eq!( + gated_body["auctionTrace"]["outcome"], + serde_json::json!("skipped"), + "disabled auction should not be fabricated as completed no-bid" + ); } #[test] diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 27a94b62d..262d3fc95 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -17,7 +17,12 @@ use crate::settings::Settings; /// /// A single source of truth so the adapter copies of the privacy downgrade /// cannot drift apart. -pub const SURROGATE_CACHE_HEADERS: &[&str] = &["surrogate-control", "fastly-surrogate-control"]; +pub const SURROGATE_CACHE_HEADERS: &[&str] = &[ + "surrogate-control", + "fastly-surrogate-control", + "cdn-cache-control", + "cloudflare-cdn-cache-control", +]; /// Forces cookie-bearing responses to stay private to shared caches. /// @@ -82,8 +87,9 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: for (key, value) in &settings.response_headers { if response_is_uncacheable && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) - || key.eq_ignore_ascii_case("surrogate-control") - || key.eq_ignore_ascii_case("fastly-surrogate-control")) + || SURROGATE_CACHE_HEADERS + .iter() + .any(|name| key.eq_ignore_ascii_case(name))) { continue; } diff --git a/crates/trusted-server-integration-tests/browser/global-setup.ts b/crates/trusted-server-integration-tests/browser/global-setup.ts index f54d92dbe..14b3b0539 100644 --- a/crates/trusted-server-integration-tests/browser/global-setup.ts +++ b/crates/trusted-server-integration-tests/browser/global-setup.ts @@ -16,12 +16,11 @@ const WASM_PATH = "../../../target/wasm32-wasip1/release/trusted-server-adapter-fastly.wasm", ); -const VICEROY_CONFIG = - process.env.VICEROY_CONFIG_PATH || - resolve( - __dirname, - "../../../target/integration-test-artifacts/configs/viceroy.toml", - ); +function viceroyConfigPath(framework: string): string { + if (process.env.VICEROY_CONFIG_PATH) return process.env.VICEROY_CONFIG_PATH; + const filename = framework === "ad-trace" ? "viceroy-ad-trace.toml" : "viceroy.toml"; + return resolve(__dirname, `../../../target/integration-test-artifacts/configs/${filename}`); +} /** Persist current state so global-teardown can always clean up. */ function writeState(state: { @@ -47,7 +46,7 @@ async function globalSetup(): Promise { writeState({ containerId, framework }); console.log(`[global-setup] Starting Viceroy (WASM: ${WASM_PATH})...`); - const viceroy = await startViceroy(WASM_PATH, VICEROY_CONFIG); + const viceroy = await startViceroy(WASM_PATH, viceroyConfigPath(framework)); viceroyPid = viceroy.process.pid; console.log(`[global-setup] Viceroy ready at ${viceroy.baseUrl}`); diff --git a/crates/trusted-server-integration-tests/browser/helpers/infra.ts b/crates/trusted-server-integration-tests/browser/helpers/infra.ts index 0402bb266..1b7682b6b 100644 --- a/crates/trusted-server-integration-tests/browser/helpers/infra.ts +++ b/crates/trusted-server-integration-tests/browser/helpers/infra.ts @@ -7,6 +7,7 @@ const ORIGIN_PORT = process.env.INTEGRATION_ORIGIN_PORT || "8888"; /** Framework-specific container configuration. */ const FRAMEWORK_CONFIG: Record = { + "ad-trace": { image: "test-ad-trace:latest", port: 80 }, nextjs: { image: "test-nextjs:latest", port: 3000 }, wordpress: { image: "test-wordpress:latest", port: 80 }, }; diff --git a/crates/trusted-server-integration-tests/browser/helpers/state.ts b/crates/trusted-server-integration-tests/browser/helpers/state.ts index b8f5d4b66..dd655d01c 100644 --- a/crates/trusted-server-integration-tests/browser/helpers/state.ts +++ b/crates/trusted-server-integration-tests/browser/helpers/state.ts @@ -8,7 +8,7 @@ export interface TestState { framework: string; } -const KNOWN_FRAMEWORKS = ["nextjs", "wordpress"] as const; +const KNOWN_FRAMEWORKS = ["ad-trace", "nextjs", "wordpress"] as const; const STATE_FILE = resolve(__dirname, "../.browser-test-state.json"); let cachedState: TestState | undefined; diff --git a/crates/trusted-server-integration-tests/browser/package.json b/crates/trusted-server-integration-tests/browser/package.json index 13282f289..72b3fb997 100644 --- a/crates/trusted-server-integration-tests/browser/package.json +++ b/crates/trusted-server-integration-tests/browser/package.json @@ -4,6 +4,7 @@ "private": true, "scripts": { "test": "npx playwright test", + "test:ad-trace": "TEST_FRAMEWORK=ad-trace npx playwright test tests/ad-trace/auction-trace.spec.ts", "test:nextjs": "TEST_FRAMEWORK=nextjs npx playwright test", "test:wordpress": "TEST_FRAMEWORK=wordpress npx playwright test" }, diff --git a/crates/trusted-server-integration-tests/browser/playwright.config.ts b/crates/trusted-server-integration-tests/browser/playwright.config.ts index 8a1ef3b5b..812c889ec 100644 --- a/crates/trusted-server-integration-tests/browser/playwright.config.ts +++ b/crates/trusted-server-integration-tests/browser/playwright.config.ts @@ -1,7 +1,13 @@ import { defineConfig } from "@playwright/test"; +const framework = process.env.TEST_FRAMEWORK || "nextjs"; + export default defineConfig({ testDir: "./tests", + testMatch: + framework === "ad-trace" + ? ["ad-trace/**/*.spec.ts"] + : ["nextjs/**/*.spec.ts", "shared/**/*.spec.ts", "wordpress/**/*.spec.ts"], globalSetup: "./global-setup.ts", globalTeardown: "./global-teardown.ts", timeout: 30_000, @@ -20,5 +26,5 @@ export default defineConfig({ }, ], reporter: [["list"], ["html", { open: "never" }]], - outputDir: "./test-results", + outputDir: `./test-results-${framework}`, }); diff --git a/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts b/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts new file mode 100644 index 000000000..9c2935d07 --- /dev/null +++ b/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts @@ -0,0 +1,439 @@ +import { expect, test, type Page } from "@playwright/test"; +import { runtimeUrl } from "../../helpers/state.js"; + +const ORIGIN_PORT = process.env.INTEGRATION_ORIGIN_PORT || "8888"; + +async function serveBuiltPrebid(page: Page): Promise { + const response = await fetch( + `http://127.0.0.1:${ORIGIN_PORT}/prebid-bundle.js`, + ); + if (!response.ok) + throw new Error(`fixture Prebid bundle returned ${response.status}`); + const body = await response.text(); + await page.route("**/integrations/prebid/bundle.js*", (route) => + route.fulfill({ + status: 200, + contentType: "application/javascript", + body, + }), + ); +} + +async function openTesterPage(page: Page): Promise { + await serveBuiltPrebid(page); + await page.goto(runtimeUrl("/?ts_console=1"), { + waitUntil: "domcontentloaded", + }); + await expect(page).toHaveURL(runtimeUrl("/")); + await expect + .poll(() => + page.evaluate(() => + ( + window as Window & { + tsjs?: { adTrace?: { export(): unknown } }; + } + ).tsjs?.adTrace?.export(), + ), + ) + .toBeTruthy(); + await expect + .poll(() => + page.evaluate(() => { + const result = ( + window as Window & { + tsjs?: { + adTrace?: { + export(): { + slots: Array<{ + slotId: string; + stages: { + creative: { outcome: string }; + }; + }>; + }; + }; + }; + } + ).tsjs?.adTrace?.export(); + return result?.slots.find( + (slot) => slot.slotId === "ad-trace-slot", + )?.stages.creative.outcome; + }), + ) + .toBe("load_acknowledged"); + await expect + .poll(() => + page.evaluate(() => + ( + window as Window & { + tsjs?: { + adTrace?: { + getEvents(): Array<{ kind: string }>; + }; + }; + } + ).tsjs?.adTrace + ?.getEvents() + .some((event) => event.kind === "gpt_slot_render_ended"), + ), + ) + .toBe(true); +} + +async function exported(page: Page) { + return page.evaluate(() => + ( + window as Window & { + tsjs: { + adTrace: { + export(): { slots: Array> }; + }; + }; + } + ).tsjs.adTrace.export(), + ); +} + +test.describe("tester-only auction trace contract", () => { + test("config without an activated console session exposes no browser trace surface", async ({ + page, + }) => { + await serveBuiltPrebid(page); + await page.goto(runtimeUrl("/"), { waitUntil: "domcontentloaded" }); + + expect( + await page.evaluate( + () => + typeof (window as Window & { tsjs?: { adTrace?: unknown } }) + .tsjs?.adTrace, + ), + ).toBe("undefined"); + }); + + test("console session supports true, persists privately, and can be disabled", async ({ + page, + }) => { + await serveBuiltPrebid(page); + const activation = await page.goto(runtimeUrl("/?ts_console=true"), { + waitUntil: "domcontentloaded", + }); + await expect(page).toHaveURL(runtimeUrl("/")); + expect(activation?.headers()["cache-control"]).toBe("private, no-store"); + await expect + .poll(() => + page.evaluate( + () => + typeof ( + window as Window & { tsjs?: { adTrace?: unknown } } + ).tsjs?.adTrace, + ), + ) + .toBe("object"); + expect( + (await page.context().cookies()).find( + (cookie) => cookie.name === "__Host-ts-console", + ), + ).toMatchObject({ + value: "1", + httpOnly: true, + secure: true, + sameSite: "Lax", + }); + + await page.reload({ waitUntil: "domcontentloaded" }); + expect( + await page.evaluate( + () => + typeof (window as Window & { tsjs?: { adTrace?: unknown } }) + .tsjs?.adTrace, + ), + ).toBe("object"); + + await page.goto(runtimeUrl("/?ts_console=0"), { + waitUntil: "domcontentloaded", + }); + await expect(page).toHaveURL(runtimeUrl("/")); + expect( + await page.evaluate( + () => + typeof (window as Window & { tsjs?: { adTrace?: unknown } }) + .tsjs?.adTrace, + ), + ).toBe("undefined"); + + await page.reload({ waitUntil: "domcontentloaded" }); + expect( + await page.evaluate( + () => + typeof (window as Window & { tsjs?: { adTrace?: unknown } }) + .tsjs?.adTrace, + ), + ).toBe("undefined"); + }); + + test("initial TS winner reaches direct GPT and source-validated creative acknowledgement", async ({ + page, + }) => { + await openTesterPage(page); + + await expect + .poll(async () => { + const result = await exported(page); + const slot = result.slots.find( + (item) => item.slotId === "ad-trace-slot", + ) as + | { + stages?: Record< + string, + { outcome?: string; confidence?: string } + >; + } + | undefined; + return { + trustedServer: slot?.stages?.trustedServer?.outcome, + prebid: slot?.stages?.prebid?.outcome, + gam: slot?.stages?.gam?.outcome, + creative: slot?.stages?.creative?.outcome, + }; + }) + .toEqual({ + trustedServer: "won", + prebid: "not_run", + gam: "trusted_server_won", + creative: "load_acknowledged", + }); + + const session = await page.context().newCDPSession(page); + const tree = (await session.send("Accessibility.getFullAXTree")) as { + nodes: Array<{ name?: { value?: string } }>; + }; + const visibleText = tree.nodes + .map((node) => node.name?.value || "") + .join("\n"); + expect(visibleText).toContain("TS winner: won · definitive"); + expect(visibleText).toContain( + "Creative: load_acknowledged · definitive", + ); + }); + + test("direct auction API render reaches an exact iframe-load acknowledgement", async ({ + page, + }) => { + await openTesterPage(page); + await page.evaluate(() => { + const direct = document.createElement("div"); + direct.id = "direct-api-slot"; + document.body.appendChild(direct); + const ts = (window as Window & { + tsjs: { + addAdUnits(unit: unknown): void; + requestAds(): void; + }; + }).tsjs; + ts.addAdUnits({ + code: "direct-api-slot", + mediaTypes: { banner: { sizes: [[300, 250]] } }, + bids: [{ bidder: "example", params: {} }], + }); + ts.requestAds(); + }); + + await expect + .poll(() => + page.evaluate(() => { + const result = ( + window as Window & { + tsjs: { + adTrace: { + export(): { + renders: Array<{ + slotId: string; + source: string; + outcome: string; + }>; + }; + }; + }; + } + ).tsjs.adTrace.export(); + return result.renders.find( + (render) => render.slotId === "direct-api-slot", + ); + }), + ) + .toMatchObject({ + slotId: "direct-api-slot", + source: "direct_auction", + outcome: "confirmed", + }); + }); + + test("actual generated Prebid selects the traced TS bid before a probable GAM result", async ({ + page, + }) => { + await openTesterPage(page); + await expect + .poll(() => + page.evaluate(() => { + const win = window as Window & { + pbjs?: { requestBids?: unknown }; + googletag?: { + pubads(): { __tsRefreshWrapped?: boolean }; + }; + }; + return ( + typeof win.pbjs?.requestBids === "function" && + win.googletag?.pubads().__tsRefreshWrapped === true + ); + }), + ) + .toBe(true); + await page.evaluate(() => { + const win = window as Window & { + adTraceFixture: { + latestSlot(): unknown; + setSuppressCreative(value: boolean): void; + }; + googletag: { pubads(): { refresh(slots: unknown[]): void } }; + }; + win.adTraceFixture.setSuppressCreative(true); + win.googletag.pubads().refresh([win.adTraceFixture.latestSlot()]); + }); + await expect + .poll(async () => { + const result = await exported(page); + const slot = result.slots.find( + (item) => item.slotId === "ad-trace-slot", + ) as + | { + stages?: Record< + string, + { outcome?: string; confidence?: string } + >; + } + | undefined; + return { + prebid: slot?.stages?.prebid, + gam: slot?.stages?.gam, + }; + }) + .toMatchObject({ + prebid: { outcome: "won", confidence: "definitive" }, + gam: { + outcome: "trusted_server_candidate", + confidence: "probable", + }, + }); + }); + + test("client selection, backfill, direct-or-unattributed, and retained generations stay independent", async ({ + page, + }) => { + await openTesterPage(page); + + await page.evaluate(() => { + const win = window as Window & { + adTraceFixture: { simulateClientSelection(): void }; + }; + win.adTraceFixture.simulateClientSelection(); + }); + await expect + .poll(async () => { + const result = await exported(page); + const slot = result.slots.find( + (item) => item.slotId === "ad-trace-slot", + ) as + | { stages?: Record } + | undefined; + return { + prebid: slot?.stages?.prebid?.outcome, + gam: slot?.stages?.gam?.outcome, + }; + }) + .toEqual({ prebid: "lost", gam: "client_prebid_candidate" }); + + await page.evaluate(() => { + const win = window as Window & { + adTraceFixture: { + latestSlot(): unknown; + setNextRender(flags: { isBackfill: boolean }): void; + requestCurrent(): void; + }; + tsjs: { + captureAdTraceRequest(slot: unknown, trigger: string): void; + }; + }; + const slot = win.adTraceFixture.latestSlot(); + win.adTraceFixture.setNextRender({ isBackfill: true }); + win.tsjs.captureAdTraceRequest(slot, "fixture_backfill"); + win.adTraceFixture.requestCurrent(); + }); + await expect + .poll(async () => { + const result = await exported(page); + const slot = result.slots.find( + (item) => item.slotId === "ad-trace-slot", + ) as + | { stages?: Record } + | undefined; + return slot?.stages?.gam?.outcome; + }) + .toBe("backfill"); + + await page.evaluate(() => { + const win = window as Window & { + adTraceFixture: { + latestSlot(): { clearTargeting(): void }; + requestCurrent(): void; + }; + tsjs: { + captureAdTraceRequest(slot: unknown, trigger: string): void; + }; + }; + const slot = win.adTraceFixture.latestSlot(); + slot.clearTargeting(); + win.tsjs.captureAdTraceRequest(slot, "fixture_direct"); + win.adTraceFixture.requestCurrent(); + }); + await expect + .poll(async () => { + const result = await exported(page); + const slot = result.slots.find( + (item) => item.slotId === "ad-trace-slot", + ) as + | { stages?: Record } + | undefined; + return slot?.stages?.gam?.outcome; + }) + .toBe("direct_or_unattributed"); + + const generations = await page.evaluate(() => { + const win = window as Window & { + adTraceFixture: { + simulateRetainedGenerationAcknowledgement(): unknown; + }; + }; + return win.adTraceFixture.simulateRetainedGenerationAcknowledgement(); + }); + const result = await exported(page); + const slot = result.slots.find( + (item) => item.slotId === "ad-trace-slot", + ) as { + latestGeneration: number; + generations: Array<{ + generation: number; + stages: { creative: { outcome: string } }; + }>; + }; + const retained = generations as { first: number; second: number }; + expect(slot.latestGeneration).toBe(retained.second); + expect( + slot.generations.find((item) => item.generation === retained.first) + ?.stages.creative.outcome, + ).toBe("load_acknowledged"); + expect( + slot.generations.find((item) => item.generation === retained.second) + ?.stages.creative.outcome, + ).not.toBe("load_acknowledged"); + }); +}); diff --git a/crates/trusted-server-integration-tests/browser/tests/shared/ad-trace-gate.spec.ts b/crates/trusted-server-integration-tests/browser/tests/shared/ad-trace-gate.spec.ts new file mode 100644 index 000000000..09ceb53d5 --- /dev/null +++ b/crates/trusted-server-integration-tests/browser/tests/shared/ad-trace-gate.spec.ts @@ -0,0 +1,20 @@ +import { expect, test } from "@playwright/test"; +import { runtimeUrl } from "../../helpers/state.js"; + +test("console query alone does not install ad trace when config is disabled", async ({ + page, +}) => { + await page.goto(runtimeUrl("/?ts_console=1"), { + waitUntil: "domcontentloaded", + }); + + await expect + .poll(() => + page.evaluate( + () => + typeof (window as Window & { tsjs?: { adTrace?: unknown } }) + .tsjs?.adTrace, + ), + ) + .toBe("undefined"); +}); diff --git a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.ad-trace.integration.toml b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.ad-trace.integration.toml new file mode 100644 index 000000000..851d50cdf --- /dev/null +++ b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.ad-trace.integration.toml @@ -0,0 +1,67 @@ +[[handlers]] +path = "^/_ts/admin" +username = "admin" +password = "integration-admin-password-32-bytes-ok" + +[publisher] +domain = "localhost" +cookie_domain = "localhost" +origin_url = "http://127.0.0.1:8888" +proxy_secret = "integration-test-proxy-secret" + +[ec] +passphrase = "integration-test-ec-secret-padded-32" +ec_store = "ec_identity_store" + +[request_signing] +enabled = false +config_store_id = "app_config" +secret_store_id = "secrets" + +[integrations.ad_trace] +enabled = true + +[integrations.prebid] +enabled = true +server_url = "http://127.0.0.1:8888/openrtb2/auction" +external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" +timeout_ms = 750 +bidders = ["example-bidder"] +client_side_bidders = [] +debug = false +test_mode = true + +[integrations.gpt] +enabled = true +script_url = "https://ads.example.com/gpt.js" +cache_ttl_seconds = 3600 +rewrite_script = false + +[proxy] +certificate_check = false +allowed_domains = ["assets.example.com"] + +[auction] +enabled = true +providers = ["prebid"] +timeout_ms = 1000 +allowed_context_keys = [] + +[creative_opportunities] +gam_network_id = "123456789" +auction_timeout_ms = 750 +price_granularity = "dense" + +[[creative_opportunities.slot]] +id = "ad-trace-slot" +div_id = "ad-trace-slot" +gam_unit_path = "/123456789/example/ad-trace" +page_patterns = ["/", "/spa*"] +formats = [{ width = 300, height = 250 }] + +[creative_opportunities.slot.providers.prebid] +bidders = { example-bidder = { placement = "example-placement" } } + +[debug] +ja4_endpoint_enabled = false +inject_adm_for_testing = true diff --git a/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/Dockerfile b/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/Dockerfile new file mode 100644 index 000000000..7996d6fde --- /dev/null +++ b/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/Dockerfile @@ -0,0 +1,15 @@ +# Deterministic publisher/PBS fixture for the tester-only ad trace journey. +FROM php:8.3-cli-alpine + +WORKDIR /var/www/html + +COPY crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/ /var/www/html/ +COPY target/integration-test-artifacts/prebid/ /opt/prebid/ + +RUN bundle="$(find /opt/prebid -maxdepth 1 -name 'trusted-prebid-*.js' -type f | head -n 1)" \ + && test -n "$bundle" \ + && cp "$bundle" /var/www/html/prebid-bundle.js + +EXPOSE 80 + +CMD ["php", "-S", "0.0.0.0:80", "router.php"] diff --git a/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/index.php b/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/index.php new file mode 100644 index 000000000..e7bfca062 --- /dev/null +++ b/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/index.php @@ -0,0 +1,201 @@ + + + + + + Trusted Server ad trace fixture + + + + +

Ad trace contract fixture

+

This page uses deterministic local PBS, GPT, and universal creative protocol mocks.

+
+ + diff --git a/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/router.php b/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/router.php new file mode 100644 index 000000000..e14e7b6e8 --- /dev/null +++ b/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/router.php @@ -0,0 +1,50 @@ + $imp) { + $slotId = is_string($imp['id'] ?? null) ? $imp['id'] : 'ad-trace-slot'; + $bids[] = [ + 'id' => 'example-bid-' . ($index + 1), + 'impid' => $slotId, + 'adid' => 'example-ad-' . ($index + 1), + 'price' => 1.25, + 'adm' => '
Example creative loaded
', + 'crid' => 'example-creative-' . ($index + 1), + 'w' => 300, + 'h' => 250, + 'adomain' => ['advertiser.example.com'], + ]; + } + + header('Content-Type: application/json'); + echo json_encode([ + 'id' => is_string($request['id'] ?? null) ? $request['id'] : 'example-auction', + 'seatbid' => $bids ? [['seat' => 'example-bidder', 'bid' => $bids]] : [], + 'cur' => 'USD', + ], JSON_UNESCAPED_SLASHES); + return; +} + +if ($path === '/prebid-bundle.js') { + header('Content-Type: application/javascript'); + readfile(__DIR__ . '/prebid-bundle.js'); + return; +} + +if ($path === '/' || $path === '/spa-one' || $path === '/spa-two') { + require __DIR__ . '/index.php'; + return; +} + +http_response_code(404); +header('Content-Type: text/plain'); +echo 'Not found'; diff --git a/crates/trusted-server-integration-tests/tests/parity.rs b/crates/trusted-server-integration-tests/tests/parity.rs index e85b1d8d1..e41d84dc9 100644 --- a/crates/trusted-server-integration-tests/tests/parity.rs +++ b/crates/trusted-server-integration-tests/tests/parity.rs @@ -44,6 +44,9 @@ fn test_settings() -> Settings { [ec] passphrase = "test-secret-key-32-bytes-minimum" + + [integrations.ad_trace] + enabled = true "#, ) .expect("should parse parity test settings") @@ -85,6 +88,24 @@ async fn axum_get(uri: &str) -> (u16, HeaderMap) { (resp.status().as_u16(), resp.headers().clone()) } +async fn axum_document_get(uri: &str) -> (u16, HeaderMap) { + let mut svc = EdgeZeroAxumService::new(axum_router()); + let req = AxumRequest::builder() + .method("GET") + .uri(uri) + .header("sec-fetch-dest", "document") + .body(AxumBody::empty()) + .expect("should build document GET request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + (resp.status().as_u16(), resp.headers().clone()) +} + /// Send a POST request to the Axum adapter and return (status, headers, body bytes). async fn axum_post(uri: &str, body: &str) -> (u16, HeaderMap, bytes::Bytes) { use http_body_util::BodyExt as _; @@ -131,6 +152,17 @@ async fn cf_get(uri: &str) -> (u16, HeaderMap) { (resp.status().as_u16(), resp.headers().clone()) } +async fn cf_document_get(uri: &str) -> (u16, HeaderMap) { + let req = request_builder() + .method("GET") + .uri(uri) + .header("sec-fetch-dest", "document") + .body(edgezero_core::body::Body::empty()) + .expect("should build document GET request"); + let resp = cf_router().oneshot(req).await.expect("should respond"); + (resp.status().as_u16(), resp.headers().clone()) +} + /// Send a POST request to the Cloudflare adapter and return (status, headers, body bytes). async fn cf_post(uri: &str, body: &str) -> (u16, HeaderMap, bytes::Bytes) { let router = cf_router(); @@ -174,6 +206,17 @@ async fn spin_get(uri: &str) -> (u16, HeaderMap) { (s, h) } +async fn spin_document_get(uri: &str) -> (u16, HeaderMap) { + let req = request_builder() + .method("GET") + .uri(uri) + .header("sec-fetch-dest", "document") + .body(edgezero_core::body::Body::empty()) + .expect("should build document GET request"); + let resp = spin_router().oneshot(req).await.expect("should respond"); + (resp.status().as_u16(), resp.headers().clone()) +} + /// Send a POST request to the Spin adapter and return (status, headers, body bytes). async fn spin_post(uri: &str, body: &str) -> (u16, HeaderMap, bytes::Bytes) { let router = spin_router(); @@ -456,6 +499,34 @@ async fn verify_signature_route_parity() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn console_activation_finalizes_auth_short_circuits() { + let uri = "/_ts/admin/keys/rotate?ts_console=1"; + let responses = [ + axum_document_get(uri).await, + cf_document_get(uri).await, + spin_document_get(uri).await, + ]; + + for (status, headers) in responses { + assert_eq!(status, 401); + assert_eq!( + headers + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("private, no-store") + ); + assert!( + headers + .get_all("set-cookie") + .iter() + .filter_map(|value| value.to_str().ok()) + .any(|value| value.starts_with("__Host-ts-console=1;")), + "auth short-circuit should preserve the console session action" + ); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_rotate_unauthenticated_parity() { // Both adapters must return 401 for unauthenticated admin requests on the diff --git a/crates/trusted-server-js/lib/src/core/ad_trace.ts b/crates/trusted-server-js/lib/src/core/ad_trace.ts new file mode 100644 index 000000000..aca037f5d --- /dev/null +++ b/crates/trusted-server-js/lib/src/core/ad_trace.ts @@ -0,0 +1,505 @@ +import type { + AdTraceApi, + AdTraceConfidence, + AdTraceEvent, + AdTraceEventKind, + AdTraceExport, + AdTraceObservation, + AdTraceStage, + AdTraceStageName, + GenerationTraceSnapshot, + RenderTraceOutcome, + RenderTraceSnapshot, + RenderTraceVisibility, + SlotTraceSnapshot, +} from './types'; + +export const AD_TRACE_MAX_EVENTS = 256; +export const AD_TRACE_MAX_SLOTS = 64; +export const AD_TRACE_MAX_GENERATIONS = 8; +export const AD_TRACE_MAX_RENDERS = 200; +export const AD_TRACE_ACK_TTL_MS = 30_000; +const AD_TRACE_MAX_LISTENERS = 32; + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const LABEL_RE = /^[\w.-]{1,64}$/; +const EVENT_KINDS = new Set([ + 'ts_auction_observed', + 'ts_winner_observed', + 'prebid_auction_init', + 'prebid_bid_response', + 'prebid_targeting_selected', + 'prebid_bid_won', + 'prebid_auction_end', + 'prebid_render_succeeded', + 'prebid_render_failed', + 'gpt_targeting_applied', + 'gpt_request_started', + 'gpt_slot_requested', + 'gpt_slot_response_received', + 'gpt_slot_render_ended', + 'gpt_slot_onload', + 'aps_display_bids_set', + 'pb_render_requested', + 'pb_render_rejected', + 'pb_render_served', + 'direct_render_rejected', + 'creative_load_acknowledged', + 'generation_superseded', +]); +const CONFIDENCES = new Set(['definitive', 'strong', 'probable', 'none']); +const EMPTY_STAGE: AdTraceStage = { outcome: 'not_observed', confidence: 'none', reason: 'none' }; + +type MutableGeneration = GenerationTraceSnapshot; +interface MutableSlot { + slotId: string; + latestGeneration: number; + baseStages: Record; + generations: MutableGeneration[]; +} + +export interface AdTraceStore extends AdTraceApi { + record(observation: AdTraceObservation): void; + nextGeneration(slotId: string): number; + subscribe(listener: () => void): () => void; + bindElement(slotId: string, generation: number, element: HTMLElement): void; + getBoundElement(slotId: string, generation: number): HTMLElement | undefined; + updateVisibility(slotId: string, generation: number, visibility: RenderTraceVisibility): void; +} + +function stages(): Record { + return { + trustedServer: { ...EMPTY_STAGE }, + prebid: { ...EMPTY_STAGE }, + gam: { ...EMPTY_STAGE }, + creative: { ...EMPTY_STAGE }, + }; +} + +function safeLabel(value: unknown): string | undefined { + return typeof value === 'string' && LABEL_RE.test(value) ? value : undefined; +} +function safeUuid(value: unknown): string | undefined { + return typeof value === 'string' && UUID_RE.test(value) ? value : undefined; +} +function cloneStages(value: Record) { + return Object.fromEntries( + Object.entries(value).map(([key, stage]) => [key, { ...stage }]) + ) as Record; +} +function cloneFreeze(value: T): T { + const clone = JSON.parse(JSON.stringify(value)) as T; + const freeze = (item: unknown): void => { + if (!item || typeof item !== 'object' || Object.isFrozen(item)) return; + Object.freeze(item); + Object.values(item as Record).forEach(freeze); + }; + freeze(clone); + return clone; +} +function newSlot(slotId: string): MutableSlot { + return { slotId, latestGeneration: 0, baseStages: stages(), generations: [] }; +} + +function updateStage(target: Record, event: AdTraceEvent): void { + const explicit = event.outcome + ? { + outcome: event.outcome, + confidence: event.confidence ?? 'none', + reason: event.reason ?? 'observed', + } + : undefined; + switch (event.kind) { + case 'ts_winner_observed': + target.trustedServer = { + outcome: 'won', + confidence: 'definitive', + reason: 'final_server_winner', + }; + break; + case 'ts_auction_observed': + target.trustedServer = explicit ?? { + outcome: 'unresolved', + confidence: 'none', + reason: 'terminal_summary', + }; + break; + case 'prebid_targeting_selected': + target.prebid = explicit ?? { + outcome: event.bidTraceId ? 'won' : 'client_bid_won', + confidence: 'definitive', + reason: 'selected_targeting', + }; + break; + case 'prebid_auction_end': + if (explicit && target.prebid.confidence !== 'definitive') target.prebid = explicit; + break; + case 'prebid_bid_won': + if (target.prebid.outcome === 'client_bid_won' || target.prebid.outcome === 'lost') { + target.prebid = { + ...target.prebid, + reason: 'selected_targeting_with_bid_won', + }; + if (target.gam.outcome === 'direct_or_unattributed') { + target.gam = { + outcome: 'client_prebid_candidate', + confidence: 'probable', + reason: 'client_bid_won_and_gpt_rendered', + }; + } + } + break; + case 'prebid_render_succeeded': + if (target.creative.confidence !== 'definitive') { + target.creative = { + outcome: 'prebid_render_succeeded', + confidence: 'strong', + reason: event.reason ?? 'prebid_render_succeeded', + }; + } + break; + case 'prebid_render_failed': + if (target.creative.confidence !== 'definitive') { + target.creative = { + outcome: 'render_failed', + confidence: 'definitive', + reason: event.reason ?? 'prebid_render_failed', + }; + } + break; + case 'gpt_slot_render_ended': + // Cooperative acknowledgement is stronger than later GPT callbacks and + // must never be downgraded to a probable candidate. + if (target.gam.confidence === 'definitive') break; + if (explicit?.outcome === 'unresolved') target.gam = explicit; + else if (event.isEmpty) + target.gam = { outcome: 'empty', confidence: 'definitive', reason: 'gpt_empty' }; + else if (event.isBackfill) + target.gam = { outcome: 'backfill', confidence: 'definitive', reason: 'gpt_backfill' }; + else if (event.bidTraceId) + target.gam = { + outcome: 'trusted_server_candidate', + confidence: 'probable', + reason: 'trace_targeting_rendered', + }; + else if ( + (target.prebid.outcome === 'client_bid_won' || target.prebid.outcome === 'lost') && + target.prebid.reason === 'selected_targeting_with_bid_won' + ) + target.gam = { + outcome: 'client_prebid_candidate', + confidence: 'probable', + reason: 'client_bid_won_and_gpt_rendered', + }; + else + target.gam = { + outcome: 'direct_or_unattributed', + confidence: 'probable', + reason: 'non_empty_unattributed', + }; + break; + case 'aps_display_bids_set': + // APS setting display bids is a handoff only. GAM attribution remains + // unobserved until a correlated non-empty GPT render arrives. + break; + case 'gpt_slot_onload': + if (target.creative.outcome === 'not_observed') + target.creative = { + outcome: 'gpt_iframe_onload', + confidence: 'probable', + reason: 'gpt_slot_onload', + }; + break; + case 'pb_render_served': + if (target.creative.confidence !== 'definitive') { + target.creative = { + outcome: 'renderer_served', + confidence: 'strong', + reason: event.reason ?? 'pb_render_response', + }; + } + break; + case 'direct_render_rejected': + if (target.creative.confidence === 'none') { + target.creative = { + outcome: 'rejected', + confidence: 'none', + reason: event.reason ?? 'direct_render_rejected', + }; + } + break; + case 'creative_load_acknowledged': + target.creative = { + outcome: 'load_acknowledged', + confidence: 'definitive', + reason: 'source_validated_load', + }; + if (event.reason !== 'direct_iframe_load') { + target.gam = { + outcome: 'trusted_server_won', + confidence: 'definitive', + reason: 'creative_load_acknowledged', + }; + } + break; + case 'generation_superseded': + // Ownership cleanup is lifecycle evidence, not contradictory render + // evidence. Preserve every previously observed stage unchanged. + break; + default: + break; + } +} + +function snapshot(slot: MutableSlot): SlotTraceSnapshot { + const latest = slot.generations.at(-1); + return { + slotId: slot.slotId, + latestGeneration: slot.latestGeneration, + generations: slot.generations.map((item) => ({ + generation: item.generation, + stages: cloneStages(item.stages), + })), + stages: cloneStages(latest?.stages ?? slot.baseStages), + }; +} + +function isRenderEvent(kind: AdTraceEventKind): boolean { + return ( + kind === 'gpt_request_started' || + kind === 'gpt_slot_render_ended' || + kind === 'prebid_render_succeeded' || + kind === 'prebid_render_failed' || + kind === 'pb_render_requested' || + kind === 'pb_render_rejected' || + kind === 'pb_render_served' || + kind === 'direct_render_rejected' || + kind === 'creative_load_acknowledged' || + kind === 'generation_superseded' + ); +} + +function renderSource(event: AdTraceEvent): RenderTraceSnapshot['source'] { + if (event.reason?.startsWith('direct_')) return 'direct_auction'; + if (event.kind.startsWith('pb_render_') || event.kind === 'creative_load_acknowledged') + return 'pb_render'; + return 'gpt'; +} + +function renderOutcome( + current: Pick, + event: AdTraceEvent +): { outcome: RenderTraceOutcome; confidence: AdTraceConfidence } { + if (current.outcome === 'confirmed') return { outcome: 'confirmed', confidence: 'definitive' }; + if (current.outcome === 'empty' && current.confidence === 'definitive') { + return { outcome: 'empty', confidence: 'definitive' }; + } + if (event.kind === 'creative_load_acknowledged') + return { outcome: 'confirmed', confidence: 'definitive' }; + if (current.outcome === 'served') return { outcome: 'served', confidence: 'strong' }; + if (event.kind === 'pb_render_served') return { outcome: 'served', confidence: 'strong' }; + if (event.kind === 'gpt_slot_render_ended') + return event.isEmpty + ? { outcome: 'empty', confidence: 'definitive' } + : { outcome: 'gam_only', confidence: 'probable' }; + if (current.outcome === 'gam_only') return { outcome: 'gam_only', confidence: 'probable' }; + return { outcome: 'unresolved', confidence: 'none' }; +} + +export function createAdTraceStore( + now: () => number = () => (typeof performance === 'undefined' ? Date.now() : performance.now()) +): AdTraceStore { + const slots = new Map(); + const events: AdTraceEvent[] = []; + const renders: RenderTraceSnapshot[] = []; + const renderByGeneration = new Map(); + const elementByGeneration = new Map(); + const listeners = new Set<() => void>(); + let sequence = 0; + let generationSequence = 0; + let renderSequence = 0; + let droppedEvents = 0; + let evictedSlots = 0; + const ensureSlot = (slotId: string): MutableSlot => { + let slot = slots.get(slotId); + if (slot) return slot; + if (slots.size >= AD_TRACE_MAX_SLOTS) { + const oldest = slots.keys().next().value as string | undefined; + if (oldest) { + slots.delete(oldest); + evictedSlots += 1; + } + } + slot = newSlot(slotId); + slots.set(slotId, slot); + return slot; + }; + const notify = (): void => listeners.forEach((listener) => listener()); + const emitRender = (render: RenderTraceSnapshot): void => { + if (typeof window === 'undefined' || typeof CustomEvent === 'undefined') return; + window.dispatchEvent(new CustomEvent('tsjs:adRendered', { detail: cloneFreeze(render) })); + }; + const updateRender = (event: AdTraceEvent, slotId: string, generation: number): void => { + if (!isRenderEvent(event.kind)) return; + const key = `${slotId}:${generation}`; + let render = renderByGeneration.get(key); + const timestamp = now(); + if (!render) { + render = { + sequence: ++renderSequence, + slotId, + generation, + source: renderSource(event), + outcome: 'unresolved', + confidence: 'none', + visibility: 'unknown', + createdAt: timestamp, + updatedAt: timestamp, + }; + renderByGeneration.set(key, render); + renders.push(render); + if (renders.length > AD_TRACE_MAX_RENDERS) { + const evicted = renders.shift(); + if (evicted) { + const evictedKey = `${evicted.slotId}:${evicted.generation}`; + renderByGeneration.delete(evictedKey); + elementByGeneration.delete(evictedKey); + } + } + } + const next = renderOutcome(render, event); + render.outcome = next.outcome; + render.confidence = next.confidence; + if (event.reason?.startsWith('direct_')) render.source = 'direct_auction'; + else if (event.kind.startsWith('pb_render_') || event.kind === 'creative_load_acknowledged') + render.source = render.source === 'direct_auction' ? render.source : 'pb_render'; + if (event.auctionTraceId) render.auctionTraceId = event.auctionTraceId; + if (event.bidTraceId) render.bidTraceId = event.bidTraceId; + render.updatedAt = timestamp; + emitRender(render); + }; + + return { + record(observation) { + if (!EVENT_KINDS.has(observation.kind)) return; + if (observation.confidence && !CONFIDENCES.has(observation.confidence)) return; + const slotId = safeLabel(observation.slotId); + const generation = + Number.isInteger(observation.generation) && (observation.generation ?? 0) > 0 + ? observation.generation + : undefined; + const event: AdTraceEvent = { + sequence: ++sequence, + timestamp: now(), + kind: observation.kind, + ...(slotId ? { slotId } : {}), + ...(generation ? { generation } : {}), + ...(safeUuid(observation.auctionTraceId) + ? { auctionTraceId: observation.auctionTraceId } + : {}), + ...(safeUuid(observation.bidTraceId) ? { bidTraceId: observation.bidTraceId } : {}), + ...(safeLabel(observation.provider) ? { provider: observation.provider } : {}), + ...(safeLabel(observation.bidder) ? { bidder: observation.bidder } : {}), + ...(safeLabel(observation.outcome) ? { outcome: observation.outcome } : {}), + ...(observation.confidence ? { confidence: observation.confidence } : {}), + ...(safeLabel(observation.reason) ? { reason: observation.reason } : {}), + ...(typeof observation.isEmpty === 'boolean' ? { isEmpty: observation.isEmpty } : {}), + ...(typeof observation.isBackfill === 'boolean' + ? { isBackfill: observation.isBackfill } + : {}), + }; + events.push(event); + if (events.length > AD_TRACE_MAX_EVENTS) { + events.shift(); + droppedEvents += 1; + } + if (slotId) { + const slot = ensureSlot(slotId); + const exact = generation + ? slot.generations.find((item) => item.generation === generation) + : undefined; + if (exact) updateStage(exact.stages, event); + else if ( + !generation && + (event.kind === 'ts_winner_observed' || event.kind === 'ts_auction_observed') + ) { + // Generationless server evidence seeds only the next request. Updating + // the latest retained generation would rewrite prior-navigation history. + updateStage(slot.baseStages, event); + } + if (generation) updateRender(event, slotId, generation); + } + notify(); + }, + nextGeneration(slotId) { + const safeSlotId = safeLabel(slotId); + if (!safeSlotId) return 0; + const slot = ensureSlot(safeSlotId); + slot.latestGeneration = ++generationSequence; + slot.generations.push({ + generation: slot.latestGeneration, + stages: cloneStages(slot.baseStages), + }); + if (slot.generations.length > AD_TRACE_MAX_GENERATIONS) slot.generations.shift(); + notify(); + return slot.latestGeneration; + }, + getSlot(slotId) { + const slot = slots.get(slotId); + return slot ? cloneFreeze(snapshot(slot)) : undefined; + }, + getEvents() { + return cloneFreeze(events); + }, + getRenderTimeline() { + return cloneFreeze(renders); + }, + export() { + const value: AdTraceExport = { + version: 1, + slots: [...slots.values()].map(snapshot), + events, + renders, + metadata: { droppedEvents, evictedSlots }, + }; + return cloneFreeze(value); + }, + subscribe(listener) { + if (listeners.size >= AD_TRACE_MAX_LISTENERS) return () => {}; + listeners.add(listener); + return () => listeners.delete(listener); + }, + bindElement(slotId, generation, element) { + const safeSlotId = safeLabel(slotId); + if (!safeSlotId || !Number.isInteger(generation) || generation <= 0) return; + const key = `${safeSlotId}:${generation}`; + if (!elementByGeneration.has(key) && elementByGeneration.size >= AD_TRACE_MAX_RENDERS) { + const oldest = elementByGeneration.keys().next().value as string | undefined; + if (oldest) elementByGeneration.delete(oldest); + } + elementByGeneration.set(key, element); + }, + getBoundElement(slotId, generation) { + const safeSlotId = safeLabel(slotId); + if (!safeSlotId || !Number.isInteger(generation) || generation <= 0) return undefined; + return elementByGeneration.get(`${safeSlotId}:${generation}`); + }, + updateVisibility(slotId, generation, visibility) { + const safeSlotId = safeLabel(slotId); + if (!safeSlotId || !Number.isInteger(generation) || generation <= 0) return; + const render = renderByGeneration.get(`${safeSlotId}:${generation}`); + if (!render || render.visibility === visibility) return; + render.visibility = visibility; + render.updatedAt = now(); + emitRender(render); + notify(); + }, + }; +} + +export function isCanonicalTraceUuid(value: unknown): value is string { + return safeUuid(value) !== undefined; +} +export function isBoundedTraceLabel(value: unknown): value is string { + return safeLabel(value) !== undefined; +} diff --git a/crates/trusted-server-js/lib/src/core/auction.ts b/crates/trusted-server-js/lib/src/core/auction.ts index 40f54367a..c9b94b7cd 100644 --- a/crates/trusted-server-js/lib/src/core/auction.ts +++ b/crates/trusted-server-js/lib/src/core/auction.ts @@ -3,6 +3,12 @@ // and the Prebid.js trustedServer adapter. import { log } from './log'; +import type { + AuctionTraceOutcome, + AuctionTraceSource, + AuctionTraceSummary, + TrustedServerBidTrace, +} from './types'; // --------------------------------------------------------------------------- // Types @@ -38,6 +44,11 @@ export interface AdRequest { } /** A parsed bid from an OpenRTB seatbid response. */ +export type AuctionClientResult = + | { kind: 'ok'; summary?: AuctionTraceSummary; bids: AuctionBid[] } + | { kind: 'transport_error'; reason: 'network' | 'http' } + | { kind: 'invalid_response'; reason: 'non_json' | 'invalid_shape' }; + export interface AuctionBid { /** Matches the `impid` in the response — corresponds to adUnit `code`. */ impid: string; @@ -55,6 +66,72 @@ export interface AuctionBid { creativeId: string; /** Advertiser domains. */ adomain: string[]; + /** Tester-gated trace joined to the validated root summary. */ + trace?: TrustedServerBidTrace; +} + +const TRACE_UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const TRACE_LABEL_RE = /^[\w.-]{1,64}$/; +const TRACE_SOURCES = new Set([ + 'initial_navigation', + 'spa_navigation', + 'auction_api', +]); +const TRACE_OUTCOMES = new Set([ + 'completed', + 'no_bid', + 'skipped', + 'failed', + 'abandoned', +]); + +/** Strictly parse the optional Trusted Server root extension. */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function parseAuctionTraceSummary(body: any): AuctionTraceSummary | undefined { + const trace = body?.ext?.trusted_server?.trace; + if ( + trace?.version !== 1 || + !TRACE_UUID_RE.test(trace.auction_trace_id) || + !TRACE_SOURCES.has(trace.source) || + !TRACE_OUTCOMES.has(trace.outcome) + ) { + return undefined; + } + return { + version: 1, + auctionTraceId: trace.auction_trace_id, + source: trace.source, + outcome: trace.outcome, + }; +} + +function parseBidTrace( + bid: any, // eslint-disable-line @typescript-eslint/no-explicit-any + root: AuctionTraceSummary | undefined +): TrustedServerBidTrace | undefined { + const trace = bid?.ext?.trusted_server?.trace; + if ( + !root || + root.outcome !== 'completed' || + trace?.version !== 1 || + !TRACE_UUID_RE.test(trace.bid_trace_id) || + typeof trace.slot_id !== 'string' || + trace.slot_id !== bid?.impid || + !TRACE_LABEL_RE.test(trace.slot_id) || + !TRACE_LABEL_RE.test(trace.provider) || + !TRACE_LABEL_RE.test(trace.bidder) + ) { + return undefined; + } + return { + version: 1, + auctionTraceId: root.auctionTraceId, + bidTraceId: trace.bid_trace_id, + source: root.source, + slotId: trace.slot_id, + provider: trace.provider, + bidder: trace.bidder, + }; } // --------------------------------------------------------------------------- @@ -123,6 +200,7 @@ export function buildAdRequest(units: any[], options?: { eids?: AuctionEid[] }): // eslint-disable-next-line @typescript-eslint/no-explicit-any export function parseAuctionResponse(body: any): AuctionBid[] { const bids: AuctionBid[] = []; + const rootTrace = parseAuctionTraceSummary(body); const seatbids = body?.seatbid; if (!Array.isArray(seatbids)) return bids; @@ -137,6 +215,7 @@ export function parseAuctionResponse(body: any): AuctionBid[] { // `if (!bid.adm)` guard. The client-side `typeof !== 'string'` check in // sanitizeCreativeHtml is a second line of defense for callers that bypass // parseAuctionResponse and pass untrusted values directly. + const trace = parseBidTrace(b, rootTrace); bids.push({ impid: b.impid ?? '', adm: b.adm ?? '', @@ -146,25 +225,45 @@ export function parseAuctionResponse(body: any): AuctionBid[] { seat, creativeId: b.crid ?? `${seat}-${b.impid ?? ''}`, adomain: Array.isArray(b.adomain) ? b.adomain : [], + ...(trace ? { trace } : {}), }); } } return bids; } +function isValidAuctionResponseShape(data: Record): boolean { + const seatbid = data.seatbid; + // Preserve the legacy valid empty response while rejecting a present but + // malformed collection that would otherwise be misreported as no-bid. + if (seatbid === undefined) return true; + if (!Array.isArray(seatbid)) return false; + return seatbid.every((seat) => { + if (!seat || typeof seat !== 'object' || Array.isArray(seat)) return false; + const bids = (seat as Record).bid; + return ( + bids === undefined || + (Array.isArray(bids) && + bids.every((bid) => !!bid && typeof bid === 'object' && !Array.isArray(bid))) + ); + }); +} + // --------------------------------------------------------------------------- // Auction HTTP call // --------------------------------------------------------------------------- /** - * POST an {@link AdRequest} to the given endpoint and return parsed bids. - * - * Returns an empty array on network or parse errors (non-throwing). + * POST an {@link AdRequest} and distinguish a valid empty auction from + * transport or response-shape failures. */ -export async function sendAuction(endpoint: string, request: AdRequest): Promise { +export async function sendAuction( + endpoint: string, + request: AdRequest +): Promise { if (typeof fetch !== 'function') { log.warn('auction: fetch not available'); - return []; + return { kind: 'transport_error', reason: 'network' }; } log.info('auction: sending request', { endpoint, units: request.adUnits.length }); @@ -179,17 +278,36 @@ export async function sendAuction(endpoint: string, request: AdRequest): Promise }); const ct = res.headers.get('content-type') || ''; - if (res.ok && ct.includes('application/json')) { - const data: unknown = await res.json(); - const bids = parseAuctionResponse(data); - log.info('auction: received bids', { count: bids.length }); - return bids; + if (!res.ok) { + log.warn('auction: unexpected response', { ok: res.ok, status: res.status, ct }); + return { kind: 'transport_error', reason: 'http' }; + } + if (!ct.includes('application/json')) { + log.warn('auction: non-json response', { status: res.status, ct }); + return { kind: 'invalid_response', reason: 'non_json' }; } - log.warn('auction: unexpected response', { ok: res.ok, status: res.status, ct }); - return []; + let data: unknown; + try { + data = await res.json(); + } catch (err) { + log.warn('auction: invalid json response', err); + return { kind: 'invalid_response', reason: 'non_json' }; + } + if ( + !data || + typeof data !== 'object' || + Array.isArray(data) || + !isValidAuctionResponseShape(data as Record) + ) { + return { kind: 'invalid_response', reason: 'invalid_shape' }; + } + const bids = parseAuctionResponse(data); + const summary = parseAuctionTraceSummary(data); + log.info('auction: received bids', { count: bids.length }); + return { kind: 'ok', ...(summary ? { summary } : {}), bids }; } catch (err) { log.warn('auction: request failed', err); - return []; + return { kind: 'transport_error', reason: 'network' }; } } diff --git a/crates/trusted-server-js/lib/src/core/global.d.ts b/crates/trusted-server-js/lib/src/core/global.d.ts index c7c8b08fb..2e753c6d9 100644 --- a/crates/trusted-server-js/lib/src/core/global.d.ts +++ b/crates/trusted-server-js/lib/src/core/global.d.ts @@ -2,6 +2,8 @@ import type { TsjsApi } from './types'; declare global { interface Window { + /** Request-scoped server bootstrap consumed synchronously by ad trace. */ + __tsjs_adTraceActive?: boolean; tsjs?: TsjsApi; pbjs?: TsjsApi; } diff --git a/crates/trusted-server-js/lib/src/core/request.ts b/crates/trusted-server-js/lib/src/core/request.ts index e39300a14..40b41d524 100644 --- a/crates/trusted-server-js/lib/src/core/request.ts +++ b/crates/trusted-server-js/lib/src/core/request.ts @@ -4,6 +4,7 @@ import { collectContext } from './context'; import { getAllUnits, firstSize } from './registry'; import { createAdIframe, findSlot, buildCreativeDocument, sanitizeCreativeHtml } from './render'; import { buildAdRequest, sendAuction } from './auction'; +import type { AuctionTraceSummary, TrustedServerBidTrace } from './types'; export type RequestAdsCallback = () => void; export interface RequestAdsOptions { @@ -11,6 +12,16 @@ export interface RequestAdsOptions { timeout?: number; } +const MAX_DIRECT_RENDER_OWNERS = 64; + +interface DirectRenderOwner { + token: symbol; + slotId: string; + generation?: number; +} + +const latestDirectOwners = new Map(); + type RenderCreativeInlineOptions = { slotId: string; // Accept unknown input here because bidder JSON is untrusted at runtime. @@ -19,8 +30,64 @@ type RenderCreativeInlineOptions = { creativeHeight?: number; seat: string; creativeId: string; + owner: DirectRenderOwner; + trace?: TrustedServerBidTrace; }; +function claimDirectOwner(slotId: string): DirectRenderOwner { + const previous = latestDirectOwners.get(slotId); + if (previous) recordDirectRejection(previous, 'direct_owner_replaced'); + const ts = window.tsjs; + const generation = ts?.recordAdTrace ? ts.nextAdTraceGeneration?.(slotId) : undefined; + const owner: DirectRenderOwner = { + token: Symbol(slotId), + slotId, + ...(generation && generation > 0 ? { generation } : {}), + }; + latestDirectOwners.delete(slotId); + latestDirectOwners.set(slotId, owner); + if (latestDirectOwners.size > MAX_DIRECT_RENDER_OWNERS) { + const oldest = latestDirectOwners.keys().next().value as string | undefined; + if (oldest) { + const evicted = latestDirectOwners.get(oldest); + if (evicted) recordDirectRejection(evicted, 'direct_owner_evicted'); + latestDirectOwners.delete(oldest); + } + } + return owner; +} + +function ownerIsCurrent(owner: DirectRenderOwner): boolean { + return latestDirectOwners.get(owner.slotId) === owner; +} + +function recordRootSummary( + summary: AuctionTraceSummary | undefined, + owner: DirectRenderOwner, + hasWinner: boolean +): void { + if (!summary || !owner.generation) return; + window.tsjs?.recordAdTrace?.({ + kind: 'ts_auction_observed', + slotId: owner.slotId, + generation: owner.generation, + auctionTraceId: summary.auctionTraceId, + outcome: summary.outcome === 'completed' && !hasWinner ? 'no_bid' : summary.outcome, + confidence: 'definitive', + reason: 'terminal_summary', + }); +} + +function recordDirectRejection(owner: DirectRenderOwner, reason: string): void { + if (!owner.generation) return; + window.tsjs?.recordAdTrace?.({ + kind: 'direct_render_rejected', + slotId: owner.slotId, + generation: owner.generation, + reason, + }); +} + // Entry point matching Prebid's requestBids signature; uses unified /auction endpoint. export function requestAds( callbackOrOpts?: RequestAdsCallback | RequestAdsOptions, @@ -39,34 +106,83 @@ export function requestAds( log.info('requestAds: called', { hasCallback: typeof callback === 'function' }); try { const adUnits = getAllUnits(); + const requestedSlotIds = [ + ...new Set( + adUnits + .map((unit) => unit.code) + .filter((code): code is string => typeof code === 'string' && code.length > 0) + ), + ]; + const owners = new Map(requestedSlotIds.map((slotId) => [slotId, claimDirectOwner(slotId)])); const config = collectContext(); const payload = { ...buildAdRequest(adUnits), config }; log.debug('requestAds: payload', { units: adUnits.length, contextKeys: Object.keys(config) }); - // Use unified auction endpoint - void sendAuction('/auction', payload) - .then((bids) => { - log.info('requestAds: got bids', { count: bids.length }); - for (const bid of bids) { - if (!bid.impid) continue; - if (!bid.adm) { - log.debug('requestAds: bid has no adm, skipping', { slotId: bid.impid }); - continue; + void sendAuction('/auction', payload).then((result) => { + if (result.kind !== 'ok') { + for (const owner of owners.values()) { + if (ownerIsCurrent(owner)) { + recordDirectRejection(owner, `${result.kind}_${result.reason}`); } - renderCreativeInline({ - slotId: bid.impid, - creativeHtml: bid.adm, - creativeWidth: bid.width, - creativeHeight: bid.height, - seat: bid.seat, - creativeId: bid.creativeId, + } + return; + } + + log.info('requestAds: got bids', { count: result.bids.length }); + const bySlot = new Map(); + for (const bid of result.bids) { + if (!owners.has(bid.impid)) continue; + const existing = bySlot.get(bid.impid) ?? []; + existing.push(bid); + bySlot.set(bid.impid, existing); + } + + for (const [slotId, owner] of owners) { + if (!ownerIsCurrent(owner)) continue; + const slotBids = bySlot.get(slotId) ?? []; + recordRootSummary(result.summary, owner, slotBids.length > 0); + if (slotBids.length === 0) continue; + if (slotBids.length !== 1) { + recordDirectRejection(owner, 'ambiguous_winner'); + continue; + } + + const bid = slotBids[0]; + const trace = + bid.trace && + result.summary && + bid.trace.slotId === slotId && + bid.trace.auctionTraceId === result.summary.auctionTraceId + ? bid.trace + : undefined; + if (trace && owner.generation) { + window.tsjs?.recordAdTrace?.({ + kind: 'ts_winner_observed', + slotId, + generation: owner.generation, + auctionTraceId: trace.auctionTraceId, + bidTraceId: trace.bidTraceId, + provider: trace.provider, + bidder: trace.bidder, }); } - log.info('requestAds: rendered creatives from response'); - }) - .catch((err) => { - log.warn('requestAds: auction failed', err); - }); + if (!bid.adm) { + recordDirectRejection(owner, 'missing_adm'); + continue; + } + renderCreativeInline({ + slotId, + creativeHtml: bid.adm, + creativeWidth: bid.width, + creativeHeight: bid.height, + seat: bid.seat, + creativeId: bid.creativeId, + owner, + ...(trace ? { trace } : {}), + }); + } + log.info('requestAds: rendered creatives from response'); + }); // Synchronously invoke callback to match test expectations try { @@ -87,16 +203,24 @@ function renderCreativeInline({ creativeHeight, seat, creativeId, + owner, + trace, }: RenderCreativeInlineOptions): void { + if (!ownerIsCurrent(owner)) return; const container = findSlot(slotId) as HTMLElement | null; if (!container) { + recordDirectRejection(owner, 'slot_missing'); log.warn('renderCreativeInline: slot not found; skipping render', { slotId, seat, creativeId }); return; } try { + if (owner.generation) { + window.tsjs?.bindAdTraceElement?.(slotId, owner.generation, container); + } const sanitization = sanitizeCreativeHtml(creativeHtml); if (sanitization.kind === 'rejected') { + recordDirectRejection(owner, 'creative_rejected'); log.warn('renderCreativeInline: rejected creative', { slotId, seat, @@ -107,6 +231,7 @@ function renderCreativeInline({ return; } + if (!ownerIsCurrent(owner)) return; // Clear the slot only after sanitization succeeds so rejected creatives never blank existing content. container.innerHTML = ''; @@ -132,8 +257,36 @@ function renderCreativeInline({ width, height, }); + iframe.addEventListener( + 'load', + () => { + if (!ownerIsCurrent(owner) || !iframe.isConnected || iframe.parentElement !== container) + return; + if (owner.generation) { + window.tsjs?.recordAdTrace?.({ + kind: 'creative_load_acknowledged', + slotId, + generation: owner.generation, + auctionTraceId: trace?.auctionTraceId, + bidTraceId: trace?.bidTraceId, + reason: 'direct_iframe_load', + }); + } + }, + { once: true } + ); iframe.srcdoc = buildCreativeDocument(sanitization.sanitizedHtml); + if (owner.generation) { + window.tsjs?.recordAdTrace?.({ + kind: 'pb_render_served', + slotId, + generation: owner.generation, + auctionTraceId: trace?.auctionTraceId, + bidTraceId: trace?.bidTraceId, + reason: 'direct_iframe_created', + }); + } log.info('renderCreativeInline: rendered', { slotId, @@ -144,6 +297,7 @@ function renderCreativeInline({ originalLength: sanitization.originalLength, }); } catch (err) { + recordDirectRejection(owner, 'render_failed'); log.warn('renderCreativeInline: failed', { slotId, seat, creativeId, err }); } } diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 360e2aa49..615f174ef 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -48,6 +48,126 @@ export interface AuctionDebugBidData { metadata?: Record; } +export type AuctionTraceSource = 'initial_navigation' | 'spa_navigation' | 'auction_api'; +export type AuctionTraceOutcome = 'completed' | 'no_bid' | 'skipped' | 'failed' | 'abandoned'; + +/** Privacy-safe summary emitted only for configured tester traffic. */ +export interface AuctionTraceSummary { + version: 1; + auctionTraceId: string; + source: AuctionTraceSource; + outcome: AuctionTraceOutcome; +} + +/** Privacy-safe trace for one final Trusted Server winning bid. */ +export interface TrustedServerBidTrace { + version: 1; + auctionTraceId: string; + bidTraceId: string; + source: AuctionTraceSource; + slotId: string; + provider: string; + bidder: string; +} + +export type AdTraceConfidence = 'definitive' | 'strong' | 'probable' | 'none'; +export type AdTraceStageName = 'trustedServer' | 'prebid' | 'gam' | 'creative'; +export interface AdTraceStage { + outcome: string; + confidence: AdTraceConfidence; + reason: string; +} + +export type AdTraceEventKind = + | 'ts_auction_observed' + | 'ts_winner_observed' + | 'prebid_auction_init' + | 'prebid_bid_response' + | 'prebid_targeting_selected' + | 'prebid_bid_won' + | 'prebid_auction_end' + | 'prebid_render_succeeded' + | 'prebid_render_failed' + | 'gpt_targeting_applied' + | 'gpt_request_started' + | 'gpt_slot_requested' + | 'gpt_slot_response_received' + | 'gpt_slot_render_ended' + | 'gpt_slot_onload' + | 'aps_display_bids_set' + | 'pb_render_requested' + | 'pb_render_rejected' + | 'pb_render_served' + | 'direct_render_rejected' + | 'creative_load_acknowledged' + | 'generation_superseded'; + +/** Sanitized observation accepted by the optional recorder. */ +export interface AdTraceObservation { + kind: AdTraceEventKind; + slotId?: string; + generation?: number; + auctionTraceId?: string; + bidTraceId?: string; + provider?: string; + bidder?: string; + outcome?: string; + confidence?: AdTraceConfidence; + reason?: string; + isEmpty?: boolean; + isBackfill?: boolean; +} + +export interface AdTraceEvent extends AdTraceObservation { + sequence: number; + timestamp: number; +} + +export interface GenerationTraceSnapshot { + generation: number; + stages: Record; +} + +export interface SlotTraceSnapshot { + slotId: string; + latestGeneration: number; + generations: GenerationTraceSnapshot[]; + /** Convenience view of only the latest retained generation. */ + stages: Record; +} + +export type RenderTraceOutcome = 'confirmed' | 'served' | 'gam_only' | 'empty' | 'unresolved'; +export type RenderTraceVisibility = 'visible' | 'hidden' | 'disconnected' | 'unknown'; + +export interface RenderTraceSnapshot { + sequence: number; + slotId: string; + generation: number; + auctionTraceId?: string; + bidTraceId?: string; + source: 'gpt' | 'pb_render' | 'direct_auction'; + outcome: RenderTraceOutcome; + confidence: AdTraceConfidence; + visibility: RenderTraceVisibility; + createdAt: number; + updatedAt: number; +} + +export interface AdTraceExport { + version: 1; + slots: SlotTraceSnapshot[]; + events: AdTraceEvent[]; + renders: RenderTraceSnapshot[]; + metadata: { droppedEvents: number; evictedSlots: number }; +} + +export interface AdTraceApi { + getSlot(slotId: string): SlotTraceSnapshot | undefined; + getEvents(): readonly AdTraceEvent[]; + getRenderTimeline(): readonly RenderTraceSnapshot[]; + export(): AdTraceExport; +} + /** Bid targeting data from the server-side auction, injected into `window.tsjs.bids`. */ export interface AuctionBidData { hb_pb?: string; @@ -57,6 +177,8 @@ export interface AuctionBidData { hb_cache_path?: string; nurl?: string; burl?: string; + /** Tester-gated trace; absent for ordinary traffic and malformed input. */ + trace?: TrustedServerBidTrace; /** Raw creative markup. Only present when `[debug] inject_adm_for_testing = true`. */ adm?: string; /** Debug-only bid field mirror. Only present when `[debug] inject_adm_for_testing = true`. */ @@ -90,6 +212,80 @@ export interface TsjsApi { adSlots?: AuctionSlot[]; /** Winning bid targeting data injected before . */ bids?: Record; + /** Tester-gated terminal auction summary. */ + auctionTrace?: AuctionTraceSummary; + /** Tester-only immutable diagnostic API. */ + adTrace?: AdTraceApi; + /** Private recorder installed only by the optional ad_trace module. */ + recordAdTrace?: (observation: AdTraceObservation) => void; + /** Private generation allocator installed only by the optional module. */ + nextAdTraceGeneration?: (slotId: string) => number; + /** Private overlay subscription installed only by the optional module. */ + subscribeAdTrace?: (listener: () => void) => () => void; + /** Bind one generation to the exact DOM element captured at its request boundary. */ + bindAdTraceElement?: (slotId: string, generation: number, element: HTMLElement) => void; + /** Resolve only that exact captured element; never searches replacement DOM. */ + getAdTraceElement?: (slotId: string, generation: number) => HTMLElement | undefined; + /** Private live visibility updater used only by the active overlay. */ + updateAdTraceVisibility?: ( + slotId: string, + generation: number, + visibility: RenderTraceVisibility + ) => void; + /** Private request-scoped Prebid correlation ledger; never exported. */ + prebidCorrelation?: Array<{ + auctionId: string; + slotId: string; + requestId: string; + bidder?: string; + adId?: string; + traceToken?: string; + serverTrace?: TrustedServerBidTrace; + events?: AdTraceEventKind[]; + }>; + /** Exact selected participants retained briefly for post-request terminal events. */ + prebidSelectedParticipants?: Array<{ + auctionId: string; + slotId: string; + requestId: string; + adId?: string; + traceToken?: string; + bidder?: string; + generation: number; + selectedAt: number; + }>; + /** Request-scoped root summaries retained until the GPT request boundary. */ + prebidServerSummaries?: Array<{ + auctionId: string; + slotId: string; + summary: AuctionTraceSummary; + }>; + /** Completed Prebid auctions used to identify request-scoped no-bid selections. */ + prebidCompletedAuctions?: Array<{ auctionId: string; slotIds: string[] }>; + /** Private bootstrap queue used until the GPT module installs its capture hook. */ + pendingAdTraceRequests?: Array<{ + slot: unknown; + trigger: string; + snapshot?: { + slotId?: string; + bidder?: string; + adId?: string; + traceToken?: string; + bid?: AuctionBidData; + }; + }>; + /** Private request-boundary hook shared with bootstrap and slim Prebid. */ + captureAdTraceRequest?: ( + slot: unknown, + trigger: string, + snapshot?: { + slotId?: string; + bidder?: string; + adId?: string; + traceToken?: string; + bid?: AuctionBidData; + } + ) => number; /** Initialises GPT slots with server-side bid targeting and calls refresh(). */ adInit?: () => void; /** GPT slot objects TS defined — used to destroy stale slots on SPA navigation. */ @@ -98,12 +294,6 @@ export interface TsjsApi { servicesEnabled?: boolean; /** Maps actualDivId → slotId for slotRenderEnded billing lookup. */ divToSlotId?: Record; - /** - * Win/billing beacons already fired, keyed by `slotId|bidIdentity|kind|url`. - * Used by the GPT render bridge so a bid's nurl/burl fire at most once even - * across repeated Prebid Universal Creative requests for the same adId. - */ - firedBeacons?: Record; /** Slot-level GPT targeting keys TS applied on the previous route. */ prevSlotTargetingKeys?: Record; /** diff --git a/crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts b/crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts new file mode 100644 index 000000000..cf6d6d33c --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts @@ -0,0 +1,98 @@ +import { createAdTraceStore, isBoundedTraceLabel, isCanonicalTraceUuid } from '../../core/ad_trace'; +import type { AdTraceApi, AuctionBidData, AuctionTraceSummary, TsjsApi } from '../../core/types'; + +import { installAdTraceOverlay } from './overlay'; + +const TRACE_SOURCES = new Set(['initial_navigation', 'spa_navigation', 'auction_api']); +const TRACE_OUTCOMES = new Set(['completed', 'no_bid', 'skipped', 'failed', 'abandoned']); + +function validSummary(value: AuctionTraceSummary | undefined): value is AuctionTraceSummary { + return ( + value?.version === 1 && + isCanonicalTraceUuid(value.auctionTraceId) && + TRACE_SOURCES.has(value.source) && + TRACE_OUTCOMES.has(value.outcome) + ); +} + +function validBid(value: AuctionBidData | undefined, slotId: string): boolean { + const trace = value?.trace; + return !!( + trace?.version === 1 && + trace.slotId === slotId && + isCanonicalTraceUuid(trace.auctionTraceId) && + isCanonicalTraceUuid(trace.bidTraceId) && + isBoundedTraceLabel(trace.provider) && + isBoundedTraceLabel(trace.bidder) + ); +} + +function consumeActiveBootstrap(): boolean { + if (window.__tsjs_adTraceActive !== true) return false; + delete window.__tsjs_adTraceActive; + return true; +} + +/** Install the session-scoped recorder, immutable API, and overlay once. */ +export function installAdTrace(): boolean { + if (typeof window === 'undefined') return false; + if (window.tsjs?.adTrace) return true; + if (!consumeActiveBootstrap()) return false; + const ts = (window.tsjs ??= {} as TsjsApi); + + const store = createAdTraceStore(); + const api: AdTraceApi = Object.freeze({ + getSlot: store.getSlot, + getEvents: store.getEvents, + getRenderTimeline: store.getRenderTimeline, + export: store.export, + }); + ts.adTrace = api; + ts.recordAdTrace = store.record; + ts.nextAdTraceGeneration = store.nextGeneration; + ts.subscribeAdTrace = store.subscribe; + ts.bindAdTraceElement = store.bindElement; + ts.getAdTraceElement = store.getBoundElement; + ts.updateAdTraceVisibility = store.updateVisibility; + if (!ts.captureAdTraceRequest) { + ts.captureAdTraceRequest = (slot, trigger, snapshot) => { + const pending = (ts.pendingAdTraceRequests ??= []); + if (pending.length < 64) pending.push({ slot, trigger, snapshot }); + return 0; + }; + } + + const summary = validSummary(ts.auctionTrace) ? ts.auctionTrace : undefined; + for (const slot of ts.adSlots ?? []) { + const bid = ts.bids?.[slot.id]; + if (validBid(bid, slot.id) && bid?.trace) { + store.record({ + kind: 'ts_winner_observed', + slotId: slot.id, + auctionTraceId: bid.trace.auctionTraceId, + bidTraceId: bid.trace.bidTraceId, + provider: bid.trace.provider, + bidder: bid.trace.bidder, + }); + } else if (summary) { + store.record({ + kind: 'ts_auction_observed', + slotId: slot.id, + auctionTraceId: summary.auctionTraceId, + outcome: + summary.outcome === 'completed' || summary.outcome === 'no_bid' + ? 'no_bid' + : summary.outcome === 'skipped' + ? 'skipped' + : 'unresolved', + confidence: 'definitive', + reason: 'terminal_summary', + }); + } + } + + installAdTraceOverlay(api, store.subscribe); + return true; +} + +if (typeof window !== 'undefined') installAdTrace(); diff --git a/crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts b/crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts new file mode 100644 index 000000000..fd08963d0 --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts @@ -0,0 +1,231 @@ +import type { + AdTraceApi, + RenderTraceSnapshot, + RenderTraceVisibility, + SlotTraceSnapshot, +} from '../../core/types'; + +const HOST_ID = 'ts-ad-trace-overlay'; +const TRACE_ATTRIBUTES = [ + 'data-ts-trace-seq', + 'data-ts-trace-generation', + 'data-ts-auction-trace-id', + 'data-ts-bid-trace-id', + 'data-ts-trace-outcome', + 'data-ts-trace-visibility', +] as const; + +function stageLine(label: string, stage: { outcome: string; confidence: string }): string { + return `${label}: ${stage.outcome} · ${stage.confidence}`; +} + +function badgeText(slot: SlotTraceSnapshot, render?: RenderTraceSnapshot): string { + return [ + render ? `#${render.sequence}: ${render.outcome} · ${render.visibility}` : undefined, + stageLine('TS winner', slot.stages.trustedServer), + stageLine('Prebid winner', slot.stages.prebid), + stageLine('GAM result', slot.stages.gam), + stageLine('Creative', slot.stages.creative), + ] + .filter(Boolean) + .join('\n'); +} + +function removeTraceAttributes(element: HTMLElement): void { + for (const attribute of TRACE_ATTRIBUTES) element.removeAttribute(attribute); +} + +function effectiveVisibility(element: HTMLElement, rect: DOMRect): RenderTraceVisibility { + if (!element.isConnected) return 'disconnected'; + if (rect.width <= 0 || rect.height <= 0) return 'hidden'; + let current: HTMLElement | null = element; + while (current) { + const style = getComputedStyle(current); + if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') { + return 'hidden'; + } + current = current.parentElement; + } + return 'visible'; +} + +function stampRender(element: HTMLElement, render: RenderTraceSnapshot): void { + removeTraceAttributes(element); + element.setAttribute('data-ts-trace-seq', String(render.sequence)); + element.setAttribute('data-ts-trace-generation', String(render.generation)); + element.setAttribute('data-ts-trace-outcome', render.outcome); + element.setAttribute('data-ts-trace-visibility', render.visibility); + if (render.auctionTraceId) + element.setAttribute('data-ts-auction-trace-id', render.auctionTraceId); + if (render.bidTraceId) element.setAttribute('data-ts-bid-trace-id', render.bidTraceId); +} + +/** Install one read-only Shadow DOM trace console. */ +export function installAdTraceOverlay( + api: AdTraceApi, + subscribe: (fn: () => void) => () => void +): void { + if (document.getElementById(HOST_ID)) return; + const host = document.createElement('div'); + host.id = HOST_ID; + const root = host.attachShadow({ mode: 'closed' }); + const style = document.createElement('style'); + style.textContent = ` + :host { all: initial; } + .badge { position: fixed; z-index: 2147483647; max-width: 300px; padding: 6px 8px; + border: 1px solid #72e0a6; border-radius: 4px; background: rgba(10,18,16,.94); + color: #eefbf4; font: 11px/1.35 ui-monospace, monospace; white-space: pre; cursor: pointer; } + .badge.probable { border-color: #67a8ff; } + .panel { position: fixed; right: 12px; bottom: 12px; z-index: 2147483647; width: 460px; + max-height: 60vh; overflow: auto; padding: 10px; background: #0a1210; color: #eefbf4; + border: 1px solid #72e0a6; font: 11px/1.4 ui-monospace, monospace; } + .controls { display: flex; gap: 6px; position: sticky; top: 0; background: #0a1210; } + .warning { color: #ffd479; margin: 6px 0; } + .row { border-top: 1px solid #29443a; padding: 6px 0; } + .row strong { color: #72e0a6; } + button { margin-bottom: 6px; } pre { white-space: pre-wrap; }`; + root.appendChild(style); + const badgeLayer = document.createElement('div'); + const panel = document.createElement('div'); + panel.className = 'panel'; + const controls = document.createElement('div'); + controls.className = 'controls'; + const collapseButton = document.createElement('button'); + collapseButton.textContent = 'Collapse'; + const exportButton = document.createElement('button'); + exportButton.textContent = 'Export trace'; + const closeButton = document.createElement('button'); + closeButton.textContent = 'Close'; + const warning = document.createElement('div'); + warning.className = 'warning'; + warning.textContent = 'A non-empty GAM response alone is not proof of a Trusted Server creative.'; + const rows = document.createElement('div'); + const details = document.createElement('pre'); + details.hidden = true; + controls.append(collapseButton, exportButton, closeButton); + panel.append(controls, warning, rows, details); + root.append(badgeLayer, panel); + document.documentElement.appendChild(host); + let cleanup = (): void => {}; + + exportButton.addEventListener('click', () => { + const blob = new Blob([JSON.stringify(api.export(), null, 2)], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = 'trusted-server-ad-trace.json'; + link.click(); + URL.revokeObjectURL(url); + }); + collapseButton.addEventListener('click', () => { + rows.hidden = !rows.hidden; + warning.hidden = rows.hidden; + collapseButton.textContent = rows.hidden ? 'Expand' : 'Collapse'; + }); + closeButton.addEventListener('click', () => { + cleanup(); + host.remove(); + }); + + let observedElements = new Set(); + const resizeObserver = + typeof ResizeObserver === 'undefined' ? undefined : new ResizeObserver(() => schedule()); + + const render = (): void => { + badgeLayer.replaceChildren(); + rows.replaceChildren(); + const exported = api.export(); + const slotById = new Map(exported.slots.map((slot) => [slot.slotId, slot])); + const latestBySlot = new Map(); + for (const item of exported.renders) latestBySlot.set(item.slotId, item); + const nextObserved = new Set(); + + for (const item of [...exported.renders].reverse()) { + const row = document.createElement('div'); + row.className = 'row'; + const title = document.createElement('strong'); + title.textContent = `#${item.sequence} ${item.slotId} · ${item.source}`; + const summary = document.createElement('div'); + summary.textContent = `${item.outcome} · ${item.confidence} · ${item.visibility}`; + row.append(title, summary); + row.addEventListener('click', () => { + details.hidden = false; + details.textContent = JSON.stringify( + { render: item, stages: slotById.get(item.slotId)?.stages }, + null, + 2 + ); + }); + rows.appendChild(row); + } + + for (const [slotId, slot] of slotById) { + const item = latestBySlot.get(slotId); + const element = item ? window.tsjs?.getAdTraceElement?.(slotId, item.generation) : undefined; + if (!element || !item) continue; + const rect = element.getBoundingClientRect(); + const visibility = effectiveVisibility(element, rect); + window.tsjs?.updateAdTraceVisibility?.(slotId, item.generation, visibility); + const effectiveItem = visibility === item.visibility ? item : { ...item, visibility }; + if (visibility === 'disconnected') { + resizeObserver?.unobserve(element); + removeTraceAttributes(element); + continue; + } + nextObserved.add(element); + if (!observedElements.has(element)) resizeObserver?.observe(element); + stampRender(element, effectiveItem); + const badge = document.createElement('div'); + badge.className = `badge ${item.outcome === 'confirmed' ? '' : 'probable'}`; + badge.textContent = badgeText(slot, effectiveItem); + badge.style.left = `${Math.max(0, rect.left)}px`; + badge.style.top = `${Math.max(0, rect.top)}px`; + badge.addEventListener('click', () => { + panel.hidden = false; + details.hidden = false; + details.textContent = JSON.stringify( + { render: effectiveItem, stages: slot.stages }, + null, + 2 + ); + }); + badgeLayer.appendChild(badge); + } + for (const element of observedElements) { + if (!nextObserved.has(element)) { + resizeObserver?.unobserve(element); + removeTraceAttributes(element); + } + } + observedElements = nextObserved; + }; + + let framePending = false; + const schedule = (): void => { + if (framePending) return; + framePending = true; + requestAnimationFrame(() => { + framePending = false; + if (host.isConnected) render(); + }); + }; + const unsubscribe = subscribe(schedule); + let cleaned = false; + cleanup = (): void => { + if (cleaned) return; + cleaned = true; + unsubscribe(); + resizeObserver?.disconnect(); + for (const element of observedElements) removeTraceAttributes(element); + window.removeEventListener('scroll', schedule); + window.removeEventListener('resize', schedule); + lifecycleObserver.disconnect(); + }; + const lifecycleObserver = new MutationObserver(() => { + if (!host.isConnected) cleanup(); + }); + lifecycleObserver.observe(document.documentElement, { childList: true, subtree: true }); + window.addEventListener('scroll', schedule, { passive: true }); + window.addEventListener('resize', schedule, { passive: true }); + render(); +} diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index ca4689684..63aca2833 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -1,5 +1,5 @@ import { log } from '../../core/log'; -import type { AuctionSlot, AuctionBidData, TsjsApi } from '../../core/types'; +import type { AuctionSlot, AuctionBidData, AuctionTraceSummary, TsjsApi } from '../../core/types'; import { installGptGuard } from './script_guard'; @@ -32,7 +32,11 @@ const TS_BID_TARGETING_KEYS = [ 'hb_cache_host', 'hb_cache_path', ] as const; -const TS_BASE_TARGETING_KEYS = [...TS_BID_TARGETING_KEYS, TS_INITIAL_TARGETING_KEY] as const; +const TS_BASE_TARGETING_KEYS = [ + ...TS_BID_TARGETING_KEYS, + TS_INITIAL_TARGETING_KEY, + 'ts_trace', +] as const; // ------------------------------------------------------------------ // googletag type stubs (minimal surface needed by the shim) @@ -48,8 +52,189 @@ interface GoogleTagSlot { } interface SlotRenderEndedEvent { - isEmpty: boolean; + isEmpty?: boolean; + isBackfill?: boolean; + slot: GoogleTagSlot; +} + +interface GptSlotEvent { slot: GoogleTagSlot; + isEmpty?: boolean; + isBackfill?: boolean; +} + +interface RenderCandidate { + slotId: string; + generation: number; + slot: GoogleTagSlot; + divId: string; + /** Renderable only when this record's own hb_adid matches the request snapshot. */ + bid?: Readonly; + adId?: string; + traceToken?: string; + createdAt: number; + terminal: boolean; + consumed: boolean; + superseded: boolean; +} + +interface ExpectedRender { + candidate: RenderCandidate; + source: MessageEventSource; + expiresAt: number; + consumed: boolean; +} + +interface AdTraceRequestBoundarySnapshot { + slotId?: string; + bidder?: string; + adId?: string; + traceToken?: string; + bid?: AuctionBidData; +} + +const requestCandidates = new Map(); +const expectedRenders = new Map(); +const fallbackGenerations = new Map(); + +const MAX_EXPECTED_RENDERS = 200; +const MAX_FALLBACK_GENERATIONS = 200; +const MAX_ACTIVE_CACHE_RENDERS = 64; +const MAX_PRIVATE_REQUEST_OWNERS = 64; +let privateNavigationGeneration = 0; + +interface PrivateRequestOwner { + slotId: string; + adId?: string; + bid?: Readonly; + generation?: number; + element: HTMLElement | null; + navigationGeneration: number; + expiresAt: number; + served: boolean; +} + +const latestPrivateRequestBySlot = new Map(); +const staleTsAdIdBits = new Uint32Array(64); + +function staleAdIdHashes(value: string): [number, number] { + let first = 2166136261; + let second = 5381; + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + first = Math.imul(first ^ code, 16777619) >>> 0; + second = (Math.imul(second, 33) ^ code) >>> 0; + } + return [first % 2048, second % 2048]; +} + +function rememberStaleAdIdBits(adId: string): void { + for (const hash of staleAdIdHashes(adId)) { + staleTsAdIdBits[hash >>> 5] |= 1 << (hash & 31); + } +} + +function staleAdIdBitsContain(adId: string): boolean { + return staleAdIdHashes(adId).every( + (hash) => (staleTsAdIdBits[hash >>> 5] & (1 << (hash & 31))) !== 0 + ); +} + +interface ActiveCacheRender { + controller: AbortController; + slotId: string; + adId: string; + source: MessageEventSource | null; + generation?: number; + candidate?: RenderCandidate; + cacheHost: string; + cachePath: string; + traceToken?: string; + navigationGeneration: number; + expiresAt: number; + expiryTimer?: ReturnType; +} + +const activeCacheRenders = new Set(); +const latestCacheRenderBySlot = new Map(); + +function rememberStaleTsOwner(owner: PrivateRequestOwner): void { + if (!owner.bid || !owner.adId) return; + rememberStaleAdIdBits(owner.adId); +} + +function retireActiveCacheRender(render: ActiveCacheRender): void { + if (render.expiryTimer) clearTimeout(render.expiryTimer); + render.controller.abort(); + activeCacheRenders.delete(render); + if (latestCacheRenderBySlot.get(render.slotId) === render) { + latestCacheRenderBySlot.delete(render.slotId); + } +} + +function invalidatePrivateRequestOwners(slotId?: string): void { + const entries = slotId + ? [[slotId, latestPrivateRequestBySlot.get(slotId)] as const] + : [...latestPrivateRequestBySlot.entries()]; + for (const [key, owner] of entries) { + if (!owner) continue; + rememberStaleTsOwner(owner); + latestPrivateRequestBySlot.delete(key); + } +} + +function abortActiveCacheRenders(slotId?: string): void { + privateNavigationGeneration += slotId ? 0 : 1; + for (const render of [...activeCacheRenders]) { + if (!slotId || render.slotId === slotId) retireActiveCacheRender(render); + } + invalidatePrivateRequestOwners(slotId); +} + +function claimPrivateRequestOwner( + slotId: string, + adId: string | undefined, + bid: Readonly | undefined, + element: HTMLElement | null +): PrivateRequestOwner { + for (const render of [...activeCacheRenders]) { + if (render.slotId === slotId) retireActiveCacheRender(render); + } + const previous = latestPrivateRequestBySlot.get(slotId); + if (previous) rememberStaleTsOwner(previous); + const owner: PrivateRequestOwner = { + slotId, + adId, + bid, + element, + navigationGeneration: privateNavigationGeneration, + expiresAt: monotonicNow() + 30_000, + served: false, + }; + latestPrivateRequestBySlot.delete(slotId); + latestPrivateRequestBySlot.set(slotId, owner); + while (latestPrivateRequestBySlot.size > MAX_PRIVATE_REQUEST_OWNERS) { + const oldest = latestPrivateRequestBySlot.keys().next().value as string | undefined; + if (!oldest) break; + const evicted = latestPrivateRequestBySlot.get(oldest); + if (evicted) rememberStaleTsOwner(evicted); + latestPrivateRequestBySlot.delete(oldest); + for (const render of [...activeCacheRenders]) { + if (render.slotId === oldest) retireActiveCacheRender(render); + } + } + return owner; +} + +function isKnownStaleTsAdId(adId: string): boolean { + // The fixed-size bitset intentionally never forgets within the page session: + // false positives fail closed, while bounded-map eviction cannot create a + // false negative that lets a stale TS Universal Creative fall through. + return staleAdIdBitsContain(adId); +} + +function monotonicNow(): number { + return typeof performance === 'undefined' ? Date.now() : performance.now(); } function findSlotElementByDivId(divId: string): HTMLElement | null { @@ -103,7 +288,7 @@ interface GoogleTagPubAdsService { setTargeting(key: string, value: string | string[]): GoogleTagPubAdsService; getTargeting(key: string): string[]; enableSingleRequest(): void; - addEventListener(event: string, fn: (e: SlotRenderEndedEvent) => void): void; + addEventListener(event: string, fn: (e: GptSlotEvent) => void): void; refresh(slots?: GoogleTagSlot[]): void; getSlots?(): GoogleTagSlot[]; disableInitialLoad?(): void; @@ -128,6 +313,39 @@ type GptWindow = Window & { __tsjs_slim_prebid_url?: string; }; +const cacheInvalidationHookedTags = new WeakSet(); +const cacheInvalidationHookedSlots = new WeakSet(); + +function installSlotCacheInvalidationHook(slot: GoogleTagSlot): void { + if (cacheInvalidationHookedSlots.has(slot) || typeof slot.clearTargeting !== 'function') return; + const original = slot.clearTargeting.bind(slot); + slot.clearTargeting = (key?: string) => { + const slotId = slotIdForGptSlot(slot); + if (slotId) abortActiveCacheRenders(slotId); + return original(key); + }; + cacheInvalidationHookedSlots.add(slot); +} + +function installGoogleTagCacheInvalidationHooks(g: Partial): void { + if (cacheInvalidationHookedTags.has(g)) return; + if (typeof g.destroySlots === 'function') { + const original = g.destroySlots.bind(g); + g.destroySlots = (slots?: GoogleTagSlot[]) => { + if (slots) { + slots.forEach((slot) => { + const slotId = slotIdForGptSlot(slot); + if (slotId) abortActiveCacheRenders(slotId); + }); + } else { + abortActiveCacheRenders(); + } + return original(slots); + }; + } + cacheInvalidationHookedTags.add(g); +} + // ------------------------------------------------------------------ // Shim implementation // ------------------------------------------------------------------ @@ -345,24 +563,40 @@ function injectAdmIntoSlot(divId: string, adm: string): void { } } -function fireWinBillingBeacons(slotId: string, bid: AuctionBidData): void { - if (!slotId || (!bid.nurl && !bid.burl)) return; +const MAX_BILLING_DEDUPE_KEYS = 512; +const BILLING_DEDUPE_TTL_MS = 30 * 60_000; +const firedBillingKeys = new Map(); - const fired = (window.tsjs!.firedBeacons ??= {}); +function billingEntries(slotId: string, bid: AuctionBidData): Array<[string, string]> { const bidIdentity = bid.hb_adid ?? bid.nurl ?? bid.burl ?? ''; - const urls = [ - ['nurl', bid.nurl], - ['burl', bid.burl], - ] as const; - - for (const [kind, url] of urls) { - if (!url) continue; + return ( + [ + ['nurl', bid.nurl], + ['burl', bid.burl], + ] as const + ).flatMap(([kind, url]) => + url ? [[`${slotId}|${bidIdentity}|${kind}|${url}`, url] as [string, string]] : [] + ); +} - const beaconKey = `${slotId}|${bidIdentity}|${kind}|${url}`; - if (fired[beaconKey]) continue; +function billingCapacityAvailable(slotId: string, bid: AuctionBidData): boolean { + const now = monotonicNow(); + for (const [key, expiresAt] of firedBillingKeys) { + if (expiresAt <= now) firedBillingKeys.delete(key); + } + const additional = billingEntries(slotId, bid).filter( + ([key]) => !firedBillingKeys.has(key) + ).length; + return firedBillingKeys.size + additional <= MAX_BILLING_DEDUPE_KEYS; +} +function fireWinBillingBeacons(slotId: string, bid: AuctionBidData): void { + if (!slotId) return; + const now = monotonicNow(); + for (const [key, url] of billingEntries(slotId, bid)) { + if (firedBillingKeys.has(key)) continue; if (queueWinBillingBeacon(url)) { - fired[beaconKey] = true; + firedBillingKeys.set(key, now + BILLING_DEDUPE_TTL_MS); } } } @@ -445,8 +679,363 @@ function installInitialLoadDetector(ts: TsjsApi): void { }); } +function slotIdForGptSlot(slot: GoogleTagSlot): string | undefined { + const divId = slot.getSlotElementId?.() ?? ''; + return ( + window.tsjs?.divToSlotId?.[divId] ?? + window.tsjs?.adSlots?.find((item) => { + return ( + divId === item.div_id || + divId === `${item.div_id}-container` || + divId.startsWith(item.div_id) + ); + })?.id + ); +} + +function firstSlotTarget(slot: GoogleTagSlot, key: string): string | undefined { + return slot.getTargeting?.(key)?.find((value) => value.length > 0); +} + +function supersedeCandidate(candidate: RenderCandidate, reason: string): void { + if (candidate.superseded) return; + candidate.superseded = true; + for (const render of [...activeCacheRenders]) { + if (render.candidate === candidate) retireActiveCacheRender(render); + } + window.tsjs?.recordAdTrace?.({ + kind: 'generation_superseded', + slotId: candidate.slotId, + generation: candidate.generation, + bidTraceId: candidate.traceToken, + reason, + }); +} + +export function supersedeAdTraceSlot(slot: GoogleTagSlot, reason: string): void { + const slotId = slotIdForGptSlot(slot); + if (slotId) { + abortActiveCacheRenders(slotId); + if (window.tsjs?.prebidSelectedParticipants) { + window.tsjs.prebidSelectedParticipants = window.tsjs.prebidSelectedParticipants.filter( + (entry) => entry.slotId !== slotId + ); + } + } + for (const candidates of requestCandidates.values()) { + candidates + .filter((candidate) => candidate.slot === slot && !candidate.superseded) + .forEach((candidate) => supersedeCandidate(candidate, reason)); + } +} + +/** Capture immutable attribution immediately before one concrete GPT request. */ +export function captureAdTraceRequest( + slot: GoogleTagSlot, + trigger: string, + snapshot?: AdTraceRequestBoundarySnapshot +): number { + const ts = window.tsjs; + const hasBoundarySnapshot = snapshot !== undefined; + const slotId = hasBoundarySnapshot ? snapshot.slotId : slotIdForGptSlot(slot); + if (!slotId) return 0; + installSlotCacheInvalidationHook(slot); + + // Private service ownership is captured for every GPT request, even when the + // diagnostic recorder is disabled. It must precede all asynchronous render + // work so a later request or navigation can invalidate the exact owner. + const bidder = hasBoundarySnapshot ? snapshot.bidder : firstSlotTarget(slot, 'hb_bidder'); + const adId = hasBoundarySnapshot ? snapshot.adId : firstSlotTarget(slot, 'hb_adid'); + const rawTraceToken = hasBoundarySnapshot + ? snapshot.traceToken + : firstSlotTarget(slot, 'ts_trace'); + const traceToken = + rawTraceToken && TRACE_TOKEN_RE.test(rawTraceToken) ? rawTraceToken : undefined; + const liveBid = hasBoundarySnapshot ? snapshot.bid : ts?.bids?.[slotId]; + const renderBidMatches = + !!liveBid && + !!adId && + liveBid.hb_adid === adId && + (!traceToken || liveBid.trace?.bidTraceId === traceToken); + const divId = slot.getSlotElementId?.() ?? ''; + const privateBid = renderBidMatches ? Object.freeze({ ...liveBid }) : undefined; + const privateOwner = claimPrivateRequestOwner( + slotId, + adId, + privateBid, + divId ? findSlotElementByDivId(divId) : null + ); + + if (!ts?.recordAdTrace) return 0; + (requestCandidates.get(slotId) ?? []) + .filter((candidate) => !candidate.superseded && !candidate.consumed) + .forEach((candidate) => supersedeCandidate(candidate, 'request_replaced')); + const generation = + ts.nextAdTraceGeneration?.(slotId) ?? (fallbackGenerations.get(slotId) ?? 0) + 1; + privateOwner.generation = generation; + fallbackGenerations.delete(slotId); + fallbackGenerations.set(slotId, generation); + while (fallbackGenerations.size > MAX_FALLBACK_GENERATIONS) { + const oldest = fallbackGenerations.keys().next().value as string | undefined; + if (!oldest) break; + fallbackGenerations.delete(oldest); + } + + // Diagnostic attribution reads the same immutable request-boundary values as + // the private owner, but remains optional and independently gated. + const ledger = ts.prebidCorrelation ?? []; + const selectedMatches = traceToken + ? ledger.filter((entry) => entry.slotId === slotId && entry.traceToken === traceToken) + : adId + ? ledger.filter((entry) => entry.slotId === slotId && entry.adId === adId) + : []; + const selectedParticipant = selectedMatches.length === 1 ? selectedMatches[0] : undefined; + const completedAuction = [...(ts.prebidCompletedAuctions ?? [])] + .reverse() + .find((entry) => entry.slotIds.includes(slotId)); + const auctionId = + selectedParticipant?.auctionId ?? (!adId ? completedAuction?.auctionId : undefined); + const participants = auctionId + ? ledger.filter((entry) => entry.slotId === slotId && entry.auctionId === auctionId) + : []; + const hasTracedTsParticipant = participants.some((entry) => !!entry.traceToken); + const tracedServerParticipant = participants.find((entry) => entry.serverTrace); + const serverSummary = auctionId + ? (ts.prebidServerSummaries ?? []).find( + (entry) => entry.auctionId === auctionId && entry.slotId === slotId + )?.summary + : undefined; + if (selectedParticipant) { + const selected = (ts.prebidSelectedParticipants ??= []).filter( + (entry) => monotonicNow() - entry.selectedAt <= 30_000 + ); + selected.push({ + auctionId: selectedParticipant.auctionId, + slotId, + requestId: selectedParticipant.requestId, + adId: selectedParticipant.adId, + traceToken: selectedParticipant.traceToken, + bidder: selectedParticipant.bidder, + generation, + selectedAt: monotonicNow(), + }); + while (selected.length > 128) selected.shift(); + ts.prebidSelectedParticipants = selected; + } + if (auctionId) { + ts.prebidCorrelation = ledger.filter( + (entry) => !(entry.slotId === slotId && entry.auctionId === auctionId) + ); + ts.prebidCompletedAuctions = (ts.prebidCompletedAuctions ?? []).filter( + (entry) => entry.auctionId !== auctionId + ); + ts.prebidServerSummaries = (ts.prebidServerSummaries ?? []).filter( + (entry) => !(entry.auctionId === auctionId && entry.slotId === slotId) + ); + } + + const candidate: RenderCandidate = { + slotId, + generation, + slot, + divId, + ...(privateBid ? { bid: privateBid } : {}), + adId, + traceToken, + createdAt: monotonicNow(), + terminal: false, + consumed: false, + superseded: false, + }; + const capturedElement = candidate.divId ? findSlotElementByDivId(candidate.divId) : null; + if (capturedElement) ts.bindAdTraceElement?.(slotId, generation, capturedElement); + if (!requestCandidates.has(slotId) && requestCandidates.size >= 64) { + const oldestSlotId = requestCandidates.keys().next().value as string | undefined; + if (oldestSlotId) { + requestCandidates + .get(oldestSlotId) + ?.forEach((item) => supersedeCandidate(item, 'slot_evicted')); + requestCandidates.delete(oldestSlotId); + } + } + const candidates = requestCandidates.get(slotId) ?? []; + candidates + .filter((item) => !item.superseded && monotonicNow() - item.createdAt > 30_000) + .forEach((item) => supersedeCandidate(item, 'generation_expired')); + candidates.push(candidate); + if (candidates.length > 8) { + const evicted = candidates.shift(); + if (evicted) supersedeCandidate(evicted, 'generation_evicted'); + } + requestCandidates.set(slotId, candidates); + + const serverTrace = tracedServerParticipant?.serverTrace; + if (serverTrace) { + ts.recordAdTrace({ + kind: 'ts_winner_observed', + slotId, + generation, + auctionTraceId: serverTrace.auctionTraceId, + bidTraceId: serverTrace.bidTraceId, + provider: serverTrace.provider, + bidder: serverTrace.bidder, + }); + } else if (serverSummary) { + ts.recordAdTrace({ + kind: 'ts_auction_observed', + slotId, + generation, + auctionTraceId: serverSummary.auctionTraceId, + outcome: serverSummary.outcome === 'completed' ? 'no_bid' : serverSummary.outcome, + confidence: 'definitive', + reason: 'terminal_summary', + }); + } + + let outcome = 'no_bid'; + let reason = 'no_selected_targeting'; + let confidence: 'definitive' | 'none' = 'definitive'; + if (selectedMatches.length > 1) { + outcome = 'unresolved'; + reason = 'ambiguous_prebid_request'; + confidence = 'none'; + } else if (selectedParticipant) { + if (traceToken && selectedParticipant.traceToken === traceToken) outcome = 'won'; + else if (!traceToken) outcome = hasTracedTsParticipant ? 'lost' : 'client_bid_won'; + else outcome = hasTracedTsParticipant ? 'lost' : 'unresolved'; + reason = 'selected_targeting'; + } else if (completedAuction && !bidder && !adId && !traceToken) { + outcome = 'no_bid'; + reason = 'prebid_no_bid'; + } else if (bidder || adId || traceToken) { + outcome = traceToken && renderBidMatches ? 'not_run' : 'client_bid_won'; + reason = traceToken && renderBidMatches ? 'direct_gpt_request' : 'unjoined_targeting'; + if (!traceToken && !renderBidMatches) confidence = 'none'; + } + ts.recordAdTrace({ + kind: 'prebid_targeting_selected', + slotId, + generation, + bidTraceId: traceToken, + bidder, + outcome, + confidence, + reason, + }); + for (const kind of selectedParticipant?.events ?? []) { + ts.recordAdTrace({ + kind, + slotId, + generation, + bidTraceId: traceToken, + bidder, + }); + } + if (liveBid?.hb_bidder === 'aps' || liveBid?.hb_bidder === 'amazon-aps') { + ts.recordAdTrace({ + kind: 'aps_display_bids_set', + slotId, + generation, + bidTraceId: traceToken, + }); + } + ts.recordAdTrace({ + kind: 'gpt_request_started', + slotId, + generation, + auctionTraceId: liveBid?.trace?.auctionTraceId ?? ts.auctionTrace?.auctionTraceId, + bidTraceId: traceToken, + provider: liveBid?.trace?.provider, + bidder, + reason: trigger, + }); + return generation; +} + +function candidateForSlot( + slot: GoogleTagSlot, + includeTerminal = false +): RenderCandidate | undefined { + const slotId = slotIdForGptSlot(slot); + if (!slotId) return undefined; + const candidates = (requestCandidates.get(slotId) ?? []).filter( + (candidate) => + candidate.slot === slot && + !candidate.superseded && + (includeTerminal || !candidate.terminal) && + monotonicNow() - candidate.createdAt <= 30_000 + ); + if (candidates.length !== 1) { + if (candidates.length > 1) { + candidates.forEach((candidate) => + window.tsjs?.recordAdTrace?.({ + kind: 'gpt_slot_render_ended', + slotId, + generation: candidate.generation, + bidTraceId: candidate.traceToken, + outcome: 'unresolved', + confidence: 'none', + reason: 'overlapping_request', + }) + ); + } else { + window.tsjs?.recordAdTrace?.({ + kind: 'gpt_slot_response_received', + slotId, + outcome: 'unresolved', + confidence: 'none', + reason: 'missing_generation', + }); + } + return undefined; + } + return candidates[0]; +} + +function installGptEvidenceListeners(service: GoogleTagPubAdsService): void { + if (!window.tsjs?.recordAdTrace) return; + const instrumented = service as GoogleTagPubAdsService & { __tsAdTraceListeners?: boolean }; + if (instrumented.__tsAdTraceListeners) return; + instrumented.__tsAdTraceListeners = true; + const record = + (kind: 'gpt_slot_requested' | 'gpt_slot_response_received' | 'gpt_slot_onload') => + (event: GptSlotEvent): void => { + const candidate = candidateForSlot(event.slot, kind === 'gpt_slot_onload'); + if (!candidate) return; + window.tsjs?.recordAdTrace?.({ + kind, + slotId: candidate.slotId, + generation: candidate.generation, + bidTraceId: candidate.traceToken, + }); + }; + service.addEventListener('slotRequested', record('gpt_slot_requested')); + service.addEventListener('slotResponseReceived', record('gpt_slot_response_received')); + service.addEventListener('slotOnload', record('gpt_slot_onload')); + service.addEventListener('slotRenderEnded', (event: GptSlotEvent) => { + const candidate = candidateForSlot(event.slot); + if (!candidate) return; + candidate.terminal = true; + window.tsjs?.recordAdTrace?.({ + kind: 'gpt_slot_render_ended', + slotId: candidate.slotId, + generation: candidate.generation, + bidTraceId: candidate.traceToken, + isEmpty: event.isEmpty, + isBackfill: event.isBackfill, + }); + }); +} + export function installTsAdInit(): void { const ts = (window.tsjs ??= {} as TsjsApi); + const pendingBootstrapRequests = ts.pendingAdTraceRequests ?? []; + ts.pendingAdTraceRequests = []; + ts.captureAdTraceRequest = (slot, trigger, snapshot) => + captureAdTraceRequest(slot as GoogleTagSlot, trigger, snapshot); + pendingBootstrapRequests.forEach(({ slot, trigger, snapshot }) => + ts.captureAdTraceRequest?.(slot, trigger, snapshot) + ); installInitialLoadDetector(ts); ts.adInit = function () { const slots = ts.adSlots ?? []; @@ -454,18 +1043,51 @@ export function installTsAdInit(): void { // The slotRenderEnded listener below reads ts.bids live so SPA navigation // updates (new ts.bids injected before ) are picked up at render time. const bids = ts.bids ?? {}; + const summary = ts.auctionTrace; + for (const slot of slots) { + const bid = bids[slot.id]; + if (bid?.trace && TRACE_TOKEN_RE.test(bid.trace.bidTraceId)) { + ts.recordAdTrace?.({ + kind: 'ts_winner_observed', + slotId: slot.id, + auctionTraceId: bid.trace.auctionTraceId, + bidTraceId: bid.trace.bidTraceId, + provider: bid.trace.provider, + bidder: bid.trace.bidder, + }); + } else if (summary) { + ts.recordAdTrace?.({ + kind: 'ts_auction_observed', + slotId: slot.id, + auctionTraceId: summary.auctionTraceId, + outcome: + summary.outcome === 'completed' || summary.outcome === 'no_bid' + ? 'no_bid' + : summary.outcome === 'skipped' + ? 'skipped' + : 'unresolved', + confidence: 'definitive', + reason: 'terminal_summary', + }); + } + } const g = (window as GptWindow).googletag; if (!g) return; g.cmd?.push(() => { + installGoogleTagCacheInvalidationHooks(g); // Destroy previously defined TS slots before redefining for the new page. if (ts.prevGptSlots && ts.prevGptSlots.length > 0) { + (ts.prevGptSlots as GoogleTagSlot[]).forEach((slot) => + supersedeAdTraceSlot(slot, 'slot_destroyed') + ); g.destroySlots?.(ts.prevGptSlots as GoogleTagSlot[]); ts.prevGptSlots = []; } // Slots TS defined itself — tracked for SPA destroy. Publisher-owned // slots are reused but never destroyed by TS on navigation. + installGptEvidenceListeners(g.pubads!()); const newSlots: GoogleTagSlot[] = []; // Publisher-owned slots TS reused — refreshed to pick up server-side // targeting. The publisher already display()ed these. @@ -493,6 +1115,7 @@ export function installTsAdInit(): void { (g.pubads!().getSlots?.() ?? []).forEach((gptSlot: GoogleTagSlot) => { const elementId = gptSlot.getSlotElementId(); if (!prevTouchedDivIds.has(elementId)) return; + supersedeAdTraceSlot(gptSlot, 'targeting_cleared'); clearTargetingKeys(gptSlot, [ ...TS_BASE_TARGETING_KEYS, ...(prevSlotTargetingKeys[elementId] ?? []), @@ -529,6 +1152,7 @@ export function installTsAdInit(): void { tsOwned = true; } + installSlotCacheInvalidationHook(gptSlot); const slotDivId2 = gptSlot.getSlotElementId?.() ?? actualDivId; clearTargetingKeys(gptSlot, [ ...TS_BASE_TARGETING_KEYS, @@ -540,7 +1164,18 @@ export function installTsAdInit(): void { TS_BID_TARGETING_KEYS.forEach((key) => { if (bid[key]) gptSlot.setTargeting(key, String(bid[key]!)); }); + if (bid.trace?.bidTraceId && TRACE_TOKEN_RE.test(bid.trace.bidTraceId)) { + gptSlot.setTargeting('ts_trace', bid.trace.bidTraceId); + } gptSlot.setTargeting(TS_INITIAL_TARGETING_KEY, '1'); + ts.recordAdTrace?.({ + kind: 'gpt_targeting_applied', + slotId: slot.id, + auctionTraceId: bid.trace?.auctionTraceId, + bidTraceId: bid.trace?.bidTraceId, + provider: bid.trace?.provider, + bidder: bid.trace?.bidder, + }); // Map both inner div and container div → slot ID so slotRenderEnded // (which reports the GPT slot's div, i.e. slotDivId/container) can look up // the slot, while adm injection (which targets the inner div) also works. @@ -562,6 +1197,11 @@ export function installTsAdInit(): void { if (bid.hb_bidder === 'aps' || bid.hb_bidder === 'amazon-aps') { // eslint-disable-next-line @typescript-eslint/no-explicit-any (window as any).apstag?.setDisplayBids?.(); + ts.recordAdTrace?.({ + kind: 'aps_display_bids_set', + slotId: slot.id, + bidTraceId: bid.trace?.bidTraceId, + }); } }); @@ -607,7 +1247,11 @@ export function installTsAdInit(): void { // called without a matching display call") and misses its impression. // Must run after enableServices(); on SPA navigation services are already // enabled, so this runs unconditionally for any newly-defined slots. - slotsToDisplay.forEach((divId) => g.display?.(divId)); + slotsToDisplay.forEach((divId) => { + const gptSlot = newSlots.find((slot) => slot.getSlotElementId() === divId); + if (gptSlot && !ts.gptInitialLoadDisabled) captureAdTraceRequest(gptSlot, 'display'); + g.display?.(divId); + }); // Slots needing an explicit ad request via refresh(). Reused // publisher-owned slots always need one to pick up the just-applied @@ -630,6 +1274,7 @@ export function installTsAdInit(): void { // the same slots still go through the wrapper normally. ts.adInitRefreshInProgress = true; try { + slotsNeedingRefresh.forEach((slot) => captureAdTraceRequest(slot, 'refresh')); g.pubads!().refresh(slotsNeedingRefresh); } finally { ts.adInitRefreshInProgress = false; @@ -640,6 +1285,7 @@ export function installTsAdInit(): void { } interface PageBidsResponse { + auctionTrace?: AuctionTraceSummary; slots: AuctionSlot[]; bids: Record; } @@ -712,16 +1358,20 @@ export function installSpaAuctionHook(): void { // same-pathname back/forward (scroll restoration), and pushState/replaceState // can be called with the current URL, so guard every entry point against // re-requesting impressions for a path we already loaded. - let currentPath = location.pathname; + let currentPath = `${location.pathname}${location.search}`; // Last path whose slots/bids were actually applied — the initial SSR page // counts. A failed navigation rolls `currentPath` back to this rather than to // the immediately-previous committed value: on rapid A→B where A was aborted // mid-flight and B then fails, rolling back to A (never loaded) would strand // it behind the no-op guard, so we roll back to the last applied route instead. - let lastAppliedPath = location.pathname; + let lastAppliedPath = `${location.pathname}${location.search}`; async function onNavigate(path: string): Promise { + // Navigation invalidates private render ownership even when the resulting + // route key is unchanged (for example a state-only replaceState call). + abortActiveCacheRenders(); if (path === currentPath) return; + ts.prebidSelectedParticipants = []; currentPath = path; inflight?.abort(); const controller = new AbortController(); @@ -752,6 +1402,7 @@ export function installSpaAuctionHook(): void { await waitForSlotElements(data.slots, controller.signal); if (inflight !== controller) return; ts.adSlots = data.slots; + ts.auctionTrace = data.auctionTrace; ts.bids = data.bids; // This route is now the committed, loaded state — a later failed // navigation rolls back here, and a return trip no-ops correctly. @@ -778,7 +1429,8 @@ export function installSpaAuctionHook(): void { const original = history[method].bind(history); history[method] = function (state: unknown, unused: string, url?: string | URL | null): void { original(state, unused, url); - const newPath = url ? new URL(String(url), location.href).pathname : location.pathname; + const parsed = url ? new URL(String(url), location.href) : location; + const newPath = `${parsed.pathname}${parsed.search}`; // onNavigate no-ops when newPath equals the last loaded path. void onNavigate(newPath); }; @@ -788,7 +1440,7 @@ export function installSpaAuctionHook(): void { patchHistoryMethod('replaceState'); window.addEventListener('popstate', () => { - void onNavigate(location.pathname); + void onNavigate(`${location.pathname}${location.search}`); }); } @@ -816,9 +1468,53 @@ export function installSlimPrebidLoader(): void { const TS_DISPLAY_RENDERER = '(function(){window.render=function(d,h,w){' + 'var f=h.mkFrame(w.document,{width:d.width||"100%",height:d.height||"100%"});' + + 'if(typeof d.traceToken==="string"){f.addEventListener("load",function(){' + + 'top.postMessage({type:"ts-creative-load",version:1,traceToken:d.traceToken},"*");},{once:true});}' + 'if(d.adUrl&&!d.ad){f.src=d.adUrl;}else{f.srcdoc=d.ad;}' + 'w.document.body.appendChild(f);};})();'; +const TRACE_TOKEN_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +function pruneExpectedRenders(): void { + const now = monotonicNow(); + for (const [token, entries] of expectedRenders) { + const retained = entries.filter((entry) => !entry.consumed && entry.expiresAt >= now); + if (retained.length > 0) expectedRenders.set(token, retained); + else expectedRenders.delete(token); + } +} + +function armExpectedRender( + candidate: RenderCandidate | undefined, + source: MessageEventSource | null +): string | undefined { + if ( + !candidate?.bid || + candidate.superseded || + !candidate.traceToken || + !TRACE_TOKEN_RE.test(candidate.traceToken) || + !source + ) { + return undefined; + } + pruneExpectedRenders(); + const expectedCount = [...expectedRenders.values()].reduce( + (count, entries) => count + entries.length, + 0 + ); + if (expectedCount >= MAX_EXPECTED_RENDERS) return undefined; + candidate.consumed = true; + const entries = expectedRenders.get(candidate.traceToken) ?? []; + entries.push({ + candidate, + source, + expiresAt: monotonicNow() + 30_000, + consumed: false, + }); + expectedRenders.set(candidate.traceToken, entries); + return candidate.traceToken; +} + /** * Install the TS → pbRender bridge. * @@ -839,13 +1535,6 @@ const TS_DISPLAY_RENDERER = export function installTsRenderBridge(): void { if (typeof window === 'undefined') return; - // adIds whose PBS Cache render is in flight. `fireWinBillingBeacons` only - // dedups after the async cache fetch resolves, so two Prebid Request messages - // for the same adId arriving before the first fetch settles would both fetch - // and both fire the nurl/burl beacons. Tracking in-flight adIds prevents the - // concurrent double-fire; the entry is cleared once the fetch settles. - const renderingAdIds = new Set(); - window.addEventListener('message', (e: MessageEvent) => { let data: Record; try { @@ -857,6 +1546,43 @@ export function installTsRenderBridge(): void { return; } + if (data['type'] === 'ts-creative-load') { + const token = data['traceToken']; + if (data['version'] !== 1 || typeof token !== 'string' || !TRACE_TOKEN_RE.test(token)) return; + const entries = expectedRenders.get(token) ?? []; + entries + .filter((entry) => !entry.consumed && entry.expiresAt < monotonicNow()) + .forEach((entry) => supersedeCandidate(entry.candidate, 'ack_expired')); + const matches = entries.filter( + (entry) => + !entry.consumed && + !entry.candidate.superseded && + entry.expiresAt >= monotonicNow() && + entry.source === e.source && + (requestCandidates.get(entry.candidate.slotId) ?? []).includes(entry.candidate) + ); + if (matches.length !== 1) { + const candidate = entries[0]?.candidate; + window.tsjs?.recordAdTrace?.({ + kind: 'pb_render_rejected', + slotId: candidate?.slotId, + generation: candidate?.generation, + bidTraceId: TRACE_TOKEN_RE.test(token) ? token : undefined, + reason: matches.length > 1 ? 'ambiguous_generation' : 'invalid_acknowledgement', + }); + return; + } + const expected = matches[0]; + expected.consumed = true; + window.tsjs?.recordAdTrace?.({ + kind: 'creative_load_acknowledged', + slotId: expected.candidate.slotId, + generation: expected.candidate.generation, + bidTraceId: token, + }); + return; + } + if (data['message'] !== 'Prebid Request') return; const adId = data['adId'] as string | undefined; if (!adId) return; @@ -866,30 +1592,65 @@ export function installTsRenderBridge(): void { const sourceSlotId = slotIdForMessageSource(e.source); if (!sourceSlotId) return; - // Build reverse map adId → slotId from live window.tsjs.bids. - const bids = window.tsjs?.bids ?? {}; - let slotId: string | undefined; - let matchedBid: (typeof bids)[string] | undefined; - for (const [sid, bid] of Object.entries(bids)) { - if (bid.hb_adid === adId) { - slotId = sid; - matchedBid = bid; - break; + const allCandidates = requestCandidates.get(sourceSlotId) ?? []; + allCandidates + .filter((candidate) => !candidate.superseded && monotonicNow() - candidate.createdAt > 30_000) + .forEach((candidate) => supersedeCandidate(candidate, 'generation_expired')); + const candidates = allCandidates.filter( + (candidate) => + candidate.adId === adId && + !candidate.consumed && + !candidate.superseded && + monotonicNow() - candidate.createdAt <= 30_000 + ); + const exactCandidate = candidates.length === 1 ? candidates[0] : undefined; + window.tsjs?.recordAdTrace?.({ + kind: candidates.length === 1 ? 'pb_render_requested' : 'pb_render_rejected', + slotId: sourceSlotId, + generation: exactCandidate?.generation, + bidTraceId: exactCandidate?.traceToken, + reason: + candidates.length === 1 + ? 'exact_generation' + : candidates.length > 1 + ? 'ambiguous_generation' + : 'missing_generation', + }); + + const slotId = sourceSlotId; + const requestOwner = latestPrivateRequestBySlot.get(slotId); + const liveBid = window.tsjs?.bids?.[slotId]; + const ownerCurrent = + !!requestOwner?.bid && + requestOwner.adId === adId && + requestOwner.bid.hb_adid === adId && + requestOwner.navigationGeneration === privateNavigationGeneration && + requestOwner.expiresAt >= monotonicNow() && + !requestOwner.served && + !!requestOwner.element?.isConnected && + findSlotElementByDivId(requestOwner.element.id) === requestOwner.element && + slotIdForMessageSource(e.source) === slotId && + liveBid?.hb_adid === requestOwner.bid.hb_adid && + liveBid.hb_cache_host === requestOwner.bid.hb_cache_host && + liveBid.hb_cache_path === requestOwner.bid.hb_cache_path && + liveBid.trace?.bidTraceId === requestOwner.bid.trace?.bidTraceId; + if (!ownerCurrent || !requestOwner?.bid) { + // A once-TS-owned message must not escape to ordinary Prebid after its + // request owner was replaced or invalidated. + if (isKnownStaleTsAdId(adId) || liveBid?.hb_adid === adId) { + e.stopImmediatePropagation(); } + return; } - - // Not a TS bid — let Prebid.js handle it. - if (!slotId || !matchedBid) return; - - // The requesting iframe's slot must own the resolved adId. Without this an - // iframe under slot A could request slot B's hb_adid and receive slot B's - // creative/dimensions while firing slot B's win/billing beacons. - if (slotId !== sourceSlotId) return; + const matchedBid = requestOwner.bid; const slot = window.tsjs?.adSlots?.find((s) => s.id === slotId); const [width, height] = slot?.formats?.[0] ?? [728, 90]; if (matchedBid.adm) { + if (!billingCapacityAvailable(slotId, matchedBid)) return; + const traceToken = armExpectedRender(exactCandidate, e.source); + requestOwner.served = true; e.stopImmediatePropagation(); port.postMessage( JSON.stringify({ @@ -899,9 +1660,16 @@ export function installTsRenderBridge(): void { renderer: TS_DISPLAY_RENDERER, width, height, + ...(traceToken ? { traceToken } : {}), }) ); fireWinBillingBeacons(slotId, matchedBid); + window.tsjs?.recordAdTrace?.({ + kind: 'pb_render_served', + slotId, + generation: exactCandidate?.generation, + bidTraceId: traceToken, + }); log.debug(`[tsjs-gpt] pbRender bridge served '${slotId}' from debug adm`); return; } @@ -909,19 +1677,114 @@ export function installTsRenderBridge(): void { // No TS render source — let Prebid.js handle it. if (!matchedBid.hb_cache_host || !matchedBid.hb_cache_path) return; - // TS owns this adId — stop Prebid from also processing it. + const capturedSource = e.source; + const capturedElement = requestOwner.element; + const capturedCacheHost = matchedBid.hb_cache_host; + const capturedCachePath = matchedBid.hb_cache_path; + const capturedTraceToken = matchedBid.trace?.bidTraceId; + + const previousOwner = latestCacheRenderBySlot.get(slotId); + if ( + previousOwner && + previousOwner.adId === adId && + previousOwner.source === capturedSource && + previousOwner.generation === requestOwner.generation && + previousOwner.cacheHost === capturedCacheHost && + previousOwner.cachePath === capturedCachePath && + previousOwner.traceToken === capturedTraceToken && + !previousOwner.controller.signal.aborted && + previousOwner.expiresAt >= monotonicNow() + ) { + // A duplicate message for the exact accepted owner must not start a + // second fetch or escape to the ordinary Prebid renderer. + e.stopImmediatePropagation(); + return; + } + if (previousOwner) retireActiveCacheRender(previousOwner); + + // Capacity overflow must not evict a different live billing owner. Leave + // the message untouched so the ordinary Prebid path can process it. + if (activeCacheRenders.size >= MAX_ACTIVE_CACHE_RENDERS) return; + + const controller = new AbortController(); + const activeRender: ActiveCacheRender = { + controller, + slotId, + adId, + source: capturedSource, + generation: requestOwner.generation, + ...(exactCandidate?.generation === requestOwner.generation + ? { candidate: exactCandidate } + : {}), + cacheHost: capturedCacheHost, + cachePath: capturedCachePath, + traceToken: capturedTraceToken, + navigationGeneration: privateNavigationGeneration, + expiresAt: requestOwner.expiresAt, + }; + + activeCacheRenders.add(activeRender); + latestCacheRenderBySlot.set(slotId, activeRender); + activeRender.expiryTimer = setTimeout( + () => retireActiveCacheRender(activeRender), + Math.max(0, activeRender.expiresAt - monotonicNow()) + ); + // TS owns this accepted render — stop Prebid from also processing it. e.stopImmediatePropagation(); - // Skip a concurrent re-render of the same adId so its win/billing beacons - // fire at most once even before the first cache fetch resolves. - if (renderingAdIds.has(adId)) return; - renderingAdIds.add(adId); + const stillCurrent = (): boolean => { + const liveBid = window.tsjs?.bids?.[slotId]; + const candidateCurrent = + !exactCandidate || + (!exactCandidate.superseded && + (requestCandidates.get(slotId) ?? []).includes(exactCandidate)); + return ( + !controller.signal.aborted && + latestCacheRenderBySlot.get(slotId) === activeRender && + latestPrivateRequestBySlot.get(slotId) === requestOwner && + requestOwner.navigationGeneration === privateNavigationGeneration && + requestOwner.expiresAt >= monotonicNow() && + !requestOwner.served && + activeRender.navigationGeneration === privateNavigationGeneration && + activeRender.expiresAt >= monotonicNow() && + candidateCurrent && + !!capturedElement?.isConnected && + findSlotElementByDivId(capturedElement.id) === capturedElement && + slotIdForMessageSource(capturedSource) === slotId && + liveBid?.hb_adid === adId && + liveBid.hb_cache_host === capturedCacheHost && + liveBid.hb_cache_path === capturedCachePath && + liveBid.trace?.bidTraceId === capturedTraceToken + ); + }; - const cacheUrl = `https://${matchedBid.hb_cache_host}${matchedBid.hb_cache_path}?uuid=${encodeURIComponent(adId)}`; + const cacheUrl = `https://${capturedCacheHost}${capturedCachePath}?uuid=${encodeURIComponent(adId)}`; - fetch(cacheUrl, { mode: 'cors' }) + fetch(cacheUrl, { mode: 'cors', signal: controller.signal }) .then((res) => (res.ok ? res.text() : Promise.reject(res.status))) .then((ad) => { + if (!stillCurrent()) { + window.tsjs?.recordAdTrace?.({ + kind: 'pb_render_rejected', + slotId, + generation: exactCandidate?.generation, + bidTraceId: exactCandidate?.traceToken, + reason: 'stale_cache_completion', + }); + return; + } + if (!billingCapacityAvailable(slotId, matchedBid)) { + window.tsjs?.recordAdTrace?.({ + kind: 'pb_render_rejected', + slotId, + generation: exactCandidate?.generation, + bidTraceId: exactCandidate?.traceToken, + reason: 'billing_capacity', + }); + return; + } + const traceToken = armExpectedRender(exactCandidate, capturedSource); + requestOwner.served = true; port.postMessage( JSON.stringify({ message: 'Prebid Response', @@ -930,16 +1793,28 @@ export function installTsRenderBridge(): void { renderer: TS_DISPLAY_RENDERER, width, height, + ...(traceToken ? { traceToken } : {}), }) ); fireWinBillingBeacons(slotId, matchedBid); + window.tsjs?.recordAdTrace?.({ + kind: 'pb_render_served', + slotId, + generation: exactCandidate?.generation, + bidTraceId: traceToken, + }); log.debug(`[tsjs-gpt] pbRender bridge served '${slotId}' from PBS Cache`); }) .catch((err) => { + if (err instanceof DOMException && err.name === 'AbortError') return; log.warn(`[tsjs-gpt] pbRender bridge: PBS Cache fetch failed for '${slotId}'`, err); }) .finally(() => { - renderingAdIds.delete(adId); + if (activeRender.expiryTimer) clearTimeout(activeRender.expiryTimer); + activeCacheRenders.delete(activeRender); + if (latestCacheRenderBySlot.get(slotId) === activeRender) { + latestCacheRenderBySlot.delete(slotId); + } }); }); } diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 342e4038d..b8784d1b4 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -27,9 +27,9 @@ import 'prebid.js/modules/userId.js'; import './_adapters.generated'; import { log } from '../../core/log'; -import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; +import { buildAdRequest, parseAuctionResponse, parseAuctionTraceSummary } from '../../core/auction'; import type { AuctionBid, AuctionEid } from '../../core/auction'; -import type { AuctionSlot } from '../../core/types'; +import type { AdTraceEventKind, AuctionSlot, TrustedServerBidTrace } from '../../core/types'; import { INCLUDED_PREBID_USER_ID_MODULES } from './_user_ids.generated'; import { PREBID_USER_ID_MODULE_REGISTRY } from './user_id_modules'; @@ -47,6 +47,7 @@ const TS_REFRESH_TARGETING_KEYS = [ 'hb_adid', 'hb_cache_host', 'hb_cache_path', + 'ts_trace', ] as const; /** Configuration options for the Prebid integration. */ @@ -220,6 +221,12 @@ export function auctionBidsToPrebidBids(auctionBids: AuctionBid[], bidRequests: meta: { advertiserDomains: bid.adomain, }, + ...(bid.trace + ? { + adserverTargeting: { ts_trace: bid.trace.bidTraceId }, + tsTrace: bid.trace, + } + : {}), }; }); } @@ -242,6 +249,7 @@ type TrustedServerBidRequest = { adUnitCode?: string; code?: string; bidId?: string; + auctionId?: string; }; type TrustedServerRequest = { method: 'POST'; @@ -458,6 +466,155 @@ function serverSideBidderParamsForRefresh( return params; } +function installAdTracePrebidObservers(): void { + const ts = window.tsjs; + if (!ts?.recordAdTrace) return; + const instrumented = pbjs as unknown as { + __tsAdTraceObserved?: boolean; + onEvent?: (event: string, handler: (data: Record) => void) => void; + setTargetingForGPTAsync?: (codes?: string[]) => unknown; + }; + if (instrumented.__tsAdTraceObserved) return; + instrumented.__tsAdTraceObserved = true; + + const record = + (kind: AdTraceEventKind) => + (data: Record = {}): void => { + const nestedBid = + data.bid && typeof data.bid === 'object' + ? (data.bid as Record) + : undefined; + const evidence = nestedBid ?? data; + const slotId = + typeof evidence.adUnitCode === 'string' + ? evidence.adUnitCode + : typeof evidence.code === 'string' + ? evidence.code + : undefined; + const bidder = + typeof evidence.bidderCode === 'string' + ? evidence.bidderCode + : typeof evidence.bidder === 'string' + ? evidence.bidder + : undefined; + const auctionId = + typeof evidence.auctionId === 'string' + ? evidence.auctionId + : typeof data.auctionId === 'string' + ? data.auctionId + : ''; + const requestId = + typeof evidence.requestId === 'string' + ? evidence.requestId + : typeof evidence.adId === 'string' + ? evidence.adId + : ''; + const adId = typeof evidence.adId === 'string' ? evidence.adId : requestId || undefined; + const targeting = evidence.adserverTargeting as Record | undefined; + const serverTrace = evidence.tsTrace as TrustedServerBidTrace | undefined; + const traceToken = + typeof targeting?.ts_trace === 'string' + ? targeting.ts_trace + : typeof (evidence.tsTrace as { bidTraceId?: unknown } | undefined)?.bidTraceId === + 'string' + ? ((evidence.tsTrace as { bidTraceId: string }).bidTraceId as string) + : undefined; + const ledger = (ts.prebidCorrelation ??= []); + if (kind === 'prebid_bid_response' && auctionId && slotId && requestId) { + ledger.push({ + auctionId, + slotId, + requestId, + bidder, + adId, + traceToken, + serverTrace, + events: [], + }); + if (ledger.length > 256) ledger.shift(); + } else if (kind === 'prebid_auction_end' && auctionId) { + const adUnits = Array.isArray(data.adUnits) + ? (data.adUnits as Array>) + : []; + const received = Array.isArray(data.bidsReceived) + ? (data.bidsReceived as Array>) + : []; + const slotIds = new Set(); + for (const unit of adUnits) { + if (typeof unit.code === 'string') slotIds.add(unit.code); + } + for (const bid of received) { + if (typeof bid.adUnitCode === 'string') slotIds.add(bid.adUnitCode); + } + for (const entry of ledger) { + if (entry.auctionId === auctionId) slotIds.add(entry.slotId); + } + const completed = (ts.prebidCompletedAuctions ??= []); + completed.push({ auctionId, slotIds: [...slotIds] }); + if (completed.length > 64) completed.shift(); + } else if (kind !== 'prebid_auction_init') { + const selected = (ts.prebidSelectedParticipants ?? []).filter( + (entry) => performance.now() - entry.selectedAt <= 30_000 + ); + ts.prebidSelectedParticipants = selected; + const selectedMatches = + auctionId && slotId && requestId + ? selected.filter( + (entry) => + entry.auctionId === auctionId && + entry.slotId === slotId && + (entry.requestId === requestId || entry.adId === adId) && + (!traceToken || entry.traceToken === traceToken) + ) + : []; + if (selectedMatches.length === 1) { + const selectedEntry = selectedMatches[0]; + ts.recordAdTrace?.({ + kind, + slotId, + generation: selectedEntry.generation, + bidTraceId: selectedEntry.traceToken, + bidder: selectedEntry.bidder ?? bidder, + }); + if (kind === 'prebid_render_succeeded' || kind === 'prebid_render_failed') { + ts.prebidSelectedParticipants = selected.filter((entry) => entry !== selectedEntry); + } + return; + } + + const matches = ledger.filter( + (entry) => + (!auctionId || entry.auctionId === auctionId) && + (!slotId || entry.slotId === slotId) && + (!requestId || entry.requestId === requestId || entry.adId === adId) + ); + if (matches.length === 1) { + const events = (matches[0].events ??= []); + events.push(kind); + while (events.length > 16) events.shift(); + } + } + ts.recordAdTrace?.({ kind, slotId, bidder }); + }; + + instrumented.onEvent?.('auctionInit', record('prebid_auction_init')); + instrumented.onEvent?.('bidResponse', record('prebid_bid_response')); + instrumented.onEvent?.('bidWon', record('prebid_bid_won')); + instrumented.onEvent?.('auctionEnd', record('prebid_auction_end')); + instrumented.onEvent?.('adRenderSucceeded', record('prebid_render_succeeded')); + instrumented.onEvent?.('adRenderFailed', record('prebid_render_failed')); + + // Observe the actual selection call once. The GPT request-boundary hook reads + // the resulting slot targeting synchronously; this wrapper never caches it. + const original = instrumented.setTargetingForGPTAsync?.bind(pbjs); + if (!original) return; + instrumented.setTargetingForGPTAsync = function (codes?: string[]) { + const result = original(codes); + ts.recordAdTrace?.({ kind: 'prebid_targeting_selected', reason: 'targeting_applied' }); + return result; + }; +} + function clearRefreshTargeting(slot: RefreshGptSlot): void { if (typeof slot.clearTargeting !== 'function') return; @@ -551,6 +708,16 @@ export function installPrebidNpm(config?: Partial): typeof pbjs log.debug('[tsjs-prebid] interpretResponse', { hasSeatbid: !!body?.seatbid }); const auctionBids = parseAuctionResponse(body); const bidRequests = request?.tsjsBidRequests ?? request?.bidRequests ?? []; + const summary = parseAuctionTraceSummary(body); + if (summary && window.tsjs?.recordAdTrace) { + const summaries = (window.tsjs.prebidServerSummaries ??= []); + for (const bidRequest of bidRequests) { + const auctionId = bidRequest.auctionId; + const slotId = bidRequest.adUnitCode ?? bidRequest.code; + if (auctionId && slotId) summaries.push({ auctionId, slotId, summary }); + } + while (summaries.length > 64) summaries.shift(); + } return auctionBidsToPrebidBids(auctionBids, bidRequests); }, }); @@ -681,6 +848,7 @@ export function installPrebidNpm(config?: Partial): typeof pbjs // prebid.js via NPM. pbjs.processQueue(); recordUserIdModuleDiagnostics(); + installAdTracePrebidObservers(); // Validate that every client-side bidder has its adapter registered. // Adapters self-register on import, so a missing adapter means the bidder @@ -811,6 +979,9 @@ export function installRefreshHandler(timeoutMs = 1500): void { adUnits, bidsBackHandler: () => { pbjs.setTargetingForGPTAsync?.(refreshAdUnitCodes); + targetSlots.forEach((slot) => + window.tsjs?.captureAdTraceRequest?.(slot, 'prebid_refresh') + ); originalRefresh(targetSlots, opts); }, timeout: timeoutMs, diff --git a/crates/trusted-server-js/lib/test/core/ad_trace.test.ts b/crates/trusted-server-js/lib/test/core/ad_trace.test.ts new file mode 100644 index 000000000..0348e8dd7 --- /dev/null +++ b/crates/trusted-server-js/lib/test/core/ad_trace.test.ts @@ -0,0 +1,332 @@ +import { describe, expect, it } from 'vitest'; +import { + AD_TRACE_MAX_EVENTS, + AD_TRACE_MAX_GENERATIONS, + AD_TRACE_MAX_RENDERS, + AD_TRACE_MAX_SLOTS, + createAdTraceStore, +} from '../../src/core/ad_trace'; + +const BID_TRACE_ID = '550e8400-e29b-41d4-a716-446655440000'; + +describe('ad trace reducer', () => { + it('bounds events, slots, and retained generations', () => { + let now = 0; + const store = createAdTraceStore(() => ++now); + for (let i = 0; i < AD_TRACE_MAX_EVENTS + 1; i++) { + store.record({ kind: 'prebid_auction_init', reason: 'observed' }); + } + for (let i = 0; i < AD_TRACE_MAX_SLOTS + 1; i++) { + store.nextGeneration(`slot-${i}`); + } + for (let i = 0; i < AD_TRACE_MAX_GENERATIONS + 1; i++) { + store.nextGeneration('latest-slot'); + } + + const exported = store.export(); + expect(exported.events).toHaveLength(AD_TRACE_MAX_EVENTS); + expect(exported.metadata.droppedEvents).toBe(1); + expect(exported.slots).toHaveLength(AD_TRACE_MAX_SLOTS); + expect(store.getSlot('latest-slot')?.generations).toHaveLength(AD_TRACE_MAX_GENERATIONS); + expect(exported.metadata.evictedSlots).toBeGreaterThan(0); + }); + + it('keeps the four stages independent and only acknowledges an exact load event', () => { + const store = createAdTraceStore(() => 10); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'ts_winner_observed', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + }); + store.record({ + kind: 'gpt_slot_render_ended', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + isEmpty: false, + }); + + expect(store.getSlot('slot-a')?.stages.gam.outcome).toBe('trusted_server_candidate'); + expect(store.getSlot('slot-a')?.stages.creative.outcome).toBe('not_observed'); + + store.record({ + kind: 'creative_load_acknowledged', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + }); + const slot = store.getSlot('slot-a'); + expect(slot?.stages.gam).toMatchObject({ + outcome: 'trusted_server_won', + confidence: 'definitive', + }); + expect(slot?.stages.creative).toMatchObject({ + outcome: 'load_acknowledged', + confidence: 'definitive', + }); + }); + + it('updates only the acknowledged retained generation, never the latest generation', () => { + const store = createAdTraceStore(() => 1); + const first = store.nextGeneration('slot-a'); + const second = store.nextGeneration('slot-a'); + store.record({ + kind: 'creative_load_acknowledged', + slotId: 'slot-a', + generation: first, + bidTraceId: BID_TRACE_ID, + }); + + const slot = store.getSlot('slot-a'); + expect(slot?.latestGeneration).toBe(second); + expect(slot?.stages.creative.outcome).toBe('not_observed'); + expect(slot?.generations[0].stages.creative.outcome).toBe('load_acknowledged'); + expect(slot?.generations[1].stages.creative.outcome).toBe('not_observed'); + }); + + it('never downgrades a definitive acknowledgement with a later GPT callback', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'creative_load_acknowledged', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + }); + store.record({ + kind: 'gpt_slot_render_ended', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + isEmpty: false, + }); + + expect(store.getSlot('slot-a')?.stages.gam.outcome).toBe('trusted_server_won'); + expect(store.getSlot('slot-a')?.stages.creative.outcome).toBe('load_acknowledged'); + }); + + it('preserves acknowledged terminal history when its generation is later cleaned up', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'creative_load_acknowledged', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + }); + store.record({ + kind: 'generation_superseded', + slotId: 'slot-a', + generation, + reason: 'slot_destroyed', + }); + + expect(store.getSlot('slot-a')?.stages.gam.outcome).toBe('trusted_server_won'); + expect(store.getSlot('slot-a')?.stages.creative.outcome).toBe('load_acknowledged'); + }); + + it('does not rewrite a retained generation when the next auction seeds server evidence', () => { + const store = createAdTraceStore(() => 1); + store.record({ + kind: 'ts_winner_observed', + slotId: 'slot-a', + bidTraceId: BID_TRACE_ID, + }); + const first = store.nextGeneration('slot-a'); + store.record({ + kind: 'ts_auction_observed', + slotId: 'slot-a', + outcome: 'no_bid', + confidence: 'definitive', + reason: 'terminal_summary', + }); + const second = store.nextGeneration('slot-a'); + + const slot = store.getSlot('slot-a'); + expect( + slot?.generations.find((item) => item.generation === first)?.stages.trustedServer.outcome + ).toBe('won'); + expect( + slot?.generations.find((item) => item.generation === second)?.stages.trustedServer.outcome + ).toBe('no_bid'); + }); + + it('classifies overlap, client Prebid, APS, no-bid, and superseded states', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'prebid_targeting_selected', + slotId: 'slot-a', + generation, + outcome: 'client_bid_won', + confidence: 'definitive', + reason: 'selected_targeting', + }); + store.record({ + kind: 'prebid_bid_won', + slotId: 'slot-a', + generation, + }); + store.record({ + kind: 'gpt_slot_render_ended', + slotId: 'slot-a', + generation, + isEmpty: false, + }); + expect(store.getSlot('slot-a')?.stages.gam.outcome).toBe('client_prebid_candidate'); + + store.record({ kind: 'aps_display_bids_set', slotId: 'slot-a', generation }); + expect(store.getSlot('slot-a')?.stages.gam.outcome).toBe('client_prebid_candidate'); + + store.record({ + kind: 'generation_superseded', + slotId: 'slot-a', + generation, + reason: 'slot_destroyed', + }); + expect(store.getSlot('slot-a')?.stages.creative.outcome).toBe('not_observed'); + expect(store.getSlot('slot-a')?.stages.gam.outcome).toBe('client_prebid_candidate'); + }); + + it('does not downgrade definitive stage evidence during service or cleanup', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ kind: 'prebid_render_failed', slotId: 'slot-a', generation }); + store.record({ kind: 'pb_render_served', slotId: 'slot-a', generation }); + store.record({ + kind: 'generation_superseded', + slotId: 'slot-a', + generation, + reason: 'navigation', + }); + + expect(store.getSlot('slot-a')?.stages.creative).toMatchObject({ + outcome: 'render_failed', + confidence: 'definitive', + }); + }); + + it('does not downgrade a definitive empty render outcome', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'gpt_slot_render_ended', + slotId: 'slot-a', + generation, + isEmpty: true, + }); + store.record({ + kind: 'generation_superseded', + slotId: 'slot-a', + generation, + reason: 'navigation', + }); + store.record({ kind: 'pb_render_served', slotId: 'slot-a', generation }); + + expect(store.getRenderTimeline()[0]).toMatchObject({ + outcome: 'empty', + confidence: 'definitive', + }); + }); + + it('enriches one bounded render record and keeps visibility independent', () => { + let now = 0; + const store = createAdTraceStore(() => ++now); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'gpt_slot_render_ended', + slotId: 'slot-a', + generation, + isEmpty: false, + }); + store.record({ + kind: 'pb_render_served', + slotId: 'slot-a', + generation, + reason: 'pb_render_response', + }); + store.record({ + kind: 'creative_load_acknowledged', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + }); + store.updateVisibility('slot-a', generation, 'hidden'); + + const timeline = store.getRenderTimeline(); + expect(timeline).toHaveLength(1); + expect(timeline[0]).toMatchObject({ + sequence: 1, + outcome: 'confirmed', + confidence: 'definitive', + visibility: 'hidden', + }); + store.updateVisibility('slot-a', generation, 'visible'); + expect(store.getRenderTimeline()[0]).toMatchObject({ + sequence: 1, + outcome: 'confirmed', + confidence: 'definitive', + visibility: 'visible', + }); + }); + + it('dispatches a frozen privacy-safe render event', () => { + const store = createAdTraceStore(() => 1); + const observed: unknown[] = []; + const listener = (event: Event) => observed.push((event as CustomEvent).detail); + window.addEventListener('tsjs:adRendered', listener); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'pb_render_served', + slotId: 'slot-a', + generation, + reason: 'pb_render_response', + rawUrl: 'https://private.example', + } as never); + window.removeEventListener('tsjs:adRendered', listener); + + expect(observed).toHaveLength(1); + expect(Object.isFrozen(observed[0])).toBe(true); + expect(JSON.stringify(observed[0])).not.toContain('private.example'); + }); + + it('bounds the render timeline without duplicating impression generations', () => { + const store = createAdTraceStore(() => 1); + for (let i = 0; i < AD_TRACE_MAX_RENDERS + 1; i++) { + const slotId = `render-${i}`; + const generation = store.nextGeneration(slotId); + store.record({ kind: 'gpt_request_started', slotId, generation }); + } + expect(store.getRenderTimeline()).toHaveLength(AD_TRACE_MAX_RENDERS); + expect(store.getRenderTimeline()[0].slotId).toBe('render-1'); + }); + + it('rejects malformed runtime event kinds and confidence values', () => { + const store = createAdTraceStore(() => 1); + store.record({ kind: 'not-a-real-kind', slotId: 'slot-a' } as never); + store.record({ + kind: 'ts_winner_observed', + slotId: 'slot-a', + confidence: 'certain', + } as never); + expect(store.getEvents()).toHaveLength(0); + }); + + it('exports an immutable sanitized clone', () => { + const store = createAdTraceStore(() => 1); + store.record({ + kind: 'pb_render_rejected', + slotId: 'slot-a', + reason: 'missing_generation', + // Ensure unknown private fields cannot enter the public export. + rawUrl: 'https://private.example/path', + } as never); + + const exported = store.export(); + expect(Object.isFrozen(exported)).toBe(true); + expect(JSON.stringify(exported)).not.toContain('private.example'); + expect(() => exported.events.push({} as never)).toThrow(); + }); +}); diff --git a/crates/trusted-server-js/lib/test/core/auction.test.ts b/crates/trusted-server-js/lib/test/core/auction.test.ts index 31e020eff..0d8c0a5a1 100644 --- a/crates/trusted-server-js/lib/test/core/auction.test.ts +++ b/crates/trusted-server-js/lib/test/core/auction.test.ts @@ -1,5 +1,10 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { buildAdRequest, parseAuctionResponse, sendAuction } from '../../src/core/auction'; +import { + buildAdRequest, + parseAuctionResponse, + parseAuctionTraceSummary, + sendAuction, +} from '../../src/core/auction'; describe('auction/buildAdRequest', () => { it('builds from tsjs AdUnit objects', () => { @@ -205,6 +210,100 @@ describe('auction/parseAuctionResponse', () => { expect(parseAuctionResponse({ seatbid: [] })).toEqual([]); }); + it('strictly joins valid root and bid traces without changing legacy fields', () => { + const body = { + ext: { + trusted_server: { + trace: { + version: 1, + auction_trace_id: '650e8400-e29b-41d4-a716-446655440000', + source: 'auction_api', + outcome: 'completed', + }, + }, + }, + seatbid: [ + { + seat: 'example-bidder', + bid: [ + { + impid: 'slot-1', + price: 1.5, + ext: { + trusted_server: { + trace: { + version: 1, + bid_trace_id: '550e8400-e29b-41d4-a716-446655440000', + slot_id: 'slot-1', + provider: 'prebid', + bidder: 'example-bidder', + }, + }, + }, + }, + ], + }, + ], + }; + + expect(parseAuctionTraceSummary(body)).toEqual({ + version: 1, + auctionTraceId: '650e8400-e29b-41d4-a716-446655440000', + source: 'auction_api', + outcome: 'completed', + }); + expect(parseAuctionResponse(body)[0].trace).toEqual({ + version: 1, + auctionTraceId: '650e8400-e29b-41d4-a716-446655440000', + bidTraceId: '550e8400-e29b-41d4-a716-446655440000', + source: 'auction_api', + slotId: 'slot-1', + provider: 'prebid', + bidder: 'example-bidder', + }); + }); + + it('ignores malformed, contradictory, mismatched, and oversized trace fields', () => { + const body = { + ext: { + trusted_server: { + trace: { + version: 1, + auction_trace_id: '650e8400-e29b-41d4-a716-446655440000', + source: 'auction_api', + outcome: 'no_bid', + }, + }, + }, + seatbid: [ + { + seat: 'seat', + bid: [ + { + impid: 'slot-1', + price: 1, + ext: { + trusted_server: { + trace: { + version: 1, + bid_trace_id: '550e8400-e29b-41d4-a716-446655440000', + slot_id: 'different-slot', + provider: 'p'.repeat(65), + bidder: 'seat', + }, + }, + }, + }, + ], + }, + ], + }; + expect(parseAuctionTraceSummary(body)?.outcome).toBe('no_bid'); + expect(parseAuctionResponse(body)[0].trace).toBeUndefined(); + body.ext.trusted_server.trace.auction_trace_id = 'not-a-uuid'; + expect(parseAuctionTraceSummary(body)).toBeUndefined(); + }); + it('defaults missing fields gracefully', () => { const body = { seatbid: [{ bid: [{ impid: 'slot-1', price: 1.5 }] }], @@ -259,7 +358,7 @@ describe('auction/sendAuction', () => { ], }; - const bids = await sendAuction('/auction', request); + const result = await sendAuction('/auction', request); expect(globalThis.fetch).toHaveBeenCalledWith( '/auction', @@ -269,18 +368,46 @@ describe('auction/sendAuction', () => { body: JSON.stringify(request), }) ); - expect(bids).toHaveLength(1); - expect(bids[0].price).toBe(2.5); + expect(result.kind).toBe('ok'); + if (result.kind !== 'ok') throw new Error('expected successful auction'); + expect(result.bids).toHaveLength(1); + expect(result.bids[0].price).toBe(2.5); }); - it('returns empty array on network error', async () => { + it('distinguishes a network error from a valid empty auction', async () => { globalThis.fetch = vi.fn().mockRejectedValue(new Error('network error')) as any; - const bids = await sendAuction('/auction', { adUnits: [] }); - expect(bids).toEqual([]); + const result = await sendAuction('/auction', { adUnits: [] }); + expect(result).toEqual({ kind: 'transport_error', reason: 'network' }); + }); + + it('accepts legacy empty but rejects malformed seatbid collections', async () => { + globalThis.fetch = vi + .fn() + .mockResolvedValueOnce({ + ok: true, + status: 200, + headers: { get: () => 'application/json' }, + json: async () => ({}), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + headers: { get: () => 'application/json' }, + json: async () => ({ seatbid: {} }), + }) as any; + + await expect(sendAuction('/auction', { adUnits: [] })).resolves.toEqual({ + kind: 'ok', + bids: [], + }); + await expect(sendAuction('/auction', { adUnits: [] })).resolves.toEqual({ + kind: 'invalid_response', + reason: 'invalid_shape', + }); }); - it('returns empty array for non-JSON response', async () => { + it('distinguishes a non-JSON response', async () => { globalThis.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, @@ -288,11 +415,11 @@ describe('auction/sendAuction', () => { json: async () => ({}), }) as any; - const bids = await sendAuction('/auction', { adUnits: [] }); - expect(bids).toEqual([]); + const result = await sendAuction('/auction', { adUnits: [] }); + expect(result).toEqual({ kind: 'invalid_response', reason: 'non_json' }); }); - it('returns empty array for non-OK response', async () => { + it('distinguishes a non-OK response', async () => { globalThis.fetch = vi.fn().mockResolvedValue({ ok: false, status: 500, @@ -300,7 +427,7 @@ describe('auction/sendAuction', () => { json: async () => ({}), }) as any; - const bids = await sendAuction('/auction', { adUnits: [] }); - expect(bids).toEqual([]); + const result = await sendAuction('/auction', { adUnits: [] }); + expect(result).toEqual({ kind: 'transport_error', reason: 'http' }); }); }); diff --git a/crates/trusted-server-js/lib/test/core/request.test.ts b/crates/trusted-server-js/lib/test/core/request.test.ts index 2c56361dc..80da9fbae 100644 --- a/crates/trusted-server-js/lib/test/core/request.test.ts +++ b/crates/trusted-server-js/lib/test/core/request.test.ts @@ -10,6 +10,7 @@ describe('request.requestAds', () => { beforeEach(async () => { await vi.resetModules(); document.body.innerHTML = ''; + delete window.tsjs; originalFetch = globalThis.fetch; }); @@ -217,9 +218,8 @@ describe('request.requestAds', () => { expect(JSON.stringify(rejectionCall)).not.toContain('[object Object]'); }); - it('does not blank the slot when a later bid for the same slot is rejected', async () => { - // Regression: multi-bid scenario where a rejected bid must not erase an earlier - // successful render into the same slot. + it('rejects an ambiguous multi-winner response without blanking the slot', async () => { + // A final auction response must contain at most one winner per requested slot. const goodCreative = '
Safe Ad
'; (globalThis as any).fetch = vi.fn().mockResolvedValue({ ok: true, @@ -243,16 +243,14 @@ describe('request.requestAds', () => { const { addAdUnits } = await import('../../src/core/registry'); const { requestAds } = await import('../../src/core/request'); - document.body.innerHTML = '
'; + document.body.innerHTML = '
existing
'; addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); requestAds(); await flushRequestAds(); - // The good creative should have rendered; the bad one should not have blanked it. - const iframe = document.querySelector('#slot1 iframe') as HTMLIFrameElement | null; - expect(iframe).toBeTruthy(); - expect(iframe!.srcdoc).toContain(goodCreative); + expect(document.querySelector('#slot1 iframe')).toBeNull(); + expect(document.querySelector('#slot1')?.textContent).toContain('existing'); }); it('rejects creatives that sanitize to empty markup', async () => { @@ -296,6 +294,134 @@ describe('request.requestAds', () => { ); }); + it('keeps the latest direct owner when overlapping responses resolve out of order', async () => { + const resolves: Array<(response: Response) => void> = []; + (globalThis as any).fetch = vi.fn().mockImplementation( + () => + new Promise((resolve) => { + resolves.push(resolve); + }) + ); + const recordAdTrace = vi.fn(); + window.tsjs = { + recordAdTrace, + nextAdTraceGeneration: vi.fn().mockReturnValueOnce(1).mockReturnValueOnce(2), + } as any; + const { addAdUnits } = await import('../../src/core/registry'); + const { requestAds } = await import('../../src/core/request'); + document.body.innerHTML = '
existing
'; + addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); + + requestAds(); + requestAds(); + expect(resolves).toHaveLength(2); + const response = (creative: string) => + ({ + ok: true, + status: 200, + headers: { get: () => 'application/json' }, + json: async () => ({ + seatbid: [{ seat: 'trusted-server', bid: [{ impid: 'slot1', adm: creative }] }], + }), + }) as Response; + + resolves[1](response('
new owner
')); + await flushRequestAds(); + resolves[0](response('
stale owner
')); + await flushRequestAds(); + + const iframe = document.querySelector('#slot1 iframe') as HTMLIFrameElement; + expect(iframe.srcdoc).toContain('new owner'); + expect(iframe.srcdoc).not.toContain('stale owner'); + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'direct_render_rejected', + generation: 1, + reason: 'direct_owner_replaced', + }) + ); + }); + + it('records an exact direct auction winner, placement, and iframe load', async () => { + const auctionTraceId = '550e8400-e29b-41d4-a716-446655440000'; + const bidTraceId = '123e4567-e89b-42d3-a456-426614174000'; + const recordAdTrace = vi.fn(); + window.tsjs = { + recordAdTrace, + nextAdTraceGeneration: vi.fn().mockReturnValue(1), + } as any; + (globalThis as any).fetch = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + headers: { get: () => 'application/json' }, + json: async () => ({ + ext: { + trusted_server: { + trace: { + version: 1, + auction_trace_id: auctionTraceId, + source: 'auction_api', + outcome: 'completed', + }, + }, + }, + seatbid: [ + { + seat: 'trusted-server', + bid: [ + { + impid: 'slot1', + adm: '
direct
', + ext: { + trusted_server: { + trace: { + version: 1, + auction_trace_id: auctionTraceId, + bid_trace_id: bidTraceId, + source: 'auction_api', + slot_id: 'slot1', + provider: 'prebid', + bidder: 'example', + }, + }, + }, + }, + ], + }, + ], + }), + }); + + const { addAdUnits } = await import('../../src/core/registry'); + const { requestAds } = await import('../../src/core/request'); + document.body.innerHTML = '
'; + addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); + + requestAds(); + await flushRequestAds(); + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'ts_winner_observed', + generation: 1, + auctionTraceId, + bidTraceId, + }) + ); + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'pb_render_served', reason: 'direct_iframe_created' }) + ); + + const iframe = document.querySelector('#slot1 iframe') as HTMLIFrameElement; + iframe.dispatchEvent(new Event('load')); + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'creative_load_acknowledged', + generation: 1, + reason: 'direct_iframe_load', + }) + ); + }); + it('skips iframe insertion when slot is missing', async () => { // mock fetch for unified auction endpoint - returns inline HTML (globalThis as any).fetch = vi.fn().mockResolvedValue({ diff --git a/crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts b/crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts new file mode 100644 index 000000000..a3eaa7d6f --- /dev/null +++ b/crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts @@ -0,0 +1,41 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +describe('ad_trace integration gate', () => { + beforeEach(() => { + vi.resetModules(); + document.getElementById('ts-ad-trace-overlay')?.remove(); + delete window.__tsjs_adTraceActive; + delete window.tsjs; + }); + + afterEach(() => { + document.getElementById('ts-ad-trace-overlay')?.remove(); + delete window.__tsjs_adTraceActive; + delete window.tsjs; + }); + + it('leaves API and private recorders absent without the server bootstrap', async () => { + const { installAdTrace } = await import('../../../src/integrations/ad_trace/index'); + expect(installAdTrace()).toBe(false); + expect(window.tsjs?.adTrace).toBeUndefined(); + expect(window.tsjs?.recordAdTrace).toBeUndefined(); + }); + + it('installs one immutable API and consumes the exact bootstrap', async () => { + window.__tsjs_adTraceActive = true; + const { installAdTrace } = await import('../../../src/integrations/ad_trace/index'); + expect(installAdTrace()).toBe(true); + expect(window.__tsjs_adTraceActive).toBeUndefined(); + expect(Object.isFrozen(window.tsjs?.adTrace)).toBe(true); + expect(typeof window.tsjs?.recordAdTrace).toBe('function'); + expect(document.querySelectorAll('#ts-ad-trace-overlay')).toHaveLength(1); + expect(installAdTrace()).toBe(true); + expect(document.querySelectorAll('#ts-ad-trace-overlay')).toHaveLength(1); + }); + + it('does not accept the legacy tester cookie without bootstrap', async () => { + document.cookie = 'ts-tester=true; Path=/'; + const { installAdTrace } = await import('../../../src/integrations/ad_trace/index'); + expect(installAdTrace()).toBe(false); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts b/crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts new file mode 100644 index 000000000..0a5aaa217 --- /dev/null +++ b/crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { installAdTraceOverlay } from '../../../src/integrations/ad_trace/overlay'; +import type { AdTraceApi } from '../../../src/core/types'; + +function api(): AdTraceApi { + const slot = { + slotId: 'slot-a', + latestGeneration: 1, + generations: [], + stages: { + trustedServer: { outcome: 'won', confidence: 'definitive', reason: 'winner' }, + prebid: { outcome: 'not_run', confidence: 'definitive', reason: 'direct' }, + gam: { outcome: 'trusted_server_candidate', confidence: 'probable', reason: 'render' }, + creative: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + }, + } as const; + const renders = [ + { + sequence: 1, + slotId: 'slot-a', + generation: 1, + source: 'gpt', + outcome: 'gam_only', + confidence: 'probable', + visibility: 'unknown', + createdAt: 1, + updatedAt: 1, + }, + ] as const; + return { + getSlot: () => slot as any, + getEvents: () => [], + getRenderTimeline: () => renders as any, + export: () => ({ + version: 1, + slots: [slot as any], + events: [], + renders: renders as any, + metadata: { droppedEvents: 0, evictedSlots: 0 }, + }), + }; +} + +describe('ad trace overlay lifecycle', () => { + afterEach(() => { + document.getElementById('ts-ad-trace-overlay')?.remove(); + document.getElementById('slot-prefix-rendered')?.remove(); + delete window.tsjs; + vi.restoreAllMocks(); + }); + + it('finds prefix slots, observes resize, and coalesces animation frames', () => { + const element = document.createElement('div'); + element.id = 'slot-prefix-rendered'; + const rect = vi.spyOn(element, 'getBoundingClientRect').mockReturnValue({ + left: 10, + top: 20, + width: 300, + height: 250, + } as DOMRect); + document.body.appendChild(element); + const updateVisibility = vi.fn(); + window.tsjs = { + adSlots: [{ id: 'slot-a', div_id: 'slot-prefix' }], + getAdTraceElement: () => element, + updateAdTraceVisibility: updateVisibility, + } as any; + + const observe = vi.fn(); + vi.stubGlobal( + 'ResizeObserver', + class { + observe = observe; + unobserve = vi.fn(); + disconnect = vi.fn(); + } + ); + const frames: FrameRequestCallback[] = []; + vi.spyOn(window, 'requestAnimationFrame').mockImplementation((callback) => { + frames.push(callback); + return frames.length; + }); + let subscriber: (() => void) | undefined; + installAdTraceOverlay(api(), (listener) => { + subscriber = listener; + return vi.fn(); + }); + + expect(rect).toHaveBeenCalledTimes(1); + expect(observe).toHaveBeenCalledWith(element); + expect(updateVisibility).toHaveBeenCalledWith('slot-a', 1, 'visible'); + expect(element.getAttribute('data-ts-trace-seq')).toBe('1'); + expect(element.getAttribute('data-ts-trace-outcome')).toBe('gam_only'); + window.dispatchEvent(new Event('scroll')); + window.dispatchEvent(new Event('scroll')); + subscriber?.(); + expect(frames).toHaveLength(1); + frames.shift()?.(1); + expect(rect).toHaveBeenCalledTimes(2); + + const replacement = document.createElement('div'); + replacement.id = element.id; + element.replaceWith(replacement); + subscriber?.(); + frames.shift()?.(2); + expect(replacement.hasAttribute('data-ts-trace-seq')).toBe(false); + expect(element.hasAttribute('data-ts-trace-seq')).toBe(false); + expect(updateVisibility).toHaveBeenCalledWith('slot-a', 1, 'disconnected'); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 4a6368768..24f900123 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -883,15 +883,30 @@ describe('installTsRenderBridge', () => { afterEach(() => { vi.unstubAllGlobals(); document.getElementById('div-header')?.remove(); + document.getElementById('div-sidebar')?.remove(); delete (window as TestWindow).tsjs; }); + function capturePrivateOwner(slotId: string, divId: string): void { + const ts = (window as TestWindow).tsjs!; + const bid = ts.bids?.[slotId]; + ts.captureAdTraceRequest?.( + { + getSlotElementId: () => divId, + getTargeting: () => [], + }, + 'test_request', + { slotId, adId: bid?.hb_adid, bid } + ); + } + function createTrustedSlotIframe(): Window { const slot = document.createElement('div'); slot.id = 'div-header'; const iframe = document.createElement('iframe'); slot.appendChild(iframe); document.body.appendChild(slot); + capturePrivateOwner('homepage_header', slot.id); return iframe.contentWindow!; } @@ -960,7 +975,7 @@ describe('installTsRenderBridge', () => { expect(fetchStub).toHaveBeenCalledWith( 'https://openads.example.com/cache?uuid=test-cache-uuid', - { mode: 'cors' } + expect.objectContaining({ mode: 'cors', signal: expect.any(AbortSignal) }) ); expect(stopSpy).toHaveBeenCalled(); expect(portMessages).toHaveLength(1); @@ -982,17 +997,15 @@ describe('installTsRenderBridge', () => { }) as unknown as MessageEvent ); await new Promise((resolve) => setTimeout(resolve, 50)); + expect(fetchStub).toHaveBeenCalledTimes(1); + expect(portMessages).toHaveLength(1); expect(beaconSpy).toHaveBeenCalledTimes(2); beaconSpy.mockRestore(); }); it('fetches PBS Cache once when two same-adId messages race before the fetch resolves', async () => { - // Concurrent render double-fire guard: two 'Prebid Request' messages for the - // same adId can arrive before the first cache fetch settles. The in-flight - // `renderingAdIds` gate must collapse them to a single fetch — the persistent - // firedBeacons dedup only engages after a fetch resolves, so it cannot stop - // the second fetch on its own. Deferring the fetch keeps both messages in the - // window where only the in-flight gate can prevent the duplicate. + // Two duplicate requests from the exact same private owner collapse to one + // fetch while it remains current. const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); const mockAd = '
Test Creative
'; let resolveFetch: (value: Response) => void = () => {}; @@ -1039,6 +1052,190 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); + it('supersedes a same-adId cache owner from a different source', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + const resolves: Array<(value: Response) => void> = []; + fetchStub.mockImplementation( + () => + new Promise((resolve) => { + resolves.push(resolve); + }) + ); + const bridgeListener = await captureBridgeListener(); + const oldPort = { postMessage: vi.fn() }; + const newPort = { postMessage: vi.fn() }; + const oldSource = createTrustedSlotIframe(); + const newFrame = document.createElement('iframe'); + document.getElementById('div-header')?.appendChild(newFrame); + const newSource = newFrame.contentWindow!; + + const dispatch = (source: Window, port: { postMessage: ReturnType }): void => { + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [port], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + }; + dispatch(oldSource, oldPort); + dispatch(newSource, newPort); + expect(fetchStub).toHaveBeenCalledTimes(2); + + resolves[0]({ ok: true, text: () => Promise.resolve('
old
') } as Response); + resolves[1]({ ok: true, text: () => Promise.resolve('
new
') } as Response); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(oldPort.postMessage).not.toHaveBeenCalled(); + expect(newPort.postMessage).toHaveBeenCalledTimes(1); + expect(beaconSpy).toHaveBeenCalledTimes(2); + beaconSpy.mockRestore(); + }); + + it('allows concurrent same-adId owners in different slots', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + const ts = (window as TestWindow).tsjs!; + ts.bids!.sidebar = { + ...ts.bids!.homepage_header, + nurl: 'https://ssp.example/sidebar-win', + burl: 'https://ssp.example/sidebar-bill', + }; + ts.adSlots!.push({ + id: 'sidebar', + formats: [[300, 250]], + gam_unit_path: '/a/b/sidebar', + div_id: 'div-sidebar', + targeting: {}, + }); + const resolves: Array<(value: Response) => void> = []; + fetchStub.mockImplementation( + () => + new Promise((resolve) => { + resolves.push(resolve); + }) + ); + const bridgeListener = await captureBridgeListener(); + const headerPort = { postMessage: vi.fn() }; + const sidebarPort = { postMessage: vi.fn() }; + const headerSource = createTrustedSlotIframe(); + const sidebar = document.createElement('div'); + sidebar.id = 'div-sidebar'; + const sidebarFrame = document.createElement('iframe'); + sidebar.appendChild(sidebarFrame); + document.body.appendChild(sidebar); + capturePrivateOwner('sidebar', sidebar.id); + + const dispatch = (source: Window, port: { postMessage: ReturnType }): void => { + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [port], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + }; + dispatch(headerSource, headerPort); + dispatch(sidebarFrame.contentWindow!, sidebarPort); + resolves.forEach((resolve, index) => + resolve({ + ok: true, + text: () => Promise.resolve(`
creative ${index}
`), + } as Response) + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(headerPort.postMessage).toHaveBeenCalledTimes(1); + expect(sidebarPort.postMessage).toHaveBeenCalledTimes(1); + expect(beaconSpy).toHaveBeenCalledTimes(4); + beaconSpy.mockRestore(); + }); + + it('blocks a late TS message after navigation before page-bids applies', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const port = { postMessage: vi.fn() }; + const stop = vi.fn(); + + window.dispatchEvent(new PopStateEvent('popstate')); + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [port], + source, + stopImmediatePropagation: stop, + }) as unknown as MessageEvent + ); + + expect(stop).toHaveBeenCalledOnce(); + expect(port.postMessage).not.toHaveBeenCalled(); + expect(fetchStub).not.toHaveBeenCalled(); + expect(beaconSpy).not.toHaveBeenCalled(); + beaconSpy.mockRestore(); + }); + + it('blocks an old traced message after a newer request capture', async () => { + const ts = (window as TestWindow).tsjs!; + ts.recordAdTrace = vi.fn(); + ts.nextAdTraceGeneration = vi.fn().mockReturnValueOnce(1).mockReturnValueOnce(2); + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + ts.bids!.homepage_header = { + ...ts.bids!.homepage_header, + hb_adid: 'new-cache-uuid', + }; + capturePrivateOwner('homepage_header', 'div-header'); + const port = { postMessage: vi.fn() }; + const stop = vi.fn(); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [port], + source, + stopImmediatePropagation: stop, + }) as unknown as MessageEvent + ); + + expect(stop).toHaveBeenCalledOnce(); + expect(port.postMessage).not.toHaveBeenCalled(); + expect(fetchStub).not.toHaveBeenCalled(); + }); + + it('drops a detached stale cache completion without responding or billing', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + let resolveFetch: (value: Response) => void = () => {}; + fetchStub.mockReturnValue( + new Promise((resolve) => { + resolveFetch = resolve; + }) + ); + const bridgeListener = await captureBridgeListener(); + const port = { postMessage: vi.fn() }; + const source = createTrustedSlotIframe(); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [port], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + document.getElementById('div-header')?.remove(); + resolveFetch({ + ok: true, + text: () => Promise.resolve('
stale
'), + } as Response); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(port.postMessage).not.toHaveBeenCalled(); + expect(beaconSpy).not.toHaveBeenCalled(); + beaconSpy.mockRestore(); + }); + it('responds with adm without fetching PBS Cache when debug adm is available', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); const debugAdm = '
Debug Creative
'; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_trace.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_trace.test.ts new file mode 100644 index 000000000..50d051eaf --- /dev/null +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_trace.test.ts @@ -0,0 +1,327 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const OLD_TOKEN = '550e8400-e29b-41d4-a716-446655440000'; +const NEW_TOKEN = '650e8400-e29b-41d4-a716-446655440000'; + +function slotWithTargeting(values: Record) { + return { + getSlotElementId: () => 'div-header', + getTargeting: (key: string) => (values[key] ? [values[key]] : []), + }; +} + +function trustedSource(): Window { + const root = document.createElement('div'); + root.id = 'div-header'; + const iframe = document.createElement('iframe'); + root.appendChild(iframe); + document.body.appendChild(root); + return iframe.contentWindow!; +} + +describe('GPT immutable ad trace render attribution', () => { + let bridge: (event: MessageEvent) => void; + let module: typeof import('../../../src/integrations/gpt/index'); + let record: ReturnType; + + beforeEach(async () => { + vi.resetModules(); + record = vi.fn(); + Object.defineProperty(navigator, 'sendBeacon', { + value: vi.fn(), + configurable: true, + writable: true, + }); + let generation = 0; + window.tsjs = { + recordAdTrace: record, + nextAdTraceGeneration: () => ++generation, + divToSlotId: { 'div-header': 'slot-a' }, + adSlots: [ + { + id: 'slot-a', + div_id: 'div-header', + gam_unit_path: '/123/example', + formats: [[300, 250]], + }, + ], + bids: { + 'slot-a': { + hb_adid: 'old-ad-id', + adm: '
Old creative
', + nurl: 'https://billing.example/win', + burl: 'https://billing.example/bill', + trace: { + version: 1, + auctionTraceId: '750e8400-e29b-41d4-a716-446655440000', + bidTraceId: OLD_TOKEN, + source: 'initial_navigation', + slotId: 'slot-a', + provider: 'prebid', + bidder: 'example-bidder', + }, + }, + }, + } as any; + const originalAdd = window.addEventListener.bind(window); + const spy = vi + .spyOn(window, 'addEventListener') + .mockImplementation((type, listener, options) => { + if (type === 'message') bridge = listener as (event: MessageEvent) => void; + originalAdd(type, listener, options); + }); + module = await import('../../../src/integrations/gpt/index'); + spy.mockRestore(); + }); + + afterEach(() => { + document.getElementById('div-header')?.remove(); + delete window.tsjs; + vi.restoreAllMocks(); + }); + + it('preserves authoritative missing values in a queued boundary snapshot', () => { + const source = trustedSource(); + const port = { postMessage: vi.fn() }; + const stop = vi.fn(); + const beacon = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + module.captureAdTraceRequest(slotWithTargeting({ hb_adid: 'old-ad-id' }) as any, 'bootstrap', { + slotId: 'slot-a', + bidder: undefined, + adId: undefined, + traceToken: undefined, + bid: undefined, + }); + + bridge( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'old-ad-id' }), + ports: [port], + source, + stopImmediatePropagation: stop, + }) as unknown as MessageEvent + ); + + expect(stop).toHaveBeenCalledOnce(); + expect(port.postMessage).not.toHaveBeenCalled(); + expect(beacon).not.toHaveBeenCalled(); + }); + + it('never pairs a new client or refreshed TS adId with the stale live bid payload', () => { + const source = trustedSource(); + const port = { postMessage: vi.fn() }; + const beacon = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + + for (const targeting of [ + { hb_adid: 'client-ad-id', hb_bidder: 'client-bidder' }, + { hb_adid: 'new-ts-ad-id', hb_bidder: 'example-bidder', ts_trace: NEW_TOKEN }, + ]) { + window.tsjs!.prebidCorrelation = [ + { + auctionId: 'auction-2', + slotId: 'slot-a', + requestId: 'request-2', + adId: targeting.hb_adid, + bidder: targeting.hb_bidder, + ...(targeting.ts_trace ? { traceToken: targeting.ts_trace } : {}), + ...(!targeting.ts_trace ? { events: ['prebid_bid_won' as const] } : {}), + }, + ]; + module.captureAdTraceRequest(slotWithTargeting(targeting) as any, 'prebid_refresh'); + bridge( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: targeting.hb_adid }), + ports: [port], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + } + + expect(port.postMessage).not.toHaveBeenCalled(); + expect(beacon).not.toHaveBeenCalled(); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'prebid_targeting_selected', outcome: 'client_bid_won' }) + ); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'prebid_bid_won', generation: 1 }) + ); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'prebid_targeting_selected', + outcome: 'won', + bidTraceId: NEW_TOKEN, + }) + ); + + window.tsjs!.prebidCorrelation = [ + { + auctionId: 'auction-3', + slotId: 'slot-a', + requestId: 'client-request', + adId: 'winning-client-ad', + bidder: 'client-bidder', + }, + { + auctionId: 'auction-3', + slotId: 'slot-a', + requestId: 'ts-request', + adId: 'losing-ts-ad', + bidder: 'trustedServer', + traceToken: NEW_TOKEN, + }, + ]; + module.captureAdTraceRequest( + slotWithTargeting({ hb_adid: 'winning-client-ad', hb_bidder: 'client-bidder' }) as any, + 'prebid_refresh' + ); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'prebid_targeting_selected', outcome: 'lost' }) + ); + }); + + it('serves, bills once, and acknowledges only the exact immutable generation/source/token', () => { + const source = trustedSource(); + const foreignSource = window; + const port = { postMessage: vi.fn() }; + const beacon = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + module.captureAdTraceRequest( + slotWithTargeting({ + hb_adid: 'old-ad-id', + hb_bidder: 'example-bidder', + ts_trace: OLD_TOKEN, + }) as any, + 'display' + ); + + bridge( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'old-ad-id' }), + ports: [port], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + const response = JSON.parse(port.postMessage.mock.calls[0][0]); + expect(response.traceToken).toBe(OLD_TOKEN); + expect(response.ad).toBe('
Old creative
'); + expect(beacon).toHaveBeenCalledTimes(2); + + // A newer generation does not steal or invalidate the retained exact ack. + const nextSlot = slotWithTargeting({ hb_adid: 'client-next', hb_bidder: 'client-bidder' }); + module.captureAdTraceRequest(nextSlot as any, 'prebid_refresh'); + + bridge( + Object.assign(new Event('message'), { + data: { type: 'ts-creative-load', version: 1, traceToken: OLD_TOKEN }, + source: foreignSource, + }) as unknown as MessageEvent + ); + expect(record).not.toHaveBeenCalledWith( + expect.objectContaining({ kind: 'creative_load_acknowledged' }) + ); + bridge( + Object.assign(new Event('message'), { + data: { type: 'ts-creative-load', version: 1, traceToken: NEW_TOKEN }, + source, + }) as unknown as MessageEvent + ); + expect(record).not.toHaveBeenCalledWith( + expect.objectContaining({ kind: 'creative_load_acknowledged' }) + ); + + bridge( + Object.assign(new Event('message'), { + data: { type: 'ts-creative-load', version: 1, traceToken: OLD_TOKEN }, + source, + }) as unknown as MessageEvent + ); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'creative_load_acknowledged', + generation: 1, + bidTraceId: OLD_TOKEN, + }) + ); + expect(beacon).toHaveBeenCalledTimes(2); + + module.supersedeAdTraceSlot(nextSlot as any, 'slot_destroyed'); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'generation_superseded', + generation: 2, + reason: 'slot_destroyed', + }) + ); + }); + + it('rejects acknowledgements after the exact slot generation is superseded', () => { + const source = trustedSource(); + const slot = slotWithTargeting({ + hb_adid: 'old-ad-id', + hb_bidder: 'example-bidder', + ts_trace: OLD_TOKEN, + }); + const port = { postMessage: vi.fn() }; + vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + module.captureAdTraceRequest(slot as any, 'display'); + bridge( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'old-ad-id' }), + ports: [port], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + module.supersedeAdTraceSlot(slot as any, 'slot_destroyed'); + record.mockClear(); + bridge( + Object.assign(new Event('message'), { + data: { type: 'ts-creative-load', version: 1, traceToken: OLD_TOKEN }, + source, + }) as unknown as MessageEvent + ); + expect(record).not.toHaveBeenCalledWith( + expect.objectContaining({ kind: 'creative_load_acknowledged' }) + ); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'pb_render_rejected', reason: 'invalid_acknowledgement' }) + ); + }); + + it('expires pending acknowledgements after thirty seconds', () => { + let now = 0; + vi.spyOn(performance, 'now').mockImplementation(() => now); + const source = trustedSource(); + const slot = slotWithTargeting({ + hb_adid: 'old-ad-id', + hb_bidder: 'example-bidder', + ts_trace: OLD_TOKEN, + }); + const port = { postMessage: vi.fn() }; + vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + module.captureAdTraceRequest(slot as any, 'display'); + bridge( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'old-ad-id' }), + ports: [port], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + now = 30_001; + record.mockClear(); + bridge( + Object.assign(new Event('message'), { + data: { type: 'ts-creative-load', version: 1, traceToken: OLD_TOKEN }, + source, + }) as unknown as MessageEvent + ); + expect(record).not.toHaveBeenCalledWith( + expect.objectContaining({ kind: 'creative_load_acknowledged' }) + ); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'generation_superseded', reason: 'ack_expired' }) + ); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts index 406c6d1f5..2f21da832 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts @@ -240,6 +240,14 @@ describe('GPT – installTsAdInit', () => { ['ts_initial', ['1']], ['pos', ['old-pos']], ]); + const clearTargeting = vi.fn((key?: string) => { + if (key) { + slotTargeting.delete(key); + } else { + slotTargeting.clear(); + } + return gptSlot; + }); const gptSlot: any = { getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), getTargeting: vi.fn((key: string) => slotTargeting.get(key) ?? []), @@ -247,14 +255,7 @@ describe('GPT – installTsAdInit', () => { slotTargeting.set(key, Array.isArray(value) ? value : [value]); return gptSlot; }), - clearTargeting: vi.fn((key?: string) => { - if (key) { - slotTargeting.delete(key); - } else { - slotTargeting.clear(); - } - return gptSlot; - }), + clearTargeting, }; const pubads = { getSlots: vi.fn(() => [gptSlot]), @@ -295,13 +296,13 @@ describe('GPT – installTsAdInit', () => { installTsAdInit(); (window as any).tsjs.adInit(); - expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_pb'); - expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_bidder'); - expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_adid'); - expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_cache_host'); - expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_cache_path'); - expect(gptSlot.clearTargeting).toHaveBeenCalledWith('ts_initial'); - expect(gptSlot.clearTargeting).toHaveBeenCalledWith('pos'); + expect(clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(clearTargeting).toHaveBeenCalledWith('hb_bidder'); + expect(clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_host'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(clearTargeting).toHaveBeenCalledWith('ts_initial'); + expect(clearTargeting).toHaveBeenCalledWith('pos'); expect(slotTargeting.get('hb_pb')).toBeUndefined(); expect(slotTargeting.get('hb_bidder')).toBeUndefined(); expect(slotTargeting.get('hb_adid')).toBeUndefined(); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 726f40b49..f0a878bfb 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -8,6 +8,7 @@ const { mockRegisterBidAdapter, mockGetUserIdsAsEids, mockGetConfig, + mockOnEvent, mockPbjs, mockGetBidAdapter, mockAdapterManager, @@ -21,6 +22,7 @@ const { () => [] as Array<{ source: string; uids?: Array<{ id: string; atype?: number }> }> ); const mockGetConfig = vi.fn(); + const mockOnEvent = vi.fn(); const mockPbjs = { setConfig: mockSetConfig, processQueue: mockProcessQueue, @@ -28,6 +30,7 @@ const { registerBidAdapter: mockRegisterBidAdapter, getUserIdsAsEids: mockGetUserIdsAsEids, getConfig: mockGetConfig, + onEvent: mockOnEvent, adUnits: [] as any[], }; const mockAdapterManager = { @@ -40,6 +43,7 @@ const { mockRegisterBidAdapter, mockGetUserIdsAsEids, mockGetConfig, + mockOnEvent, mockPbjs, mockGetBidAdapter, mockAdapterManager, @@ -149,6 +153,39 @@ describe('prebid/auctionBidsToPrebidBids', () => { }); }); + it('adds adapter targeting only for a validated Trusted Server trace', () => { + const traced: AuctionBid = { + impid: 'slot-traced', + adm: '
Ad
', + price: 2, + width: 300, + height: 250, + seat: 'example-bidder', + creativeId: 'creative-1', + adomain: [], + trace: { + version: 1, + auctionTraceId: '650e8400-e29b-41d4-a716-446655440000', + bidTraceId: '550e8400-e29b-41d4-a716-446655440000', + source: 'auction_api', + slotId: 'slot-traced', + provider: 'prebid', + bidder: 'example-bidder', + }, + }; + + const [bid] = auctionBidsToPrebidBids( + [traced], + [{ adUnitCode: 'slot-traced', bidId: 'request-1' }] + ); + expect(bid.adserverTargeting).toEqual({ + ts_trace: '550e8400-e29b-41d4-a716-446655440000', + }); + expect(auctionBidsToPrebidBids([{ ...traced, trace: undefined }], [])[0]).not.toHaveProperty( + 'adserverTargeting' + ); + }); + it('falls back to impid when no matching bidRequest found', () => { const auctionBids: AuctionBid[] = [ { @@ -218,6 +255,8 @@ describe('prebid/installPrebidNpm', () => { document.cookie = 'ts-eids=; Path=/; Max-Age=0'; delete (window as any).__tsjs_prebid; delete (window as any).__tsjs_prebid_diagnostics; + delete (mockPbjs as any).__tsAdTraceObserved; + delete window.tsjs; }); afterEach(() => { @@ -800,6 +839,50 @@ describe('prebid/installPrebidNpm', () => { expect(document.cookie).toBe(''); }); + + it('joins late winner and render events to the retained selected generation', () => { + const recordAdTrace = vi.fn(); + window.tsjs = { + recordAdTrace, + prebidSelectedParticipants: [ + { + auctionId: 'auction-1', + slotId: 'slot-a', + requestId: 'request-1', + adId: 'ad-1', + bidder: 'client-bidder', + generation: 7, + selectedAt: performance.now(), + }, + ], + } as any; + installPrebidNpm(); + const handlers = new Map) => void>( + mockOnEvent.mock.calls.map(([event, handler]) => [event, handler]) + ); + const bid = { + auctionId: 'auction-1', + adUnitCode: 'slot-a', + requestId: 'request-1', + adId: 'ad-1', + bidderCode: 'client-bidder', + }; + + handlers.get('bidWon')?.(bid); + handlers.get('adRenderSucceeded')?.({ bid }); + + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'prebid_bid_won', generation: 7, slotId: 'slot-a' }) + ); + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'prebid_render_succeeded', + generation: 7, + slotId: 'slot-a', + }) + ); + expect(window.tsjs.prebidSelectedParticipants).toEqual([]); + }); }); }); diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index b975590c6..35384e13f 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -998,6 +998,23 @@ apply when the integration section exists in `trusted-server.toml`. | --------- | ------- | ------------------------------ | | `enabled` | Boolean | Enable/disable the integration | +### Ad Trace Integration + +**Section**: `[integrations.ad_trace]` + +| Field | Type | Default | Description | +| --------- | ------- | ------- | ------------------------------------------------ | +| `enabled` | Boolean | `false` | Include tester-only auction trace browser support | + +Browser-visible auction IDs, bid IDs, targeting, API state, and the console require this setting plus an activated browser session. Visit a publisher page with the exact query `?ts_console=true` or `?ts_console=1`; Trusted Server enables the first response and sets a host-only session cookie automatically. Use `?ts_console=false` or `?ts_console=0` to clear the session. The reserved query is removed from downstream requests and cleaned from eligible HTML URLs. Active trace responses are private and non-storeable. + +The integration is disabled by default. The query is a self-service diagnostic toggle, not authorization, and does nothing without the explicit configuration gate. The console never exposes the internal auction request ID, identity data, consent strings, page URLs, partner notification URLs, cache coordinates, raw targeting, or creative markup. A creative marked `confirmed` means its exact Trusted Server renderer iframe load was acknowledged; it does not claim viewability or arbitrary advertiser JavaScript completion. + +```toml +[integrations.ad_trace] +enabled = false +``` + ### Prebid Integration **Section**: `[integrations.prebid]` diff --git a/scripts/generate-integration-viceroy-configs.sh b/scripts/generate-integration-viceroy-configs.sh index 761d06926..97ee870a0 100755 --- a/scripts/generate-integration-viceroy-configs.sh +++ b/scripts/generate-integration-viceroy-configs.sh @@ -13,6 +13,7 @@ ARTIFACTS_DIR="${ARTIFACTS_DIR:-$REPO_ROOT/target/integration-test-artifacts}" CONFIG_DIR="$ARTIFACTS_DIR/configs" TEMPLATE_PATH="crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml" APP_CONFIG_PATH="crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml" +AD_TRACE_APP_CONFIG_PATH="crates/trusted-server-integration-tests/fixtures/configs/trusted-server.ad-trace.integration.toml" INTEGRATION_TARGET_DIR="crates/trusted-server-integration-tests/target" ORIGIN_URL="http://127.0.0.1:$ORIGIN_PORT" HOST_TARGET="$(rustc -vV | sed -n 's/^host: //p')" @@ -41,3 +42,9 @@ fi --app-config "$APP_CONFIG_PATH" \ --output "$CONFIG_DIR/viceroy.toml" \ --origin-url "$ORIGIN_URL" + +"$GENERATOR_BIN" \ + --template "$TEMPLATE_PATH" \ + --app-config "$AD_TRACE_APP_CONFIG_PATH" \ + --output "$CONFIG_DIR/viceroy-ad-trace.toml" \ + --origin-url "$ORIGIN_URL" diff --git a/scripts/integration-tests-browser.sh b/scripts/integration-tests-browser.sh index ce5e64387..0ec16089e 100755 --- a/scripts/integration-tests-browser.sh +++ b/scripts/integration-tests-browser.sh @@ -37,6 +37,19 @@ TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK=false \ echo "==> Generating Viceroy configs..." INTEGRATION_ORIGIN_PORT="$ORIGIN_PORT" ./scripts/generate-integration-viceroy-configs.sh GENERATED_VICEROY_CONFIG_PATH="$REPO_ROOT/target/integration-test-artifacts/configs/viceroy.toml" +GENERATED_AD_TRACE_CONFIG_PATH="$REPO_ROOT/target/integration-test-artifacts/configs/viceroy-ad-trace.toml" + +# Build the actual external Prebid bundle consumed by the isolated ad-trace +# fixture. The browser routes its first-party managed URL to this local asset; +# no public ad network is contacted. +echo "==> Building deterministic external Prebid fixture bundle..." +rm -rf "$REPO_ROOT/target/integration-test-artifacts/prebid" +mkdir -p "$REPO_ROOT/target/integration-test-artifacts/prebid" +npm ci --prefix crates/trusted-server-js/lib +npm run --prefix crates/trusted-server-js/lib build:prebid-external -- \ + --adapters=rubicon \ + --user-id-modules=sharedIdSystem \ + --out "$REPO_ROOT/target/integration-test-artifacts/prebid" # --- Build Docker images --- echo "==> Building WordPress test container..." @@ -49,6 +62,12 @@ docker build \ -t test-nextjs:latest \ crates/trusted-server-integration-tests/fixtures/frameworks/nextjs/ +echo "==> Building ad-trace test container..." +docker build \ + -f crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/Dockerfile \ + -t test-ad-trace:latest \ + . + # --- Install Playwright --- echo "==> Installing Playwright dependencies..." cd "$REPO_ROOT/$BROWSER_DIR" @@ -71,15 +90,22 @@ stop_matching_containers() { } cleanup() { + stop_matching_containers test-ad-trace:latest stop_matching_containers test-nextjs:latest stop_matching_containers test-wordpress:latest } trap cleanup EXIT # --- Run tests for each framework --- -for framework in nextjs wordpress; do +for framework in nextjs wordpress ad-trace; do echo "==> Running Playwright tests for $framework..." - TEST_FRAMEWORK="$framework" npx playwright test "$@" + if [ "$framework" = "ad-trace" ]; then + TEST_FRAMEWORK="$framework" VICEROY_CONFIG_PATH="$GENERATED_AD_TRACE_CONFIG_PATH" \ + npx playwright test "$@" + else + TEST_FRAMEWORK="$framework" VICEROY_CONFIG_PATH="$GENERATED_VICEROY_CONFIG_PATH" \ + npx playwright test "$@" + fi done echo "==> All browser tests passed." diff --git a/tinybird/datasources/auction_events_raw.datasource b/tinybird/datasources/auction_events_raw.datasource index d62f8ae5d..592158713 100644 --- a/tinybird/datasources/auction_events_raw.datasource +++ b/tinybird/datasources/auction_events_raw.datasource @@ -32,6 +32,7 @@ SCHEMA > `price_cpm` Nullable(Float64), `currency` LowCardinality(Nullable(String)), `is_win` Nullable(UInt8), + `bid_trace_id` Nullable(UUID), `ad_domain` Nullable(String), `ad_id` Nullable(String), `event_date` Date DEFAULT toDate(event_ts) diff --git a/tinybird/fixtures/auction_events_raw.ndjson b/tinybird/fixtures/auction_events_raw.ndjson index 078d0c533..10626e3ad 100644 --- a/tinybird/fixtures/auction_events_raw.ndjson +++ b/tinybird/fixtures/auction_events_raw.ndjson @@ -1,7 +1,7 @@ {"event_ts":"2026-06-23 12:00:00.000","event_kind":"summary","auction_id":"550e8400-e29b-41d4-a716-446655440000","auction_source":"auction_api","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":"completed","terminal_reason":null,"slot_count":2,"total_time_ms":120,"winning_bid_count":1,"provider":null,"provider_role":null,"status":null,"provider_response_time_ms":null,"provider_bid_count":null,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} {"event_ts":"2026-06-23 12:00:00.000","event_kind":"provider_call","auction_id":"550e8400-e29b-41d4-a716-446655440000","auction_source":"auction_api","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"prebid","provider_role":"bidder","status":"success","provider_response_time_ms":80,"provider_bid_count":2,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} {"event_ts":"2026-06-23 12:00:00.000","event_kind":"provider_call","auction_id":"550e8400-e29b-41d4-a716-446655440000","auction_source":"auction_api","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"aps","provider_role":"bidder","status":"nobid","provider_response_time_ms":95,"provider_bid_count":0,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} -{"event_ts":"2026-06-23 12:00:00.000","event_kind":"bid","auction_id":"550e8400-e29b-41d4-a716-446655440000","auction_source":"auction_api","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"prebid","provider_role":null,"status":null,"provider_response_time_ms":null,"provider_bid_count":null,"slot_id":"slot-1","slot_w":300,"slot_h":250,"media_type":"banner","seat":"kargo","price_cpm":1.25,"currency":"USD","is_win":1,"ad_domain":"advertiser.example","ad_id":"ad-1"} +{"event_ts":"2026-06-23 12:00:00.000","event_kind":"bid","auction_id":"550e8400-e29b-41d4-a716-446655440000","auction_source":"auction_api","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"prebid","provider_role":null,"status":null,"provider_response_time_ms":null,"provider_bid_count":null,"slot_id":"slot-1","slot_w":300,"slot_h":250,"media_type":"banner","seat":"kargo","price_cpm":1.25,"currency":"USD","is_win":1,"bid_trace_id":"950e8400-e29b-41d4-a716-446655440000","ad_domain":"advertiser.example","ad_id":"ad-1"} {"event_ts":"2026-06-23 12:01:00.000","event_kind":"summary","auction_id":"650e8400-e29b-41d4-a716-446655440000","auction_source":"initial_navigation","publisher_domain":"test-publisher.example","page_path":"/sports","country":"US","region":"CA","is_mobile":1,"is_known_browser":1,"gdpr_applies":0,"consent_present":1,"terminal_status":"abandoned","terminal_reason":"pass_through_response","slot_count":1,"total_time_ms":35,"winning_bid_count":0,"provider":null,"provider_role":null,"status":null,"provider_response_time_ms":null,"provider_bid_count":null,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} {"event_ts":"2026-06-23 12:01:00.000","event_kind":"provider_call","auction_id":"650e8400-e29b-41d4-a716-446655440000","auction_source":"initial_navigation","publisher_domain":"test-publisher.example","page_path":"/sports","country":"US","region":"CA","is_mobile":1,"is_known_browser":1,"gdpr_applies":0,"consent_present":1,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"prebid","provider_role":"bidder","status":"abandoned","provider_response_time_ms":35,"provider_bid_count":0,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} {"event_ts":"2026-06-23 12:02:00.000","event_kind":"summary","auction_id":"750e8400-e29b-41d4-a716-446655440000","auction_source":"spa_navigation","publisher_domain":"test-publisher.example","page_path":"/privacy","country":"DE","region":null,"is_mobile":2,"is_known_browser":2,"gdpr_applies":1,"consent_present":1,"terminal_status":"skipped","terminal_reason":"consent_denied","slot_count":1,"total_time_ms":0,"winning_bid_count":0,"provider":null,"provider_role":null,"status":null,"provider_response_time_ms":null,"provider_bid_count":null,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 26d95d681..c1ae6ddb0 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -59,6 +59,11 @@ enabled = false rewrite_attributes = ["href", "link", "siteBaseUrl", "siteProductionDomain", "url"] max_combined_payload_bytes = 10485760 +# Session-scoped auction-to-creative trace diagnostics. When enabled, visit a +# publisher page with `?ts_console=1` or `?ts_console=true` to open the console. +[integrations.ad_trace] +enabled = false + [integrations.testlight] enabled = false endpoint = "https://testlight.example.com/openrtb2/auction" From 3df591fd9dee9803e885171b78322df671c9ff5c Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 23 Jul 2026 12:06:46 -0500 Subject: [PATCH 103/198] Revert "Merge pull request #922 from IABTechLab/trace-auction-winners-to-creatives" This reverts commit 58706b49e2200c34b8071132f3aa598aec537364, reversing changes made to 96fec883eb8e0899ba4e5604932b801edaf7f8af. --- .../src/auction/formats.rs | 34 ------ crates/trusted-server-core/src/auction/mod.rs | 1 - .../src/auction/orchestrator.rs | 28 ----- .../trusted-server-core/src/auction/types.rs | 79 ------------ .../src/integrations/prebid.rs | 43 ------- crates/trusted-server-core/src/publisher.rs | 79 ++---------- .../trusted-server-js/lib/src/core/auction.ts | 12 -- .../trusted-server-js/lib/src/core/request.ts | 37 ------ .../trusted-server-js/lib/src/core/trace.ts | 71 ----------- .../trusted-server-js/lib/src/core/types.ts | 43 ------- .../lib/src/integrations/gpt/index.ts | 56 +-------- .../lib/test/core/auction.test.ts | 34 ------ .../lib/test/core/request.test.ts | 107 ---------------- .../lib/test/core/trace.test.ts | 114 ------------------ .../lib/test/integrations/gpt/ad_init.test.ts | 86 ------------- 15 files changed, 8 insertions(+), 816 deletions(-) delete mode 100644 crates/trusted-server-js/lib/src/core/trace.ts delete mode 100644 crates/trusted-server-js/lib/test/core/trace.test.ts diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index 2754861d1..284db6242 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -14,7 +14,6 @@ use url::Url; use uuid::Uuid; use crate::auction::context::ContextValue; -use crate::auction::types::adm_trace_hash; use crate::consent::ConsentContext; use crate::constants::{HEADER_X_TS_EC_CONSENT, HEADER_X_TS_EIDS, HEADER_X_TS_EIDS_TRUNCATED}; use crate::creative; @@ -342,39 +341,6 @@ pub fn convert_to_openrtb_response( })); }; - // Trace hash over the exact markup delivered to the client (post - // sanitize/rewrite) so the client can stamp the rendered creative with - // a value that matches this response byte-for-byte. Logged at info so - // server logs join against the DOM markers without debug logging. - let adm_hash = adm - .as_deref() - .filter(|markup| !markup.is_empty()) - .map(adm_trace_hash); - if let Some(ref hash) = adm_hash { - log::info!( - "auction delivered creative: auction_id={} slot_id={} bidder={} crid={:?} adm_hash={}", - auction_request.id, - slot_id, - bid.bidder, - bid.creative_id, - hash, - ); - } - let mut ts_ext = serde_json::Map::new(); - ts_ext.insert( - "auction_id".to_string(), - serde_json::Value::String(auction_request.id.clone()), - ); - if let Some(ref hash) = adm_hash { - ts_ext.insert( - "adm_hash".to_string(), - serde_json::Value::String(hash.clone()), - ); - } - let mut ext = ext.unwrap_or_default(); - ext.insert("ts".to_string(), JsonValue::Object(ts_ext)); - let ext = Some(ext); - let openrtb_bid = OpenRtbBid { id: bid .bid_id diff --git a/crates/trusted-server-core/src/auction/mod.rs b/crates/trusted-server-core/src/auction/mod.rs index ea39d8739..986beb984 100644 --- a/crates/trusted-server-core/src/auction/mod.rs +++ b/crates/trusted-server-core/src/auction/mod.rs @@ -35,7 +35,6 @@ pub use telemetry::{ }; pub use types::{ AdFormat, AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus, MediaType, - adm_trace_hash, }; /// Type alias for provider builder functions. diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index 9a17553c9..eb9b1d138 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -166,29 +166,6 @@ fn remaining_budget_ms(start: Instant, timeout_ms: u32) -> u32 { timeout_ms.saturating_sub(elapsed) } -/// Log one structured trace line per winning bid. -/// -/// Emits the full trace tuple — auction ID, slot, bidder, ad/cache/creative -/// IDs, and the creative trace hash — so a rendered creative on the page -/// (carrying the same tuple in its DOM markers) can be joined back to this -/// auction in server logs. -fn log_winning_bids(auction_id: &str, winning_bids: &HashMap) { - for (slot_id, bid) in winning_bids { - log::info!( - "auction winner: auction_id={} slot_id={} bidder={} price={:?} bid_id={:?} ad_id={:?} cache_id={:?} crid={:?} adm_hash={:?}", - auction_id, - slot_id, - bid.bidder, - bid.price, - bid.bid_id, - bid.ad_id, - bid.cache_id, - bid.creative_id, - bid.creative_trace_hash(), - ); - } -} - fn snapshot_context_request(request: &Request) -> Request { let mut snapshot = Request::new(EdgeBody::empty()); *snapshot.method_mut() = request.method().clone(); @@ -303,8 +280,6 @@ impl AuctionOrchestrator { strategy_name ); - log_winning_bids(&request.id, &result.winning_bids); - Ok(OrchestrationResult { total_time_ms: start_time.elapsed().as_millis() as u64, ..result @@ -1325,7 +1300,6 @@ impl AuctionOrchestrator { responses.len(), ); let winning = self.select_winning_bids(&responses, &floor_prices); - log_winning_bids(&request.id, &winning); return OrchestrationResult { provider_responses: responses, mediator_response: None, @@ -1443,8 +1417,6 @@ impl AuctionOrchestrator { (None, self.select_winning_bids(&responses, &floor_prices)) }; - log_winning_bids(&request.id, &winning_bids); - OrchestrationResult { provider_responses: responses, mediator_response, diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index 31f4a6341..a6ad61f3a 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -313,49 +313,6 @@ impl From<&AuctionResponse> for ProviderSummary { } } -/// Length of the hex-encoded creative trace hash. -/// -/// 16 hex chars (64 bits of SHA-256) — short enough for a DOM attribute and a -/// log field, long enough that collisions across a page's creatives are not a -/// practical concern for tracing. -const ADM_TRACE_HASH_LEN: usize = 16; - -/// Compute the trace hash for a creative markup string. -/// -/// The hash is the first [`ADM_TRACE_HASH_LEN`] hex characters of the SHA-256 -/// of the exact bytes handed to the client. It is a correlation key for -/// tracing a winning bid to the creative rendered on the page — server logs, -/// the injected bid payload, and DOM markers all carry the same value — not an -/// integrity mechanism. -/// -/// # Examples -/// -/// ``` -/// use trusted_server_core::auction::adm_trace_hash; -/// -/// let hash = adm_trace_hash("
example creative
"); -/// assert_eq!(hash.len(), 16); -/// ``` -#[must_use] -pub fn adm_trace_hash(adm: &str) -> String { - use sha2::{Digest as _, Sha256}; - - let digest = Sha256::digest(adm.as_bytes()); - let mut hex = hex::encode(digest); - hex.truncate(ADM_TRACE_HASH_LEN); - hex -} - -impl Bid { - /// Trace hash of this bid's creative markup, when present. - /// - /// See [`adm_trace_hash`] for the hash definition. - #[must_use] - pub fn creative_trace_hash(&self) -> Option { - self.creative.as_deref().map(adm_trace_hash) - } -} - /// `OpenRTB` response metadata for the orchestrator. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct OrchestratorExt { @@ -451,42 +408,6 @@ mod tests { } } - #[test] - fn adm_trace_hash_is_sha256_prefix() { - // SHA-256("abc") = ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad - assert_eq!( - adm_trace_hash("abc"), - "ba7816bf8f01cfea", - "should be the first 16 hex chars of the SHA-256 digest" - ); - } - - #[test] - fn adm_trace_hash_distinguishes_creatives() { - assert_ne!( - adm_trace_hash("
creative a
"), - adm_trace_hash("
creative b
"), - "should produce different hashes for different markup" - ); - } - - #[test] - fn creative_trace_hash_follows_creative_presence() { - let mut bid = make_bid("kargo"); - assert_eq!( - bid.creative_trace_hash(), - None, - "should be None without creative markup" - ); - - bid.creative = Some("
example creative
".to_owned()); - assert_eq!( - bid.creative_trace_hash(), - Some(adm_trace_hash("
example creative
")), - "should hash the creative markup when present" - ); - } - #[test] fn provider_summary_from_successful_response() { let response = AuctionResponse::success( diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index e50717fb9..1d17bd0a2 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -6844,49 +6844,6 @@ set = { networkId = 42 } ); } - #[test] - fn parse_bid_extracts_crid() { - let bid_json = serde_json::json!({ - "id": "bid-id-321", - "impid": "atf_sidebar_ad", - "price": 1.25, - "adm": "
ad
", - "crid": "cr-98765", - "w": 300, - "h": 250 - }); - let provider = PrebidAuctionProvider::new(base_config()); - let bid = provider - .parse_bid(&bid_json, "kargo") - .expect("should parse bid"); - assert_eq!( - bid.creative_id.as_deref(), - Some("cr-98765"), - "should extract the OpenRTB creative ID" - ); - assert_eq!( - bid.bid_id.as_deref(), - Some("bid-id-321"), - "should extract the OpenRTB bid ID" - ); - } - - #[test] - fn parse_bid_sets_crid_to_none_when_absent() { - let bid_json = serde_json::json!({ - "id": "bid-id-322", - "impid": "atf_sidebar_ad", - "price": 1.25, - "w": 300, - "h": 250 - }); - let provider = PrebidAuctionProvider::new(base_config()); - let bid = provider - .parse_bid(&bid_json, "kargo") - .expect("should parse bid"); - assert!(bid.creative_id.is_none(), "should be None when crid absent"); - } - #[test] fn parse_bid_sets_cache_fields_to_none_when_no_cache_entry() { let bid_json = serde_json::json!({ diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index ba35a8818..699dcbc97 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -1550,7 +1550,6 @@ pub async fn stream_publisher_body_async( ) .await; } - return stream_publisher_body(body, output, params, settings, integration_registry); } @@ -1709,9 +1708,6 @@ fn request_origin(scheme: &str, host: &str) -> String { } /// Write winning bids from an auction result into the shared `ad_bids_state` lock. -/// -/// `auction_id` propagates into each bid entry as `hb_auction_id` so the -/// injected `tsjs.bids` payload can be traced back to the server-side auction. pub(crate) fn write_bids_to_state( winning_bids: &std::collections::HashMap, price_granularity: PriceGranularity, @@ -1719,7 +1715,6 @@ pub(crate) fn write_bids_to_state( settings: &Settings, request_origin: &str, include_debug_bid: bool, - auction_id: Option<&str>, ) { log::debug!( "write_bids_to_state: {} winning bid(s): [{}]", @@ -1732,7 +1727,6 @@ pub(crate) fn write_bids_to_state( settings, request_origin, include_debug_bid, - auction_id, ); let bids_script = build_bids_script(&bid_map); *ad_bids_state.lock().expect("should lock bid state") = Some(bids_script); @@ -2374,10 +2368,6 @@ async fn collect_non_html_auction( settings, &request_origin(¶ms.request_scheme, ¶ms.request_host), settings.debug.inject_adm_for_testing, - telemetry - .auction_request - .as_ref() - .map(|request| request.id.as_str()), ); } @@ -2426,7 +2416,6 @@ async fn collect_stream_auction( settings, request_origin, settings.debug.inject_adm_for_testing, - telemetry.auction_request.as_ref().map(|r| r.id.as_str()), ); if settings.debug.auction_html_comment { @@ -3170,19 +3159,12 @@ fn html_escape_for_script(s: &str) -> String { /// /// Returns a JSON object map of slot ID → bid metadata including the bucketed /// CPM (`hb_pb`), bidder (`hb_bidder`), and optional ad ID, nurl, and burl. -/// -/// Every entry also carries the trace fields `hb_auction_id` (when -/// `auction_id` is known), `hb_crid` (when the bidder returned a creative ID), -/// and `hb_adm_hash` (when the bid has creative markup) so the client can -/// stamp rendered creatives with a tuple that joins back to the server-side -/// `auction winner:` log lines. pub(crate) fn build_bid_map( winning_bids: &std::collections::HashMap, granularity: crate::price_bucket::PriceGranularity, settings: &Settings, request_origin: &str, include_debug_bid: bool, - auction_id: Option<&str>, ) -> serde_json::Map { // Inline creatives render in a foreign origin (PUC's srcdoc under GAM), so // their proxy/click URLs must be absolute against the origin the visitor is @@ -3205,24 +3187,6 @@ pub(crate) fn build_bid_map( "hb_bidder".to_string(), serde_json::Value::String(bid.bidder.clone()), ); - if let Some(auction_id) = auction_id { - obj.insert( - "hb_auction_id".to_string(), - serde_json::Value::String(auction_id.to_string()), - ); - } - if let Some(creative_id) = &bid.creative_id { - obj.insert( - "hb_crid".to_string(), - serde_json::Value::String(creative_id.clone()), - ); - } - if let Some(adm_hash) = bid.creative_trace_hash() { - obj.insert( - "hb_adm_hash".to_string(), - serde_json::Value::String(adm_hash), - ); - } // Winning creative dimensions — the bridge sizes the inline // render from these, falling back to the first configured slot // format only when absent, which mis-sizes a multi-size slot. @@ -3653,8 +3617,8 @@ pub async fn handle_page_bids( // skip the live auction, matching the existing bot/prefetch behaviour. let ad_stack_enabled = auction_enabled && consent_allows_auction; - let (winning_bids, page_auction_id) = if matched_slots.is_empty() { - (std::collections::HashMap::new(), None) + let winning_bids = if matched_slots.is_empty() { + std::collections::HashMap::new() } else { // Same publisher identity as the outbound bid request — see the // matching note on the initial-navigation observation above. @@ -3720,7 +3684,7 @@ pub async fn handle_page_bids( ) }) .await; - (winning_bids, Some(auction_request.id.clone())) + winning_bids } Err(e) => { log::warn!("page-bids auction failed: {e:?}"); @@ -3737,7 +3701,7 @@ pub async fn handle_page_bids( ) }) .await; - (std::collections::HashMap::new(), None) + std::collections::HashMap::new() } } } else { @@ -3763,7 +3727,7 @@ pub async fn handle_page_bids( ) }) .await; - (std::collections::HashMap::new(), None) + std::collections::HashMap::new() } }; @@ -3773,7 +3737,6 @@ pub async fn handle_page_bids( settings, &page_bids_request_origin, settings.debug.inject_adm_for_testing, - page_auction_id.as_deref(), ); // Gate slots on the ad-stack kill switch / consent: when disabled, return no @@ -7476,7 +7439,6 @@ mod tests { &test_settings(), "", false, - None, ); let entry = map.get("atf_sidebar_ad").expect("should have bid entry"); let obj = entry.as_object().expect("should be object"); @@ -7532,7 +7494,6 @@ mod tests { &test_settings(), "", false, - None, ); let obj = map["atf_sidebar_ad"] .as_object() @@ -7575,7 +7536,6 @@ mod tests { &test_settings(), "", false, - None, ); let obj = map .get("atf_sidebar_ad") @@ -7609,7 +7569,6 @@ mod tests { &test_settings(), "", false, - None, ); let obj = map .get("atf_sidebar_ad") @@ -7651,7 +7610,6 @@ mod tests { &test_settings(), "", false, - None, ); let obj = map .get("atf_sidebar_ad") @@ -7698,7 +7656,6 @@ mod tests { &test_settings(), "", false, - None, ); let adm = map .get("atf_sidebar_ad") @@ -7749,7 +7706,6 @@ mod tests { &test_settings(), "", false, - None, ); let obj = map .get("atf_sidebar_ad") @@ -7786,14 +7742,7 @@ mod tests { ); winning_bids.insert("atf_sidebar_ad".to_string(), bid); - let map = build_bid_map( - &winning_bids, - PriceGranularity::Dense, - &settings, - "", - false, - None, - ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); let adm = map .get("atf_sidebar_ad") .and_then(|v| v.as_object()) @@ -7848,7 +7797,6 @@ mod tests { &settings, "http://localhost:7676", false, - None, ); let adm = map .get("atf_sidebar_ad") @@ -7894,14 +7842,7 @@ mod tests { ); winning_bids.insert("atf_sidebar_ad".to_string(), bid); - let map = build_bid_map( - &winning_bids, - PriceGranularity::Dense, - &settings, - "", - false, - None, - ); + let map = build_bid_map(&winning_bids, PriceGranularity::Dense, &settings, "", false); let adm = map .get("atf_sidebar_ad") .and_then(|v| v.as_object()) @@ -7942,7 +7883,6 @@ mod tests { &test_settings(), "", false, - None, ); let script = build_bids_script(&map); assert!( @@ -7983,7 +7923,6 @@ mod tests { &test_settings(), "", true, - None, ); let obj = map .get("atf_sidebar_ad") @@ -8054,7 +7993,6 @@ mod tests { &test_settings(), "", false, - None, ); let obj = map .get("atf_sidebar_ad") @@ -8110,7 +8048,6 @@ mod tests { &test_settings(), "", false, - None, ); let obj = map .get("atf_sidebar_ad") @@ -8164,7 +8101,6 @@ mod tests { &test_settings(), "", false, - None, ); let obj = map .get("atf_sidebar_ad") @@ -8209,7 +8145,6 @@ mod tests { &test_settings(), "", false, - None, ); assert!( map.is_empty(), diff --git a/crates/trusted-server-js/lib/src/core/auction.ts b/crates/trusted-server-js/lib/src/core/auction.ts index aeea99992..a02684362 100644 --- a/crates/trusted-server-js/lib/src/core/auction.ts +++ b/crates/trusted-server-js/lib/src/core/auction.ts @@ -60,10 +60,6 @@ export interface AuctionBid { creativeId: string; /** Advertiser domains. */ adomain: string[]; - /** Server-side auction ID (response top-level `id` / `ext.ts.auction_id`). */ - auctionId?: string; - /** Trace hash of the delivered adm (`ext.ts.adm_hash`, 16 hex chars of SHA-256). */ - admHash?: string; } // --------------------------------------------------------------------------- @@ -132,7 +128,6 @@ export function parseAuctionResponse(body: any): AuctionBid[] { const bids: AuctionBid[] = []; const seatbids = body?.seatbid; if (!Array.isArray(seatbids)) return bids; - const responseAuctionId = typeof body?.id === 'string' && body.id !== '' ? body.id : undefined; for (const seatbid of seatbids) { const seat: string = typeof seatbid?.seat === 'string' ? seatbid.seat : 'unknown'; @@ -140,7 +135,6 @@ export function parseAuctionResponse(body: any): AuctionBid[] { if (!Array.isArray(seatBids)) continue; for (const bid of seatBids) { - const trace = bid?.ext?.ts; const impid = typeof bid?.impid === 'string' ? bid.impid : ''; const renderer = parseApsRendererDescriptor(bid?.ext?.trusted_server?.renderer); const width = typeof bid?.w === 'number' ? bid.w : (renderer?.width ?? 300); @@ -160,12 +154,6 @@ export function parseAuctionResponse(body: any): AuctionBid[] { height, seat, creativeId, - auctionId: - typeof trace?.auction_id === 'string' && trace.auction_id !== '' - ? trace.auction_id - : responseAuctionId, - admHash: - typeof trace?.adm_hash === 'string' && trace.adm_hash !== '' ? trace.adm_hash : undefined, adomain: Array.isArray(bid?.adomain) ? bid.adomain.filter((domain: unknown): domain is string => typeof domain === 'string') : [], diff --git a/crates/trusted-server-js/lib/src/core/request.ts b/crates/trusted-server-js/lib/src/core/request.ts index 4b7880e73..ae352c58a 100644 --- a/crates/trusted-server-js/lib/src/core/request.ts +++ b/crates/trusted-server-js/lib/src/core/request.ts @@ -2,7 +2,6 @@ import { renderApsCreative } from '../integrations/aps/render'; import { buildAdRequest, sendAuction } from './auction'; -import { recordRender, stampCreativeTrace } from './trace'; import { collectContext } from './context'; import { log } from './log'; import { getAllUnits, firstSize } from './registry'; @@ -22,8 +21,6 @@ type RenderCreativeInlineOptions = { creativeHeight?: number; seat: string; creativeId: string; - auctionId?: string; - admHash?: string; }; // Entry point matching Prebid's requestBids signature; uses unified /auction endpoint. @@ -69,8 +66,6 @@ export function requestAds( creativeHeight: bid.height, seat: bid.seat, creativeId: bid.creativeId, - auctionId: bid.auctionId, - admHash: bid.admHash, }); } log.info('requestAds: rendered creatives from response'); @@ -98,22 +93,10 @@ function renderCreativeInline({ creativeHeight, seat, creativeId, - auctionId, - admHash, }: RenderCreativeInlineOptions): void { - const trace = { - slotId, - path: 'auction' as const, - auctionId, - bidder: seat, - creativeId, - admHash, - servedFrom: 'inline' as const, - }; const container = findSlot(slotId) as HTMLElement | null; if (!container) { log.warn('renderCreativeInline: slot not found; skipping render', { slotId, seat, creativeId }); - recordRender({ ...trace, rendered: false }); return; } @@ -127,14 +110,6 @@ function renderCreativeInline({ originalLength: sanitization.originalLength, rejectionReason: sanitization.rejectionReason, }); - // Stamp rendered:false so the DOM marker semantics match the SSAT path - // (explicit false on a failed render, not just an absent attribute). - const rejectedRecord = recordRender({ - ...trace, - rendered: false, - elementId: container.id || undefined, - }); - stampCreativeTrace(container, rejectedRecord); return; } @@ -166,22 +141,10 @@ function renderCreativeInline({ iframe.srcdoc = buildCreativeDocument(sanitization.sanitizedHtml); - // Trace: registry entry + DOM markers joining this creative back to the - // server-side auction (matches the `auction delivered creative:` log line). - const record = recordRender({ - ...trace, - rendered: true, - elementId: container.id || undefined, - }); - stampCreativeTrace(container, record); - stampCreativeTrace(iframe, record); - log.info('renderCreativeInline: rendered', { slotId, seat, creativeId, - auctionId, - admHash, width, height, originalLength: sanitization.originalLength, diff --git a/crates/trusted-server-js/lib/src/core/trace.ts b/crates/trusted-server-js/lib/src/core/trace.ts deleted file mode 100644 index 94edd9fc2..000000000 --- a/crates/trusted-server-js/lib/src/core/trace.ts +++ /dev/null @@ -1,71 +0,0 @@ -// Render-trace registry and DOM markers: joins a creative rendered on the -// page back to the winning server-side auction bid. Every render writes a -// RenderRecord to window.tsjs.renders (keyed by slot ID), stamps the slot -// element with data-ts-* attributes carrying the same trace tuple, and fires -// a 'tsjs:adRendered' CustomEvent so tests and tooling can await renders. -import { log } from './log'; -import type { RenderRecord, TsjsApi } from './types'; - -/** CustomEvent fired on window after each render-trace record is written. */ -export const RENDER_EVENT_NAME = 'tsjs:adRendered'; - -/** - * Write a render record into `window.tsjs.renders` and fire the render event. - * - * Repeated records for the same slot (SPA navigation, GPT refresh) overwrite - * the previous entry and increment `count`, so the registry always reflects - * the latest render while preserving how many renders the slot has seen. - */ -export function recordRender(record: Omit): RenderRecord { - const full: RenderRecord = { ...record, count: 1, at: Date.now() }; - try { - const ts = (window.tsjs ??= {} as TsjsApi); - const renders = (ts.renders ??= {}); - const prev = renders[record.slotId]; - if (prev) full.count = prev.count + 1; - renders[record.slotId] = full; - } catch (err) { - log.warn('trace: failed to write render record', { slotId: record.slotId, err }); - } - try { - window.dispatchEvent(new CustomEvent(RENDER_EVENT_NAME, { detail: full })); - } catch (err) { - // CustomEvent unavailable — registry entry above is still written. - log.debug('trace: failed to dispatch render event', { slotId: record.slotId, err }); - } - return full; -} - -/** - * Stamp an element with `data-ts-*` attributes carrying the trace tuple, so - * a creative in the DOM can be joined to the server-side `auction winner:` / - * `auction delivered creative:` log lines by inspection alone. - * - * Attributes whose record field is absent are removed, so a re-render of the - * same element (SPA navigation, GPT refresh) never leaves stale values from a - * previous auction next to the new ones. - */ -export function stampCreativeTrace(el: Element, record: RenderRecord): void { - const attrs: Array<[string, string | undefined]> = [ - ['data-ts-slot-id', record.slotId], - ['data-ts-render-path', record.path], - ['data-ts-rendered', String(record.rendered)], - ['data-ts-auction-id', record.auctionId], - ['data-ts-bidder', record.bidder], - ['data-ts-ad-id', record.adId], - ['data-ts-creative-id', record.creativeId], - ['data-ts-adm-hash', record.admHash], - ['data-ts-served-from', record.servedFrom], - ]; - try { - for (const [name, value] of attrs) { - if (value !== undefined && value !== '') { - el.setAttribute(name, value); - } else { - el.removeAttribute(name); - } - } - } catch (err) { - log.warn('trace: failed to stamp element', { slotId: record.slotId, err }); - } -} diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 859095a63..7b81e78a6 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -87,12 +87,6 @@ export interface AuctionBidData { hb_adid?: string; hb_cache_host?: string; hb_cache_path?: string; - /** Server-side auction ID — trace key joining this bid to server logs. */ - hb_auction_id?: string; - /** Upstream creative ID (OpenRTB `crid`), when the bidder returned one. */ - hb_crid?: string; - /** Trace hash of the bid's raw creative markup (16 hex chars of SHA-256). */ - hb_adm_hash?: string; /** Winning creative width; the bridge sizes the inline render from this. */ w?: number; /** Winning creative height; the bridge sizes the inline render from this. */ @@ -119,41 +113,6 @@ export interface AuctionBidData { debug_bid?: AuctionDebugBidData; } -/** How a creative reached the page for a [`RenderRecord`]. */ -export type RenderServedFrom = 'inline' | 'gam' | 'debug-adm' | 'pbs-cache'; - -/** - * One entry in `window.tsjs.renders` — the client-side half of the render - * trace. Field values mirror the server-side `auction winner:` log line so - * the two can be joined on (auctionId, slotId). - */ -export interface RenderRecord { - /** Slot the creative was rendered for. */ - slotId: string; - /** Which render path produced this record. */ - path: 'auction' | 'ssat'; - /** Whether a creative actually rendered (false for empty/rejected). */ - rendered: boolean; - /** Actual DOM element ID the slot resolved to (div_id may be a prefix). */ - elementId?: string; - /** Server-side auction ID. */ - auctionId?: string; - /** Winning bidder / seat. */ - bidder?: string; - /** hb_adid (PBS cache UUID or OpenRTB adid). */ - adId?: string; - /** Upstream creative ID (OpenRTB crid). */ - creativeId?: string; - /** Trace hash of the creative markup (16 hex chars of SHA-256). */ - admHash?: string; - /** Mechanism that delivered the creative. */ - servedFrom?: RenderServedFrom; - /** How many renders this slot has seen (SPA navigations, refreshes). */ - count: number; - /** Epoch ms when the record was written. */ - at: number; -} - export interface TsjsApi { version: string; que: Array<() => void>; @@ -188,8 +147,6 @@ export interface TsjsApi { apsPrebidRenderers?: Record; /** Initialises GPT slots with server-side bid targeting and calls refresh(). */ adInit?: () => void; - /** Render-trace registry: latest render per slot (see [`RenderRecord`]). */ - renders?: Record; /** GPT slot objects TS defined — used to destroy stale slots on SPA navigation. */ prevGptSlots?: unknown[]; /** Guards one-time-per-page enableSingleRequest/enableServices calls. */ diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index b7aa9b8ac..8c0dcacba 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -1,6 +1,5 @@ import { log } from '../../core/log'; -import { recordRender, stampCreativeTrace } from '../../core/trace'; -import type { AuctionSlot, AuctionBidData, RenderServedFrom, TsjsApi } from '../../core/types'; +import type { AuctionSlot, AuctionBidData, TsjsApi } from '../../core/types'; import { APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, @@ -369,7 +368,6 @@ function injectAdmIntoSlot(divId: string, adm: string): void { f.width = String(slotEl.offsetWidth || 728); f.height = String(slotEl.offsetHeight || 90); f.setAttribute('sandbox', ADM_IFRAME_SANDBOX); - f.setAttribute('data-ts-injected-adm', 'true'); f.srcdoc = adm; slotEl.appendChild(f); log.debug(`[tsjs-gpt] gam-intercept: replaced slot content for '${divId}'`); @@ -638,20 +636,6 @@ export function installTsAdInit(): void { if (!slotId) return; // Read ts.bids live (not the snapshot above) so post-navigation bid data is used. const bid = (ts.bids ?? {})[slotId] ?? {}; - const record = recordRender({ - slotId, - path: 'ssat', - rendered: !event.isEmpty, - elementId: divId, - auctionId: bid.hb_auction_id, - bidder: bid.hb_bidder, - adId: bid.hb_adid, - creativeId: bid.hb_crid, - admHash: bid.hb_adm_hash, - servedFrom: 'gam', - }); - const slotElement = document.getElementById(divId); - if (slotElement) stampCreativeTrace(slotElement, record); // GAM interceptor (testing bypass): directly replace the GAM creative. // `adm` is now always injected in production, so it can no longer gate @@ -974,39 +958,6 @@ export function parseCachedBid(body: string): CachedBid | undefined { * Lives in gpt/index.ts (not prebid/index.ts) to avoid pulling the full * Prebid bundle into tsjs-gpt.js via inlineDynamicImports. */ -/** - * Trace a creative served by the pbRender bridge: registry entry + DOM markers - * on the slot element. `servedFrom` distinguishes debug adm injection from a - * PBS Cache fetch so verification tooling knows which mechanism delivered the - * markup into the GAM iframe. - * - * `el` must be resolved by the caller at message-receipt time: the PBS Cache - * path stamps only after an async fetch, and re-resolving from live - * `tsjs.adSlots`/DOM at that point could stamp a *new* route's slot with the - * previous page's trace data after an SPA navigation. The connectivity check - * below drops the stamp when the captured element has left the document. - */ -function recordBridgeRender( - slotId: string, - bid: AuctionBidData, - servedFrom: RenderServedFrom, - el: HTMLElement | null -): void { - const record = recordRender({ - slotId, - path: 'ssat', - rendered: true, - elementId: el?.id, - auctionId: bid.hb_auction_id, - bidder: bid.hb_bidder, - adId: bid.hb_adid, - creativeId: bid.hb_crid, - admHash: bid.hb_adm_hash, - servedFrom, - }); - if (el && el.isConnected) stampCreativeTrace(el, record); -} - export function installTsRenderBridge(): void { if (typeof window === 'undefined') return; @@ -1139,10 +1090,6 @@ export function installTsRenderBridge(): void { const [fallbackWidth, fallbackHeight] = slot?.formats?.[0] ?? [728, 90]; const width = matchedBid.w ?? fallbackWidth; const height = matchedBid.h ?? fallbackHeight; - // Resolve the slot element now, at message-receipt time: the PBS Cache - // branch stamps after an async fetch, and by then an SPA navigation may - // have swapped tsjs.adSlots/DOM for a new route with the same slot IDs. - const slotEl = slot ? findSlotElementByDivId(slot.div_id) : null; if (matchedBid.renderer !== undefined) { const renderer = validateApsRenderer(matchedBid.renderer); @@ -1240,7 +1187,6 @@ export function installTsRenderBridge(): void { }) ); fireWinBillingBeacons(slotId, matchedBid); - recordBridgeRender(slotId, matchedBid, 'pbs-cache', slotEl); log.debug(`[tsjs-gpt] pbRender bridge served '${slotId}' from PBS Cache`); }) .catch((err) => { diff --git a/crates/trusted-server-js/lib/test/core/auction.test.ts b/crates/trusted-server-js/lib/test/core/auction.test.ts index 276f819a5..7e9bb2947 100644 --- a/crates/trusted-server-js/lib/test/core/auction.test.ts +++ b/crates/trusted-server-js/lib/test/core/auction.test.ts @@ -312,40 +312,6 @@ describe('auction/parseAuctionResponse', () => { expect(bids[0].height).toBe(250); expect(bids[0].adomain).toEqual([]); }); - - it('retains the auction id from the response top-level id', () => { - const body = { - id: 'auction-uuid-1', - seatbid: [{ seat: 'kargo', bid: [{ impid: 'slot-1', price: 1.0, adm: '
A
' }] }], - }; - - const bids = parseAuctionResponse(body); - expect(bids[0].auctionId).toBe('auction-uuid-1'); - expect(bids[0].admHash).toBeUndefined(); - }); - - it('prefers bid-level ext.ts trace fields over the top-level id', () => { - const body = { - id: 'auction-uuid-1', - seatbid: [ - { - seat: 'kargo', - bid: [ - { - impid: 'slot-1', - price: 1.0, - adm: '
A
', - ext: { ts: { auction_id: 'auction-uuid-2', adm_hash: 'a1b2c3d4e5f60718' } }, - }, - ], - }, - ], - }; - - const bids = parseAuctionResponse(body); - expect(bids[0].auctionId).toBe('auction-uuid-2'); - expect(bids[0].admHash).toBe('a1b2c3d4e5f60718'); - }); }); describe('auction/sendAuction', () => { diff --git a/crates/trusted-server-js/lib/test/core/request.test.ts b/crates/trusted-server-js/lib/test/core/request.test.ts index 62db24003..8dffd825b 100644 --- a/crates/trusted-server-js/lib/test/core/request.test.ts +++ b/crates/trusted-server-js/lib/test/core/request.test.ts @@ -398,113 +398,6 @@ describe('request.requestAds', () => { ); }); - it('stamps trace markers and records the render in window.tsjs.renders', async () => { - const creativeHtml = '
Traced Creative
'; - (globalThis as any).fetch = vi.fn().mockResolvedValue({ - ok: true, - status: 200, - headers: { get: () => 'application/json' }, - json: async () => ({ - id: 'auction-trace-1', - seatbid: [ - { - seat: 'kargo', - bid: [ - { - impid: 'slot1', - adm: creativeHtml, - crid: 'cr-777', - ext: { ts: { auction_id: 'auction-trace-1', adm_hash: 'a1b2c3d4e5f60718' } }, - }, - ], - }, - ], - }), - }); - - const { addAdUnits } = await import('../../src/core/registry'); - const { requestAds } = await import('../../src/core/request'); - const { RENDER_EVENT_NAME } = await import('../../src/core/trace'); - const eventListener = vi.fn(); - window.addEventListener(RENDER_EVENT_NAME, eventListener); - - document.body.innerHTML = '
'; - addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); - - requestAds(); - await flushRequestAds(); - - const container = document.querySelector('#slot1') as HTMLElement; - expect(container.getAttribute('data-ts-slot-id')).toBe('slot1'); - expect(container.getAttribute('data-ts-render-path')).toBe('auction'); - expect(container.getAttribute('data-ts-rendered')).toBe('true'); - expect(container.getAttribute('data-ts-auction-id')).toBe('auction-trace-1'); - expect(container.getAttribute('data-ts-bidder')).toBe('kargo'); - expect(container.getAttribute('data-ts-creative-id')).toBe('cr-777'); - expect(container.getAttribute('data-ts-adm-hash')).toBe('a1b2c3d4e5f60718'); - - const iframe = container.querySelector('iframe') as HTMLIFrameElement; - expect(iframe.getAttribute('data-ts-slot-id')).toBe('slot1'); - expect(iframe.getAttribute('data-ts-adm-hash')).toBe('a1b2c3d4e5f60718'); - - const record = (window as any).tsjs?.renders?.['slot1']; - expect(record).toEqual( - expect.objectContaining({ - slotId: 'slot1', - path: 'auction', - rendered: true, - elementId: 'slot1', - auctionId: 'auction-trace-1', - bidder: 'kargo', - creativeId: 'cr-777', - admHash: 'a1b2c3d4e5f60718', - servedFrom: 'inline', - }) - ); - expect(eventListener).toHaveBeenCalledTimes(1); - - window.removeEventListener(RENDER_EVENT_NAME, eventListener); - }); - - it('records a rendered:false trace entry when the creative is rejected', async () => { - (globalThis as any).fetch = vi.fn().mockResolvedValue({ - ok: true, - status: 200, - headers: { get: () => 'application/json' }, - json: async () => ({ - id: 'auction-trace-2', - seatbid: [ - { - seat: 'appnexus', - bid: [{ impid: 'slot1', adm: ' ', crid: 'creative-empty' }], - }, - ], - }), - }); - - const { addAdUnits } = await import('../../src/core/registry'); - const { requestAds } = await import('../../src/core/request'); - - document.body.innerHTML = '
'; - addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); - - requestAds(); - await flushRequestAds(); - - const record = (window as any).tsjs?.renders?.['slot1']; - expect(record).toEqual( - expect.objectContaining({ - slotId: 'slot1', - path: 'auction', - rendered: false, - auctionId: 'auction-trace-2', - }) - ); - // Rejected creative must stamp an explicit rendered:false marker, - // matching the SSAT path's empty-render semantics. - expect(document.querySelector('#slot1')?.getAttribute('data-ts-rendered')).toBe('false'); - }); - it('skips iframe insertion when slot is missing', async () => { // mock fetch for unified auction endpoint - returns inline HTML (globalThis as any).fetch = vi.fn().mockResolvedValue({ diff --git a/crates/trusted-server-js/lib/test/core/trace.test.ts b/crates/trusted-server-js/lib/test/core/trace.test.ts deleted file mode 100644 index ba7a640e2..000000000 --- a/crates/trusted-server-js/lib/test/core/trace.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest'; -import { recordRender, stampCreativeTrace, RENDER_EVENT_NAME } from '../../src/core/trace'; -import type { RenderRecord, TsjsApi } from '../../src/core/types'; - -describe('trace/recordRender', () => { - beforeEach(() => { - delete (window as { tsjs?: TsjsApi }).tsjs; - }); - - it('writes a render record into window.tsjs.renders', () => { - const record = recordRender({ - slotId: 'slot-1', - path: 'auction', - rendered: true, - elementId: 'slot-1', - auctionId: 'auction-abc', - bidder: 'kargo', - creativeId: 'cr-1', - admHash: 'a1b2c3d4e5f60718', - servedFrom: 'inline', - }); - - expect(window.tsjs?.renders?.['slot-1']).toEqual(record); - expect(record.count).toBe(1); - expect(record.at).toBeGreaterThan(0); - }); - - it('overwrites the previous record and increments count on re-render', () => { - recordRender({ slotId: 'slot-1', path: 'ssat', rendered: true, auctionId: 'a-1' }); - const second = recordRender({ - slotId: 'slot-1', - path: 'ssat', - rendered: true, - auctionId: 'a-2', - }); - - const entry = window.tsjs?.renders?.['slot-1']; - expect(entry?.auctionId).toBe('a-2'); - expect(entry?.count).toBe(2); - expect(second.count).toBe(2); - }); - - it('fires a tsjs:adRendered CustomEvent with the record as detail', () => { - const listener = vi.fn(); - window.addEventListener(RENDER_EVENT_NAME, listener); - - const record = recordRender({ slotId: 'slot-ev', path: 'auction', rendered: true }); - - expect(listener).toHaveBeenCalledTimes(1); - const event = listener.mock.calls[0][0] as CustomEvent; - expect(event.detail).toEqual(record); - - window.removeEventListener(RENDER_EVENT_NAME, listener); - }); -}); - -describe('trace/stampCreativeTrace', () => { - it('stamps data-ts-* attributes for present fields only', () => { - const el = document.createElement('div'); - const record: RenderRecord = { - slotId: 'slot-1', - path: 'ssat', - rendered: true, - auctionId: 'ts-req-abc', - bidder: 'kargo', - adId: 'cache-uuid-1', - admHash: 'a1b2c3d4e5f60718', - count: 1, - at: 1, - }; - - stampCreativeTrace(el, record); - - expect(el.getAttribute('data-ts-slot-id')).toBe('slot-1'); - expect(el.getAttribute('data-ts-render-path')).toBe('ssat'); - expect(el.getAttribute('data-ts-rendered')).toBe('true'); - expect(el.getAttribute('data-ts-auction-id')).toBe('ts-req-abc'); - expect(el.getAttribute('data-ts-bidder')).toBe('kargo'); - expect(el.getAttribute('data-ts-ad-id')).toBe('cache-uuid-1'); - expect(el.getAttribute('data-ts-adm-hash')).toBe('a1b2c3d4e5f60718'); - // creativeId absent — attribute must not exist. - expect(el.hasAttribute('data-ts-creative-id')).toBe(false); - }); - - it('removes stale attributes when a re-render lacks a field', () => { - const el = document.createElement('div'); - const first: RenderRecord = { - slotId: 'slot-1', - path: 'ssat', - rendered: true, - auctionId: 'auction-old', - admHash: 'a1b2c3d4e5f60718', - servedFrom: 'gam', - count: 1, - at: 1, - }; - stampCreativeTrace(el, first); - - const second: RenderRecord = { - slotId: 'slot-1', - path: 'ssat', - rendered: true, - auctionId: 'auction-new', - count: 2, - at: 2, - }; - stampCreativeTrace(el, second); - - expect(el.getAttribute('data-ts-auction-id')).toBe('auction-new'); - // The previous auction's hash and mechanism must not survive the re-stamp. - expect(el.hasAttribute('data-ts-adm-hash')).toBe(false); - expect(el.hasAttribute('data-ts-served-from')).toBe(false); - }); -}); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 3d7dfa978..c790d37a0 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -635,92 +635,6 @@ describe('installTsAdInit', () => { beaconSpy.mockRestore(); }); - it('stamps trace markers and records the render on slotRenderEnded', async () => { - let capturedListener: ((e: SlotRenderEvent) => void) | undefined; - - const mockSlot = { - addService: vi.fn().mockReturnThis(), - setTargeting: vi.fn().mockReturnThis(), - getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), - getTargeting: vi.fn().mockReturnValue([]), - }; - const mockPubads = { - enableSingleRequest: vi.fn(), - getSlots: vi.fn().mockReturnValue([mockSlot]), - refresh: vi.fn(), - addEventListener: vi.fn((event: string, fn: (e: SlotRenderEvent) => void) => { - if (event === 'slotRenderEnded') capturedListener = fn; - }), - }; - (window as TestWindow).googletag = { - cmd: { push: vi.fn((fn: () => void) => fn()) }, - defineSlot: vi.fn().mockReturnValue(mockSlot), - pubads: vi.fn().mockReturnValue(mockPubads), - enableServices: vi.fn(), - }; - (window as TestWindow).tsjs = { - adSlots: [ - { - id: 'atf_sidebar_ad', - gam_unit_path: '/123/atf', - div_id: 'div-atf-sidebar', - formats: [[300, 250]], - targeting: {}, - }, - ], - bids: { - atf_sidebar_ad: { - hb_pb: '1.00', - hb_bidder: 'kargo', - hb_adid: 'cache-uuid-9', - hb_auction_id: 'ts-req-trace9', - hb_crid: 'cr-98765', - hb_adm_hash: 'a1b2c3d4e5f60718', - }, - }, - }; - - const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); - installTsAdInit(); - (window as TestWindow).tsjs!.adInit!(); - - expect(capturedListener).toBeDefined(); - capturedListener!({ isEmpty: false, slot: mockSlot }); - - const el = document.getElementById('div-atf-sidebar')!; - expect(el.getAttribute('data-ts-slot-id')).toBe('atf_sidebar_ad'); - expect(el.getAttribute('data-ts-render-path')).toBe('ssat'); - expect(el.getAttribute('data-ts-rendered')).toBe('true'); - expect(el.getAttribute('data-ts-auction-id')).toBe('ts-req-trace9'); - expect(el.getAttribute('data-ts-bidder')).toBe('kargo'); - expect(el.getAttribute('data-ts-ad-id')).toBe('cache-uuid-9'); - expect(el.getAttribute('data-ts-creative-id')).toBe('cr-98765'); - expect(el.getAttribute('data-ts-adm-hash')).toBe('a1b2c3d4e5f60718'); - - const record = (window as TestWindow).tsjs!.renders?.['atf_sidebar_ad']; - expect(record).toEqual( - expect.objectContaining({ - slotId: 'atf_sidebar_ad', - path: 'ssat', - rendered: true, - elementId: 'div-atf-sidebar', - auctionId: 'ts-req-trace9', - bidder: 'kargo', - adId: 'cache-uuid-9', - creativeId: 'cr-98765', - admHash: 'a1b2c3d4e5f60718', - servedFrom: 'gam', - }) - ); - - // An empty render must record rendered:false and bump the count. - capturedListener!({ isEmpty: true, slot: mockSlot }); - const second = (window as TestWindow).tsjs!.renders?.['atf_sidebar_ad']; - expect(second?.rendered).toBe(false); - expect(second?.count).toBe(2); - expect(el.getAttribute('data-ts-rendered')).toBe('false'); - }); - it('does not fire beacons for an APS-style bid that carries no hb_adid', async () => { const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); let capturedListener: ((e: SlotRenderEvent) => void) | undefined; From 3b02916a04cb67d24a976223aca4956544c36e2c Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 23 Jul 2026 12:33:47 -0500 Subject: [PATCH 104/198] Track effective GPT initial-load configuration --- .../src/integrations/gpt.rs | 4 + .../src/integrations/gpt_bootstrap.js | 37 +++- .../lib/src/integrations/gpt/index.ts | 41 +++-- .../lib/test/integrations/gpt/ad_init.test.ts | 158 +++++++++++++++++- 4 files changed, 215 insertions(+), 25 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 539d01610..0bd72bb6b 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -1265,6 +1265,10 @@ mod tests { combined.contains("gpt.setConfig"), "bootstrap should wrap googletag.setConfig() to detect the disabled state" ); + assert!( + combined.contains("gpt.getConfig"), + "bootstrap should read GPT's modern initial-load configuration" + ); assert!( combined.contains("pubads.disableInitialLoad"), "bootstrap should wrap legacy disableInitialLoad() calls" diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 9abf556c8..1ea331793 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -19,25 +19,42 @@ var ts = (window.tsjs = window.tsjs || {}); if (ts.adInit) return; - // Track whether the publisher disabled GPT initial load. GPT exposes no - // getter for this, so wrap both googletag.setConfig() and the legacy - // pubads().disableInitialLoad() method to record it. With initial load + // Track whether the publisher disabled GPT initial load. Read the modern + // googletag.getConfig() value when available, and wrap googletag.setConfig() + // and the legacy pubads().disableInitialLoad() method as fallbacks because + // getConfig() may not report the legacy API's state. With initial load // disabled, display() only registers a slot and the ad request must come from // a later refresh(); adInit() reads this to refresh its own freshly defined // slots so they are not left blank. Pushed onto the command queue so it runs // before the publisher's own GPT configuration. + function syncInitialLoadDisabled(gpt) { + if (typeof gpt.getConfig !== "function") return false; + var config = gpt.getConfig("disableInitialLoad"); + if (!config || typeof config.disableInitialLoad === "undefined") { + return false; + } + ts.gptInitialLoadDisabled = config.disableInitialLoad === true; + return true; + } + (window.googletag = window.googletag || { cmd: [] }).cmd.push(function () { var gpt = window.googletag; + syncInitialLoadDisabled(gpt); if ( typeof gpt.setConfig === "function" && !gpt.__tsInitialLoadConfigHooked ) { var originalSetConfig = gpt.setConfig.bind(gpt); gpt.setConfig = function (config) { - if (config && config.disableInitialLoad === true) { - ts.gptInitialLoadDisabled = true; + var result = originalSetConfig.apply(gpt, arguments); + if ( + !syncInitialLoadDisabled(gpt) && + config && + "disableInitialLoad" in config + ) { + ts.gptInitialLoadDisabled = config.disableInitialLoad === true; } - return originalSetConfig(config); + return result; }; gpt.__tsInitialLoadConfigHooked = true; } @@ -52,8 +69,11 @@ } var originalDisableInitialLoad = pubads.disableInitialLoad.bind(pubads); pubads.disableInitialLoad = function () { - ts.gptInitialLoadDisabled = true; - return originalDisableInitialLoad(); + var result = originalDisableInitialLoad.apply(pubads, arguments); + if (!syncInitialLoadDisabled(gpt)) { + ts.gptInitialLoadDisabled = true; + } + return result; }; pubads.__tsInitialLoadHooked = true; }); @@ -166,6 +186,7 @@ // unless the publisher disabled initial load, in which case display() only // registers them and refresh() must request the ad — otherwise they render // blank. Only add them in that case to avoid double-requesting. + syncInitialLoadDisabled(window.googletag); var slotsNeedingRefresh = ts.gptInitialLoadDisabled ? slotsToRefresh.concat(newSlots) : slotsToRefresh; diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 8d5263b9b..c82e16f68 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -110,7 +110,7 @@ interface GoogleTagPubAdsService { } interface GoogleTagConfig extends Record { - disableInitialLoad?: boolean; + disableInitialLoad?: boolean | null; } interface GoogleTag { @@ -124,7 +124,8 @@ interface GoogleTag { destroySlots(slots?: GoogleTagSlot[]): boolean; enableServices(): void; display(elementId: string): void; - setConfig(config: GoogleTagConfig): void; + setConfig?(config: GoogleTagConfig): void; + getConfig?(key: 'disableInitialLoad'): GoogleTagConfig | undefined; _loaded_?: boolean; } @@ -414,9 +415,9 @@ function queueWinBillingBeacon(url: string): boolean { /** * Track whether the publisher disabled GPT initial load. * - * GPT exposes no getter for the initial-load-disabled flag, so wrap both the - * modern `googletag.setConfig({ disableInitialLoad: true })` API and the legacy - * `pubads().disableInitialLoad()` method to record it on `window.tsjs`. With + * GPT's modern `getConfig()` getter may not report state set through the legacy + * `pubads().disableInitialLoad()` API, so read it when available and wrap both + * configuration APIs to record the state on `window.tsjs`. With * initial load disabled, `display()` only registers a slot — the ad request * must come from a later `refresh()`. adInit() reads this to refresh its own * freshly defined slots so they are not left blank. @@ -431,6 +432,16 @@ function queueWinBillingBeacon(url: string): boolean { * `installTsAdInit` runs, so the detector is still queued ahead of the * publisher's GPT setup. */ +function syncInitialLoadDisabled(gpt: Partial, ts: TsjsApi): boolean { + if (typeof gpt.getConfig !== 'function') return false; + + const config = gpt.getConfig('disableInitialLoad'); + if (!config || config.disableInitialLoad === undefined) return false; + + ts.gptInitialLoadDisabled = config.disableInitialLoad === true; + return true; +} + function installInitialLoadDetector(ts: TsjsApi): void { const win = window as GptWindow; const cmd = win.googletag?.cmd; @@ -441,13 +452,17 @@ function installInitialLoadDetector(ts: TsjsApi): void { | undefined; if (!gpt) return; + syncInitialLoadDisabled(gpt, ts); + if (typeof gpt.setConfig === 'function' && !gpt.__tsInitialLoadConfigHooked) { const originalSetConfig = gpt.setConfig.bind(gpt); - gpt.setConfig = function (config: GoogleTagConfig) { - if (config?.disableInitialLoad === true) { - ts.gptInitialLoadDisabled = true; + gpt.setConfig = function (...args: Parameters) { + const config = args[0]; + const result = originalSetConfig(...args); + if (!syncInitialLoadDisabled(gpt, ts) && config && 'disableInitialLoad' in config) { + ts.gptInitialLoadDisabled = config.disableInitialLoad === true; } - return originalSetConfig(config); + return result; }; gpt.__tsInitialLoadConfigHooked = true; } @@ -460,8 +475,11 @@ function installInitialLoadDetector(ts: TsjsApi): void { } const originalDisableInitialLoad = service.disableInitialLoad.bind(service); service.disableInitialLoad = function () { - ts.gptInitialLoadDisabled = true; - return originalDisableInitialLoad(); + const result = originalDisableInitialLoad(); + if (!syncInitialLoadDisabled(gpt, ts)) { + ts.gptInitialLoadDisabled = true; + } + return result; }; service.__tsInitialLoadHooked = true; }); @@ -640,6 +658,7 @@ export function installTsAdInit(): void { // first-impression slot renders blank on initial-load-disabled pages. Only // add them in that case; otherwise display() + refresh() would // double-request the impression. + syncInitialLoadDisabled(g, ts); const slotsNeedingRefresh = ts.gptInitialLoadDisabled ? slotsToRefresh.concat(newSlots) : slotsToRefresh; diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 8f943e08c..a21d82297 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -1,3 +1,6 @@ +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + import { describe, it, expect, vi, beforeEach, afterEach, afterAll } from 'vitest'; // Track every 'message' EventListener added to window across the entire test @@ -205,6 +208,7 @@ describe('installTsAdInit', () => { refresh: vi.fn(), disableInitialLoad: vi.fn(), }; + const getConfigMock = vi.fn().mockReturnValue(undefined); const displayMock = vi.fn(); (window as TestWindow).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, @@ -212,6 +216,8 @@ describe('installTsAdInit', () => { display: displayMock, pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), + // GPT's modern getter does not report legacy disableInitialLoad() state. + getConfig: getConfigMock, }; (window as TestWindow).tsjs = { adSlots: [ @@ -243,7 +249,65 @@ describe('installTsAdInit', () => { expect(mockPubads.refresh).toHaveBeenCalledWith([mockSlot]); }); - it('refreshes TS-defined slots when setConfig disables GPT initial load', async () => { + it('keeps the legacy disabled state in the edge bootstrap when getConfig is unavailable', async () => { + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + }; + const disableInitialLoadMock = vi.fn(); + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([]), + refresh: vi.fn(), + disableInitialLoad: disableInitialLoadMock, + }; + const displayMock = vi.fn(); + const getConfigMock = vi.fn().mockReturnValue(undefined); + const googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + display: displayMock, + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + getConfig: getConfigMock, + }; + (window as TestWindow).googletag = googletag; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + const bootstrap = await readFile( + path.resolve(process.cwd(), '../../trusted-server-core/src/integrations/gpt_bootstrap.js'), + 'utf8' + ); + const runBootstrap = new Function('window', 'googletag', bootstrap) as ( + window: Window, + googletag: object + ) => void; + runBootstrap(window, googletag); + + mockPubads.disableInitialLoad(); + expect(disableInitialLoadMock).toHaveBeenCalledOnce(); + expect((window as TestWindow).tsjs!.gptInitialLoadDisabled).toBe(true); + + (window as TestWindow).tsjs!.adInit!(); + + expect(displayMock).toHaveBeenCalledWith('div-atf-sidebar'); + expect(mockPubads.refresh).toHaveBeenCalledWith([mockSlot]); + }); + + it('tracks the effective initial-load state from setConfig', async () => { // Modern GPT configuration uses googletag.setConfig() rather than the // legacy pubads().disableInitialLoad() method. TS must detect both forms. const mockSlot = { @@ -260,13 +324,24 @@ describe('installTsAdInit', () => { refresh: vi.fn(), }; const displayMock = vi.fn(); - const setConfigMock = vi.fn(); + type InitialLoadConfig = { + disableInitialLoad?: boolean | null; + singleRequest?: boolean; + }; + let effectiveConfig: InitialLoadConfig = {}; + const setConfigMock = vi.fn((config: InitialLoadConfig) => { + if ('disableInitialLoad' in config) { + effectiveConfig = { disableInitialLoad: config.disableInitialLoad }; + } + }); + const getConfigMock = vi.fn(() => effectiveConfig); (window as TestWindow).googletag = { cmd: { push: vi.fn((fn: () => void) => fn()) }, defineSlot: vi.fn().mockReturnValue(mockSlot), display: displayMock, pubads: vi.fn().mockReturnValue(mockPubads), enableServices: vi.fn(), + getConfig: getConfigMock, setConfig: setConfigMock, }; (window as TestWindow).tsjs = { @@ -285,12 +360,83 @@ describe('installTsAdInit', () => { const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); installTsAdInit(); + installTsAdInit(); + + const gpt = (window as TestWindow).googletag as { + setConfig(config: InitialLoadConfig): void; + }; + gpt.setConfig({ singleRequest: true }); + expect(setConfigMock).toHaveBeenCalledOnce(); + expect((window as TestWindow).tsjs!.gptInitialLoadDisabled).not.toBe(true); + + (window as TestWindow).tsjs!.adInit!(); + + expect(displayMock).toHaveBeenCalledWith('div-atf-sidebar'); + expect(mockPubads.refresh).not.toHaveBeenCalled(); const config = { disableInitialLoad: true, singleRequest: true }; - ((window as TestWindow).googletag as { setConfig(value: typeof config): void }).setConfig( - config - ); - expect(setConfigMock).toHaveBeenCalledWith(config); + gpt.setConfig(config); + expect(setConfigMock).toHaveBeenCalledTimes(2); + expect(setConfigMock).toHaveBeenLastCalledWith(config); + expect(getConfigMock).toHaveBeenCalledWith('disableInitialLoad'); + expect((window as TestWindow).tsjs!.gptInitialLoadDisabled).toBe(true); + + (window as TestWindow).tsjs!.adInit!(); + + expect(mockPubads.refresh).toHaveBeenCalledWith([mockSlot]); + + mockPubads.refresh.mockClear(); + gpt.setConfig({ disableInitialLoad: false }); + expect((window as TestWindow).tsjs!.gptInitialLoadDisabled).toBe(false); + gpt.setConfig({ disableInitialLoad: null }); + expect((window as TestWindow).tsjs!.gptInitialLoadDisabled).toBe(false); + + (window as TestWindow).tsjs!.adInit!(); + + expect(mockPubads.refresh).not.toHaveBeenCalled(); + }); + + it('reads initial-load configuration effective before detector installation', async () => { + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([]), + addEventListener: vi.fn(), + refresh: vi.fn(), + }; + const displayMock = vi.fn(); + const getConfigMock = vi.fn().mockReturnValue({ disableInitialLoad: true }); + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + display: displayMock, + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + getConfig: getConfigMock, + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + + expect(getConfigMock).toHaveBeenCalledWith('disableInitialLoad'); expect((window as TestWindow).tsjs!.gptInitialLoadDisabled).toBe(true); (window as TestWindow).tsjs!.adInit!(); From b9ce68b345e36abaa6be50f021c369637edb1ff5 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 23 Jul 2026 12:35:26 -0500 Subject: [PATCH 105/198] Document GPT initial-load getter fallback --- crates/trusted-server-js/lib/src/core/types.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index e791355e6..193f6912f 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -116,7 +116,8 @@ export interface TsjsApi { /** * True once the publisher has disabled GPT initial load through * `googletag.setConfig()` or `googletag.pubads().disableInitialLoad()`. - * GPT exposes no getter for this state, so TS tracks both configuration APIs. + * GPT's getter may not report state set through the legacy API, so TS tracks + * both configuration APIs. * When set, `display()` only registers a slot and the ad request must come * from a `refresh()`; adInit() uses this to refresh its own freshly defined * slots so they are not left blank. From 050ad9caa3b265ae279aa2e4d702e2dcba133113 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 23 Jul 2026 11:56:34 -0500 Subject: [PATCH 106/198] Add privacy-safe auction-to-creative tracing Introduce query-activated diagnostic sessions, trace identities and telemetry, bounded GPT and Prebid render ownership, creative acknowledgements, and the browser timeline overlay. Add deterministic integration fixtures and coverage across edge adapters and browser render paths. --- .../setup-integration-test-env/action.yml | 29 + .github/workflows/integration-tests.yml | 23 +- crates/trusted-server-adapter-axum/src/app.rs | 3 +- .../src/middleware.rs | 46 + .../src/app.rs | 3 +- .../src/middleware.rs | 45 + .../trusted-server-adapter-fastly/src/app.rs | 3 +- .../trusted-server-adapter-fastly/src/main.rs | 2 + .../src/middleware.rs | 44 + .../src/tinybird.rs | 1 + crates/trusted-server-adapter-spin/src/app.rs | 9 +- .../src/middleware.rs | 38 +- .../benches/html_processor_bench.rs | 1 + .../src/auction/endpoints.rs | 42 +- .../src/auction/formats.rs | 244 +++-- .../src/auction/orchestrator.rs | 519 ++++++--- .../src/auction/telemetry.rs | 222 ++-- .../src/auction/test_support.rs | 10 +- .../trusted-server-core/src/auction/types.rs | 147 +++ crates/trusted-server-core/src/config.rs | 11 +- crates/trusted-server-core/src/constants.rs | 4 + .../trusted-server-core/src/html_processor.rs | 28 +- .../src/integrations/ad_trace.rs | 620 +++++++++++ .../src/integrations/aps.rs | 3 + .../src/integrations/gpt_bootstrap.js | 54 + .../src/integrations/mod.rs | 5 + .../src/integrations/prebid.rs | 23 +- crates/trusted-server-core/src/openrtb.rs | 33 +- crates/trusted-server-core/src/publisher.rs | 476 +++++++-- .../src/response_privacy.rs | 12 +- .../browser/global-setup.ts | 13 +- .../browser/helpers/infra.ts | 1 + .../browser/helpers/state.ts | 2 +- .../browser/package.json | 1 + .../browser/playwright.config.ts | 8 +- .../tests/ad-trace/auction-trace.spec.ts | 439 ++++++++ .../tests/shared/ad-trace-gate.spec.ts | 20 + .../trusted-server.ad-trace.integration.toml | 67 ++ .../fixtures/frameworks/ad-trace/Dockerfile | 15 + .../frameworks/ad-trace/public/index.php | 201 ++++ .../frameworks/ad-trace/public/router.php | 50 + .../tests/parity.rs | 71 ++ .../lib/src/core/ad_trace.ts | 505 +++++++++ .../trusted-server-js/lib/src/core/auction.ts | 154 ++- .../lib/src/core/global.d.ts | 2 + .../trusted-server-js/lib/src/core/request.ts | 216 +++- .../trusted-server-js/lib/src/core/types.ts | 202 +++- .../lib/src/integrations/ad_trace/index.ts | 98 ++ .../lib/src/integrations/ad_trace/overlay.ts | 231 ++++ .../lib/src/integrations/gpt/index.ts | 985 +++++++++++++++++- .../lib/src/integrations/prebid/index.ts | 175 +++- .../lib/test/core/ad_trace.test.ts | 332 ++++++ .../lib/test/core/auction.test.ts | 153 ++- .../lib/test/core/request.test.ts | 142 ++- .../test/integrations/ad_trace/index.test.ts | 41 + .../integrations/ad_trace/overlay.test.ts | 110 ++ .../lib/test/integrations/gpt/ad_init.test.ts | 287 ++++- .../test/integrations/gpt/ad_trace.test.ts | 327 ++++++ .../lib/test/integrations/gpt/index.test.ts | 31 +- .../test/integrations/prebid/index.test.ts | 80 +- docs/guide/configuration.md | 17 + .../generate-integration-viceroy-configs.sh | 7 + scripts/integration-tests-browser.sh | 30 +- .../datasources/auction_events_raw.datasource | 1 + tinybird/fixtures/auction_events_raw.ndjson | 2 +- trusted-server.example.toml | 5 + 66 files changed, 7104 insertions(+), 617 deletions(-) create mode 100644 crates/trusted-server-core/src/integrations/ad_trace.rs create mode 100644 crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts create mode 100644 crates/trusted-server-integration-tests/browser/tests/shared/ad-trace-gate.spec.ts create mode 100644 crates/trusted-server-integration-tests/fixtures/configs/trusted-server.ad-trace.integration.toml create mode 100644 crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/Dockerfile create mode 100644 crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/index.php create mode 100644 crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/router.php create mode 100644 crates/trusted-server-js/lib/src/core/ad_trace.ts create mode 100644 crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts create mode 100644 crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts create mode 100644 crates/trusted-server-js/lib/test/core/ad_trace.test.ts create mode 100644 crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts create mode 100644 crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts create mode 100644 crates/trusted-server-js/lib/test/integrations/gpt/ad_trace.test.ts diff --git a/.github/actions/setup-integration-test-env/action.yml b/.github/actions/setup-integration-test-env/action.yml index 841d8d5bd..12c41502a 100644 --- a/.github/actions/setup-integration-test-env/action.yml +++ b/.github/actions/setup-integration-test-env/action.yml @@ -93,6 +93,26 @@ runs: TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK: "false" run: cargo build -p trusted-server-adapter-axum + - name: Set up Node.js for browser fixtures + if: ${{ inputs.build-test-images == 'true' }} + uses: actions/setup-node@v4 + with: + node-version: ${{ steps.node-version.outputs.node-version }} + cache: npm + cache-dependency-path: crates/trusted-server-js/lib/package-lock.json + + - name: Build external Prebid fixture bundle + if: ${{ inputs.build-test-images == 'true' }} + shell: bash + run: | + rm -rf "$GITHUB_WORKSPACE/target/integration-test-artifacts/prebid" + mkdir -p "$GITHUB_WORKSPACE/target/integration-test-artifacts/prebid" + npm ci --prefix crates/trusted-server-js/lib + npm run --prefix crates/trusted-server-js/lib build:prebid-external -- \ + --adapters=rubicon \ + --user-id-modules=sharedIdSystem \ + --out "$GITHUB_WORKSPACE/target/integration-test-artifacts/prebid" + - name: Build WordPress test container if: ${{ inputs.build-test-images == 'true' }} shell: bash @@ -109,6 +129,15 @@ runs: -t test-nextjs:latest \ crates/trusted-server-integration-tests/fixtures/frameworks/nextjs/ + - name: Build ad-trace test container + if: ${{ inputs.build-test-images == 'true' }} + shell: bash + run: | + docker build \ + -f crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/Dockerfile \ + -t test-ad-trace:latest \ + . + - name: Add wasm32-unknown-unknown target for Cloudflare build if: ${{ inputs.build-cloudflare == 'true' }} shell: bash diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 4973afe44..36c4a8f6d 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -46,7 +46,7 @@ jobs: cp -r crates/trusted-server-adapter-cloudflare/build/. "$CF_BUILD_ARTIFACT_PATH/" docker save \ --output "$DOCKER_ARTIFACT_PATH" \ - test-wordpress:latest test-nextjs:latest + test-ad-trace:latest test-wordpress:latest test-nextjs:latest - name: Upload integration test artifacts uses: actions/upload-artifact@v4 @@ -237,10 +237,29 @@ jobs: path: crates/trusted-server-integration-tests/browser/playwright-report-wordpress/ retention-days: 7 + - name: Run browser tests (ad trace contract) + if: always() + working-directory: crates/trusted-server-integration-tests/browser + env: + WASM_BINARY_PATH: ${{ env.WASM_ARTIFACT_PATH }} + INTEGRATION_ORIGIN_PORT: ${{ env.ORIGIN_PORT }} + VICEROY_CONFIG_PATH: ${{ env.ARTIFACTS_DIR }}/configs/viceroy-ad-trace.toml + TEST_FRAMEWORK: ad-trace + PLAYWRIGHT_HTML_REPORT: playwright-report-ad-trace + run: npx playwright test tests/ad-trace/auction-trace.spec.ts + + - name: Upload Playwright report (ad trace contract) + uses: actions/upload-artifact@v4 + if: always() + with: + name: playwright-report-ad-trace + path: crates/trusted-server-integration-tests/browser/playwright-report-ad-trace/ + retention-days: 7 + - name: Upload Playwright traces and screenshots uses: actions/upload-artifact@v4 if: failure() with: name: playwright-traces - path: crates/trusted-server-integration-tests/browser/test-results/ + path: crates/trusted-server-integration-tests/browser/test-results-*/ retention-days: 7 diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 2f4329574..fe18e3604 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -32,7 +32,7 @@ use trusted_server_core::settings_data::{ use trusted_server_core::platform::RuntimeServices; -use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; +use crate::middleware::{AdTracePrepareMiddleware, AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{AxumPlatformConfigStore, build_runtime_services}; // --------------------------------------------------------------------------- @@ -540,6 +540,7 @@ fn build_router(state: &Arc) -> RouterService { let mut router = RouterService::builder() .middleware(FinalizeResponseMiddleware::new(Arc::clone(&state.settings))) + .middleware(AdTracePrepareMiddleware::new(Arc::clone(&state.settings))) .middleware(AuthMiddleware::new(Arc::clone(&state.settings))); router = router.route("/health", Method::GET, |_ctx: RequestContext| async { diff --git a/crates/trusted-server-adapter-axum/src/middleware.rs b/crates/trusted-server-adapter-axum/src/middleware.rs index 45cbedc2c..f3ea0d198 100644 --- a/crates/trusted-server-adapter-axum/src/middleware.rs +++ b/crates/trusted-server-adapter-axum/src/middleware.rs @@ -5,6 +5,7 @@ use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; +use edgezero_core::response::IntoResponse; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; use trusted_server_core::settings::Settings; @@ -38,6 +39,51 @@ impl Middleware for FinalizeResponseMiddleware { async fn handle(&self, ctx: RequestContext, next: Next<'_>) -> Result { let mut response = next.run(ctx).await?; apply_finalize_headers(&self.settings, &mut response); + trusted_server_core::integrations::ad_trace::finalize_response(&mut response); + Ok(response) + } +} + +// --------------------------------------------------------------------------- +// AdTracePrepareMiddleware +// --------------------------------------------------------------------------- + +/// Prepares and sanitizes the request before auth, routing, or downstream use. +pub struct AdTracePrepareMiddleware { + settings: Arc, +} + +impl AdTracePrepareMiddleware { + #[must_use] + pub fn new(settings: Arc) -> Self { + Self { settings } + } +} + +#[async_trait(?Send)] +impl Middleware for AdTracePrepareMiddleware { + async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { + let decision = match trusted_server_core::integrations::ad_trace::prepare_request( + &self.settings, + ctx.request_mut(), + ) { + Ok(decision) => decision, + Err(report) => { + log::error!("ad trace request preparation failed: {report:?}"); + return Ok(crate::app::http_error(&report)); + } + }; + let mut response = match next.run(ctx).await { + Ok(response) => response, + Err(error) => { + log::error!("request handler failed after ad trace preparation: {error:?}"); + error.into_response()? + } + }; + trusted_server_core::integrations::ad_trace::attach_response_decision( + &decision, + &mut response, + ); Ok(response) } } diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index c931360f6..798e2a2e0 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -29,7 +29,7 @@ use trusted_server_core::request_signing::{ }; use trusted_server_core::settings::Settings; -use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; +use crate::middleware::{AdTracePrepareMiddleware, AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::build_runtime_services; // --------------------------------------------------------------------------- @@ -432,6 +432,7 @@ fn build_router(state: &Arc) -> RouterService { let mut router = RouterService::builder() .middleware(FinalizeResponseMiddleware::new(Arc::clone(&state.settings))) + .middleware(AdTracePrepareMiddleware::new(Arc::clone(&state.settings))) .middleware(AuthMiddleware::new(Arc::clone(&state.settings))) .get( "/.well-known/trusted-server.json", diff --git a/crates/trusted-server-adapter-cloudflare/src/middleware.rs b/crates/trusted-server-adapter-cloudflare/src/middleware.rs index 5b605bcff..de15ac62b 100644 --- a/crates/trusted-server-adapter-cloudflare/src/middleware.rs +++ b/crates/trusted-server-adapter-cloudflare/src/middleware.rs @@ -5,6 +5,7 @@ use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; +use edgezero_core::response::IntoResponse; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; use trusted_server_core::settings::Settings; @@ -46,6 +47,50 @@ impl Middleware for FinalizeResponseMiddleware { let mut response = next.run(ctx).await?; apply_finalize_headers(&self.settings, geo_available, &mut response); + trusted_server_core::integrations::ad_trace::finalize_response(&mut response); + Ok(response) + } +} + +// --------------------------------------------------------------------------- +// AdTracePrepareMiddleware +// --------------------------------------------------------------------------- + +pub struct AdTracePrepareMiddleware { + settings: Arc, +} + +impl AdTracePrepareMiddleware { + #[must_use] + pub fn new(settings: Arc) -> Self { + Self { settings } + } +} + +#[async_trait(?Send)] +impl Middleware for AdTracePrepareMiddleware { + async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { + let decision = match trusted_server_core::integrations::ad_trace::prepare_request( + &self.settings, + ctx.request_mut(), + ) { + Ok(decision) => decision, + Err(report) => { + log::error!("ad trace request preparation failed: {report:?}"); + return Ok(crate::app::http_error(&report)); + } + }; + let mut response = match next.run(ctx).await { + Ok(response) => response, + Err(error) => { + log::error!("request handler failed after ad trace preparation: {error:?}"); + error.into_response()? + } + }; + trusted_server_core::integrations::ad_trace::attach_response_decision( + &decision, + &mut response, + ); Ok(response) } } diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index ae4ca421f..1700c3969 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -129,7 +129,7 @@ use trusted_server_core::settings_data::{ }; use trusted_server_core::tester_cookie::{handle_clear_tester, handle_set_tester}; -use crate::middleware::{AuthMiddleware, FinalizeResponseMiddleware}; +use crate::middleware::{AdTracePrepareMiddleware, AuthMiddleware, FinalizeResponseMiddleware}; use crate::platform::{ FastlyPlatformBackend, FastlyPlatformConfigStore, FastlyPlatformGeo, FastlyPlatformHttpClient, FastlyPlatformSecretStore, UnavailableKvStore, open_kv_store, @@ -1159,6 +1159,7 @@ impl TrustedServerApp { Arc::clone(&state.settings), Arc::new(FastlyPlatformGeo), )) + .middleware(AdTracePrepareMiddleware::new(Arc::clone(&state.settings))) .middleware(AuthMiddleware::new(Arc::clone(&state.settings))); let fallback_handler = fallback_route_handler(Arc::clone(state)); diff --git a/crates/trusted-server-adapter-fastly/src/main.rs b/crates/trusted-server-adapter-fastly/src/main.rs index 8fec21435..e6cda086f 100644 --- a/crates/trusted-server-adapter-fastly/src/main.rs +++ b/crates/trusted-server-adapter-fastly/src/main.rs @@ -336,6 +336,8 @@ fn send_edgezero_response( // added a per-user Set-Cookie after `apply_finalize_headers` ran, so // re-apply the privacy downgrade before send. crate::middleware::enforce_set_cookie_cache_privacy(&mut response); + // Reassert console no-store after asset/EC/filter response mutations. + trusted_server_core::integrations::ad_trace::finalize_response(&mut response); let (parts, body) = response.into_parts(); diff --git a/crates/trusted-server-adapter-fastly/src/middleware.rs b/crates/trusted-server-adapter-fastly/src/middleware.rs index 2c00ac2ff..f6a674301 100644 --- a/crates/trusted-server-adapter-fastly/src/middleware.rs +++ b/crates/trusted-server-adapter-fastly/src/middleware.rs @@ -85,6 +85,7 @@ impl Middleware for FinalizeResponseMiddleware { }); apply_finalize_headers(&self.settings, geo_info.as_ref(), &mut response); + trusted_server_core::integrations::ad_trace::finalize_response(&mut response); response .headers_mut() .insert(HEADER_X_TS_FINALIZED, HeaderValue::from_static("1")); @@ -93,6 +94,49 @@ impl Middleware for FinalizeResponseMiddleware { } } +// --------------------------------------------------------------------------- +// AdTracePrepareMiddleware +// --------------------------------------------------------------------------- + +/// Sanitizes and snapshots the console decision before auth and route dispatch. +pub struct AdTracePrepareMiddleware { + settings: Arc, +} + +impl AdTracePrepareMiddleware { + pub fn new(settings: Arc) -> Self { + Self { settings } + } +} + +#[async_trait(?Send)] +impl Middleware for AdTracePrepareMiddleware { + async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { + let decision = match trusted_server_core::integrations::ad_trace::prepare_request( + &self.settings, + ctx.request_mut(), + ) { + Ok(decision) => decision, + Err(report) => { + log::error!("ad trace request preparation failed: {report:?}"); + return Ok(crate::app::http_error(&report)); + } + }; + let mut response = match next.run(ctx).await { + Ok(response) => response, + Err(error) => { + log::error!("request handler failed after ad trace preparation: {error:?}"); + error.into_response()? + } + }; + trusted_server_core::integrations::ad_trace::attach_response_decision( + &decision, + &mut response, + ); + Ok(response) + } +} + // --------------------------------------------------------------------------- // AuthMiddleware // --------------------------------------------------------------------------- diff --git a/crates/trusted-server-adapter-fastly/src/tinybird.rs b/crates/trusted-server-adapter-fastly/src/tinybird.rs index f2df61744..8ca1bdc16 100644 --- a/crates/trusted-server-adapter-fastly/src/tinybird.rs +++ b/crates/trusted-server-adapter-fastly/src/tinybird.rs @@ -419,6 +419,7 @@ mod tests { price_cpm: None, currency: None, is_win: None, + bid_trace_id: None, ad_domain: None, ad_id: None, } diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 2291fce74..dbfb5c4fd 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -705,13 +705,10 @@ fn build_router(state: &Arc) -> RouterService { let mut builder = RouterService::builder() .middleware(FinalizeResponseMiddleware::new(Arc::clone(&state.settings))) + // Normalize and sanitize outside auth so even auth short-circuits + // cannot forward reserved console inputs or skip response actions. + .middleware(NormalizeMiddleware::new(Arc::clone(&state.settings))) .middleware(AuthMiddleware::new(Arc::clone(&state.settings))) - // Innermost middleware: normalize every routed request (strip - // spoofable forwarded headers, derive the trusted Host/scheme/client-IP - // from Spin's synthetic runtime headers) so no handler can opt out of - // the de-spoofing invariant. Runs after auth so the basic-auth gate - // continues to see the original request, matching prior behaviour. - .middleware(NormalizeMiddleware::new()) // Cheap liveness probe, matching the Fastly/Axum adapters. Registered // explicitly so it is not absorbed by the publisher `/{*rest}` fallback. .get("/health", |_ctx: RequestContext| async { diff --git a/crates/trusted-server-adapter-spin/src/middleware.rs b/crates/trusted-server-adapter-spin/src/middleware.rs index 1bcede1fc..1f9178057 100644 --- a/crates/trusted-server-adapter-spin/src/middleware.rs +++ b/crates/trusted-server-adapter-spin/src/middleware.rs @@ -5,6 +5,7 @@ use edgezero_core::context::RequestContext; use edgezero_core::error::EdgeError; use edgezero_core::http::{HeaderValue, Response}; use edgezero_core::middleware::{Middleware, Next}; +use edgezero_core::response::IntoResponse; use trusted_server_core::auth::enforce_basic_auth; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; use trusted_server_core::settings::Settings; @@ -39,6 +40,7 @@ impl Middleware for FinalizeResponseMiddleware { let mut response = next.run(ctx).await?; apply_finalize_headers(&self.settings, geo_available, &mut response); + trusted_server_core::integrations::ad_trace::finalize_response(&mut response); Ok(response) } } @@ -95,16 +97,17 @@ impl Middleware for AuthMiddleware { /// signing handler that begins deriving an issuer/audience from `RequestInfo`, /// cannot silently trust spoofable input by forgetting to opt in. /// -/// Registered after [`AuthMiddleware`] (innermost) so the basic-auth gate still -/// evaluates the original request, preserving prior behaviour. -#[derive(Default)] -pub struct NormalizeMiddleware; +/// Registered outside [`AuthMiddleware`] so de-spoofing and console sanitation +/// also apply when auth short-circuits the request. +pub struct NormalizeMiddleware { + settings: Arc, +} impl NormalizeMiddleware { /// Creates a new [`NormalizeMiddleware`]. #[must_use] - pub fn new() -> Self { - Self + pub fn new(settings: Arc) -> Self { + Self { settings } } } @@ -112,7 +115,28 @@ impl NormalizeMiddleware { impl Middleware for NormalizeMiddleware { async fn handle(&self, mut ctx: RequestContext, next: Next<'_>) -> Result { crate::app::normalize_spin_request(ctx.request_mut()); - next.run(ctx).await + let decision = match trusted_server_core::integrations::ad_trace::prepare_request( + &self.settings, + ctx.request_mut(), + ) { + Ok(decision) => decision, + Err(report) => { + log::error!("ad trace request preparation failed: {report:?}"); + return Ok(crate::app::http_error(&report)); + } + }; + let mut response = match next.run(ctx).await { + Ok(response) => response, + Err(error) => { + log::error!("request handler failed after ad trace preparation: {error:?}"); + error.into_response()? + } + }; + trusted_server_core::integrations::ad_trace::attach_response_decision( + &decision, + &mut response, + ); + Ok(response) } } diff --git a/crates/trusted-server-core/benches/html_processor_bench.rs b/crates/trusted-server-core/benches/html_processor_bench.rs index 6c7a397b0..24b034ec9 100644 --- a/crates/trusted-server-core/benches/html_processor_bench.rs +++ b/crates/trusted-server-core/benches/html_processor_bench.rs @@ -9,6 +9,7 @@ fn make_config() -> HtmlProcessorConfig { request_host: "proxy.bench.example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::default(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index e5796323f..74a2b7ab1 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -1,7 +1,5 @@ //! HTTP endpoint handlers for auction requests. -use std::collections::HashMap; - use edgezero_core::body::Body as EdgeBody; use error_stack::{Report, ResultExt}; use http::{Request, Response, StatusCode, header}; @@ -24,12 +22,12 @@ use crate::platform::RuntimeServices; use crate::settings::Settings; use super::AuctionOrchestrator; -use super::formats::{convert_to_openrtb_response, convert_tsjs_to_auction_request}; +use super::formats::{convert_to_openrtb_response_with_trace, convert_tsjs_to_auction_request}; use super::telemetry::{ AuctionObservationContext, AuctionSource, AuctionTerminalOutcome, build_auction_events, emit_auction_events_best_effort_lazy, }; -use super::types::AuctionContext; +use super::types::{AuctionContext, AuctionPublicOutcome, AuctionTraceContext}; const MAX_CLIENT_EID_SOURCES: usize = 64; const MAX_CLIENT_UIDS_PER_SOURCE: usize = 32; @@ -160,6 +158,8 @@ pub async fn handle_auction( ); let http_req = Request::from_parts(parts, EdgeBody::empty()); + let trace_enabled = crate::integrations::ad_trace::browser_trace_enabled(&http_req); + let trace = AuctionTraceContext::new(AuctionSource::AuctionApi); // Story 5 middleware contract: auction is a read-only EC route. // It must not generate EC IDs; it only consumes pre-routed context. @@ -193,11 +193,8 @@ pub async fn handle_auction( ec_id, None, )?; - let observation = AuctionObservationContext::from_auction_request( - AuctionSource::AuctionApi, - &auction_request, - ec_context, - ); + let observation = + AuctionObservationContext::from_auction_request(&trace, &auction_request, ec_context); emit_auction_events_best_effort_lazy(services, || { build_auction_events( observation, @@ -209,18 +206,13 @@ pub async fn handle_auction( }) .await; - let empty_result = OrchestrationResult { - provider_responses: Vec::new(), - mediator_response: None, - winning_bids: HashMap::new(), - total_time_ms: 0, - metadata: HashMap::new(), - }; - return convert_to_openrtb_response( + let empty_result = OrchestrationResult::empty(trace, AuctionPublicOutcome::Skipped); + return convert_to_openrtb_response_with_trace( &empty_result, settings, &auction_request, ec_context.ec_allowed(), + trace_enabled, ); } @@ -282,6 +274,7 @@ pub async fn handle_auction( // Create auction context let context = AuctionContext { + trace: &trace, settings, request: &http_req, timeout_ms: settings.auction.timeout_ms, @@ -289,11 +282,8 @@ pub async fn handle_auction( services, }; - let observation = AuctionObservationContext::from_auction_request( - AuctionSource::AuctionApi, - &auction_request, - ec_context, - ); + let observation = + AuctionObservationContext::from_auction_request(&trace, &auction_request, ec_context); // Run the auction let result = match orchestrator.run_auction(&auction_request, &context).await { @@ -337,7 +327,13 @@ pub async fn handle_auction( ); // Convert to OpenRTB response format with inline creative HTML - convert_to_openrtb_response(&result, settings, &auction_request, ec_context.ec_allowed()) + convert_to_openrtb_response_with_trace( + &result, + settings, + &auction_request, + ec_context.ec_allowed(), + trace_enabled, + ) } /// Resolves partner EIDs from the KV identity graph for bidstream decoration. diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index 284db6242..62fbef1f9 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -21,8 +21,8 @@ use crate::ec::eids::encode_eids_header; use crate::error::TrustedServerError; use crate::geo::GeoInfo; use crate::openrtb::{ - BidExt, BidTrustedServerExt, OpenRtbBid, OpenRtbResponse, ResponseExt, SeatBid, ToExt, - to_openrtb_i32, + AuctionTraceWire, BidExt, BidTraceWire, BidTrustedServerExt, OpenRtbBid, OpenRtbResponse, + ResponseExt, SeatBid, ToExt, TrustedServerResponseExt, to_openrtb_i32, }; use crate::platform::RuntimeServices; use crate::settings::Settings; @@ -263,6 +263,21 @@ pub fn convert_to_openrtb_response( settings: &Settings, auction_request: &AuctionRequest, ec_allowed: bool, +) -> Result, Report> { + convert_to_openrtb_response_with_trace(result, settings, auction_request, ec_allowed, false) +} + +/// Convert an auction result with optional tester-gated trace extensions. +/// +/// # Errors +/// +/// Returns the same errors as [`convert_to_openrtb_response`]. +pub fn convert_to_openrtb_response_with_trace( + result: &OrchestrationResult, + settings: &Settings, + auction_request: &AuctionRequest, + ec_allowed: bool, + trace_enabled: bool, ) -> Result, Report> { // Build OpenRTB-style seatbid array let mut seatbids = Vec::with_capacity(result.winning_bids.len()); @@ -286,7 +301,7 @@ pub fn convert_to_openrtb_response( // Ordinary markup remains on the mandatory sanitize/rewrite path. A // typed renderer is serialized separately and never enters the HTML sanitizer. - let (adm, ext) = if let Some(ref raw_creative) = bid.creative { + let (adm, renderer) = if let Some(ref raw_creative) = bid.creative { let sanitize_creatives = settings.auction.sanitize_creatives; let sanitized = if sanitize_creatives { creative::sanitize_creative_html(raw_creative) @@ -325,13 +340,7 @@ pub fn convert_to_openrtb_response( (Some(processed), None) } else if let Some(renderer) = bid.renderer.as_ref() { - ( - None, - BidExt { - trusted_server: BidTrustedServerExt { renderer }, - } - .to_ext(), - ) + (None, Some(renderer)) } else { return Err(Report::new(TrustedServerError::Auction { message: format!( @@ -341,6 +350,28 @@ pub fn convert_to_openrtb_response( })); }; + let bid_trace = trace_enabled + .then(|| result.trace.winning_bids.get(slot_id)) + .flatten() + .map(|trace| BidTraceWire { + version: 1, + bid_trace_id: trace.bid_trace_id.to_string(), + slot_id: slot_id.clone(), + provider: trace.provider.clone(), + bidder: trace.bidder.clone(), + }); + let ext = (renderer.is_some() || bid_trace.is_some()) + .then(|| { + BidExt { + trusted_server: BidTrustedServerExt { + renderer, + trace: bid_trace, + }, + } + .to_ext() + }) + .flatten(); + let openrtb_bid = OpenRtbBid { id: bid .bid_id @@ -390,6 +421,14 @@ pub fn convert_to_openrtb_response( time_ms: result.total_time_ms, provider_details, }, + trusted_server: trace_enabled.then(|| TrustedServerResponseExt { + trace: AuctionTraceWire { + version: 1, + auction_trace_id: result.trace.summary.auction.auction_trace_id.to_string(), + source: result.trace.summary.auction.source.as_str(), + outcome: result.trace.summary.outcome.as_str(), + }, + }), } .to_ext(), ..Default::default() @@ -489,13 +528,14 @@ mod tests { } fn make_empty_result() -> OrchestrationResult { - OrchestrationResult { - provider_responses: Vec::new(), - mediator_response: None, - winning_bids: HashMap::new(), - total_time_ms: 10, - metadata: HashMap::new(), - } + let mut result = OrchestrationResult::empty( + crate::auction::types::AuctionTraceContext::new( + crate::auction::types::AuctionSource::AuctionApi, + ), + crate::auction::types::AuctionPublicOutcome::NoBid, + ); + result.total_time_ms = 10; + result } fn make_bid(slot_id: &str, bidder: &str, price: Option) -> Bid { @@ -531,19 +571,34 @@ mod tests { } fn make_result(bid: Bid) -> OrchestrationResult { - OrchestrationResult { - provider_responses: vec![AuctionResponse { - provider: "prebid".to_string(), - bids: vec![bid.clone()], - status: BidStatus::Success, - response_time_ms: 42, - metadata: HashMap::new(), - }], - mediator_response: None, - winning_bids: HashMap::from([(bid.slot_id.clone(), bid)]), - total_time_ms: 50, + let mut result = make_empty_result(); + result.trace.summary.outcome = crate::auction::types::AuctionPublicOutcome::Completed; + result.trace.winning_bids.insert( + bid.slot_id.clone(), + crate::auction::types::WinningBidTrace { + bid_trace_id: crate::auction::types::BidTraceId::new(), + provider: "prebid".to_owned(), + bidder: bid.bidder.clone(), + }, + ); + result.winning_bid_origins.insert( + bid.slot_id.clone(), + crate::auction::types::WinningBidOrigin { + response_index: 0, + bid_index: 0, + mediated: false, + }, + ); + result.provider_responses = vec![AuctionResponse { + provider: "prebid".to_string(), + bids: vec![bid.clone()], + status: BidStatus::Success, + response_time_ms: 42, metadata: HashMap::new(), - } + }]; + result.winning_bids = HashMap::from([(bid.slot_id.clone(), bid)]); + result.total_time_ms = 50; + result } fn response_json(response: Response) -> JsonValue { @@ -1222,19 +1277,21 @@ mod tests { }] } }); - let result = OrchestrationResult { - provider_responses: vec![AuctionResponse { - provider: "aps".to_string(), - bids: vec![bid.clone()], - status: BidStatus::Success, - response_time_ms: 42, - metadata: HashMap::from([("debug".to_string(), debug.clone())]), - }], - mediator_response: None, - winning_bids: HashMap::from([(bid.slot_id.clone(), bid)]), - total_time_ms: 50, - metadata: HashMap::new(), - }; + let mut result = OrchestrationResult::empty( + crate::auction::types::AuctionTraceContext::new( + crate::auction::types::AuctionSource::AuctionApi, + ), + crate::auction::types::AuctionPublicOutcome::NoBid, + ); + result.provider_responses = vec![AuctionResponse { + provider: "aps".to_string(), + bids: vec![bid.clone()], + status: BidStatus::Success, + response_time_ms: 42, + metadata: HashMap::from([("debug".to_string(), debug.clone())]), + }]; + result.winning_bids = HashMap::from([(bid.slot_id.clone(), bid)]); + result.total_time_ms = 50; let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) .expect("should convert APS response with debug metadata"); @@ -1331,17 +1388,69 @@ mod tests { ); } + #[test] + fn gated_response_adds_namespaced_root_and_winning_bid_trace() { + let settings = make_settings(); + let auction_request = make_auction_request(); + let result = make_result(make_bid("div-gpt-top", "appnexus", Some(2.75))); + + let response = convert_to_openrtb_response_with_trace( + &result, + &settings, + &auction_request, + false, + true, + ) + .expect("should convert traced response"); + let json = response_json(response); + + assert_eq!( + json["ext"]["trusted_server"]["trace"]["auction_trace_id"], + json!(result.trace.summary.auction.auction_trace_id.to_string()), + "should expose the shared trace identity" + ); + assert_eq!( + json["seatbid"][0]["bid"][0]["ext"]["trusted_server"]["trace"]["bid_trace_id"], + json!( + result.trace.winning_bids["div-gpt-top"] + .bid_trace_id + .to_string() + ), + "should expose only the final winner trace" + ); + assert_ne!( + json["ext"]["trusted_server"]["trace"]["auction_trace_id"], + json!(auction_request.id), + "should never expose the internal request ID as trace identity" + ); + } + + #[test] + fn ungated_response_omits_all_trace_extensions() { + let settings = make_settings(); + let auction_request = make_auction_request(); + let result = make_result(make_bid("div-gpt-top", "appnexus", Some(2.75))); + + let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) + .expect("should convert legacy response"); + let json = response_json(response); + + assert!( + json["ext"].get("trusted_server").is_none(), + "ungated root should omit trace" + ); + assert!( + json["seatbid"][0]["bid"][0].get("ext").is_none(), + "ungated bid should omit trace" + ); + } + #[test] fn convert_to_openrtb_response_allows_empty_winning_bids() { let settings = make_settings(); let auction_request = make_auction_request(); - let result = OrchestrationResult { - provider_responses: vec![], - mediator_response: None, - winning_bids: HashMap::new(), - total_time_ms: 50, - metadata: HashMap::new(), - }; + let mut result = make_empty_result(); + result.total_time_ms = 50; let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) .expect("should convert auction result without winning bids"); @@ -1366,22 +1475,27 @@ mod tests { let top_bid = make_bid("div-gpt-top", "appnexus", Some(2.75)); let mut sidebar_bid = make_bid("div-gpt-sidebar", "rubicon", Some(1.25)); sidebar_bid.creative = Some("
Sidebar
".to_string()); - let result = OrchestrationResult { - provider_responses: vec![AuctionResponse { - provider: "prebid".to_string(), - bids: vec![top_bid.clone(), sidebar_bid.clone()], - status: BidStatus::Success, - response_time_ms: 42, - metadata: HashMap::new(), - }], - mediator_response: None, - winning_bids: HashMap::from([ - (top_bid.slot_id.clone(), top_bid), - (sidebar_bid.slot_id.clone(), sidebar_bid), - ]), - total_time_ms: 50, - metadata: HashMap::new(), - }; + let mut result = make_result(top_bid.clone()); + result.provider_responses[0].bids.push(sidebar_bid.clone()); + result.trace.winning_bids.insert( + sidebar_bid.slot_id.clone(), + crate::auction::types::WinningBidTrace { + bid_trace_id: crate::auction::types::BidTraceId::new(), + provider: "prebid".to_owned(), + bidder: sidebar_bid.bidder.clone(), + }, + ); + result.winning_bid_origins.insert( + sidebar_bid.slot_id.clone(), + crate::auction::types::WinningBidOrigin { + response_index: 0, + bid_index: 1, + mediated: false, + }, + ); + result + .winning_bids + .insert(sidebar_bid.slot_id.clone(), sidebar_bid); let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) .expect("should convert multiple winning bids"); diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index eb9b1d138..0dfa13008 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -14,7 +14,11 @@ use crate::platform::{PlatformPendingRequest, RuntimeServices}; use super::config::AuctionConfig; use super::provider::AuctionProvider; use super::telemetry::AbandonedProviderCall; -use super::types::{AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStatus}; +use super::types::{ + AuctionContext, AuctionPublicOutcome, AuctionRequest, AuctionResponse, AuctionResultTrace, + AuctionTraceContext, AuctionTraceSummary, Bid, BidStatus, BidTraceId, WinningBidOrigin, + WinningBidTrace, +}; /// In-flight auction requests dispatched to SSP backends. /// @@ -24,6 +28,7 @@ use super::types::{AuctionContext, AuctionRequest, AuctionResponse, Bid, BidStat /// race in Fastly's native layer, enabling TTFB ≈ origin latency rather than /// TTFB ≈ auction timeout. pub struct DispatchedAuction { + trace: AuctionTraceContext, pending_requests: Vec, backend_to_provider: HashMap, u32)>, launch_responses: Vec, @@ -53,6 +58,12 @@ pub enum DispatchAuctionOutcome { } impl DispatchedAuction { + /// Return the trace context retained for split-phase collection. + #[must_use] + pub fn trace(&self) -> &AuctionTraceContext { + &self.trace + } + /// Consume the dispatch token without collecting provider responses. #[must_use] pub fn abandon( @@ -82,6 +93,7 @@ impl DispatchedAuction { impl DispatchedAuction { pub(crate) fn empty_for_test(request: AuctionRequest, timeout_ms: u32) -> Self { Self { + trace: AuctionTraceContext::new(super::types::AuctionSource::InitialNavigation), pending_requests: Vec::new(), backend_to_provider: HashMap::new(), launch_responses: Vec::new(), @@ -150,6 +162,39 @@ fn provider_transport_failed_response( .with_metadata("message", serde_json::json!("Provider request failed")) } +fn build_winning_bid_traces( + winning_bids: &HashMap, + origins: &HashMap, + provider_responses: &[AuctionResponse], + mediator_response: Option<&AuctionResponse>, + mut id_source: impl FnMut() -> BidTraceId, +) -> HashMap { + let mut traces = HashMap::with_capacity(winning_bids.len()); + for (slot_id, bid) in winning_bids { + let provider = origins + .get(slot_id) + .and_then(|origin| { + if origin.mediated { + mediator_response.map(|response| response.provider.clone()) + } else { + provider_responses + .get(origin.response_index) + .map(|response| response.provider.clone()) + } + }) + .unwrap_or_else(|| "unattributed".to_owned()); + traces.insert( + slot_id.clone(), + WinningBidTrace { + bid_trace_id: id_source(), + provider, + bidder: bid.bidder.clone(), + }, + ); + } + traces +} + fn provider_timeout_response(provider_name: &str, response_time_ms: u64) -> AuctionResponse { AuctionResponse::error(provider_name, response_time_ms) .with_metadata("error_type", serde_json::json!(ERROR_TYPE_TIMEOUT)) @@ -301,120 +346,103 @@ impl AuctionOrchestrator { let provider_responses = self.run_providers_parallel(request, context).await?; let floor_prices = self.floor_prices_by_slot(request); - let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { - let mediator = self.get_provider(mediator_name)?; + let (mediator_response, winning_bids, winning_bid_origins) = + if let Some(mediator_name) = &self.config.mediator { + let mediator = self.get_provider(mediator_name)?; + + log::info!( + "Sending {} provider responses to mediator: {}", + provider_responses.len(), + mediator.provider_name() + ); - log::info!( - "Sending {} provider responses to mediator: {}", - provider_responses.len(), - mediator.provider_name() - ); + // Give the mediator only the remaining time from the auction + // deadline, not the full timeout — the bidding phase already + // consumed part of it. Canonicalize the value for backend-name + // stability without exceeding the remaining budget. + let remaining_ms = remaining_budget_ms(mediation_start, context.timeout_ms); + let mediator_timeout = context + .services + .backend() + .canonicalize_transport_timeout_ms(remaining_ms, mediator.timeout_ms()); + + if mediator_timeout == 0 { + log::warn!("Auction timeout exhausted during bidding phase; skipping mediator"); + let (winning_bids, winning_bid_origins) = + self.select_winning_bids(&provider_responses, &floor_prices); + return Ok(self.finalize_result( + context.trace, + provider_responses, + None, + winning_bids, + winning_bid_origins, + 0, + )); + } - // Give the mediator only the remaining time from the auction - // deadline, not the full timeout — the bidding phase already - // consumed part of it, and the mediator has no select-loop - // deadline backstop. The platform canonicalizes the value for - // backend-name stability (see - // `PlatformBackend::canonicalize_transport_timeout_ms`); it never - // exceeds the remaining budget. See the transport-deadline note on - // `run_providers_parallel` for the limits of this bound. - let remaining_ms = remaining_budget_ms(mediation_start, context.timeout_ms); - let mediator_timeout = context - .services - .backend() - .canonicalize_transport_timeout_ms(remaining_ms, mediator.timeout_ms()); - - if mediator_timeout == 0 { - log::warn!("Auction timeout exhausted during bidding phase; skipping mediator"); - let winning = self.select_winning_bids(&provider_responses, &floor_prices); - return Ok(OrchestrationResult { - provider_responses, - mediator_response: None, - winning_bids: winning, - total_time_ms: 0, - metadata: HashMap::new(), - }); - } + let mediator_context = AuctionContext { + trace: context.trace, + settings: context.settings, + request: context.request, + timeout_ms: mediator_timeout, + provider_responses: Some(&provider_responses), + services: context.services, + }; - let mediator_context = AuctionContext { - settings: context.settings, - request: context.request, - timeout_ms: mediator_timeout, - provider_responses: Some(&provider_responses), - services: context.services, + let start_time = Instant::now(); + let pending = mediator + .request_bids(request, &mediator_context) + .await + .change_context(TrustedServerError::Auction { + message: format!("Mediator {} failed to launch", mediator.provider_name()), + })?; + + let platform_resp = mediator_context + .services + .http_client() + .wait(pending) + .await + .change_context(TrustedServerError::Auction { + message: format!("Mediator {} request failed", mediator.provider_name()), + })?; + + let response_time_ms = start_time.elapsed().as_millis() as u64; + // Use the context-aware parse so mediators (e.g. adserver_mock) can + // restore nurl/burl/ad_id and PBS cache fields from the collected SSP + // responses. The dispatched collect path already does this; the + // synchronous mediation path used by POST /auction and + // /__ts/page-bids must match or mediated cache bids lose the metadata + // needed for creative rendering and win/billing beacons. + let mediator_resp = mediator + .parse_response_with_context( + platform_resp, + response_time_ms, + request, + &mediator_context, + ) + .await + .change_context(TrustedServerError::Auction { + message: format!("Mediator {} parse failed", mediator.provider_name()), + })?; + + let (winning_bids, winning_bid_origins) = + self.select_mediator_winning_bids(&mediator_resp, &floor_prices); + (Some(mediator_resp), winning_bids, winning_bid_origins) + } else { + // No mediator - select best bid per slot from bidder responses + let (winning_bids, winning_bid_origins) = + self.select_winning_bids(&provider_responses, &floor_prices); + (None, winning_bids, winning_bid_origins) }; - let start_time = Instant::now(); - let pending = mediator - .request_bids(request, &mediator_context) - .await - .change_context(TrustedServerError::Auction { - message: format!("Mediator {} failed to launch", mediator.provider_name()), - })?; - - let platform_resp = mediator_context - .services - .http_client() - .wait(pending) - .await - .change_context(TrustedServerError::Auction { - message: format!("Mediator {} request failed", mediator.provider_name()), - })?; - - let response_time_ms = start_time.elapsed().as_millis() as u64; - // Use the context-aware parse so mediators (e.g. adserver_mock) can - // restore nurl/burl/ad_id and PBS cache fields from the collected SSP - // responses. The dispatched collect path already does this; the - // synchronous mediation path used by POST /auction and - // /__ts/page-bids must match or mediated cache bids lose the metadata - // needed for creative rendering and win/billing beacons. - let mediator_resp = mediator - .parse_response_with_context( - platform_resp, - response_time_ms, - request, - &mediator_context, - ) - .await - .change_context(TrustedServerError::Auction { - message: format!("Mediator {} parse failed", mediator.provider_name()), - })?; - - // Extract only mediator bids with comparable numeric prices. - let winning = mediator_resp - .bids - .iter() - .filter_map(|bid| { - if bid.price.is_none() { - log::warn!( - "Mediator '{}' returned bid for slot '{}' without a price - skipping", - mediator.provider_name(), - bid.slot_id - ); - None - } else { - Some((bid.slot_id.clone(), bid.clone())) - } - }) - .collect(); - - ( - Some(mediator_resp), - self.apply_floor_prices(winning, &floor_prices), - ) - } else { - // No mediator - select best bid per slot from bidder responses - let winning = self.select_winning_bids(&provider_responses, &floor_prices); - (None, winning) - }; - - Ok(OrchestrationResult { + Ok(self.finalize_result( + context.trace, provider_responses, mediator_response, winning_bids, - total_time_ms: 0, // Will be set by caller - metadata: HashMap::new(), - }) + winning_bid_origins, + 0, + )) } /// Run auction with only parallel bidding (no mediation). @@ -425,15 +453,17 @@ impl AuctionOrchestrator { ) -> Result> { let provider_responses = self.run_providers_parallel(request, context).await?; let floor_prices = self.floor_prices_by_slot(request); - let winning_bids = self.select_winning_bids(&provider_responses, &floor_prices); + let (winning_bids, winning_bid_origins) = + self.select_winning_bids(&provider_responses, &floor_prices); - Ok(OrchestrationResult { + Ok(self.finalize_result( + context.trace, provider_responses, - mediator_response: None, + None, winning_bids, - total_time_ms: 0, - metadata: HashMap::new(), - }) + winning_bid_origins, + 0, + )) } /// Run all providers in parallel and collect responses. @@ -549,6 +579,7 @@ impl AuctionOrchestrator { } let provider_context = AuctionContext { + trace: context.trace, settings: context.settings, request: context.request, timeout_ms: effective_timeout, @@ -693,6 +724,7 @@ impl AuctionOrchestrator { { let response_time_ms = start_time.elapsed().as_millis() as u64; let provider_context = AuctionContext { + trace: context.trace, settings: context.settings, request: context.request, timeout_ms: effective_timeout, @@ -794,20 +826,24 @@ impl AuctionOrchestrator { Ok(responses) } - /// Select the best decoded-price bid for each slot from all responses. + /// Select the best decoded-price bid for each slot while retaining its exact origin. + /// + /// Bids with no decoded price (for example, encoded APS bids) are skipped when + /// no mediator is configured because they cannot be compared. fn select_winning_bids( &self, responses: &[AuctionResponse], floor_prices: &HashMap, - ) -> HashMap { + ) -> (HashMap, HashMap) { let mut winning_bids: HashMap = HashMap::new(); + let mut origins = HashMap::new(); - for response in responses { + for (response_index, response) in responses.iter().enumerate() { if response.status != BidStatus::Success { continue; } - for bid in &response.bids { + for (bid_index, bid) in response.bids.iter().enumerate() { let bid_price = match bid.price { Some(p) => p, None => { @@ -828,12 +864,91 @@ impl AuctionOrchestrator { }; if should_replace { + origins.insert( + bid.slot_id.clone(), + WinningBidOrigin { + response_index, + bid_index, + mediated: false, + }, + ); winning_bids.insert(bid.slot_id.clone(), bid.clone()); } } } - self.apply_floor_prices(winning_bids, floor_prices) + let winning_bids = self.apply_floor_prices(winning_bids, floor_prices); + origins.retain(|slot_id, _| winning_bids.contains_key(slot_id)); + (winning_bids, origins) + } + + fn select_mediator_winning_bids( + &self, + response: &AuctionResponse, + floor_prices: &HashMap, + ) -> (HashMap, HashMap) { + let mut winning_bids = HashMap::new(); + let mut origins = HashMap::new(); + for (bid_index, bid) in response.bids.iter().enumerate() { + if bid.price.is_none() { + log::warn!( + "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", + response.provider, + bid.slot_id + ); + continue; + } + origins.insert( + bid.slot_id.clone(), + WinningBidOrigin { + response_index: 0, + bid_index, + mediated: true, + }, + ); + winning_bids.insert(bid.slot_id.clone(), bid.clone()); + } + let winning_bids = self.apply_floor_prices(winning_bids, floor_prices); + origins.retain(|slot_id, _| winning_bids.contains_key(slot_id)); + (winning_bids, origins) + } + + fn finalize_result( + &self, + trace: &AuctionTraceContext, + provider_responses: Vec, + mediator_response: Option, + winning_bids: HashMap, + winning_bid_origins: HashMap, + total_time_ms: u64, + ) -> OrchestrationResult { + let outcome = if winning_bids.is_empty() { + AuctionPublicOutcome::NoBid + } else { + AuctionPublicOutcome::Completed + }; + let trace_bids = build_winning_bid_traces( + &winning_bids, + &winning_bid_origins, + &provider_responses, + mediator_response.as_ref(), + BidTraceId::new, + ); + OrchestrationResult { + trace: AuctionResultTrace { + summary: AuctionTraceSummary { + auction: trace.clone(), + outcome, + }, + winning_bids: trace_bids, + }, + winning_bid_origins, + provider_responses, + mediator_response, + winning_bids, + total_time_ms, + metadata: HashMap::new(), + } } fn apply_floor_prices( @@ -1017,6 +1132,7 @@ impl AuctionOrchestrator { } let provider_context = AuctionContext { + trace: context.trace, settings: context.settings, request: context.request, timeout_ms: effective_timeout, @@ -1096,6 +1212,7 @@ impl AuctionOrchestrator { ); DispatchAuctionOutcome::Dispatched(DispatchedAuction { + trace: context.trace.clone(), pending_requests, backend_to_provider, launch_responses, @@ -1124,6 +1241,7 @@ impl AuctionOrchestrator { context: &AuctionContext<'_>, ) -> OrchestrationResult { let DispatchedAuction { + trace, pending_requests, mut backend_to_provider, launch_responses, @@ -1175,6 +1293,7 @@ impl AuctionOrchestrator { { let response_time_ms = start_time.elapsed().as_millis() as u64; let provider_context = AuctionContext { + trace: context.trace, settings: context.settings, request: &provider_request_context, timeout_ms: effective_timeout, @@ -1268,7 +1387,7 @@ impl AuctionOrchestrator { } backend_to_provider.clear(); - let (mediator_response, winning_bids) = if let Some(mediator_name) = &self.config.mediator { + let (mediator_response, selection) = if let Some(mediator_name) = &self.config.mediator { match self.providers.get(mediator_name.as_str()) { Some(mediator) => { // Cap the mediator at whichever is tighter: its own configured @@ -1299,14 +1418,16 @@ impl AuctionOrchestrator { mediator.provider_name(), responses.len(), ); - let winning = self.select_winning_bids(&responses, &floor_prices); - return OrchestrationResult { - provider_responses: responses, - mediator_response: None, - winning_bids: winning, - total_time_ms: auction_start.elapsed().as_millis() as u64, - metadata: HashMap::new(), - }; + let (winning_bids, winning_bid_origins) = + self.select_winning_bids(&responses, &floor_prices); + return self.finalize_result( + &trace, + responses, + None, + winning_bids, + winning_bid_origins, + auction_start.elapsed().as_millis() as u64, + ); } let mediator_start = Instant::now(); log::info!( @@ -1327,6 +1448,7 @@ impl AuctionOrchestrator { .body(edgezero_core::body::Body::empty()) .unwrap_or_else(|_| http::Request::new(edgezero_core::body::Body::empty())); let mediator_context = AuctionContext { + trace: &trace, settings: context.settings, request: &placeholder, timeout_ms: mediator_timeout, @@ -1358,25 +1480,11 @@ impl AuctionOrchestrator { .await { Ok(mediator_resp) => { - let winning = mediator_resp - .bids - .iter() - .filter_map(|bid| { - if bid.price.is_none() { - log::warn!( - "Mediator '{}' returned bid for slot '{}' without decoded price - skipping", - mediator.provider_name(), - bid.slot_id - ); - None - } else { - Some((bid.slot_id.clone(), bid.clone())) - } - }) - .collect(); - let winning = - self.apply_floor_prices(winning, &floor_prices); - (Some(mediator_resp), winning) + let selection = self.select_mediator_winning_bids( + &mediator_resp, + &floor_prices, + ); + (Some(mediator_resp), selection) } Err(e) => { log::warn!( @@ -1417,13 +1525,15 @@ impl AuctionOrchestrator { (None, self.select_winning_bids(&responses, &floor_prices)) }; - OrchestrationResult { - provider_responses: responses, + let (winning_bids, winning_bid_origins) = selection; + self.finalize_result( + &trace, + responses, mediator_response, winning_bids, - total_time_ms: auction_start.elapsed().as_millis() as u64, - metadata: HashMap::new(), - } + winning_bid_origins, + auction_start.elapsed().as_millis() as u64, + ) } /// Check if orchestrator is enabled. @@ -1436,6 +1546,10 @@ impl AuctionOrchestrator { /// Result of an orchestrated auction. #[derive(Debug, Clone)] pub struct OrchestrationResult { + /// Privacy-safe tester trace for this finalized result. + pub trace: AuctionResultTrace, + /// Exact internal origin of each final winning bid. + pub(crate) winning_bid_origins: HashMap, /// All responses from providers pub provider_responses: Vec, /// Final response from mediator (if used) @@ -1449,6 +1563,32 @@ pub struct OrchestrationResult { } impl OrchestrationResult { + /// Build a no-bid result for a terminal path that already returns a response. + #[must_use] + pub fn empty(trace: AuctionTraceContext, outcome: AuctionPublicOutcome) -> Self { + Self { + trace: AuctionResultTrace { + summary: AuctionTraceSummary { + auction: trace, + outcome, + }, + winning_bids: HashMap::new(), + }, + winning_bid_origins: HashMap::new(), + provider_responses: Vec::new(), + mediator_response: None, + winning_bids: HashMap::new(), + total_time_ms: 0, + metadata: HashMap::new(), + } + } + + /// Return the exact provider/bid location for a final winning slot. + #[must_use] + pub(crate) fn winning_origin(&self, slot_id: &str) -> Option { + self.winning_bid_origins.get(slot_id).copied() + } + /// Get the winning bid for a specific slot. #[must_use] pub fn get_winning_bid(&self, slot_id: &str) -> Option<&Bid> { @@ -1483,7 +1623,8 @@ mod tests { use crate::auction::test_support::create_test_auction_context; use crate::auction::types::{ AdFormat, AdSlot, ApsRendererV1, ApsTagType, AuctionContext, AuctionRequest, - AuctionResponse, Bid, BidRenderer, BidStatus, MediaType, PublisherInfo, UserInfo, + AuctionResponse, Bid, BidRenderer, BidStatus, BidTraceId, MediaType, PublisherInfo, + UserInfo, WinningBidOrigin, }; use crate::error::TrustedServerError; use crate::platform::test_support::{ @@ -1499,7 +1640,7 @@ mod tests { use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex}; - use super::AuctionOrchestrator; + use super::{AuctionOrchestrator, build_winning_bid_traces}; // --------------------------------------------------------------------------- // Minimal test double for AuctionProvider @@ -1694,6 +1835,62 @@ mod tests { } } + #[test] + fn mediated_selection_retains_the_mediator_response_origin() { + let orchestrator = AuctionOrchestrator::new(AuctionConfig::default()); + let selected = mediated_bid(None); + let mediator_response = AuctionResponse::success("mediator", vec![selected], 1); + + let (_, origins) = + orchestrator.select_mediator_winning_bids(&mediator_response, &HashMap::new()); + + let origin = origins["header-banner"]; + assert!( + origin.mediated, + "mediated selection should retain mediator origin" + ); + assert_eq!( + origin.bid_index, 0, + "should retain exact mediator bid index" + ); + } + + #[test] + fn winning_trace_builder_uses_supplied_id_source_only_for_final_winners() { + let mut winner = mediated_bid(None); + winner.bidder = "example-bidder".to_owned(); + let provider_responses = vec![AuctionResponse::success( + "provider-a", + vec![winner.clone()], + 1, + )]; + let winning_bids = HashMap::from([("header-banner".to_owned(), winner)]); + let origins = HashMap::from([( + "header-banner".to_owned(), + WinningBidOrigin { + response_index: 0, + bid_index: 0, + mediated: false, + }, + )]); + let fixed = uuid::Uuid::parse_str("650e8400-e29b-41d4-a716-446655440000") + .expect("should parse fixed UUID"); + let mut calls = 0; + + let traces = + build_winning_bid_traces(&winning_bids, &origins, &provider_responses, None, || { + calls += 1; + BidTraceId::from_uuid(fixed) + }); + + assert_eq!(calls, 1, "should allocate one ID for one final winner"); + assert_eq!( + traces["header-banner"].bid_trace_id.to_string(), + fixed.to_string(), + "should use the supplied deterministic ID" + ); + } + #[async_trait::async_trait(?Send)] impl AuctionProvider for CacheRestoringMediator { fn provider_name(&self) -> &'static str { @@ -1795,6 +1992,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &req, timeout_ms: 2000, @@ -2317,6 +2515,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &req, timeout_ms: 2000, @@ -2395,6 +2594,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &req, timeout_ms: 2000, @@ -2456,6 +2656,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &req, timeout_ms: 2000, @@ -2538,6 +2739,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &req, timeout_ms: 2000, @@ -2635,6 +2837,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &req, timeout_ms: 2000, @@ -2712,6 +2915,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &req, timeout_ms: 2000, @@ -2781,6 +2985,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &req, timeout_ms: 2000, @@ -2850,6 +3055,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build downstream request"); let dispatch_context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &downstream, timeout_ms: 750, @@ -2868,6 +3074,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build placeholder request"); let collect_context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &placeholder, timeout_ms: 750, @@ -2933,6 +3140,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &req, timeout_ms: 2000, @@ -2998,6 +3206,7 @@ mod tests { .body(edgezero_core::body::Body::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &req, timeout_ms: 2000, @@ -3026,7 +3235,7 @@ mod tests { let floor_prices = HashMap::new(); let response = |provider: &str, bid: Bid| AuctionResponse::success(provider, vec![bid], 1); - let aps_wins = orchestrator.select_winning_bids( + let (aps_wins, _) = orchestrator.select_winning_bids( &[ response("aps", auction_bid("aps", 2.0)), response("ordinary", auction_bid("ordinary", 1.0)), @@ -3038,7 +3247,7 @@ mod tests { assert!(winner.renderer.is_some()); assert!(winner.creative.is_none()); - let ordinary_wins = orchestrator.select_winning_bids( + let (ordinary_wins, _) = orchestrator.select_winning_bids( &[ response("aps", auction_bid("aps", 2.0)), response("ordinary", auction_bid("ordinary", 3.0)), diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index c252f5351..34e80b8e0 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -3,7 +3,6 @@ //! Core owns the privacy-preserving auction observation model and pure row //! builder. Platform adapters provide the concrete sink implementation. -use std::collections::HashSet; use std::time::Instant; use chrono::Utc; @@ -12,7 +11,10 @@ use serde::Serialize; use uuid::Uuid; use crate::auction::orchestrator::OrchestrationResult; -use crate::auction::types::{AuctionRequest, AuctionResponse, Bid, BidStatus, MediaType}; +pub use crate::auction::types::AuctionSource; +use crate::auction::types::{ + AuctionRequest, AuctionResponse, AuctionTraceContext, Bid, BidStatus, MediaType, +}; use crate::ec::EcContext; use crate::error::TrustedServerError; use crate::platform::RuntimeServices; @@ -20,27 +22,6 @@ use crate::platform::RuntimeServices; const MAX_PAGE_PATH_BYTES: usize = 256; const DYNAMIC_SEGMENT_REPLACEMENT: &str = ":id"; -/// Source path that initiated an auction candidate. -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub enum AuctionSource { - /// Initial publisher navigation using server-side ad templates. - InitialNavigation, - /// SPA navigation through `GET /__ts/page-bids`. - SpaNavigation, - /// Explicit `POST /auction` API. - AuctionApi, -} - -impl AuctionSource { - fn as_str(self) -> &'static str { - match self { - Self::InitialNavigation => "initial_navigation", - Self::SpaNavigation => "spa_navigation", - Self::AuctionApi => "auction_api", - } - } -} - /// Terminal status for one auction observation. #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub enum AuctionTerminalStatus { @@ -123,7 +104,7 @@ impl AuctionObservationContext { /// Build an observation context from an auction request. #[must_use] pub fn from_auction_request( - auction_source: AuctionSource, + trace: &AuctionTraceContext, request: &AuctionRequest, ec_context: &EcContext, ) -> Self { @@ -135,7 +116,7 @@ impl AuctionObservationContext { .map(|url| url.path().to_owned()) .unwrap_or_else(|| "/".to_owned()); Self::from_parts( - auction_source, + trace, &request.publisher.domain, &raw_path, request.slots.len(), @@ -146,7 +127,7 @@ impl AuctionObservationContext { /// Build an observation context from publisher request parts. #[must_use] pub fn from_parts( - auction_source: AuctionSource, + trace: &AuctionTraceContext, publisher_domain: &str, raw_page_path: &str, slot_count: usize, @@ -157,8 +138,8 @@ impl AuctionObservationContext { let consent = ec_context.consent(); let slot_count = u16::try_from(slot_count).unwrap_or(u16::MAX); Self { - auction_id: Uuid::new_v4(), - auction_source, + auction_id: trace.auction_trace_id.as_uuid(), + auction_source: trace.source, publisher_domain: publisher_domain.to_owned(), page_path: normalize_page_path(raw_page_path), country: geo @@ -264,7 +245,7 @@ pub struct AuctionEventRow { pub event_ts: String, /// `summary`, `provider_call`, or `bid`. pub event_kind: String, - /// Fresh telemetry auction UUID. + /// Privacy-safe UUID shared with tester-gated trace output. pub auction_id: String, /// Source path label. pub auction_source: String, @@ -320,6 +301,8 @@ pub struct AuctionEventRow { pub currency: Option, /// Whether this is the canonical winning row for its slot. pub is_win: Option, + /// Trace UUID for the canonical winning bid only. + pub bid_trace_id: Option, /// Advertiser domain. pub ad_domain: Option, /// Creative/ad ID. @@ -359,6 +342,7 @@ impl AuctionEventRow { price_cpm: None, currency: None, is_win: None, + bid_trace_id: None, ad_domain: None, ad_id: None, } @@ -683,68 +667,84 @@ fn push_bid_rows( request: &AuctionRequest, result: &OrchestrationResult, ) { - let mut matched_wins = HashSet::new(); - - for response in &result.provider_responses { - for bid in &response.bids { - let matched_slot = result - .winning_bids - .iter() - .find(|(slot_id, winning)| { - !matched_wins.contains(*slot_id) && bid_matches_winning_bid(bid, winning) + for (response_index, response) in result.provider_responses.iter().enumerate() { + for (bid_index, bid) in response.bids.iter().enumerate() { + let winning_slot = result.winning_bids.keys().find(|slot_id| { + result.winning_origin(slot_id).is_some_and(|origin| { + !origin.mediated + && origin.response_index == response_index + && origin.bid_index == bid_index }) - .map(|(slot_id, winning)| (slot_id.clone(), winning)); - let (is_win, price) = if let Some((slot_id, winning)) = matched_slot { - matched_wins.insert(slot_id); - (1, bid.price.or(winning.price)) - } else { - (0, bid.price) - }; + }); + let trace_id = winning_slot.and_then(|slot_id| { + result + .trace + .winning_bids + .get(slot_id) + .map(|trace| trace.bid_trace_id.to_string()) + }); + let price = winning_slot + .and_then(|slot_id| result.winning_bids.get(slot_id)) + .and_then(|winning| winning.price) + .or(bid.price); rows.push(bid_row( observation, event_ts, request, &response.provider, bid, - is_win, - price, + BidRowOutcome { + is_win: u8::from(winning_slot.is_some()), + price, + bid_trace_id: trace_id, + }, )); } } if let Some(mediator_response) = &result.mediator_response { - for (slot_id, winning) in &result.winning_bids { - if matched_wins.contains(slot_id) { - continue; - } - if mediator_response - .bids - .iter() - .any(|bid| bid_matches_winning_bid(bid, winning)) - { + for (bid_index, bid) in mediator_response.bids.iter().enumerate() { + let winning_slot = result.winning_bids.keys().find(|slot_id| { + result + .winning_origin(slot_id) + .is_some_and(|origin| origin.mediated && origin.bid_index == bid_index) + }); + if let Some(slot_id) = winning_slot { + let trace_id = result + .trace + .winning_bids + .get(slot_id) + .map(|trace| trace.bid_trace_id.to_string()); rows.push(bid_row( observation, event_ts, request, &mediator_response.provider, - winning, - 1, - winning.price, + bid, + BidRowOutcome { + is_win: 1, + price: bid.price, + bid_trace_id: trace_id, + }, )); - matched_wins.insert(slot_id.clone()); } } } } +struct BidRowOutcome { + is_win: u8, + price: Option, + bid_trace_id: Option, +} + fn bid_row( observation: &AuctionObservationContext, event_ts: &str, request: &AuctionRequest, provider: &str, bid: &Bid, - is_win: u8, - price: Option, + outcome: BidRowOutcome, ) -> AuctionEventRow { let mut row = AuctionEventRow::base(observation, "bid", event_ts); row.provider = Some(provider.to_owned()); @@ -753,9 +753,10 @@ fn bid_row( row.slot_h = Some(u16::try_from(bid.height).unwrap_or(u16::MAX)); row.media_type = media_type_for_slot(request, &bid.slot_id).map(str::to_owned); row.seat = Some(bid.bidder.clone()); - row.price_cpm = price; + row.price_cpm = outcome.price; row.currency = Some(bid.currency.clone()); - row.is_win = Some(is_win); + row.is_win = Some(outcome.is_win); + row.bid_trace_id = outcome.bid_trace_id; row.ad_domain = bid .adomain .as_ref() @@ -764,16 +765,6 @@ fn bid_row( row } -fn bid_matches_winning_bid(candidate: &Bid, winning: &Bid) -> bool { - if candidate.slot_id != winning.slot_id || candidate.bidder != winning.bidder { - return false; - } - match winning.ad_id.as_deref() { - Some(winning_ad_id) => candidate.ad_id.as_deref() == Some(winning_ad_id), - None => true, - } -} - fn media_type_for_slot<'a>(request: &'a AuctionRequest, slot_id: &str) -> Option<&'a str> { request .slots @@ -948,6 +939,15 @@ mod tests { } } + fn empty_result(total_time_ms: u64) -> OrchestrationResult { + let mut result = OrchestrationResult::empty( + AuctionTraceContext::new(AuctionSource::AuctionApi), + crate::auction::types::AuctionPublicOutcome::NoBid, + ); + result.total_time_ms = total_time_ms; + result + } + fn bid(slot_id: &str, bidder: &str, ad_id: Option<&str>, price: Option) -> Bid { Bid { slot_id: slot_id.to_owned(), @@ -1027,13 +1027,27 @@ mod tests { let provider_error = AuctionResponse::error("mock", 12).with_metadata("error_type", json!("parse_response")); let winning = provider_success.bids[0].clone(); - let result = OrchestrationResult { - provider_responses: vec![provider_success, provider_no_bid, provider_error], - mediator_response: None, - winning_bids: HashMap::from([("slot-1".to_owned(), winning)]), - total_time_ms: 99, - metadata: HashMap::new(), - }; + let mut result = empty_result(99); + result.provider_responses = vec![provider_success, provider_no_bid, provider_error]; + result + .winning_bids + .insert("slot-1".to_owned(), winning.clone()); + result.winning_bid_origins.insert( + "slot-1".to_owned(), + crate::auction::types::WinningBidOrigin { + response_index: 0, + bid_index: 0, + mediated: false, + }, + ); + result.trace.winning_bids.insert( + "slot-1".to_owned(), + crate::auction::types::WinningBidTrace { + bid_trace_id: crate::auction::types::BidTraceId::new(), + provider: "prebid".to_owned(), + bidder: winning.bidder, + }, + ); let observation = AuctionObservationContext::for_test(AuctionSource::AuctionApi, "/article/1", 1); @@ -1091,13 +1105,8 @@ mod tests { let provider_http_error = AuctionResponse::error("prebid", 12) .with_metadata("error_type", json!("http_status")) .with_metadata("status", json!(403)); - let result = OrchestrationResult { - provider_responses: vec![provider_http_error], - mediator_response: None, - winning_bids: HashMap::new(), - total_time_ms: 12, - metadata: HashMap::new(), - }; + let mut result = empty_result(12); + result.provider_responses = vec![provider_http_error]; let observation = AuctionObservationContext::for_test(AuctionSource::AuctionApi, "/article/1", 1); @@ -1128,13 +1137,28 @@ mod tests { let mediator_bid = bid("slot-1", "kargo", Some("ad-1"), Some(2.0)); let mediator_response = AuctionResponse::success("adserver_mock", vec![mediator_bid.clone()], 15); - let result = OrchestrationResult { - provider_responses: vec![provider_success], - mediator_response: Some(mediator_response), - winning_bids: HashMap::from([("slot-1".to_owned(), mediator_bid)]), - total_time_ms: 80, - metadata: HashMap::new(), - }; + let mut result = empty_result(80); + result.provider_responses = vec![provider_success]; + result.mediator_response = Some(mediator_response); + result + .winning_bids + .insert("slot-1".to_owned(), mediator_bid.clone()); + result.winning_bid_origins.insert( + "slot-1".to_owned(), + crate::auction::types::WinningBidOrigin { + response_index: 0, + bid_index: 0, + mediated: false, + }, + ); + result.trace.winning_bids.insert( + "slot-1".to_owned(), + crate::auction::types::WinningBidTrace { + bid_trace_id: crate::auction::types::BidTraceId::new(), + provider: "prebid".to_owned(), + bidder: mediator_bid.bidder, + }, + ); let observation = AuctionObservationContext::for_test(AuctionSource::InitialNavigation, "/", 1); @@ -1167,13 +1191,7 @@ mod tests { #[test] fn ndjson_serialization_has_one_json_object_per_line_and_no_private_ids() { let request = test_request("ts-ec-derived-id"); - let result = OrchestrationResult { - provider_responses: Vec::new(), - mediator_response: None, - winning_bids: HashMap::new(), - total_time_ms: 1, - metadata: HashMap::new(), - }; + let result = empty_result(1); let observation = AuctionObservationContext::for_test(AuctionSource::AuctionApi, "/auction", 1); diff --git a/crates/trusted-server-core/src/auction/test_support.rs b/crates/trusted-server-core/src/auction/test_support.rs index e4b953e05..45d90731e 100644 --- a/crates/trusted-server-core/src/auction/test_support.rs +++ b/crates/trusted-server-core/src/auction/test_support.rs @@ -3,11 +3,18 @@ use std::sync::LazyLock; use edgezero_core::body::Body as EdgeBody; use http::Request; -use super::AuctionContext; +use super::{AuctionContext, AuctionSource}; +use crate::auction::types::AuctionTraceContext; use crate::platform::{RuntimeServices, test_support::noop_services}; use crate::settings::Settings; static TEST_SERVICES: LazyLock = LazyLock::new(noop_services); +static TEST_TRACE: LazyLock = + LazyLock::new(|| AuctionTraceContext::new(AuctionSource::AuctionApi)); + +pub(crate) fn test_trace() -> &'static AuctionTraceContext { + &TEST_TRACE +} pub(crate) fn create_test_auction_context<'a>( settings: &'a Settings, @@ -16,6 +23,7 @@ pub(crate) fn create_test_auction_context<'a>( ) -> AuctionContext<'a> { let services: &'static RuntimeServices = &TEST_SERVICES; AuctionContext { + trace: test_trace(), settings, request, timeout_ms, diff --git a/crates/trusted-server-core/src/auction/types.rs b/crates/trusted-server-core/src/auction/types.rs index a6ad61f3a..e101a7245 100644 --- a/crates/trusted-server-core/src/auction/types.rs +++ b/crates/trusted-server-core/src/auction/types.rs @@ -4,12 +4,157 @@ use edgezero_core::body::Body as EdgeBody; use http::Request; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +use uuid::Uuid; use crate::auction::context::ContextValue; use crate::geo::GeoInfo; use crate::platform::RuntimeServices; use crate::settings::Settings; +/// Source path that initiated an auction candidate. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuctionSource { + /// Initial publisher navigation using server-side ad templates. + InitialNavigation, + /// SPA navigation through `GET /__ts/page-bids`. + SpaNavigation, + /// Explicit `POST /auction` API. + AuctionApi, +} + +impl AuctionSource { + /// Return the stable wire label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::InitialNavigation => "initial_navigation", + Self::SpaNavigation => "spa_navigation", + Self::AuctionApi => "auction_api", + } + } +} + +/// Privacy-safe public identity for one auction candidate. +#[derive(Debug, Clone, Copy, Eq, Hash, PartialEq, derive_more::Display)] +pub struct AuctionTraceId(Uuid); + +impl AuctionTraceId { + /// Generate a fresh random trace identity. + #[must_use] + pub fn new() -> Self { + Self(Uuid::new_v4()) + } + + /// Return the underlying UUID. + #[must_use] + pub const fn as_uuid(self) -> Uuid { + self.0 + } +} + +impl Default for AuctionTraceId { + fn default() -> Self { + Self::new() + } +} + +/// Privacy-safe public identity for one final winning bid. +#[derive(Debug, Clone, Copy, Eq, Hash, PartialEq, derive_more::Display)] +pub struct BidTraceId(Uuid); + +impl BidTraceId { + /// Generate a fresh random trace identity. + #[must_use] + pub fn new() -> Self { + Self(Uuid::new_v4()) + } + + #[cfg(test)] + pub(crate) const fn from_uuid(value: Uuid) -> Self { + Self(value) + } +} + +impl Default for BidTraceId { + fn default() -> Self { + Self::new() + } +} + +/// Trace identity and source shared throughout one auction lifecycle. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct AuctionTraceContext { + pub auction_trace_id: AuctionTraceId, + pub source: AuctionSource, +} + +impl AuctionTraceContext { + /// Generate a context for an auction candidate. + #[must_use] + pub fn new(source: AuctionSource) -> Self { + Self { + auction_trace_id: AuctionTraceId::new(), + source, + } + } +} + +/// Privacy-safe terminal state exposed to tester traffic. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuctionPublicOutcome { + Completed, + NoBid, + Skipped, + Failed, + Abandoned, +} + +impl AuctionPublicOutcome { + /// Return the stable wire label. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Completed => "completed", + Self::NoBid => "no_bid", + Self::Skipped => "skipped", + Self::Failed => "failed", + Self::Abandoned => "abandoned", + } + } +} + +/// Result-independent public summary for one auction candidate. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct AuctionTraceSummary { + pub auction: AuctionTraceContext, + pub outcome: AuctionPublicOutcome, +} + +/// Public trace metadata for one final winning bid. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct WinningBidTrace { + pub bid_trace_id: BidTraceId, + pub provider: String, + pub bidder: String, +} + +/// Trace data attached to a finalized auction result. +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct AuctionResultTrace { + pub summary: AuctionTraceSummary, + pub winning_bids: HashMap, +} + +/// Exact internal location of a final winning bid. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub struct WinningBidOrigin { + pub response_index: usize, + pub bid_index: usize, + pub mediated: bool, +} + /// Represents a unified auction request across all providers. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AuctionRequest { @@ -140,6 +285,8 @@ pub struct SiteInfo { /// [dispatch]: crate::auction::AuctionOrchestrator::dispatch_auction /// [collect]: crate::auction::AuctionOrchestrator::collect_dispatched_auction pub struct AuctionContext<'a> { + /// Trace identity owned by the auction entry point. + pub trace: &'a AuctionTraceContext, pub settings: &'a Settings, pub request: &'a Request, pub timeout_ms: u32, diff --git a/crates/trusted-server-core/src/config.rs b/crates/trusted-server-core/src/config.rs index 7bbecd747..e5a59cda0 100644 --- a/crates/trusted-server-core/src/config.rs +++ b/crates/trusted-server-core/src/config.rs @@ -16,16 +16,18 @@ use validator::{Validate, ValidationError, ValidationErrors}; use crate::ec::registry::PartnerRegistry; use crate::error::TrustedServerError; use crate::integrations::{ - adserver_mock::AdServerMockConfig, aps::ApsConfig, datadome::DataDomeConfig, - didomi::DidomiIntegrationConfig, google_tag_manager::GoogleTagManagerConfig, gpt::GptConfig, - lockr::LockrConfig, nextjs::NextJsIntegrationConfig, osano::OsanoConfig, - permutive::PermutiveConfig, prebid, sourcepoint::SourcepointConfig, testlight::TestlightConfig, + ad_trace::AdTraceConfig, adserver_mock::AdServerMockConfig, aps::ApsConfig, + datadome::DataDomeConfig, didomi::DidomiIntegrationConfig, + google_tag_manager::GoogleTagManagerConfig, gpt::GptConfig, lockr::LockrConfig, + nextjs::NextJsIntegrationConfig, osano::OsanoConfig, permutive::PermutiveConfig, prebid, + sourcepoint::SourcepointConfig, testlight::TestlightConfig, }; use crate::settings::{IntegrationConfig, Settings}; const DEPLOY_VALIDATION_FIELD: &str = "trusted_server"; #[cfg(test)] const DEPLOY_VALIDATED_INTEGRATION_IDS: &[&str] = &[ + "ad_trace", "prebid", "aps", "adserver_mock", @@ -136,6 +138,7 @@ fn validate_enabled_integrations( ) -> Result, Report> { let mut enabled_auction_providers = HashSet::new(); + validate_integration::(settings, "ad_trace")?; if validate_prebid(settings)? { enabled_auction_providers.insert("prebid"); } diff --git a/crates/trusted-server-core/src/constants.rs b/crates/trusted-server-core/src/constants.rs index ffcf4f034..03b5b6d24 100644 --- a/crates/trusted-server-core/src/constants.rs +++ b/crates/trusted-server-core/src/constants.rs @@ -5,6 +5,10 @@ pub const COOKIE_TS_EC: &str = "ts-ec"; /// JSON array of Extended User IDs (`[{ source, uids }]`) from identity providers. pub const COOKIE_TS_EIDS: &str = "ts-eids"; pub const COOKIE_TS_TESTER: &str = "ts-tester"; +/// Host-only browser-session cookie activated by the ad trace console query. +pub const COOKIE_TS_CONSOLE: &str = "__Host-ts-console"; +/// Reserved self-service query parameter for the ad trace console. +pub const QUERY_TS_CONSOLE: &str = "ts_console"; pub const COOKIE_SHAREDID: &str = "sharedId"; pub const HEADER_X_PUB_USER_ID: HeaderName = HeaderName::from_static("x-pub-user-id"); diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index ad69e51c5..9ef9ee0fd 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -161,6 +161,8 @@ pub struct HtmlProcessorConfig { pub request_host: String, pub request_scheme: String, pub integrations: IntegrationRegistry, + /// Request-scoped console bootstrap injected before the unified bundle. + pub head_bootstrap_script: Option, /// Pre-computed ``. /// Injected at `` open. `None` when no slots matched. pub ad_slots_script: Option, @@ -189,6 +191,7 @@ impl HtmlProcessorConfig { request_host: request_host.to_owned(), request_scheme: request_scheme.to_owned(), integrations: integrations.clone(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: settings.publisher.max_buffered_body_bytes, @@ -205,9 +208,11 @@ impl HtmlProcessorConfig { #[must_use] pub fn with_ad_state( mut self, + head_bootstrap_script: Option, ad_slots_script: Option, ad_bids_state: std::sync::Arc>>, ) -> Self { + self.head_bootstrap_script = head_bootstrap_script; self.ad_slots_script = ad_slots_script; self.ad_bids_state = ad_bids_state; self @@ -292,6 +297,7 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let injected_bids = Arc::new(AtomicBool::new(false)); let integration_registry = config.integrations.clone(); let script_rewriters = integration_registry.script_rewriters(); + let head_bootstrap_script = config.head_bootstrap_script.clone(); let ad_slots_script = config.ad_slots_script.clone(); let ad_bids_state = config.ad_bids_state.clone(); @@ -302,10 +308,15 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso let integrations = integration_registry.clone(); let patterns = patterns.clone(); let document_state = document_state.clone(); + let head_bootstrap_script = head_bootstrap_script.clone(); let ad_slots_script = ad_slots_script.clone(); move |el| { if !injected_tsjs.get() { let mut snippet = String::new(); + // Request-scoped activation must run before every TSJS module. + if let Some(ref bootstrap) = head_bootstrap_script { + snippet.push_str(bootstrap); + } // Inject ad slots script first so it appears before tsjs bundle. if let Some(ref slots_script) = ad_slots_script { snippet.push_str(slots_script); @@ -661,6 +672,7 @@ mod tests { request_host: "test.example.com".to_owned(), request_scheme: "https".to_owned(), integrations: IntegrationRegistry::default(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, @@ -738,6 +750,8 @@ mod tests { let html = "Test"; let mut config = create_test_config(); + config.head_bootstrap_script = + Some("".to_owned()); config.integrations = IntegrationRegistry::from_rewriters_with_head_injectors( Vec::new(), Vec::new(), @@ -759,6 +773,7 @@ mod tests { let processed = String::from_utf8(output).expect("output should be valid UTF-8"); let tsjs_marker = "id=\"trustedserver-js\""; + let bootstrap_marker = "window.__tsjs_adTraceActive=true"; let head_marker = "window.__testHeadInjector=true"; assert_eq!( @@ -775,6 +790,9 @@ mod tests { let tsjs_index = processed .find(tsjs_marker) .expect("should include unified tsjs tag"); + let bootstrap_index = processed + .find(bootstrap_marker) + .expect("should include request bootstrap"); let head_index = processed .find(head_marker) .expect("should include head snippet"); @@ -783,8 +801,8 @@ mod tests { .expect("should keep existing head content"); assert!( - head_index < tsjs_index, - "should inject config before tsjs bundle so auto-init can read it" + bootstrap_index < head_index && head_index < tsjs_index, + "should inject request bootstrap and config before tsjs auto-init" ); assert!( tsjs_index < title_index, @@ -1430,6 +1448,7 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), + head_bootstrap_script: None, ad_slots_script: Some( r#""# .to_string(), @@ -1504,6 +1523,7 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), + head_bootstrap_script: None, ad_slots_script: Some( r#""#.to_string(), ), @@ -1539,6 +1559,7 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), + head_bootstrap_script: None, ad_slots_script: Some( r#""#.to_string(), ), @@ -1575,6 +1596,7 @@ mod tests { request_host: request_host.to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::default(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), max_buffered_body_bytes: 16 * 1024 * 1024, @@ -1625,6 +1647,7 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), + head_bootstrap_script: None, ad_slots_script: Some( r#""#.to_string(), ), @@ -1653,6 +1676,7 @@ mod tests { request_host: "example.com".to_string(), request_scheme: "https".to_string(), integrations: IntegrationRegistry::empty_for_tests(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: state, max_buffered_body_bytes: 16 * 1024 * 1024, diff --git a/crates/trusted-server-core/src/integrations/ad_trace.rs b/crates/trusted-server-core/src/integrations/ad_trace.rs new file mode 100644 index 000000000..41099d4aa --- /dev/null +++ b/crates/trusted-server-core/src/integrations/ad_trace.rs @@ -0,0 +1,620 @@ +//! Query-activated, session-scoped auction trace integration. + +use edgezero_core::body::Body as EdgeBody; +use error_stack::{Report, ResultExt}; +use http::{HeaderValue, Method, Request, Response, Uri, header, uri::PathAndQuery}; +use serde::Deserialize; +use validator::Validate; + +use crate::constants::{COOKIE_TS_CONSOLE, QUERY_TS_CONSOLE}; +use crate::error::TrustedServerError; +use crate::http_util::is_navigation_request; +use crate::integrations::IntegrationRegistration; +use crate::settings::{IntegrationConfig, Settings}; + +/// Stable integration identifier. +pub const AD_TRACE_INTEGRATION_ID: &str = "ad_trace"; + +const SET_CONSOLE_COOKIE: &str = "__Host-ts-console=1; Path=/; Secure; HttpOnly; SameSite=Lax"; +const CLEAR_CONSOLE_COOKIE: &str = + "__Host-ts-console=; Path=/; Secure; HttpOnly; SameSite=Lax; Max-Age=0"; + +/// Configuration for the optional browser console. +#[derive(Debug, Default, Deserialize, Validate)] +#[serde(deny_unknown_fields)] +pub struct AdTraceConfig { + /// Enable the optional ad trace browser module and console activation. + #[serde(default)] + pub enabled: bool, +} + +impl IntegrationConfig for AdTraceConfig { + fn is_enabled(&self) -> bool { + self.enabled + } +} + +/// Cookie mutation attached to an eligible console-navigation response. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ConsoleCookieAction { + #[default] + None, + SetSession, + ClearSession, +} + +/// Immutable request-scoped console decision. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AdTraceRequestDecision { + enabled: bool, + browser_bootstrap: bool, + private_response: bool, + clean_browser_path_and_query: Option, + cookie_action: ConsoleCookieAction, +} + +impl AdTraceRequestDecision { + /// Whether browser-visible trace fields and targeting are enabled. + #[must_use] + pub fn enabled(&self) -> bool { + self.enabled + } + + /// Whether this response must be private and non-storeable. + #[must_use] + pub fn requires_private_no_store(&self) -> bool { + self.private_response + || self.cookie_action != ConsoleCookieAction::None + || self.clean_browser_path_and_query.is_some() + } + + /// Build the synchronous bootstrap inserted before the unified TSJS bundle. + #[must_use] + pub fn bootstrap_script(&self) -> Option { + if !self.browser_bootstrap && self.clean_browser_path_and_query.is_none() { + return None; + } + + let mut script = String::from(""); + Some(script) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum QueryDirective { + Absent, + Enable, + Disable, + Invalid, +} + +#[derive(Clone, Copy, Debug, Default)] +struct ConsoleCookieState { + occurrences: usize, + canonical: bool, +} + +#[derive(Clone, Copy, Debug, Default)] +struct AdTraceCookieApplied; + +/// Register the optional browser module. +/// +/// # Errors +/// +/// Returns a configuration error when the integration settings are invalid. +pub fn register( + settings: &Settings, +) -> Result, Report> { + let Some(_config) = settings.integration_config::(AD_TRACE_INTEGRATION_ID)? + else { + return Ok(None); + }; + Ok(Some( + IntegrationRegistration::builder(AD_TRACE_INTEGRATION_ID).build(), + )) +} + +/// Evaluate and sanitize the console request before routing or downstream use. +/// +/// The original query and cookie are inspected first. Every reserved query pair +/// and console cookie is then removed from the request. The immutable decision +/// is stored in request extensions for handlers to consume after sanitation. +/// +/// # Errors +/// +/// Returns an error when integration configuration or URI reconstruction fails. +pub fn prepare_request( + settings: &Settings, + request: &mut Request, +) -> Result> { + let integration_enabled = settings + .integration_config::(AD_TRACE_INTEGRATION_ID)? + .is_some(); + let (directive, clean_path, had_reserved_query) = console_query(request.uri()); + let cookie_state = console_cookie_state(request); + let eligible_navigation = is_eligible_console_navigation(request); + + sanitize_console_cookie(request); + if had_reserved_query { + replace_path_and_query(request, &clean_path)?; + } + + let mut decision = AdTraceRequestDecision::default(); + if integration_enabled && eligible_navigation && had_reserved_query { + decision.clean_browser_path_and_query = Some(clean_path); + match directive { + QueryDirective::Enable => { + decision.enabled = true; + decision.browser_bootstrap = true; + decision.cookie_action = ConsoleCookieAction::SetSession; + } + QueryDirective::Disable => { + decision.cookie_action = ConsoleCookieAction::ClearSession; + } + QueryDirective::Invalid | QueryDirective::Absent => {} + } + } else if integration_enabled + && directive == QueryDirective::Absent + && cookie_state.occurrences == 1 + && cookie_state.canonical + { + decision.enabled = true; + decision.browser_bootstrap = eligible_navigation; + } + + decision.private_response = + decision.enabled && trace_payload_request(request, eligible_navigation); + request.extensions_mut().insert(decision.clone()); + Ok(decision) +} + +/// Read the previously prepared request decision. +#[must_use] +pub fn request_decision(request: &Request) -> AdTraceRequestDecision { + request + .extensions() + .get::() + .cloned() + .unwrap_or_default() +} + +/// Return whether browser-visible trace output is active for this request. +#[must_use] +pub fn browser_trace_enabled(request: &Request) -> bool { + request_decision(request).enabled() +} + +/// Copy the prepared request decision onto a response for outer finalization. +pub fn attach_response_decision( + decision: &AdTraceRequestDecision, + response: &mut Response, +) { + response.extensions_mut().insert(decision.clone()); +} + +/// Apply the response-side session mutation and cache policy. +/// +/// Safe to call more than once. The cookie is appended once, while the +/// private/no-store policy is reasserted so later adapter cache policy cannot +/// weaken it. +pub fn finalize_response(response: &mut Response) { + let Some(decision) = response + .extensions() + .get::() + .cloned() + else { + return; + }; + + if decision.cookie_action != ConsoleCookieAction::None + && response + .extensions() + .get::() + .is_none() + { + let value = match decision.cookie_action { + ConsoleCookieAction::None => None, + ConsoleCookieAction::SetSession => Some(HeaderValue::from_static(SET_CONSOLE_COOKIE)), + ConsoleCookieAction::ClearSession => { + Some(HeaderValue::from_static(CLEAR_CONSOLE_COOKIE)) + } + }; + if let Some(value) = value { + response.headers_mut().append(header::SET_COOKIE, value); + response.extensions_mut().insert(AdTraceCookieApplied); + } + } + + if decision.requires_private_no_store() { + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("private, no-store"), + ); + for name in crate::response_privacy::SURROGATE_CACHE_HEADERS { + response.headers_mut().remove(*name); + } + } +} + +fn trace_payload_request(request: &Request, eligible_navigation: bool) -> bool { + eligible_navigation + || request.uri().path() == "/auction" + || request.uri().path() == "/__ts/page-bids" +} + +fn is_eligible_console_navigation(request: &Request) -> bool { + request.method() == Method::GET + && is_navigation_request(request) + && !crate::publisher::is_prefetch_request(request) + && !crate::publisher::is_bot_user_agent(request) +} + +fn console_query(uri: &Uri) -> (QueryDirective, String, bool) { + let mut console_values = Vec::new(); + let mut retained = Vec::new(); + for pair in uri.query().unwrap_or_default().split('&') { + let (name, value) = pair.split_once('=').unwrap_or((pair, "")); + if name == QUERY_TS_CONSOLE { + console_values.push(value); + } else { + retained.push(pair); + } + } + + let directive = match console_values.as_slice() { + [] => QueryDirective::Absent, + ["true" | "1"] => QueryDirective::Enable, + ["false" | "0"] => QueryDirective::Disable, + _ => QueryDirective::Invalid, + }; + let mut clean = uri.path().to_owned(); + let retained_query = retained.join("&"); + if !retained_query.is_empty() { + clean.push('?'); + clean.push_str(&retained_query); + } + (directive, clean, !console_values.is_empty()) +} + +fn console_cookie_state(request: &Request) -> ConsoleCookieState { + let mut state = ConsoleCookieState::default(); + for value in request.headers().get_all(header::COOKIE) { + let Ok(value) = value.to_str() else { + continue; + }; + for cookie in value.split(';') { + let cookie = cookie.trim(); + match cookie.split_once('=') { + Some((name, value)) if name.trim() == COOKIE_TS_CONSOLE => { + state.occurrences += 1; + state.canonical |= value.trim() == "1"; + } + None if cookie == COOKIE_TS_CONSOLE => state.occurrences += 1, + _ => {} + } + } + } + state +} + +fn sanitize_console_cookie(request: &mut Request) { + let retained = request + .headers() + .get_all(header::COOKIE) + .iter() + .filter_map(|value| value.to_str().ok()) + .flat_map(|value| value.split(';')) + .map(str::trim) + .filter(|cookie| match cookie.split_once('=') { + Some((name, _)) => name.trim() != COOKIE_TS_CONSOLE, + None => *cookie != COOKIE_TS_CONSOLE, + }) + .filter(|cookie| !cookie.is_empty()) + .map(str::to_owned) + .collect::>(); + + request.headers_mut().remove(header::COOKIE); + if !retained.is_empty() { + let value = HeaderValue::from_str(&retained.join("; ")) + .expect("should preserve already-valid cookie header values"); + request.headers_mut().insert(header::COOKIE, value); + } +} + +fn replace_path_and_query( + request: &mut Request, + clean_path_and_query: &str, +) -> Result<(), Report> { + let mut parts = request.uri().clone().into_parts(); + parts.path_and_query = Some( + clean_path_and_query + .parse::() + .change_context(TrustedServerError::Proxy { + message: "ad trace console query produced invalid URI".to_owned(), + })?, + ); + *request.uri_mut() = Uri::from_parts(parts).change_context(TrustedServerError::Proxy { + message: "ad trace console query produced invalid URI".to_owned(), + })?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use http::{Request, Response, header}; + + use crate::test_support::tests::create_test_settings; + + use super::*; + + fn settings(enabled: bool) -> Settings { + let mut settings = create_test_settings(); + settings.integrations.insert( + AD_TRACE_INTEGRATION_ID.to_owned(), + serde_json::json!({ "enabled": enabled }), + ); + settings + } + + fn request(uri: &str, cookie: Option<&str>) -> Request { + let mut builder = Request::builder() + .method(Method::GET) + .uri(uri) + .header("sec-fetch-dest", "document"); + if let Some(cookie) = cookie { + builder = builder.header(header::COOKIE, cookie); + } + builder + .body(EdgeBody::empty()) + .expect("should build request") + } + + #[test] + fn rejects_unknown_gate_configuration() { + let mut settings = create_test_settings(); + settings.integrations.insert( + AD_TRACE_INTEGRATION_ID.to_owned(), + serde_json::json!({ "enabled": true, "enabledd": true }), + ); + + let error = settings + .integration_config::(AD_TRACE_INTEGRATION_ID) + .expect_err("should reject unknown gate field"); + assert!( + error.to_string().contains("could not be parsed"), + "should reject invalid configuration: {error}" + ); + } + + #[test] + fn query_enables_first_response_and_sanitizes_request() { + let mut req = request( + "https://publisher.example/page?x=%2F&ts_console=1&y=2", + Some("session=abc; __Host-ts-console=1"), + ); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + + assert!(decision.enabled()); + assert_eq!(decision.cookie_action, ConsoleCookieAction::SetSession); + assert_eq!( + req.uri().to_string(), + "https://publisher.example/page?x=%2F&y=2" + ); + assert_eq!( + req.headers() + .get(header::COOKIE) + .expect("should retain unrelated cookie"), + "session=abc" + ); + assert!(browser_trace_enabled(&req)); + let script = decision.bootstrap_script().expect("should bootstrap"); + assert!(script.contains("__tsjs_adTraceActive=true")); + assert!(script.contains("/page?x=%2F&y=2")); + + let mut separators = request( + "https://publisher.example/page?a=1&&ts_console=1&b=2&", + None, + ); + prepare_request(&settings(true), &mut separators).expect("should prepare"); + assert_eq!(separators.uri().query(), Some("a=1&&b=2&")); + } + + #[test] + fn exact_enable_and_disable_values_are_supported() { + for value in ["true", "1"] { + let mut req = request( + &format!("https://publisher.example/?ts_console={value}"), + None, + ); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + assert!(decision.enabled(), "{value} should enable"); + assert_eq!(decision.cookie_action, ConsoleCookieAction::SetSession); + } + for value in ["false", "0"] { + let mut req = request( + &format!("https://publisher.example/?ts_console={value}"), + Some("__Host-ts-console=1"), + ); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + assert!(!decision.enabled(), "{value} should disable"); + assert_eq!(decision.cookie_action, ConsoleCookieAction::ClearSession); + } + } + + #[test] + fn invalid_or_duplicate_query_fails_closed_without_cookie_mutation() { + for query in [ + "ts_console=True", + "ts_console=", + "ts_console=1&ts_console=true", + ] { + let mut req = request( + &format!("https://publisher.example/?{query}&keep=1"), + Some("__Host-ts-console=1"), + ); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + assert!(!decision.enabled(), "{query} should fail closed"); + assert_eq!(decision.cookie_action, ConsoleCookieAction::None); + assert_eq!(req.uri().query(), Some("keep=1")); + } + } + + #[test] + fn disabled_config_sanitizes_but_never_activates() { + let mut req = request( + "https://publisher.example/?ts_console=1&keep=1", + Some("__Host-ts-console=1; other=value; ts-tester=true"), + ); + let decision = prepare_request(&settings(false), &mut req).expect("should prepare"); + assert!(!decision.enabled()); + assert_eq!(decision.cookie_action, ConsoleCookieAction::None); + assert_eq!(decision.clean_browser_path_and_query, None); + assert_eq!(req.uri().query(), Some("keep=1")); + assert_eq!( + req.headers() + .get(header::COOKIE) + .expect("should retain unrelated cookies"), + "other=value; ts-tester=true" + ); + } + + #[test] + fn exact_session_cookie_gates_api_but_query_cannot_activate_it() { + let mut active = Request::builder() + .method(Method::POST) + .uri("https://publisher.example/auction") + .header(header::COOKIE, "__Host-ts-console=1") + .body(EdgeBody::empty()) + .expect("should build request"); + assert!( + prepare_request(&settings(true), &mut active) + .expect("should prepare") + .enabled() + ); + + let mut query_only = Request::builder() + .method(Method::POST) + .uri("https://publisher.example/auction?ts_console=1") + .body(EdgeBody::empty()) + .expect("should build request"); + let decision = prepare_request(&settings(true), &mut query_only).expect("should prepare"); + assert!(!decision.enabled()); + assert_eq!(decision.cookie_action, ConsoleCookieAction::None); + assert_eq!(query_only.uri().query(), None); + } + + #[test] + fn active_session_does_not_make_static_bundle_response_private() { + let mut req = Request::builder() + .method(Method::GET) + .uri("https://publisher.example/static/tsjs=tsjs-unified.min.js") + .header("sec-fetch-dest", "script") + .header(header::COOKIE, "__Host-ts-console=1") + .body(EdgeBody::empty()) + .expect("should build request"); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + assert!(decision.enabled()); + assert!(!decision.requires_private_no_store()); + assert_eq!(decision.bootstrap_script(), None); + } + + #[test] + fn invalid_api_query_fails_closed_even_with_session_cookie() { + let mut req = Request::builder() + .method(Method::POST) + .uri("https://publisher.example/auction?ts_console=invalid") + .header(header::COOKIE, "__Host-ts-console=1") + .body(EdgeBody::empty()) + .expect("should build request"); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + assert!(!decision.enabled()); + assert_eq!(req.uri().query(), None); + assert!(!req.headers().contains_key(header::COOKIE)); + } + + #[test] + fn duplicate_console_cookie_fails_closed_and_all_copies_are_removed() { + let mut req = request( + "https://publisher.example/", + Some("__Host-ts-console=1; a=b; __Host-ts-console=1"), + ); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + assert!(!decision.enabled()); + assert_eq!( + req.headers() + .get(header::COOKIE) + .expect("should retain unrelated cookie"), + "a=b" + ); + + let mut bare = request( + "https://publisher.example/", + Some("__Host-ts-console=1; __Host-ts-console; a=b"), + ); + let decision = prepare_request(&settings(true), &mut bare).expect("should prepare"); + assert!(!decision.enabled()); + assert_eq!( + bare.headers() + .get(header::COOKIE) + .expect("should retain unrelated cookie"), + "a=b" + ); + } + + #[test] + fn ts_tester_cookie_no_longer_activates_console() { + let mut req = request("https://publisher.example/", Some("ts-tester=true")); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + assert!(!decision.enabled()); + } + + #[test] + fn response_finalization_appends_cookie_once_and_reasserts_no_store() { + let mut req = request("https://publisher.example/?ts_console=1", None); + let decision = prepare_request(&settings(true), &mut req).expect("should prepare"); + let mut response = Response::builder() + .header(header::SET_COOKIE, "existing=value") + .header(header::CACHE_CONTROL, "public, max-age=60") + .header("surrogate-control", "max-age=60") + .header("cloudflare-cdn-cache-control", "public, max-age=60") + .body(EdgeBody::empty()) + .expect("should build response"); + attach_response_decision(&decision, &mut response); + + finalize_response(&mut response); + response.headers_mut().insert( + header::CACHE_CONTROL, + HeaderValue::from_static("public, max-age=60"), + ); + finalize_response(&mut response); + + assert_eq!( + response + .headers() + .get_all(header::SET_COOKIE) + .iter() + .count(), + 2 + ); + assert_eq!( + response.headers()[header::CACHE_CONTROL], + "private, no-store" + ); + assert!(!response.headers().contains_key("surrogate-control")); + assert!( + !response + .headers() + .contains_key("cloudflare-cdn-cache-control") + ); + } +} diff --git a/crates/trusted-server-core/src/integrations/aps.rs b/crates/trusted-server-core/src/integrations/aps.rs index 20e986156..5ce9f60a9 100644 --- a/crates/trusted-server-core/src/integrations/aps.rs +++ b/crates/trusted-server-core/src/integrations/aps.rs @@ -1262,6 +1262,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build downstream request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &downstream, timeout_ms: 321, @@ -1394,6 +1395,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build downstream request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &downstream, timeout_ms: 321, @@ -1477,6 +1479,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build downstream request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &downstream, timeout_ms: 321, diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 9abf556c8..51af415c8 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -58,6 +58,48 @@ pubads.__tsInitialLoadHooked = true; }); + function captureRequest(slot, trigger) { + function firstTarget(key) { + if (!slot || typeof slot.getTargeting !== "function") return undefined; + var values = slot.getTargeting(key); + return values && values.length ? String(values[0]) : undefined; + } + var divId = + slot && typeof slot.getSlotElementId === "function" + ? slot.getSlotElementId() + : ""; + var slotId = (ts.divToSlotId || {})[divId]; + var liveBid = slotId && ts.bids ? ts.bids[slotId] : undefined; + var bidSnapshot = liveBid + ? Object.freeze( + Object.assign({}, liveBid, { + trace: liveBid.trace ? Object.freeze(Object.assign({}, liveBid.trace)) : undefined, + }), + ) + : undefined; + // Freeze request-boundary attribution before display()/refresh(). If the + // optional module loads later, draining this queue never rereads mutable GPT + // targeting or the current route's bid object. + var snapshot = Object.freeze({ + slotId: slotId, + bidder: firstTarget("hb_bidder"), + adId: firstTarget("hb_adid"), + traceToken: firstTarget("ts_trace"), + bid: bidSnapshot, + }); + if (typeof ts.captureAdTraceRequest === "function") { + ts.captureAdTraceRequest(slot, trigger, snapshot); + return; + } + // The unified bundle may load after this bootstrap. Queue private request + // ownership unconditionally so trace-off traffic receives the same stale + // render and billing protection; diagnostic fields remain independently gated. + ts.pendingAdTraceRequests = ts.pendingAdTraceRequests || []; + if (ts.pendingAdTraceRequests.length < 64) { + ts.pendingAdTraceRequests.push({ slot: slot, trigger: trigger, snapshot: snapshot }); + } + } + ts.adInit = function () { var slots = ts.adSlots || []; var bids = ts.bids || {}; @@ -127,6 +169,9 @@ ].forEach(function (k) { if (b[k]) s.setTargeting(k, b[k]); }); + if (b.trace && b.trace.bidTraceId) { + s.setTargeting("ts_trace", b.trace.bidTraceId); + } // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts s.setTargeting("ts_initial", "1"); // Map both the inner div and the GPT slot's element ID (the @@ -159,6 +204,12 @@ // impression. Runs after enableServices(); on SPA navigation services are // already enabled, so this runs unconditionally for new slots. slotsToDisplay.forEach(function (divId) { + var requestSlot = newSlots.find(function (slot) { + return slot.getSlotElementId() === divId; + }); + if (requestSlot && !ts.gptInitialLoadDisabled) { + captureRequest(requestSlot, "bootstrap_display"); + } googletag.display(divId); }); // Reused publisher-owned slots always need a refresh to pick up the @@ -177,6 +228,9 @@ // bundle's adInit() in crates/trusted-server-js/lib/src/integrations/gpt/index.ts. ts.adInitRefreshInProgress = true; try { + slotsNeedingRefresh.forEach(function (slot) { + captureRequest(slot, "bootstrap_refresh"); + }); googletag.pubads().refresh(slotsNeedingRefresh); } finally { ts.adInitRefreshInProgress = false; diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 7bfaf27df..8d67f5c94 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -11,6 +11,7 @@ use crate::error::TrustedServerError; use crate::platform::{DEFAULT_FIRST_BYTE_TIMEOUT, PlatformBackendSpec, RuntimeServices}; use crate::settings::Settings; +pub mod ad_trace; pub mod adserver_mock; pub mod aps; pub mod datadome; @@ -292,6 +293,10 @@ pub(crate) fn builders() -> &'static [IntegrationBuilder] { id: "aps", build: aps::register, }, + IntegrationBuilder { + id: "ad_trace", + build: ad_trace::register, + }, IntegrationBuilder { id: "prebid", build: prebid::register, diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index 1d17bd0a2..e683cf214 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -2699,6 +2699,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &http_req, timeout_ms: 500, @@ -2742,6 +2743,7 @@ mod tests { .body(EdgeBody::empty()) .expect("should build request"); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &http_req, timeout_ms: 500, @@ -2763,6 +2765,11 @@ mod tests { request_host, auction_request.publisher.domain, "request_host should be the publisher domain, not the edge Host header" ); + assert!( + !String::from_utf8_lossy(&bodies[0]) + .contains(&context.trace.auction_trace_id.to_string()), + "internal trace UUID should never be serialized upstream" + ); } fn create_test_auction_context<'a>( @@ -5406,13 +5413,14 @@ external_bundle_sri = "sha384-AAAA" prebid_platform_response(StatusCode::BAD_REQUEST, Some("application/json"), body); let provider_response = futures::executor::block_on(provider.parse_response(response, 42)) .expect("should classify upstream HTTP error"); - let result = OrchestrationResult { - provider_responses: vec![provider_response], - mediator_response: None, - winning_bids: HashMap::new(), - total_time_ms: 42, - metadata: HashMap::new(), - }; + let mut result = OrchestrationResult::empty( + crate::auction::types::AuctionTraceContext::new( + crate::auction::types::AuctionSource::AuctionApi, + ), + crate::auction::types::AuctionPublicOutcome::NoBid, + ); + result.provider_responses = vec![provider_response]; + result.total_time_ms = 42; let response = convert_to_openrtb_response( &result, &make_settings(), @@ -5529,6 +5537,7 @@ external_bundle_sri = "sha384-AAAA" .expect("should build request"); let services = noop_services(); let context = AuctionContext { + trace: crate::auction::test_support::test_trace(), settings: &settings, request: &http_req, timeout_ms: 1000, diff --git a/crates/trusted-server-core/src/openrtb.rs b/crates/trusted-server-core/src/openrtb.rs index 9b7533505..dce748d46 100644 --- a/crates/trusted-server-core/src/openrtb.rs +++ b/crates/trusted-server-core/src/openrtb.rs @@ -180,16 +180,46 @@ impl ToExt for BidExt<'_> {} #[derive(Debug, Serialize)] pub struct BidTrustedServerExt<'a> { - pub renderer: &'a BidRenderer, + #[serde(skip_serializing_if = "Option::is_none")] + pub renderer: Option<&'a BidRenderer>, + #[serde(skip_serializing_if = "Option::is_none")] + pub trace: Option, } #[derive(Debug, Serialize)] pub struct ResponseExt { pub orchestrator: OrchestratorExt, + #[serde(skip_serializing_if = "Option::is_none")] + pub trusted_server: Option, } impl ToExt for ResponseExt {} +/// Namespaced Trusted Server response extensions. +#[derive(Debug, Serialize)] +pub struct TrustedServerResponseExt { + pub trace: AuctionTraceWire, +} + +/// Privacy-safe root trace extension. +#[derive(Debug, Serialize)] +pub struct AuctionTraceWire { + pub version: u8, + pub auction_trace_id: String, + pub source: &'static str, + pub outcome: &'static str, +} + +/// Privacy-safe final-winning-bid trace extension. +#[derive(Debug, Serialize)] +pub struct BidTraceWire { + pub version: u8, + pub bid_trace_id: String, + pub slot_id: String, + pub provider: String, + pub bidder: String, +} + #[cfg(test)] mod tests { use super::*; @@ -223,6 +253,7 @@ mod tests { time_ms: 12, provider_details: vec![], }, + trusted_server: None, } .to_ext(); diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 699dcbc97..f36487a5d 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -37,7 +37,7 @@ use crate::auction::endpoints::{ merge_auction_eids, resolve_auction_eids, resolve_client_auction_eids, }; use crate::auction::orchestrator::{ - AuctionOrchestrator, DispatchAuctionOutcome, DispatchedAuction, + AuctionOrchestrator, DispatchAuctionOutcome, DispatchedAuction, OrchestrationResult, }; use crate::auction::telemetry::{ AuctionObservationContext, AuctionSource, AuctionTerminalOutcome, build_auction_events, @@ -348,6 +348,7 @@ struct ProcessResponseParams<'a> { settings: &'a Settings, content_type: &'a str, integration_registry: &'a IntegrationRegistry, + head_bootstrap_script: Option<&'a str>, ad_slots_script: Option<&'a str>, ad_bids_state: &'a Arc>>, } @@ -372,8 +373,14 @@ impl PublisherBodyProcessor { ¶ms.request_scheme, settings, integration_registry, - params.ad_slots_script.as_deref().map(str::to_string), - Arc::clone(¶ms.ad_bids_state), + HtmlAdState { + head_bootstrap_script: params + .head_bootstrap_script + .as_deref() + .map(str::to_string), + ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), + ad_bids_state: Arc::clone(¶ms.ad_bids_state), + }, )?) } else if is_rsc_flight { Box::new(RscFlightUrlRewriter::new( @@ -441,8 +448,11 @@ fn process_response_streaming( params.request_scheme, params.settings, params.integration_registry, - params.ad_slots_script.map(str::to_string), - params.ad_bids_state.clone(), + HtmlAdState { + head_bootstrap_script: params.head_bootstrap_script.map(str::to_string), + ad_slots_script: params.ad_slots_script.map(str::to_string), + ad_bids_state: params.ad_bids_state.clone(), + }, )?; StreamingPipeline::new(config, processor).process(body_as_reader(body)?, output)?; } else if is_rsc_flight { @@ -767,12 +777,15 @@ async fn hold_collect_close_tail( collect_stream_auction( dispatched, state.telemetry.take(), - collect_refs.price_granularity, - collect_refs.ad_bids_state, - collect_refs.orchestrator, - collect_refs.services, - collect_refs.settings, - collect_refs.request_origin, + StreamAuctionFinalizeContext { + price_granularity: collect_refs.price_granularity, + ad_bids_state: collect_refs.ad_bids_state, + orchestrator: collect_refs.orchestrator, + services: collect_refs.services, + settings: collect_refs.settings, + request_origin: collect_refs.request_origin, + trace_enabled: collect_refs.trace_enabled, + }, ) .await; // Collection reached a terminal result; disarm only now so a drop while the @@ -903,14 +916,19 @@ async fn hold_finish_segments( /// `use<>` states that explicitly: without it, Rust 2024 would have the opaque /// type capture every input lifetime, forcing callers to keep the settings and /// registry alive for as long as the processor. +struct HtmlAdState { + head_bootstrap_script: Option, + ad_slots_script: Option, + ad_bids_state: Arc>>, +} + fn create_html_stream_processor( origin_host: &str, request_host: &str, request_scheme: &str, settings: &Settings, integration_registry: &IntegrationRegistry, - ad_slots_script: Option, - ad_bids_state: Arc>>, + ad_state: HtmlAdState, ) -> Result, Report> { use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; @@ -921,7 +939,11 @@ fn create_html_stream_processor( request_host, request_scheme, ) - .with_ad_state(ad_slots_script, ad_bids_state); + .with_ad_state( + ad_state.head_bootstrap_script, + ad_state.ad_slots_script, + ad_state.ad_bids_state, + ); Ok(create_html_processor(config)) } @@ -1030,6 +1052,7 @@ pub struct OwnedProcessResponseParams { pub(crate) request_host: String, pub(crate) request_scheme: String, pub(crate) content_type: String, + pub(crate) head_bootstrap_script: Option, pub(crate) ad_slots_script: Option, pub(crate) ad_bids_state: Arc>>, /// Observation context for the in-flight auction. @@ -1042,6 +1065,8 @@ pub struct OwnedProcessResponseParams { pub(crate) dispatched_auction: Option, /// Price granularity used to bucket bids when building `tsjs.bids`. pub(crate) price_granularity: PriceGranularity, + /// Whether the config and exact tester cookie permit browser trace output. + pub(crate) ad_trace_enabled: bool, } /// Buffers a [`PublisherResponse`] into a single [`Response`], collecting the @@ -1284,6 +1309,7 @@ pub async fn publisher_response_into_streaming_response( services: &services, settings: &settings, request_origin: &request_origin, + trace_enabled: params.ad_trace_enabled, }; while let Some(step) = hold_step_next_chunk( @@ -1470,6 +1496,7 @@ pub fn stream_publisher_body( settings, content_type: ¶ms.content_type, integration_registry, + head_bootstrap_script: params.head_bootstrap_script.as_deref(), ad_slots_script: params.ad_slots_script.as_deref(), ad_bids_state: ¶ms.ad_bids_state, }; @@ -1562,8 +1589,11 @@ pub async fn stream_publisher_body_async( ¶ms.request_scheme, settings, integration_registry, - params.ad_slots_script.as_deref().map(str::to_string), - params.ad_bids_state.clone(), + HtmlAdState { + head_bootstrap_script: params.head_bootstrap_script.as_deref().map(str::to_string), + ad_slots_script: params.ad_slots_script.as_deref().map(str::to_string), + ad_bids_state: params.ad_bids_state.clone(), + }, ) { Ok(processor) => processor, Err(err) => { @@ -1593,6 +1623,7 @@ pub async fn stream_publisher_body_async( services, settings, request_origin: request_origin(¶ms.request_scheme, ¶ms.request_host), + trace_enabled: params.ad_trace_enabled, }, ) .await @@ -1621,6 +1652,7 @@ fn mediator_placeholder_request() -> Request { /// this argument is plumbing for the (presently unused) case where the /// orchestrator needs the caller's request shape. fn make_collect_context<'a>( + trace: &'a crate::auction::types::AuctionTraceContext, settings: &'a Settings, services: &'a RuntimeServices, placeholder: &'a Request, @@ -1632,6 +1664,7 @@ fn make_collect_context<'a>( callers must not forward a real client request through the collect path" ); AuctionContext { + trace, settings, request: placeholder, timeout_ms: 0, @@ -1709,26 +1742,36 @@ fn request_origin(scheme: &str, host: &str) -> String { /// Write winning bids from an auction result into the shared `ad_bids_state` lock. pub(crate) fn write_bids_to_state( - winning_bids: &std::collections::HashMap, + result: &crate::auction::orchestrator::OrchestrationResult, price_granularity: PriceGranularity, ad_bids_state: &Arc>>, settings: &Settings, request_origin: &str, include_debug_bid: bool, + trace_enabled: bool, ) { log::debug!( "write_bids_to_state: {} winning bid(s): [{}]", - winning_bids.len(), - winning_bids.keys().cloned().collect::>().join(", ") + result.winning_bids.len(), + result + .winning_bids + .keys() + .cloned() + .collect::>() + .join(", ") ); - let bid_map = build_bid_map( - winning_bids, + let bid_map = build_bid_map_with_trace( + result, price_granularity, settings, request_origin, include_debug_bid, + trace_enabled, + ); + let bids_script = build_bids_script_with_trace( + &bid_map, + trace_enabled.then(|| auction_trace_json(&result.trace.summary)), ); - let bids_script = build_bids_script(&bid_map); *ad_bids_state.lock().expect("should lock bid state") = Some(bids_script); } @@ -1939,6 +1982,7 @@ struct AuctionCollectCtx<'a> { settings: &'a Settings, /// Trusted request origin (`scheme://host`) for absolute inline creative URLs. request_origin: String, + trace_enabled: bool, } struct AuctionHoldCollectRefs<'a> { @@ -1949,6 +1993,7 @@ struct AuctionHoldCollectRefs<'a> { settings: &'a Settings, /// Trusted request origin (`scheme://host`) for absolute inline creative URLs. request_origin: &'a str, + trace_enabled: bool, } /// Run the close-body hold loop for HTML bodies, collecting the auction before @@ -2038,6 +2083,7 @@ async fn body_close_hold_loop_stream( services, settings, request_origin, + trace_enabled, } = ctx; let mut decoder = BodyStreamDecoder::new(compression, max_body_bytes); let mut encoder = BodyStreamEncoder::new(compression); @@ -2050,6 +2096,7 @@ async fn body_close_hold_loop_stream( services, settings, request_origin: &request_origin, + trace_enabled, }; while let Some(step) = hold_step_next_chunk( @@ -2168,6 +2215,7 @@ async fn body_close_hold_loop( services, settings, request_origin, + trace_enabled, } = ctx; let mut buffer = vec![0u8; STREAM_CHUNK_SIZE]; let mut hold = Some(BodyCloseHoldBuffer::new()); @@ -2183,12 +2231,15 @@ async fn body_close_hold_loop( collect_stream_auction( dispatched, telemetry.take(), - price_granularity, - ad_bids_state, - orchestrator, - services, - settings, - &request_origin, + StreamAuctionFinalizeContext { + price_granularity, + ad_bids_state, + orchestrator, + services, + settings, + request_origin: &request_origin, + trace_enabled, + }, ) .await; @@ -2247,12 +2298,15 @@ async fn body_close_hold_loop( collect_stream_auction( dispatched, telemetry.take(), - price_granularity, - ad_bids_state, - orchestrator, - services, - settings, - &request_origin, + StreamAuctionFinalizeContext { + price_granularity, + ad_bids_state, + orchestrator, + services, + settings, + request_origin: &request_origin, + trace_enabled, + }, ) .await; @@ -2340,11 +2394,12 @@ async fn collect_non_html_auction( settings: &Settings, ) { let placeholder = mediator_placeholder_request(); + let trace = dispatched.trace().clone(); let result = orchestrator .collect_dispatched_auction( dispatched, services, - &make_collect_context(settings, services, &placeholder), + &make_collect_context(&trace, settings, services, &placeholder), ) .await; if let (Some(observation), Some(auction_request)) = @@ -2362,32 +2417,44 @@ async fn collect_non_html_auction( .await; } write_bids_to_state( - &result.winning_bids, + &result, params.price_granularity, ¶ms.ad_bids_state, settings, &request_origin(¶ms.request_scheme, ¶ms.request_host), settings.debug.inject_adm_for_testing, + params.ad_trace_enabled, ); } -// Private orchestration helper called only from `body_close_hold_loop`, whose -// arguments mirror the fields of `AuctionCollectCtx` it destructures; a separate -// parameter struct would just duplicate that context. -#[allow(clippy::too_many_arguments)] +struct StreamAuctionFinalizeContext<'a> { + price_granularity: PriceGranularity, + ad_bids_state: &'a Arc>>, + orchestrator: &'a AuctionOrchestrator, + services: &'a RuntimeServices, + settings: &'a Settings, + request_origin: &'a str, + trace_enabled: bool, +} + async fn collect_stream_auction( dispatched: DispatchedAuction, telemetry: AuctionTelemetryCarry, - price_granularity: PriceGranularity, - ad_bids_state: &Arc>>, - orchestrator: &AuctionOrchestrator, - services: &RuntimeServices, - settings: &Settings, - request_origin: &str, + context: StreamAuctionFinalizeContext<'_>, ) { + let StreamAuctionFinalizeContext { + price_granularity, + ad_bids_state, + orchestrator, + services, + settings, + request_origin, + trace_enabled, + } = context; log::info!("body_close_hold_loop: collecting dispatched auction before held body tail"); let placeholder = mediator_placeholder_request(); - let collect_ctx = make_collect_context(settings, services, &placeholder); + let trace = dispatched.trace().clone(); + let collect_ctx = make_collect_context(&trace, settings, services, &placeholder); let result = orchestrator .collect_dispatched_auction(dispatched, services, &collect_ctx) .await; @@ -2410,12 +2477,13 @@ async fn collect_stream_auction( result.winning_bids.len() ); write_bids_to_state( - &result.winning_bids, + &result, price_granularity, ad_bids_state, settings, request_origin, settings.debug.inject_adm_for_testing, + trace_enabled, ); if settings.debug.auction_html_comment { @@ -2520,6 +2588,9 @@ pub async fn handle_publisher_request( ); let consent_context = ec_context.consent().clone(); + let ad_trace_decision = crate::integrations::ad_trace::request_decision(&req); + let ad_trace_enabled = ad_trace_decision.enabled(); + let ad_trace_bootstrap = ad_trace_decision.bootstrap_script(); let ec_id = ec_context.ec_value().filter(|_| ec_allowed); let cookie_jar = handle_request_cookies(&req)?; let geo = ec_context.geo_info().cloned(); @@ -2638,13 +2709,15 @@ pub async fn handle_publisher_request( let mut dispatched_auction = if matched_slots.is_empty() { None } else { + let trace = + crate::auction::types::AuctionTraceContext::new(AuctionSource::InitialNavigation); // Telemetry attribution must use the same publisher identity as the // outbound bid request. On the navigation path `request_host` is the // trusted-server edge host, so using it here would attribute navigation // rows to the edge/staging domain while `/auction` rows (built from // `AuctionRequest::publisher.domain`) use the configured domain. let observation = AuctionObservationContext::from_parts( - AuctionSource::InitialNavigation, + &trace, &settings.publisher.domain, &request_path, matched_slots.len(), @@ -2680,6 +2753,7 @@ pub async fn handle_publisher_request( }, ); let auction_context = AuctionContext { + trace: &trace, settings, request: &req, timeout_ms: auction_timeout_ms, @@ -2701,6 +2775,21 @@ pub async fn handle_publisher_request( provider_responses, elapsed_ms, } => { + if ad_trace_enabled { + let terminal = OrchestrationResult::empty( + trace.clone(), + crate::auction::types::AuctionPublicOutcome::Failed, + ); + write_bids_to_state( + &terminal, + price_granularity, + &ad_bids_state, + settings, + &request_origin(request_scheme, request_host), + settings.debug.inject_adm_for_testing, + true, + ); + } emit_auction_events_best_effort_lazy(services, || { build_auction_events( observation, @@ -2716,6 +2805,21 @@ pub async fn handle_publisher_request( None } DispatchAuctionOutcome::NotStarted => { + if ad_trace_enabled { + let terminal = OrchestrationResult::empty( + trace.clone(), + crate::auction::types::AuctionPublicOutcome::Failed, + ); + write_bids_to_state( + &terminal, + price_granularity, + &ad_bids_state, + settings, + &request_origin(request_scheme, request_host), + settings.debug.inject_adm_for_testing, + true, + ); + } let elapsed_ms = observation.elapsed_ms(); emit_auction_events_best_effort_lazy(services, || { build_auction_events( @@ -2989,12 +3093,14 @@ pub async fn handle_publisher_request( request_host: request_host.to_string(), request_scheme: request_scheme.to_string(), content_type, + head_bootstrap_script: ad_trace_bootstrap.clone(), ad_slots_script: ad_slots_script.clone(), ad_bids_state: ad_bids_state.clone(), auction_observation, auction_request: auction_request_for_telemetry, dispatched_auction, price_granularity, + ad_trace_enabled, }), }) } @@ -3307,14 +3413,80 @@ pub(crate) fn build_bid_map( .collect() } +fn auction_trace_json(summary: &crate::auction::types::AuctionTraceSummary) -> serde_json::Value { + serde_json::json!({ + "version": 1, + "auctionTraceId": summary.auction.auction_trace_id.to_string(), + "source": summary.auction.source.as_str(), + "outcome": summary.outcome.as_str(), + }) +} + +fn apply_bid_traces( + bid_map: &mut serde_json::Map, + result_trace: &crate::auction::types::AuctionResultTrace, +) { + for (slot_id, trace) in &result_trace.winning_bids { + if let Some(serde_json::Value::Object(bid)) = bid_map.get_mut(slot_id) { + bid.insert( + "trace".to_owned(), + serde_json::json!({ + "version": 1, + "auctionTraceId": result_trace.summary.auction.auction_trace_id.to_string(), + "bidTraceId": trace.bid_trace_id.to_string(), + "source": result_trace.summary.auction.source.as_str(), + "slotId": slot_id, + "provider": trace.provider, + "bidder": trace.bidder, + }), + ); + } + } +} + +fn build_bid_map_with_trace( + result: &crate::auction::orchestrator::OrchestrationResult, + granularity: crate::price_bucket::PriceGranularity, + settings: &Settings, + request_origin: &str, + include_debug_bid: bool, + trace_enabled: bool, +) -> serde_json::Map { + let mut bid_map = build_bid_map( + &result.winning_bids, + granularity, + settings, + request_origin, + include_debug_bid, + ); + if !trace_enabled { + return bid_map; + } + apply_bid_traces(&mut bid_map, &result.trace); + bid_map +} + /// Build the `tsjs.bids` `` sequences inside the string. pub(crate) fn build_bids_script(bid_map: &serde_json::Map) -> String { + build_bids_script_with_trace(bid_map, None) +} + +fn build_bids_script_with_trace( + bid_map: &serde_json::Map, + auction_trace: Option, +) -> String { let json = serde_json::to_string(bid_map) .expect("serde_json::to_string of Map should be infallible"); let escaped = html_escape_for_script(&json); + let trace_assignment = auction_trace.map_or_else(String::new, |trace| { + let trace_json = serde_json::to_string(&trace) + .expect("serde_json::to_string of trace should be infallible"); + let escaped_trace = html_escape_for_script(&trace_json); + format!("window.tsjs.auctionTrace=JSON.parse(\"{escaped_trace}\");") + }); // adInit() defines GPT slots on the publisher's `-container` wrappers, which // mutates those ad-slot subtrees. Calling it synchronously here (this script // runs at body-parse time) lands those mutations inside React's hydration @@ -3326,11 +3498,10 @@ pub(crate) fn build_bids_script(bid_map: &serde_json::Map(window.tsjs=window.tsjs||{{}}).bids=JSON.parse(\"{}\");\ + "", - escaped +}})();" ) } @@ -3556,6 +3726,7 @@ pub async fn handle_page_bids( ); return Ok(page_bids_preflight_denied()); } + let trace_enabled = crate::integrations::ad_trace::browser_trace_enabled(&req); let requested_page = req .uri() @@ -3617,13 +3788,21 @@ pub async fn handle_page_bids( // skip the live auction, matching the existing bot/prefetch behaviour. let ad_stack_enabled = auction_enabled && consent_allows_auction; - let winning_bids = if matched_slots.is_empty() { - std::collections::HashMap::new() + let trace = crate::auction::types::AuctionTraceContext::new(AuctionSource::SpaNavigation); + let mut result_trace = crate::auction::types::AuctionResultTrace { + summary: crate::auction::types::AuctionTraceSummary { + auction: trace.clone(), + outcome: crate::auction::types::AuctionPublicOutcome::Skipped, + }, + winning_bids: std::collections::HashMap::new(), + }; + let completed_result: Option = if matched_slots.is_empty() { + None } else { // Same publisher identity as the outbound bid request — see the // matching note on the initial-navigation observation above. let observation = AuctionObservationContext::from_parts( - AuctionSource::SpaNavigation, + &trace, &settings.publisher.domain, &path_param, matched_slots.len(), @@ -3661,6 +3840,7 @@ pub async fn handle_page_bids( .auction_timeout_ms .unwrap_or(settings.auction.timeout_ms); let auction_context = AuctionContext { + trace: &trace, settings, request: &req, timeout_ms, @@ -3673,7 +3853,7 @@ pub async fn handle_page_bids( .await { Ok(result) => { - let winning_bids = result.winning_bids.clone(); + result_trace = result.trace.clone(); emit_auction_events_best_effort_lazy(services, || { build_auction_events( observation, @@ -3684,10 +3864,12 @@ pub async fn handle_page_bids( ) }) .await; - winning_bids + Some(result) } Err(e) => { log::warn!("page-bids auction failed: {e:?}"); + result_trace.summary.outcome = + crate::auction::types::AuctionPublicOutcome::Failed; let elapsed_ms = observation.elapsed_ms(); emit_auction_events_best_effort_lazy(services, || { build_auction_events( @@ -3701,7 +3883,7 @@ pub async fn handle_page_bids( ) }) .await; - std::collections::HashMap::new() + None } } } else { @@ -3727,17 +3909,25 @@ pub async fn handle_page_bids( ) }) .await; - std::collections::HashMap::new() + None } }; - let bid_map = build_bid_map( + let winning_bids = completed_result + .as_ref() + .map(|result| &result.winning_bids) + .cloned() + .unwrap_or_default(); + let mut bid_map = build_bid_map( &winning_bids, co_config.price_granularity, settings, &page_bids_request_origin, settings.debug.inject_adm_for_testing, ); + if trace_enabled { + apply_bid_traces(&mut bid_map, &result_trace); + } // Gate slots on the ad-stack kill switch / consent: when disabled, return no // slots so the SPA hook does not call `adInit()` / create GPT slots. @@ -3750,10 +3940,13 @@ pub async fn handle_page_bids( Vec::new() }; - let body = serde_json::json!({ + let mut body = serde_json::json!({ "slots": slots_json, "bids": bid_map, }); + if trace_enabled && !matched_slots.is_empty() { + body["auctionTrace"] = auction_trace_json(&result_trace.summary); + } let json_str = serde_json::to_string(&body).change_context(TrustedServerError::Proxy { message: "Failed to serialize page-bids response".to_string(), @@ -3825,16 +4018,15 @@ mod tests { fn dump_comment_for_creative(creative: &str) -> String { let mut bid = make_test_bid_with_creative(creative); bid.slot_id = "ad-header-0".to_string(); - let result = OrchestrationResult { - provider_responses: vec![ - AuctionResponse::no_bid("prebid", 665), - AuctionResponse::success("aps", vec![bid], 42), - ], - mediator_response: None, - winning_bids: std::collections::HashMap::new(), - total_time_ms: 665, - metadata: std::collections::HashMap::new(), - }; + let mut result = OrchestrationResult::empty( + crate::auction::types::AuctionTraceContext::new(AuctionSource::InitialNavigation), + crate::auction::types::AuctionPublicOutcome::NoBid, + ); + result.provider_responses = vec![ + AuctionResponse::no_bid("prebid", 665), + AuctionResponse::success("aps", vec![bid], 42), + ]; + result.total_time_ms = 665; let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); prepend_auction_debug_comment("stream", &result, &state); let comment = state @@ -3889,13 +4081,12 @@ mod tests { ) // An allowlisted key must still survive. .with_metadata("error_type", serde_json::json!("http_status")); - let result = OrchestrationResult { - provider_responses: vec![response], - mediator_response: None, - winning_bids: std::collections::HashMap::new(), - total_time_ms: 12, - metadata: std::collections::HashMap::new(), - }; + let mut result = OrchestrationResult::empty( + crate::auction::types::AuctionTraceContext::new(AuctionSource::InitialNavigation), + crate::auction::types::AuctionPublicOutcome::NoBid, + ); + result.provider_responses = vec![response]; + result.total_time_ms = 12; let state = Arc::new(Mutex::new(Some("BIDS_SCRIPT".to_string()))); prepend_auction_debug_comment("stream", &result, &state); let comment = state @@ -4070,12 +4261,14 @@ mod tests { request_host: settings.publisher.domain.clone(), request_scheme: "https".to_owned(), content_type: "application/json".to_owned(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: std::sync::Arc::new(std::sync::Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: Default::default(), + ad_trace_enabled: false, } } @@ -5100,6 +5293,7 @@ mod tests { services: &services, settings: &settings, request_origin: String::new(), + trace_enabled: false, }; let mut output = Vec::new(); @@ -5148,6 +5342,7 @@ mod tests { services: &services, settings: &settings, request_origin: "", + trace_enabled: false, }; // Passthrough processor: the ordering contract is about collection, not // HTML rewriting, so keep the emitted bytes verbatim. @@ -5819,12 +6014,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/css".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let mut output = Vec::new(); @@ -5866,12 +6063,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let mut output = Vec::new(); @@ -5902,12 +6101,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let body = EdgeBody::from_stream(futures::stream::iter(vec![Ok::<_, io::Error>( bytes::Bytes::from_static(b"live"), @@ -6016,12 +6217,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/css".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"body{background:url('https://origin.example.com/"), @@ -6068,12 +6271,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/css".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let compressed = gzip_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6123,12 +6328,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/css".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let compressed = deflate_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6178,12 +6385,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/css".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6233,12 +6442,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/css".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let compressed = brotli_encode(b"body{background:url('https://origin.example.com/asset.png')}"); @@ -6276,12 +6487,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/css".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, } } @@ -6463,6 +6676,7 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), + head_bootstrap_script: None, ad_slots_script: Some( r#""# .to_string(), @@ -6475,6 +6689,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![ bytes::Bytes::from_static(b"hello"), @@ -6525,6 +6740,7 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/css".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, @@ -6534,6 +6750,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let body = EdgeBody::stream(futures::stream::iter(vec![bytes::Bytes::from_static( b"body{background:url('https://origin.example.com/asset.png')}", @@ -6583,12 +6800,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/css".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let publisher_response = PublisherResponse::Stream { response, @@ -6806,10 +7025,11 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: Some(AuctionObservationContext::from_parts( - AuctionSource::SpaNavigation, + &crate::auction::types::AuctionTraceContext::new(AuctionSource::SpaNavigation), "proxy.example.com", "/article", 1, @@ -6821,6 +7041,7 @@ mod tests { 10, )), price_granularity: PriceGranularity::default(), + ad_trace_enabled: false, } }; let make_stream_response = || PublisherResponse::Stream { @@ -6987,6 +7208,7 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), + head_bootstrap_script: None, ad_slots_script: Some( r#""# .to_string(), @@ -6999,6 +7221,7 @@ mod tests { 10, )), price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let publisher_response = PublisherResponse::Stream { response, @@ -7056,6 +7279,7 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "Text/HTML; Charset=utf-8".to_string(), + head_bootstrap_script: None, ad_slots_script: Some( r#""# .to_string(), @@ -7065,6 +7289,7 @@ mod tests { auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let mut output = Vec::new(); @@ -7108,12 +7333,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let bogus_body = EdgeBody::from(b"not gzip".to_vec()); @@ -7215,12 +7442,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html; charset=utf-8".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let mut output = Vec::new(); stream_publisher_body(body, &mut output, ¶ms, &settings, ®istry) @@ -7271,12 +7500,14 @@ mod tests { request_host: "proxy.example.com".to_string(), request_scheme: "https".to_string(), content_type: "text/html".to_string(), + head_bootstrap_script: None, ad_slots_script: None, ad_bids_state: Arc::new(Mutex::new(None)), auction_observation: None, auction_request: None, dispatched_auction: None, price_granularity: crate::price_bucket::PriceGranularity::default(), + ad_trace_enabled: false, }; let mut output = Vec::new(); @@ -7308,7 +7539,7 @@ mod tests { mod creative_opportunities_tests { use super::super::{ MatchedSlotsContext, build_ad_slots_script, build_auction_request, build_bid_map, - build_bids_script, html_escape_for_script, + build_bids_script, build_bids_script_with_trace, html_escape_for_script, }; use crate::auction::types::{ApsRendererV1, ApsTagType, Bid, BidRenderer, MediaType}; use crate::consent::ConsentContext; @@ -8164,6 +8395,30 @@ mod tests { assert!(!inner.contains('>'), "no unescaped > in bids script"); } + #[test] + fn traced_bids_script_assigns_summary_and_bids_before_ad_init() { + let mut map = serde_json::Map::new(); + map.insert("atf".to_string(), serde_json::json!({"hb_pb": "1.00"})); + let trace = serde_json::json!({ + "version": 1, + "auctionTraceId": "550e8400-e29b-41d4-a716-446655440000", + "source": "initial_navigation", + "outcome": "completed", + }); + + let script = build_bids_script_with_trace(&map, Some(trace)); + + let trace_pos = script + .find(".auctionTrace=JSON.parse") + .expect("should assign trace"); + let bids_pos = script.find(".bids=JSON.parse").expect("should assign bids"); + let init_pos = script.find("adInit").expect("should invoke adInit"); + assert!( + trace_pos < bids_pos && bids_pos < init_pos, + "should atomically assign trace and bids before adInit" + ); + } + #[test] fn bids_script_calls_ad_init_without_retry_timer() { let mut map = serde_json::Map::new(); @@ -8515,8 +8770,10 @@ mod tests { orchestrator: &AuctionOrchestrator, slots: &[CreativeOpportunitySlot], ec_context: &EcContext, - req: Request, + mut req: Request, ) -> Response { + crate::integrations::ad_trace::prepare_request(settings, &mut req) + .expect("should prepare ad trace request"); let services = noop_services(); handle_page_bids( settings, @@ -8700,11 +8957,17 @@ mod tests { #[tokio::test] async fn url_not_matching_any_pattern_returns_empty_response() { - // Slots exist but request path does not match — no auction, no injection. - let settings = settings_with_co(); + // Slots exist but request path does not match — no auction, no injection, + // and no unjoinable trace identity even when the tester gate is open. + let mut settings = settings_with_co(); + settings + .integrations + .insert_config("ad_trace", &serde_json::json!({ "enabled": true })) + .expect("should configure ad trace"); let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); let slots = article_slot(); // slot matches /20** only - let req = make_page_bids_request("/about"); // does not match + let mut req = make_page_bids_request("/about"); // does not match + set_test_header(&mut req, "cookie", "__Host-ts-console=1"); let body = run_page_bids(&settings, &orchestrator, &slots, req).await; @@ -8724,6 +8987,45 @@ mod tests { 0, "non-matching URL should produce zero bids" ); + assert!( + body.get("auctionTrace").is_none(), + "non-matching URL should not expose an identity without telemetry" + ); + } + + #[tokio::test] + async fn page_bids_trace_requires_config_and_console_session() { + let mut settings = settings_with_co_auction_disabled(); + settings + .integrations + .insert_config("ad_trace", &serde_json::json!({ "enabled": true })) + .expect("should configure ad trace"); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + let slots = article_slot(); + + let without_cookie = make_page_bids_request("/2024/01/my-article/"); + let without_cookie_body = + run_page_bids_consent_allowed(&settings, &orchestrator, &slots, without_cookie) + .await; + assert!( + without_cookie_body.get("auctionTrace").is_none(), + "config alone should not expose trace" + ); + + let mut gated = make_page_bids_request("/2024/01/my-article/"); + set_test_header(&mut gated, "cookie", "__Host-ts-console=1"); + let gated_body = + run_page_bids_consent_allowed(&settings, &orchestrator, &slots, gated).await; + assert_eq!( + gated_body["auctionTrace"]["source"], + serde_json::json!("spa_navigation"), + "both gates should expose generic SPA trace" + ); + assert_eq!( + gated_body["auctionTrace"]["outcome"], + serde_json::json!("skipped"), + "disabled auction should not be fabricated as completed no-bid" + ); } #[test] diff --git a/crates/trusted-server-core/src/response_privacy.rs b/crates/trusted-server-core/src/response_privacy.rs index 27a94b62d..262d3fc95 100644 --- a/crates/trusted-server-core/src/response_privacy.rs +++ b/crates/trusted-server-core/src/response_privacy.rs @@ -17,7 +17,12 @@ use crate::settings::Settings; /// /// A single source of truth so the adapter copies of the privacy downgrade /// cannot drift apart. -pub const SURROGATE_CACHE_HEADERS: &[&str] = &["surrogate-control", "fastly-surrogate-control"]; +pub const SURROGATE_CACHE_HEADERS: &[&str] = &[ + "surrogate-control", + "fastly-surrogate-control", + "cdn-cache-control", + "cloudflare-cdn-cache-control", +]; /// Forces cookie-bearing responses to stay private to shared caches. /// @@ -82,8 +87,9 @@ pub fn apply_response_headers_with_cache_privacy(settings: &Settings, response: for (key, value) in &settings.response_headers { if response_is_uncacheable && (key.eq_ignore_ascii_case(header::CACHE_CONTROL.as_str()) - || key.eq_ignore_ascii_case("surrogate-control") - || key.eq_ignore_ascii_case("fastly-surrogate-control")) + || SURROGATE_CACHE_HEADERS + .iter() + .any(|name| key.eq_ignore_ascii_case(name))) { continue; } diff --git a/crates/trusted-server-integration-tests/browser/global-setup.ts b/crates/trusted-server-integration-tests/browser/global-setup.ts index f54d92dbe..14b3b0539 100644 --- a/crates/trusted-server-integration-tests/browser/global-setup.ts +++ b/crates/trusted-server-integration-tests/browser/global-setup.ts @@ -16,12 +16,11 @@ const WASM_PATH = "../../../target/wasm32-wasip1/release/trusted-server-adapter-fastly.wasm", ); -const VICEROY_CONFIG = - process.env.VICEROY_CONFIG_PATH || - resolve( - __dirname, - "../../../target/integration-test-artifacts/configs/viceroy.toml", - ); +function viceroyConfigPath(framework: string): string { + if (process.env.VICEROY_CONFIG_PATH) return process.env.VICEROY_CONFIG_PATH; + const filename = framework === "ad-trace" ? "viceroy-ad-trace.toml" : "viceroy.toml"; + return resolve(__dirname, `../../../target/integration-test-artifacts/configs/${filename}`); +} /** Persist current state so global-teardown can always clean up. */ function writeState(state: { @@ -47,7 +46,7 @@ async function globalSetup(): Promise { writeState({ containerId, framework }); console.log(`[global-setup] Starting Viceroy (WASM: ${WASM_PATH})...`); - const viceroy = await startViceroy(WASM_PATH, VICEROY_CONFIG); + const viceroy = await startViceroy(WASM_PATH, viceroyConfigPath(framework)); viceroyPid = viceroy.process.pid; console.log(`[global-setup] Viceroy ready at ${viceroy.baseUrl}`); diff --git a/crates/trusted-server-integration-tests/browser/helpers/infra.ts b/crates/trusted-server-integration-tests/browser/helpers/infra.ts index 0402bb266..1b7682b6b 100644 --- a/crates/trusted-server-integration-tests/browser/helpers/infra.ts +++ b/crates/trusted-server-integration-tests/browser/helpers/infra.ts @@ -7,6 +7,7 @@ const ORIGIN_PORT = process.env.INTEGRATION_ORIGIN_PORT || "8888"; /** Framework-specific container configuration. */ const FRAMEWORK_CONFIG: Record = { + "ad-trace": { image: "test-ad-trace:latest", port: 80 }, nextjs: { image: "test-nextjs:latest", port: 3000 }, wordpress: { image: "test-wordpress:latest", port: 80 }, }; diff --git a/crates/trusted-server-integration-tests/browser/helpers/state.ts b/crates/trusted-server-integration-tests/browser/helpers/state.ts index b8f5d4b66..dd655d01c 100644 --- a/crates/trusted-server-integration-tests/browser/helpers/state.ts +++ b/crates/trusted-server-integration-tests/browser/helpers/state.ts @@ -8,7 +8,7 @@ export interface TestState { framework: string; } -const KNOWN_FRAMEWORKS = ["nextjs", "wordpress"] as const; +const KNOWN_FRAMEWORKS = ["ad-trace", "nextjs", "wordpress"] as const; const STATE_FILE = resolve(__dirname, "../.browser-test-state.json"); let cachedState: TestState | undefined; diff --git a/crates/trusted-server-integration-tests/browser/package.json b/crates/trusted-server-integration-tests/browser/package.json index 13282f289..72b3fb997 100644 --- a/crates/trusted-server-integration-tests/browser/package.json +++ b/crates/trusted-server-integration-tests/browser/package.json @@ -4,6 +4,7 @@ "private": true, "scripts": { "test": "npx playwright test", + "test:ad-trace": "TEST_FRAMEWORK=ad-trace npx playwright test tests/ad-trace/auction-trace.spec.ts", "test:nextjs": "TEST_FRAMEWORK=nextjs npx playwright test", "test:wordpress": "TEST_FRAMEWORK=wordpress npx playwright test" }, diff --git a/crates/trusted-server-integration-tests/browser/playwright.config.ts b/crates/trusted-server-integration-tests/browser/playwright.config.ts index 8a1ef3b5b..812c889ec 100644 --- a/crates/trusted-server-integration-tests/browser/playwright.config.ts +++ b/crates/trusted-server-integration-tests/browser/playwright.config.ts @@ -1,7 +1,13 @@ import { defineConfig } from "@playwright/test"; +const framework = process.env.TEST_FRAMEWORK || "nextjs"; + export default defineConfig({ testDir: "./tests", + testMatch: + framework === "ad-trace" + ? ["ad-trace/**/*.spec.ts"] + : ["nextjs/**/*.spec.ts", "shared/**/*.spec.ts", "wordpress/**/*.spec.ts"], globalSetup: "./global-setup.ts", globalTeardown: "./global-teardown.ts", timeout: 30_000, @@ -20,5 +26,5 @@ export default defineConfig({ }, ], reporter: [["list"], ["html", { open: "never" }]], - outputDir: "./test-results", + outputDir: `./test-results-${framework}`, }); diff --git a/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts b/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts new file mode 100644 index 000000000..9c2935d07 --- /dev/null +++ b/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts @@ -0,0 +1,439 @@ +import { expect, test, type Page } from "@playwright/test"; +import { runtimeUrl } from "../../helpers/state.js"; + +const ORIGIN_PORT = process.env.INTEGRATION_ORIGIN_PORT || "8888"; + +async function serveBuiltPrebid(page: Page): Promise { + const response = await fetch( + `http://127.0.0.1:${ORIGIN_PORT}/prebid-bundle.js`, + ); + if (!response.ok) + throw new Error(`fixture Prebid bundle returned ${response.status}`); + const body = await response.text(); + await page.route("**/integrations/prebid/bundle.js*", (route) => + route.fulfill({ + status: 200, + contentType: "application/javascript", + body, + }), + ); +} + +async function openTesterPage(page: Page): Promise { + await serveBuiltPrebid(page); + await page.goto(runtimeUrl("/?ts_console=1"), { + waitUntil: "domcontentloaded", + }); + await expect(page).toHaveURL(runtimeUrl("/")); + await expect + .poll(() => + page.evaluate(() => + ( + window as Window & { + tsjs?: { adTrace?: { export(): unknown } }; + } + ).tsjs?.adTrace?.export(), + ), + ) + .toBeTruthy(); + await expect + .poll(() => + page.evaluate(() => { + const result = ( + window as Window & { + tsjs?: { + adTrace?: { + export(): { + slots: Array<{ + slotId: string; + stages: { + creative: { outcome: string }; + }; + }>; + }; + }; + }; + } + ).tsjs?.adTrace?.export(); + return result?.slots.find( + (slot) => slot.slotId === "ad-trace-slot", + )?.stages.creative.outcome; + }), + ) + .toBe("load_acknowledged"); + await expect + .poll(() => + page.evaluate(() => + ( + window as Window & { + tsjs?: { + adTrace?: { + getEvents(): Array<{ kind: string }>; + }; + }; + } + ).tsjs?.adTrace + ?.getEvents() + .some((event) => event.kind === "gpt_slot_render_ended"), + ), + ) + .toBe(true); +} + +async function exported(page: Page) { + return page.evaluate(() => + ( + window as Window & { + tsjs: { + adTrace: { + export(): { slots: Array> }; + }; + }; + } + ).tsjs.adTrace.export(), + ); +} + +test.describe("tester-only auction trace contract", () => { + test("config without an activated console session exposes no browser trace surface", async ({ + page, + }) => { + await serveBuiltPrebid(page); + await page.goto(runtimeUrl("/"), { waitUntil: "domcontentloaded" }); + + expect( + await page.evaluate( + () => + typeof (window as Window & { tsjs?: { adTrace?: unknown } }) + .tsjs?.adTrace, + ), + ).toBe("undefined"); + }); + + test("console session supports true, persists privately, and can be disabled", async ({ + page, + }) => { + await serveBuiltPrebid(page); + const activation = await page.goto(runtimeUrl("/?ts_console=true"), { + waitUntil: "domcontentloaded", + }); + await expect(page).toHaveURL(runtimeUrl("/")); + expect(activation?.headers()["cache-control"]).toBe("private, no-store"); + await expect + .poll(() => + page.evaluate( + () => + typeof ( + window as Window & { tsjs?: { adTrace?: unknown } } + ).tsjs?.adTrace, + ), + ) + .toBe("object"); + expect( + (await page.context().cookies()).find( + (cookie) => cookie.name === "__Host-ts-console", + ), + ).toMatchObject({ + value: "1", + httpOnly: true, + secure: true, + sameSite: "Lax", + }); + + await page.reload({ waitUntil: "domcontentloaded" }); + expect( + await page.evaluate( + () => + typeof (window as Window & { tsjs?: { adTrace?: unknown } }) + .tsjs?.adTrace, + ), + ).toBe("object"); + + await page.goto(runtimeUrl("/?ts_console=0"), { + waitUntil: "domcontentloaded", + }); + await expect(page).toHaveURL(runtimeUrl("/")); + expect( + await page.evaluate( + () => + typeof (window as Window & { tsjs?: { adTrace?: unknown } }) + .tsjs?.adTrace, + ), + ).toBe("undefined"); + + await page.reload({ waitUntil: "domcontentloaded" }); + expect( + await page.evaluate( + () => + typeof (window as Window & { tsjs?: { adTrace?: unknown } }) + .tsjs?.adTrace, + ), + ).toBe("undefined"); + }); + + test("initial TS winner reaches direct GPT and source-validated creative acknowledgement", async ({ + page, + }) => { + await openTesterPage(page); + + await expect + .poll(async () => { + const result = await exported(page); + const slot = result.slots.find( + (item) => item.slotId === "ad-trace-slot", + ) as + | { + stages?: Record< + string, + { outcome?: string; confidence?: string } + >; + } + | undefined; + return { + trustedServer: slot?.stages?.trustedServer?.outcome, + prebid: slot?.stages?.prebid?.outcome, + gam: slot?.stages?.gam?.outcome, + creative: slot?.stages?.creative?.outcome, + }; + }) + .toEqual({ + trustedServer: "won", + prebid: "not_run", + gam: "trusted_server_won", + creative: "load_acknowledged", + }); + + const session = await page.context().newCDPSession(page); + const tree = (await session.send("Accessibility.getFullAXTree")) as { + nodes: Array<{ name?: { value?: string } }>; + }; + const visibleText = tree.nodes + .map((node) => node.name?.value || "") + .join("\n"); + expect(visibleText).toContain("TS winner: won · definitive"); + expect(visibleText).toContain( + "Creative: load_acknowledged · definitive", + ); + }); + + test("direct auction API render reaches an exact iframe-load acknowledgement", async ({ + page, + }) => { + await openTesterPage(page); + await page.evaluate(() => { + const direct = document.createElement("div"); + direct.id = "direct-api-slot"; + document.body.appendChild(direct); + const ts = (window as Window & { + tsjs: { + addAdUnits(unit: unknown): void; + requestAds(): void; + }; + }).tsjs; + ts.addAdUnits({ + code: "direct-api-slot", + mediaTypes: { banner: { sizes: [[300, 250]] } }, + bids: [{ bidder: "example", params: {} }], + }); + ts.requestAds(); + }); + + await expect + .poll(() => + page.evaluate(() => { + const result = ( + window as Window & { + tsjs: { + adTrace: { + export(): { + renders: Array<{ + slotId: string; + source: string; + outcome: string; + }>; + }; + }; + }; + } + ).tsjs.adTrace.export(); + return result.renders.find( + (render) => render.slotId === "direct-api-slot", + ); + }), + ) + .toMatchObject({ + slotId: "direct-api-slot", + source: "direct_auction", + outcome: "confirmed", + }); + }); + + test("actual generated Prebid selects the traced TS bid before a probable GAM result", async ({ + page, + }) => { + await openTesterPage(page); + await expect + .poll(() => + page.evaluate(() => { + const win = window as Window & { + pbjs?: { requestBids?: unknown }; + googletag?: { + pubads(): { __tsRefreshWrapped?: boolean }; + }; + }; + return ( + typeof win.pbjs?.requestBids === "function" && + win.googletag?.pubads().__tsRefreshWrapped === true + ); + }), + ) + .toBe(true); + await page.evaluate(() => { + const win = window as Window & { + adTraceFixture: { + latestSlot(): unknown; + setSuppressCreative(value: boolean): void; + }; + googletag: { pubads(): { refresh(slots: unknown[]): void } }; + }; + win.adTraceFixture.setSuppressCreative(true); + win.googletag.pubads().refresh([win.adTraceFixture.latestSlot()]); + }); + await expect + .poll(async () => { + const result = await exported(page); + const slot = result.slots.find( + (item) => item.slotId === "ad-trace-slot", + ) as + | { + stages?: Record< + string, + { outcome?: string; confidence?: string } + >; + } + | undefined; + return { + prebid: slot?.stages?.prebid, + gam: slot?.stages?.gam, + }; + }) + .toMatchObject({ + prebid: { outcome: "won", confidence: "definitive" }, + gam: { + outcome: "trusted_server_candidate", + confidence: "probable", + }, + }); + }); + + test("client selection, backfill, direct-or-unattributed, and retained generations stay independent", async ({ + page, + }) => { + await openTesterPage(page); + + await page.evaluate(() => { + const win = window as Window & { + adTraceFixture: { simulateClientSelection(): void }; + }; + win.adTraceFixture.simulateClientSelection(); + }); + await expect + .poll(async () => { + const result = await exported(page); + const slot = result.slots.find( + (item) => item.slotId === "ad-trace-slot", + ) as + | { stages?: Record } + | undefined; + return { + prebid: slot?.stages?.prebid?.outcome, + gam: slot?.stages?.gam?.outcome, + }; + }) + .toEqual({ prebid: "lost", gam: "client_prebid_candidate" }); + + await page.evaluate(() => { + const win = window as Window & { + adTraceFixture: { + latestSlot(): unknown; + setNextRender(flags: { isBackfill: boolean }): void; + requestCurrent(): void; + }; + tsjs: { + captureAdTraceRequest(slot: unknown, trigger: string): void; + }; + }; + const slot = win.adTraceFixture.latestSlot(); + win.adTraceFixture.setNextRender({ isBackfill: true }); + win.tsjs.captureAdTraceRequest(slot, "fixture_backfill"); + win.adTraceFixture.requestCurrent(); + }); + await expect + .poll(async () => { + const result = await exported(page); + const slot = result.slots.find( + (item) => item.slotId === "ad-trace-slot", + ) as + | { stages?: Record } + | undefined; + return slot?.stages?.gam?.outcome; + }) + .toBe("backfill"); + + await page.evaluate(() => { + const win = window as Window & { + adTraceFixture: { + latestSlot(): { clearTargeting(): void }; + requestCurrent(): void; + }; + tsjs: { + captureAdTraceRequest(slot: unknown, trigger: string): void; + }; + }; + const slot = win.adTraceFixture.latestSlot(); + slot.clearTargeting(); + win.tsjs.captureAdTraceRequest(slot, "fixture_direct"); + win.adTraceFixture.requestCurrent(); + }); + await expect + .poll(async () => { + const result = await exported(page); + const slot = result.slots.find( + (item) => item.slotId === "ad-trace-slot", + ) as + | { stages?: Record } + | undefined; + return slot?.stages?.gam?.outcome; + }) + .toBe("direct_or_unattributed"); + + const generations = await page.evaluate(() => { + const win = window as Window & { + adTraceFixture: { + simulateRetainedGenerationAcknowledgement(): unknown; + }; + }; + return win.adTraceFixture.simulateRetainedGenerationAcknowledgement(); + }); + const result = await exported(page); + const slot = result.slots.find( + (item) => item.slotId === "ad-trace-slot", + ) as { + latestGeneration: number; + generations: Array<{ + generation: number; + stages: { creative: { outcome: string } }; + }>; + }; + const retained = generations as { first: number; second: number }; + expect(slot.latestGeneration).toBe(retained.second); + expect( + slot.generations.find((item) => item.generation === retained.first) + ?.stages.creative.outcome, + ).toBe("load_acknowledged"); + expect( + slot.generations.find((item) => item.generation === retained.second) + ?.stages.creative.outcome, + ).not.toBe("load_acknowledged"); + }); +}); diff --git a/crates/trusted-server-integration-tests/browser/tests/shared/ad-trace-gate.spec.ts b/crates/trusted-server-integration-tests/browser/tests/shared/ad-trace-gate.spec.ts new file mode 100644 index 000000000..09ceb53d5 --- /dev/null +++ b/crates/trusted-server-integration-tests/browser/tests/shared/ad-trace-gate.spec.ts @@ -0,0 +1,20 @@ +import { expect, test } from "@playwright/test"; +import { runtimeUrl } from "../../helpers/state.js"; + +test("console query alone does not install ad trace when config is disabled", async ({ + page, +}) => { + await page.goto(runtimeUrl("/?ts_console=1"), { + waitUntil: "domcontentloaded", + }); + + await expect + .poll(() => + page.evaluate( + () => + typeof (window as Window & { tsjs?: { adTrace?: unknown } }) + .tsjs?.adTrace, + ), + ) + .toBe("undefined"); +}); diff --git a/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.ad-trace.integration.toml b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.ad-trace.integration.toml new file mode 100644 index 000000000..851d50cdf --- /dev/null +++ b/crates/trusted-server-integration-tests/fixtures/configs/trusted-server.ad-trace.integration.toml @@ -0,0 +1,67 @@ +[[handlers]] +path = "^/_ts/admin" +username = "admin" +password = "integration-admin-password-32-bytes-ok" + +[publisher] +domain = "localhost" +cookie_domain = "localhost" +origin_url = "http://127.0.0.1:8888" +proxy_secret = "integration-test-proxy-secret" + +[ec] +passphrase = "integration-test-ec-secret-padded-32" +ec_store = "ec_identity_store" + +[request_signing] +enabled = false +config_store_id = "app_config" +secret_store_id = "secrets" + +[integrations.ad_trace] +enabled = true + +[integrations.prebid] +enabled = true +server_url = "http://127.0.0.1:8888/openrtb2/auction" +external_bundle_url = "https://assets.example.com/prebid/trusted-prebid.js" +timeout_ms = 750 +bidders = ["example-bidder"] +client_side_bidders = [] +debug = false +test_mode = true + +[integrations.gpt] +enabled = true +script_url = "https://ads.example.com/gpt.js" +cache_ttl_seconds = 3600 +rewrite_script = false + +[proxy] +certificate_check = false +allowed_domains = ["assets.example.com"] + +[auction] +enabled = true +providers = ["prebid"] +timeout_ms = 1000 +allowed_context_keys = [] + +[creative_opportunities] +gam_network_id = "123456789" +auction_timeout_ms = 750 +price_granularity = "dense" + +[[creative_opportunities.slot]] +id = "ad-trace-slot" +div_id = "ad-trace-slot" +gam_unit_path = "/123456789/example/ad-trace" +page_patterns = ["/", "/spa*"] +formats = [{ width = 300, height = 250 }] + +[creative_opportunities.slot.providers.prebid] +bidders = { example-bidder = { placement = "example-placement" } } + +[debug] +ja4_endpoint_enabled = false +inject_adm_for_testing = true diff --git a/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/Dockerfile b/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/Dockerfile new file mode 100644 index 000000000..7996d6fde --- /dev/null +++ b/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/Dockerfile @@ -0,0 +1,15 @@ +# Deterministic publisher/PBS fixture for the tester-only ad trace journey. +FROM php:8.3-cli-alpine + +WORKDIR /var/www/html + +COPY crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/ /var/www/html/ +COPY target/integration-test-artifacts/prebid/ /opt/prebid/ + +RUN bundle="$(find /opt/prebid -maxdepth 1 -name 'trusted-prebid-*.js' -type f | head -n 1)" \ + && test -n "$bundle" \ + && cp "$bundle" /var/www/html/prebid-bundle.js + +EXPOSE 80 + +CMD ["php", "-S", "0.0.0.0:80", "router.php"] diff --git a/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/index.php b/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/index.php new file mode 100644 index 000000000..e7bfca062 --- /dev/null +++ b/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/index.php @@ -0,0 +1,201 @@ + + + + + + Trusted Server ad trace fixture + + + + +

Ad trace contract fixture

+

This page uses deterministic local PBS, GPT, and universal creative protocol mocks.

+
+ + diff --git a/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/router.php b/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/router.php new file mode 100644 index 000000000..e14e7b6e8 --- /dev/null +++ b/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/router.php @@ -0,0 +1,50 @@ + $imp) { + $slotId = is_string($imp['id'] ?? null) ? $imp['id'] : 'ad-trace-slot'; + $bids[] = [ + 'id' => 'example-bid-' . ($index + 1), + 'impid' => $slotId, + 'adid' => 'example-ad-' . ($index + 1), + 'price' => 1.25, + 'adm' => '
Example creative loaded
', + 'crid' => 'example-creative-' . ($index + 1), + 'w' => 300, + 'h' => 250, + 'adomain' => ['advertiser.example.com'], + ]; + } + + header('Content-Type: application/json'); + echo json_encode([ + 'id' => is_string($request['id'] ?? null) ? $request['id'] : 'example-auction', + 'seatbid' => $bids ? [['seat' => 'example-bidder', 'bid' => $bids]] : [], + 'cur' => 'USD', + ], JSON_UNESCAPED_SLASHES); + return; +} + +if ($path === '/prebid-bundle.js') { + header('Content-Type: application/javascript'); + readfile(__DIR__ . '/prebid-bundle.js'); + return; +} + +if ($path === '/' || $path === '/spa-one' || $path === '/spa-two') { + require __DIR__ . '/index.php'; + return; +} + +http_response_code(404); +header('Content-Type: text/plain'); +echo 'Not found'; diff --git a/crates/trusted-server-integration-tests/tests/parity.rs b/crates/trusted-server-integration-tests/tests/parity.rs index e85b1d8d1..e41d84dc9 100644 --- a/crates/trusted-server-integration-tests/tests/parity.rs +++ b/crates/trusted-server-integration-tests/tests/parity.rs @@ -44,6 +44,9 @@ fn test_settings() -> Settings { [ec] passphrase = "test-secret-key-32-bytes-minimum" + + [integrations.ad_trace] + enabled = true "#, ) .expect("should parse parity test settings") @@ -85,6 +88,24 @@ async fn axum_get(uri: &str) -> (u16, HeaderMap) { (resp.status().as_u16(), resp.headers().clone()) } +async fn axum_document_get(uri: &str) -> (u16, HeaderMap) { + let mut svc = EdgeZeroAxumService::new(axum_router()); + let req = AxumRequest::builder() + .method("GET") + .uri(uri) + .header("sec-fetch-dest", "document") + .body(AxumBody::empty()) + .expect("should build document GET request"); + let resp = svc + .ready() + .await + .expect("should be ready") + .call(req) + .await + .expect("should respond"); + (resp.status().as_u16(), resp.headers().clone()) +} + /// Send a POST request to the Axum adapter and return (status, headers, body bytes). async fn axum_post(uri: &str, body: &str) -> (u16, HeaderMap, bytes::Bytes) { use http_body_util::BodyExt as _; @@ -131,6 +152,17 @@ async fn cf_get(uri: &str) -> (u16, HeaderMap) { (resp.status().as_u16(), resp.headers().clone()) } +async fn cf_document_get(uri: &str) -> (u16, HeaderMap) { + let req = request_builder() + .method("GET") + .uri(uri) + .header("sec-fetch-dest", "document") + .body(edgezero_core::body::Body::empty()) + .expect("should build document GET request"); + let resp = cf_router().oneshot(req).await.expect("should respond"); + (resp.status().as_u16(), resp.headers().clone()) +} + /// Send a POST request to the Cloudflare adapter and return (status, headers, body bytes). async fn cf_post(uri: &str, body: &str) -> (u16, HeaderMap, bytes::Bytes) { let router = cf_router(); @@ -174,6 +206,17 @@ async fn spin_get(uri: &str) -> (u16, HeaderMap) { (s, h) } +async fn spin_document_get(uri: &str) -> (u16, HeaderMap) { + let req = request_builder() + .method("GET") + .uri(uri) + .header("sec-fetch-dest", "document") + .body(edgezero_core::body::Body::empty()) + .expect("should build document GET request"); + let resp = spin_router().oneshot(req).await.expect("should respond"); + (resp.status().as_u16(), resp.headers().clone()) +} + /// Send a POST request to the Spin adapter and return (status, headers, body bytes). async fn spin_post(uri: &str, body: &str) -> (u16, HeaderMap, bytes::Bytes) { let router = spin_router(); @@ -456,6 +499,34 @@ async fn verify_signature_route_parity() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn console_activation_finalizes_auth_short_circuits() { + let uri = "/_ts/admin/keys/rotate?ts_console=1"; + let responses = [ + axum_document_get(uri).await, + cf_document_get(uri).await, + spin_document_get(uri).await, + ]; + + for (status, headers) in responses { + assert_eq!(status, 401); + assert_eq!( + headers + .get("cache-control") + .and_then(|value| value.to_str().ok()), + Some("private, no-store") + ); + assert!( + headers + .get_all("set-cookie") + .iter() + .filter_map(|value| value.to_str().ok()) + .any(|value| value.starts_with("__Host-ts-console=1;")), + "auth short-circuit should preserve the console session action" + ); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn admin_rotate_unauthenticated_parity() { // Both adapters must return 401 for unauthenticated admin requests on the diff --git a/crates/trusted-server-js/lib/src/core/ad_trace.ts b/crates/trusted-server-js/lib/src/core/ad_trace.ts new file mode 100644 index 000000000..aca037f5d --- /dev/null +++ b/crates/trusted-server-js/lib/src/core/ad_trace.ts @@ -0,0 +1,505 @@ +import type { + AdTraceApi, + AdTraceConfidence, + AdTraceEvent, + AdTraceEventKind, + AdTraceExport, + AdTraceObservation, + AdTraceStage, + AdTraceStageName, + GenerationTraceSnapshot, + RenderTraceOutcome, + RenderTraceSnapshot, + RenderTraceVisibility, + SlotTraceSnapshot, +} from './types'; + +export const AD_TRACE_MAX_EVENTS = 256; +export const AD_TRACE_MAX_SLOTS = 64; +export const AD_TRACE_MAX_GENERATIONS = 8; +export const AD_TRACE_MAX_RENDERS = 200; +export const AD_TRACE_ACK_TTL_MS = 30_000; +const AD_TRACE_MAX_LISTENERS = 32; + +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const LABEL_RE = /^[\w.-]{1,64}$/; +const EVENT_KINDS = new Set([ + 'ts_auction_observed', + 'ts_winner_observed', + 'prebid_auction_init', + 'prebid_bid_response', + 'prebid_targeting_selected', + 'prebid_bid_won', + 'prebid_auction_end', + 'prebid_render_succeeded', + 'prebid_render_failed', + 'gpt_targeting_applied', + 'gpt_request_started', + 'gpt_slot_requested', + 'gpt_slot_response_received', + 'gpt_slot_render_ended', + 'gpt_slot_onload', + 'aps_display_bids_set', + 'pb_render_requested', + 'pb_render_rejected', + 'pb_render_served', + 'direct_render_rejected', + 'creative_load_acknowledged', + 'generation_superseded', +]); +const CONFIDENCES = new Set(['definitive', 'strong', 'probable', 'none']); +const EMPTY_STAGE: AdTraceStage = { outcome: 'not_observed', confidence: 'none', reason: 'none' }; + +type MutableGeneration = GenerationTraceSnapshot; +interface MutableSlot { + slotId: string; + latestGeneration: number; + baseStages: Record; + generations: MutableGeneration[]; +} + +export interface AdTraceStore extends AdTraceApi { + record(observation: AdTraceObservation): void; + nextGeneration(slotId: string): number; + subscribe(listener: () => void): () => void; + bindElement(slotId: string, generation: number, element: HTMLElement): void; + getBoundElement(slotId: string, generation: number): HTMLElement | undefined; + updateVisibility(slotId: string, generation: number, visibility: RenderTraceVisibility): void; +} + +function stages(): Record { + return { + trustedServer: { ...EMPTY_STAGE }, + prebid: { ...EMPTY_STAGE }, + gam: { ...EMPTY_STAGE }, + creative: { ...EMPTY_STAGE }, + }; +} + +function safeLabel(value: unknown): string | undefined { + return typeof value === 'string' && LABEL_RE.test(value) ? value : undefined; +} +function safeUuid(value: unknown): string | undefined { + return typeof value === 'string' && UUID_RE.test(value) ? value : undefined; +} +function cloneStages(value: Record) { + return Object.fromEntries( + Object.entries(value).map(([key, stage]) => [key, { ...stage }]) + ) as Record; +} +function cloneFreeze(value: T): T { + const clone = JSON.parse(JSON.stringify(value)) as T; + const freeze = (item: unknown): void => { + if (!item || typeof item !== 'object' || Object.isFrozen(item)) return; + Object.freeze(item); + Object.values(item as Record).forEach(freeze); + }; + freeze(clone); + return clone; +} +function newSlot(slotId: string): MutableSlot { + return { slotId, latestGeneration: 0, baseStages: stages(), generations: [] }; +} + +function updateStage(target: Record, event: AdTraceEvent): void { + const explicit = event.outcome + ? { + outcome: event.outcome, + confidence: event.confidence ?? 'none', + reason: event.reason ?? 'observed', + } + : undefined; + switch (event.kind) { + case 'ts_winner_observed': + target.trustedServer = { + outcome: 'won', + confidence: 'definitive', + reason: 'final_server_winner', + }; + break; + case 'ts_auction_observed': + target.trustedServer = explicit ?? { + outcome: 'unresolved', + confidence: 'none', + reason: 'terminal_summary', + }; + break; + case 'prebid_targeting_selected': + target.prebid = explicit ?? { + outcome: event.bidTraceId ? 'won' : 'client_bid_won', + confidence: 'definitive', + reason: 'selected_targeting', + }; + break; + case 'prebid_auction_end': + if (explicit && target.prebid.confidence !== 'definitive') target.prebid = explicit; + break; + case 'prebid_bid_won': + if (target.prebid.outcome === 'client_bid_won' || target.prebid.outcome === 'lost') { + target.prebid = { + ...target.prebid, + reason: 'selected_targeting_with_bid_won', + }; + if (target.gam.outcome === 'direct_or_unattributed') { + target.gam = { + outcome: 'client_prebid_candidate', + confidence: 'probable', + reason: 'client_bid_won_and_gpt_rendered', + }; + } + } + break; + case 'prebid_render_succeeded': + if (target.creative.confidence !== 'definitive') { + target.creative = { + outcome: 'prebid_render_succeeded', + confidence: 'strong', + reason: event.reason ?? 'prebid_render_succeeded', + }; + } + break; + case 'prebid_render_failed': + if (target.creative.confidence !== 'definitive') { + target.creative = { + outcome: 'render_failed', + confidence: 'definitive', + reason: event.reason ?? 'prebid_render_failed', + }; + } + break; + case 'gpt_slot_render_ended': + // Cooperative acknowledgement is stronger than later GPT callbacks and + // must never be downgraded to a probable candidate. + if (target.gam.confidence === 'definitive') break; + if (explicit?.outcome === 'unresolved') target.gam = explicit; + else if (event.isEmpty) + target.gam = { outcome: 'empty', confidence: 'definitive', reason: 'gpt_empty' }; + else if (event.isBackfill) + target.gam = { outcome: 'backfill', confidence: 'definitive', reason: 'gpt_backfill' }; + else if (event.bidTraceId) + target.gam = { + outcome: 'trusted_server_candidate', + confidence: 'probable', + reason: 'trace_targeting_rendered', + }; + else if ( + (target.prebid.outcome === 'client_bid_won' || target.prebid.outcome === 'lost') && + target.prebid.reason === 'selected_targeting_with_bid_won' + ) + target.gam = { + outcome: 'client_prebid_candidate', + confidence: 'probable', + reason: 'client_bid_won_and_gpt_rendered', + }; + else + target.gam = { + outcome: 'direct_or_unattributed', + confidence: 'probable', + reason: 'non_empty_unattributed', + }; + break; + case 'aps_display_bids_set': + // APS setting display bids is a handoff only. GAM attribution remains + // unobserved until a correlated non-empty GPT render arrives. + break; + case 'gpt_slot_onload': + if (target.creative.outcome === 'not_observed') + target.creative = { + outcome: 'gpt_iframe_onload', + confidence: 'probable', + reason: 'gpt_slot_onload', + }; + break; + case 'pb_render_served': + if (target.creative.confidence !== 'definitive') { + target.creative = { + outcome: 'renderer_served', + confidence: 'strong', + reason: event.reason ?? 'pb_render_response', + }; + } + break; + case 'direct_render_rejected': + if (target.creative.confidence === 'none') { + target.creative = { + outcome: 'rejected', + confidence: 'none', + reason: event.reason ?? 'direct_render_rejected', + }; + } + break; + case 'creative_load_acknowledged': + target.creative = { + outcome: 'load_acknowledged', + confidence: 'definitive', + reason: 'source_validated_load', + }; + if (event.reason !== 'direct_iframe_load') { + target.gam = { + outcome: 'trusted_server_won', + confidence: 'definitive', + reason: 'creative_load_acknowledged', + }; + } + break; + case 'generation_superseded': + // Ownership cleanup is lifecycle evidence, not contradictory render + // evidence. Preserve every previously observed stage unchanged. + break; + default: + break; + } +} + +function snapshot(slot: MutableSlot): SlotTraceSnapshot { + const latest = slot.generations.at(-1); + return { + slotId: slot.slotId, + latestGeneration: slot.latestGeneration, + generations: slot.generations.map((item) => ({ + generation: item.generation, + stages: cloneStages(item.stages), + })), + stages: cloneStages(latest?.stages ?? slot.baseStages), + }; +} + +function isRenderEvent(kind: AdTraceEventKind): boolean { + return ( + kind === 'gpt_request_started' || + kind === 'gpt_slot_render_ended' || + kind === 'prebid_render_succeeded' || + kind === 'prebid_render_failed' || + kind === 'pb_render_requested' || + kind === 'pb_render_rejected' || + kind === 'pb_render_served' || + kind === 'direct_render_rejected' || + kind === 'creative_load_acknowledged' || + kind === 'generation_superseded' + ); +} + +function renderSource(event: AdTraceEvent): RenderTraceSnapshot['source'] { + if (event.reason?.startsWith('direct_')) return 'direct_auction'; + if (event.kind.startsWith('pb_render_') || event.kind === 'creative_load_acknowledged') + return 'pb_render'; + return 'gpt'; +} + +function renderOutcome( + current: Pick, + event: AdTraceEvent +): { outcome: RenderTraceOutcome; confidence: AdTraceConfidence } { + if (current.outcome === 'confirmed') return { outcome: 'confirmed', confidence: 'definitive' }; + if (current.outcome === 'empty' && current.confidence === 'definitive') { + return { outcome: 'empty', confidence: 'definitive' }; + } + if (event.kind === 'creative_load_acknowledged') + return { outcome: 'confirmed', confidence: 'definitive' }; + if (current.outcome === 'served') return { outcome: 'served', confidence: 'strong' }; + if (event.kind === 'pb_render_served') return { outcome: 'served', confidence: 'strong' }; + if (event.kind === 'gpt_slot_render_ended') + return event.isEmpty + ? { outcome: 'empty', confidence: 'definitive' } + : { outcome: 'gam_only', confidence: 'probable' }; + if (current.outcome === 'gam_only') return { outcome: 'gam_only', confidence: 'probable' }; + return { outcome: 'unresolved', confidence: 'none' }; +} + +export function createAdTraceStore( + now: () => number = () => (typeof performance === 'undefined' ? Date.now() : performance.now()) +): AdTraceStore { + const slots = new Map(); + const events: AdTraceEvent[] = []; + const renders: RenderTraceSnapshot[] = []; + const renderByGeneration = new Map(); + const elementByGeneration = new Map(); + const listeners = new Set<() => void>(); + let sequence = 0; + let generationSequence = 0; + let renderSequence = 0; + let droppedEvents = 0; + let evictedSlots = 0; + const ensureSlot = (slotId: string): MutableSlot => { + let slot = slots.get(slotId); + if (slot) return slot; + if (slots.size >= AD_TRACE_MAX_SLOTS) { + const oldest = slots.keys().next().value as string | undefined; + if (oldest) { + slots.delete(oldest); + evictedSlots += 1; + } + } + slot = newSlot(slotId); + slots.set(slotId, slot); + return slot; + }; + const notify = (): void => listeners.forEach((listener) => listener()); + const emitRender = (render: RenderTraceSnapshot): void => { + if (typeof window === 'undefined' || typeof CustomEvent === 'undefined') return; + window.dispatchEvent(new CustomEvent('tsjs:adRendered', { detail: cloneFreeze(render) })); + }; + const updateRender = (event: AdTraceEvent, slotId: string, generation: number): void => { + if (!isRenderEvent(event.kind)) return; + const key = `${slotId}:${generation}`; + let render = renderByGeneration.get(key); + const timestamp = now(); + if (!render) { + render = { + sequence: ++renderSequence, + slotId, + generation, + source: renderSource(event), + outcome: 'unresolved', + confidence: 'none', + visibility: 'unknown', + createdAt: timestamp, + updatedAt: timestamp, + }; + renderByGeneration.set(key, render); + renders.push(render); + if (renders.length > AD_TRACE_MAX_RENDERS) { + const evicted = renders.shift(); + if (evicted) { + const evictedKey = `${evicted.slotId}:${evicted.generation}`; + renderByGeneration.delete(evictedKey); + elementByGeneration.delete(evictedKey); + } + } + } + const next = renderOutcome(render, event); + render.outcome = next.outcome; + render.confidence = next.confidence; + if (event.reason?.startsWith('direct_')) render.source = 'direct_auction'; + else if (event.kind.startsWith('pb_render_') || event.kind === 'creative_load_acknowledged') + render.source = render.source === 'direct_auction' ? render.source : 'pb_render'; + if (event.auctionTraceId) render.auctionTraceId = event.auctionTraceId; + if (event.bidTraceId) render.bidTraceId = event.bidTraceId; + render.updatedAt = timestamp; + emitRender(render); + }; + + return { + record(observation) { + if (!EVENT_KINDS.has(observation.kind)) return; + if (observation.confidence && !CONFIDENCES.has(observation.confidence)) return; + const slotId = safeLabel(observation.slotId); + const generation = + Number.isInteger(observation.generation) && (observation.generation ?? 0) > 0 + ? observation.generation + : undefined; + const event: AdTraceEvent = { + sequence: ++sequence, + timestamp: now(), + kind: observation.kind, + ...(slotId ? { slotId } : {}), + ...(generation ? { generation } : {}), + ...(safeUuid(observation.auctionTraceId) + ? { auctionTraceId: observation.auctionTraceId } + : {}), + ...(safeUuid(observation.bidTraceId) ? { bidTraceId: observation.bidTraceId } : {}), + ...(safeLabel(observation.provider) ? { provider: observation.provider } : {}), + ...(safeLabel(observation.bidder) ? { bidder: observation.bidder } : {}), + ...(safeLabel(observation.outcome) ? { outcome: observation.outcome } : {}), + ...(observation.confidence ? { confidence: observation.confidence } : {}), + ...(safeLabel(observation.reason) ? { reason: observation.reason } : {}), + ...(typeof observation.isEmpty === 'boolean' ? { isEmpty: observation.isEmpty } : {}), + ...(typeof observation.isBackfill === 'boolean' + ? { isBackfill: observation.isBackfill } + : {}), + }; + events.push(event); + if (events.length > AD_TRACE_MAX_EVENTS) { + events.shift(); + droppedEvents += 1; + } + if (slotId) { + const slot = ensureSlot(slotId); + const exact = generation + ? slot.generations.find((item) => item.generation === generation) + : undefined; + if (exact) updateStage(exact.stages, event); + else if ( + !generation && + (event.kind === 'ts_winner_observed' || event.kind === 'ts_auction_observed') + ) { + // Generationless server evidence seeds only the next request. Updating + // the latest retained generation would rewrite prior-navigation history. + updateStage(slot.baseStages, event); + } + if (generation) updateRender(event, slotId, generation); + } + notify(); + }, + nextGeneration(slotId) { + const safeSlotId = safeLabel(slotId); + if (!safeSlotId) return 0; + const slot = ensureSlot(safeSlotId); + slot.latestGeneration = ++generationSequence; + slot.generations.push({ + generation: slot.latestGeneration, + stages: cloneStages(slot.baseStages), + }); + if (slot.generations.length > AD_TRACE_MAX_GENERATIONS) slot.generations.shift(); + notify(); + return slot.latestGeneration; + }, + getSlot(slotId) { + const slot = slots.get(slotId); + return slot ? cloneFreeze(snapshot(slot)) : undefined; + }, + getEvents() { + return cloneFreeze(events); + }, + getRenderTimeline() { + return cloneFreeze(renders); + }, + export() { + const value: AdTraceExport = { + version: 1, + slots: [...slots.values()].map(snapshot), + events, + renders, + metadata: { droppedEvents, evictedSlots }, + }; + return cloneFreeze(value); + }, + subscribe(listener) { + if (listeners.size >= AD_TRACE_MAX_LISTENERS) return () => {}; + listeners.add(listener); + return () => listeners.delete(listener); + }, + bindElement(slotId, generation, element) { + const safeSlotId = safeLabel(slotId); + if (!safeSlotId || !Number.isInteger(generation) || generation <= 0) return; + const key = `${safeSlotId}:${generation}`; + if (!elementByGeneration.has(key) && elementByGeneration.size >= AD_TRACE_MAX_RENDERS) { + const oldest = elementByGeneration.keys().next().value as string | undefined; + if (oldest) elementByGeneration.delete(oldest); + } + elementByGeneration.set(key, element); + }, + getBoundElement(slotId, generation) { + const safeSlotId = safeLabel(slotId); + if (!safeSlotId || !Number.isInteger(generation) || generation <= 0) return undefined; + return elementByGeneration.get(`${safeSlotId}:${generation}`); + }, + updateVisibility(slotId, generation, visibility) { + const safeSlotId = safeLabel(slotId); + if (!safeSlotId || !Number.isInteger(generation) || generation <= 0) return; + const render = renderByGeneration.get(`${safeSlotId}:${generation}`); + if (!render || render.visibility === visibility) return; + render.visibility = visibility; + render.updatedAt = now(); + emitRender(render); + notify(); + }, + }; +} + +export function isCanonicalTraceUuid(value: unknown): value is string { + return safeUuid(value) !== undefined; +} +export function isBoundedTraceLabel(value: unknown): value is string { + return safeLabel(value) !== undefined; +} diff --git a/crates/trusted-server-js/lib/src/core/auction.ts b/crates/trusted-server-js/lib/src/core/auction.ts index a02684362..50b87c955 100644 --- a/crates/trusted-server-js/lib/src/core/auction.ts +++ b/crates/trusted-server-js/lib/src/core/auction.ts @@ -5,7 +5,13 @@ import { parseApsRendererDescriptor } from '../integrations/aps/render'; import { log } from './log'; -import type { ApsRendererV1 } from './types'; +import type { + ApsRendererV1, + AuctionTraceOutcome, + AuctionTraceSource, + AuctionTraceSummary, + TrustedServerBidTrace, +} from './types'; // --------------------------------------------------------------------------- // Types @@ -41,6 +47,11 @@ export interface AdRequest { } /** A parsed bid from an OpenRTB seatbid response. */ +export type AuctionClientResult = + | { kind: 'ok'; summary?: AuctionTraceSummary; bids: AuctionBid[] } + | { kind: 'transport_error'; reason: 'network' | 'http' } + | { kind: 'invalid_response'; reason: 'non_json' | 'invalid_shape' }; + export interface AuctionBid { /** Matches the `impid` in the response — corresponds to adUnit `code`. */ impid: string; @@ -60,6 +71,72 @@ export interface AuctionBid { creativeId: string; /** Advertiser domains. */ adomain: string[]; + /** Tester-gated trace joined to the validated root summary. */ + trace?: TrustedServerBidTrace; +} + +const TRACE_UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const TRACE_LABEL_RE = /^[\w.-]{1,64}$/; +const TRACE_SOURCES = new Set([ + 'initial_navigation', + 'spa_navigation', + 'auction_api', +]); +const TRACE_OUTCOMES = new Set([ + 'completed', + 'no_bid', + 'skipped', + 'failed', + 'abandoned', +]); + +/** Strictly parse the optional Trusted Server root extension. */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function parseAuctionTraceSummary(body: any): AuctionTraceSummary | undefined { + const trace = body?.ext?.trusted_server?.trace; + if ( + trace?.version !== 1 || + !TRACE_UUID_RE.test(trace.auction_trace_id) || + !TRACE_SOURCES.has(trace.source) || + !TRACE_OUTCOMES.has(trace.outcome) + ) { + return undefined; + } + return { + version: 1, + auctionTraceId: trace.auction_trace_id, + source: trace.source, + outcome: trace.outcome, + }; +} + +function parseBidTrace( + bid: any, // eslint-disable-line @typescript-eslint/no-explicit-any + root: AuctionTraceSummary | undefined +): TrustedServerBidTrace | undefined { + const trace = bid?.ext?.trusted_server?.trace; + if ( + !root || + root.outcome !== 'completed' || + trace?.version !== 1 || + !TRACE_UUID_RE.test(trace.bid_trace_id) || + typeof trace.slot_id !== 'string' || + trace.slot_id !== bid?.impid || + !TRACE_LABEL_RE.test(trace.slot_id) || + !TRACE_LABEL_RE.test(trace.provider) || + !TRACE_LABEL_RE.test(trace.bidder) + ) { + return undefined; + } + return { + version: 1, + auctionTraceId: root.auctionTraceId, + bidTraceId: trace.bid_trace_id, + source: root.source, + slotId: trace.slot_id, + provider: trace.provider, + bidder: trace.bidder, + }; } // --------------------------------------------------------------------------- @@ -126,6 +203,7 @@ export function buildAdRequest(units: any[], options?: { eids?: AuctionEid[] }): // eslint-disable-next-line @typescript-eslint/no-explicit-any export function parseAuctionResponse(body: any): AuctionBid[] { const bids: AuctionBid[] = []; + const rootTrace = parseAuctionTraceSummary(body); const seatbids = body?.seatbid; if (!Array.isArray(seatbids)) return bids; @@ -142,6 +220,7 @@ export function parseAuctionResponse(body: any): AuctionBid[] { const creativeId = typeof bid?.crid === 'string' ? bid.crid : (renderer?.creativeId ?? `${seat}-${impid}`); + const trace = parseBidTrace(bid, rootTrace); bids.push({ impid, // Preserve non-string untrusted values so the render-time sanitizer @@ -157,25 +236,45 @@ export function parseAuctionResponse(body: any): AuctionBid[] { adomain: Array.isArray(bid?.adomain) ? bid.adomain.filter((domain: unknown): domain is string => typeof domain === 'string') : [], + ...(trace ? { trace } : {}), }); } } return bids; } +function isValidAuctionResponseShape(data: Record): boolean { + const seatbid = data.seatbid; + // Preserve the legacy valid empty response while rejecting a present but + // malformed collection that would otherwise be misreported as no-bid. + if (seatbid === undefined) return true; + if (!Array.isArray(seatbid)) return false; + return seatbid.every((seat) => { + if (!seat || typeof seat !== 'object' || Array.isArray(seat)) return false; + const bids = (seat as Record).bid; + return ( + bids === undefined || + (Array.isArray(bids) && + bids.every((bid) => !!bid && typeof bid === 'object' && !Array.isArray(bid))) + ); + }); +} + // --------------------------------------------------------------------------- // Auction HTTP call // --------------------------------------------------------------------------- /** - * POST an {@link AdRequest} to the given endpoint and return parsed bids. - * - * Returns an empty array on network or parse errors (non-throwing). + * POST an {@link AdRequest} and distinguish a valid empty auction from + * transport or response-shape failures. */ -export async function sendAuction(endpoint: string, request: AdRequest): Promise { +export async function sendAuction( + endpoint: string, + request: AdRequest +): Promise { if (typeof fetch !== 'function') { log.warn('auction: fetch not available'); - return []; + return { kind: 'transport_error', reason: 'network' }; } log.info('auction: sending request', { endpoint, units: request.adUnits.length }); @@ -190,21 +289,40 @@ export async function sendAuction(endpoint: string, request: AdRequest): Promise }); const contentType = response.headers.get('content-type') || ''; - if (response.ok && contentType.includes('application/json')) { - const data: unknown = await response.json(); - const bids = parseAuctionResponse(data); - log.info('auction: received bids', { count: bids.length }); - return bids; + if (!response.ok) { + log.warn('auction: unexpected response', { + ok: response.ok, + status: response.status, + ct: contentType, + }); + return { kind: 'transport_error', reason: 'http' }; + } + if (!contentType.includes('application/json')) { + log.warn('auction: non-json response', { status: response.status, ct: contentType }); + return { kind: 'invalid_response', reason: 'non_json' }; } - log.warn('auction: unexpected response', { - ok: response.ok, - status: response.status, - ct: contentType, - }); - return []; + let data: unknown; + try { + data = await response.json(); + } catch (error) { + log.warn('auction: invalid json response', error); + return { kind: 'invalid_response', reason: 'non_json' }; + } + if ( + !data || + typeof data !== 'object' || + Array.isArray(data) || + !isValidAuctionResponseShape(data as Record) + ) { + return { kind: 'invalid_response', reason: 'invalid_shape' }; + } + const bids = parseAuctionResponse(data); + const summary = parseAuctionTraceSummary(data); + log.info('auction: received bids', { count: bids.length }); + return { kind: 'ok', ...(summary ? { summary } : {}), bids }; } catch (error) { log.warn('auction: request failed', error); - return []; + return { kind: 'transport_error', reason: 'network' }; } } diff --git a/crates/trusted-server-js/lib/src/core/global.d.ts b/crates/trusted-server-js/lib/src/core/global.d.ts index c7c8b08fb..2e753c6d9 100644 --- a/crates/trusted-server-js/lib/src/core/global.d.ts +++ b/crates/trusted-server-js/lib/src/core/global.d.ts @@ -2,6 +2,8 @@ import type { TsjsApi } from './types'; declare global { interface Window { + /** Request-scoped server bootstrap consumed synchronously by ad trace. */ + __tsjs_adTraceActive?: boolean; tsjs?: TsjsApi; pbjs?: TsjsApi; } diff --git a/crates/trusted-server-js/lib/src/core/request.ts b/crates/trusted-server-js/lib/src/core/request.ts index ae352c58a..b7429d01c 100644 --- a/crates/trusted-server-js/lib/src/core/request.ts +++ b/crates/trusted-server-js/lib/src/core/request.ts @@ -6,6 +6,7 @@ import { collectContext } from './context'; import { log } from './log'; import { getAllUnits, firstSize } from './registry'; import { createAdIframe, findSlot, buildCreativeDocument, sanitizeCreativeHtml } from './render'; +import type { AuctionTraceSummary, TrustedServerBidTrace } from './types'; export type RequestAdsCallback = () => void; export interface RequestAdsOptions { @@ -13,6 +14,16 @@ export interface RequestAdsOptions { timeout?: number; } +const MAX_DIRECT_RENDER_OWNERS = 64; + +interface DirectRenderOwner { + token: symbol; + slotId: string; + generation?: number; +} + +const latestDirectOwners = new Map(); + type RenderCreativeInlineOptions = { slotId: string; // Accept unknown input here because bidder JSON is untrusted at runtime. @@ -21,8 +32,64 @@ type RenderCreativeInlineOptions = { creativeHeight?: number; seat: string; creativeId: string; + owner: DirectRenderOwner; + trace?: TrustedServerBidTrace; }; +function claimDirectOwner(slotId: string): DirectRenderOwner { + const previous = latestDirectOwners.get(slotId); + if (previous) recordDirectRejection(previous, 'direct_owner_replaced'); + const ts = window.tsjs; + const generation = ts?.recordAdTrace ? ts.nextAdTraceGeneration?.(slotId) : undefined; + const owner: DirectRenderOwner = { + token: Symbol(slotId), + slotId, + ...(generation && generation > 0 ? { generation } : {}), + }; + latestDirectOwners.delete(slotId); + latestDirectOwners.set(slotId, owner); + if (latestDirectOwners.size > MAX_DIRECT_RENDER_OWNERS) { + const oldest = latestDirectOwners.keys().next().value as string | undefined; + if (oldest) { + const evicted = latestDirectOwners.get(oldest); + if (evicted) recordDirectRejection(evicted, 'direct_owner_evicted'); + latestDirectOwners.delete(oldest); + } + } + return owner; +} + +function ownerIsCurrent(owner: DirectRenderOwner): boolean { + return latestDirectOwners.get(owner.slotId) === owner; +} + +function recordRootSummary( + summary: AuctionTraceSummary | undefined, + owner: DirectRenderOwner, + hasWinner: boolean +): void { + if (!summary || !owner.generation) return; + window.tsjs?.recordAdTrace?.({ + kind: 'ts_auction_observed', + slotId: owner.slotId, + generation: owner.generation, + auctionTraceId: summary.auctionTraceId, + outcome: summary.outcome === 'completed' && !hasWinner ? 'no_bid' : summary.outcome, + confidence: 'definitive', + reason: 'terminal_summary', + }); +} + +function recordDirectRejection(owner: DirectRenderOwner, reason: string): void { + if (!owner.generation) return; + window.tsjs?.recordAdTrace?.({ + kind: 'direct_render_rejected', + slotId: owner.slotId, + generation: owner.generation, + reason, + }); +} + // Entry point matching Prebid's requestBids signature; uses unified /auction endpoint. export function requestAds( callbackOrOpts?: RequestAdsCallback | RequestAdsOptions, @@ -41,38 +108,99 @@ export function requestAds( log.info('requestAds: called', { hasCallback: typeof callback === 'function' }); try { const adUnits = getAllUnits(); + const requestedSlotIds = [ + ...new Set( + adUnits + .map((unit) => unit.code) + .filter((code): code is string => typeof code === 'string' && code.length > 0) + ), + ]; + const owners = new Map(requestedSlotIds.map((slotId) => [slotId, claimDirectOwner(slotId)])); const config = collectContext(); const payload = { ...buildAdRequest(adUnits), config }; log.debug('requestAds: payload', { units: adUnits.length, contextKeys: Object.keys(config) }); - // Use unified auction endpoint - void sendAuction('/auction', payload) - .then((bids) => { - log.info('requestAds: got bids', { count: bids.length }); - for (const bid of bids) { - if (!bid.impid) continue; - if (bid.renderer) { - renderApsCreative({ slotId: bid.impid, renderer: bid.renderer }); - continue; - } - if (!bid.adm) { - log.debug('requestAds: bid has no adm, skipping', { slotId: bid.impid }); - continue; + void sendAuction('/auction', payload).then((result) => { + if (result.kind !== 'ok') { + for (const owner of owners.values()) { + if (ownerIsCurrent(owner)) { + recordDirectRejection(owner, `${result.kind}_${result.reason}`); } - renderCreativeInline({ - slotId: bid.impid, - creativeHtml: bid.adm, - creativeWidth: bid.width, - creativeHeight: bid.height, - seat: bid.seat, - creativeId: bid.creativeId, + } + return; + } + + log.info('requestAds: got bids', { count: result.bids.length }); + const bySlot = new Map(); + for (const bid of result.bids) { + if (!owners.has(bid.impid)) continue; + const existing = bySlot.get(bid.impid) ?? []; + existing.push(bid); + bySlot.set(bid.impid, existing); + } + + for (const [slotId, owner] of owners) { + if (!ownerIsCurrent(owner)) continue; + const slotBids = bySlot.get(slotId) ?? []; + recordRootSummary(result.summary, owner, slotBids.length > 0); + if (slotBids.length === 0) continue; + if (slotBids.length !== 1) { + recordDirectRejection(owner, 'ambiguous_winner'); + continue; + } + + const bid = slotBids[0]; + const trace = + bid.trace && + result.summary && + bid.trace.slotId === slotId && + bid.trace.auctionTraceId === result.summary.auctionTraceId + ? bid.trace + : undefined; + if (trace && owner.generation) { + window.tsjs?.recordAdTrace?.({ + kind: 'ts_winner_observed', + slotId, + generation: owner.generation, + auctionTraceId: trace.auctionTraceId, + bidTraceId: trace.bidTraceId, + provider: trace.provider, + bidder: trace.bidder, }); } - log.info('requestAds: rendered creatives from response'); - }) - .catch((err) => { - log.warn('requestAds: auction failed', err); - }); + if (bid.renderer) { + if (!ownerIsCurrent(owner)) continue; + if (!renderApsCreative({ slotId, renderer: bid.renderer })) { + recordDirectRejection(owner, 'aps_render_rejected'); + } else if (owner.generation) { + window.tsjs?.recordAdTrace?.({ + kind: 'pb_render_served', + slotId, + generation: owner.generation, + auctionTraceId: trace?.auctionTraceId, + bidTraceId: trace?.bidTraceId, + reason: 'direct_aps_renderer', + }); + } + continue; + } + if (!bid.adm) { + recordDirectRejection(owner, 'missing_adm'); + continue; + } + renderCreativeInline({ + slotId, + creativeHtml: bid.adm, + creativeWidth: bid.width, + creativeHeight: bid.height, + seat: bid.seat, + creativeId: bid.creativeId, + owner, + ...(trace ? { trace } : {}), + }); + } + log.info('requestAds: rendered creatives from response'); + }); // Synchronously invoke callback to match test expectations try { @@ -93,16 +221,24 @@ function renderCreativeInline({ creativeHeight, seat, creativeId, + owner, + trace, }: RenderCreativeInlineOptions): void { + if (!ownerIsCurrent(owner)) return; const container = findSlot(slotId) as HTMLElement | null; if (!container) { + recordDirectRejection(owner, 'slot_missing'); log.warn('renderCreativeInline: slot not found; skipping render', { slotId, seat, creativeId }); return; } try { + if (owner.generation) { + window.tsjs?.bindAdTraceElement?.(slotId, owner.generation, container); + } const sanitization = sanitizeCreativeHtml(creativeHtml); if (sanitization.kind === 'rejected') { + recordDirectRejection(owner, 'creative_rejected'); log.warn('renderCreativeInline: rejected creative', { slotId, seat, @@ -113,6 +249,7 @@ function renderCreativeInline({ return; } + if (!ownerIsCurrent(owner)) return; // Clear the slot only after sanitization succeeds so rejected creatives never blank existing content. container.innerHTML = ''; @@ -138,8 +275,36 @@ function renderCreativeInline({ width, height, }); + iframe.addEventListener( + 'load', + () => { + if (!ownerIsCurrent(owner) || !iframe.isConnected || iframe.parentElement !== container) + return; + if (owner.generation) { + window.tsjs?.recordAdTrace?.({ + kind: 'creative_load_acknowledged', + slotId, + generation: owner.generation, + auctionTraceId: trace?.auctionTraceId, + bidTraceId: trace?.bidTraceId, + reason: 'direct_iframe_load', + }); + } + }, + { once: true } + ); iframe.srcdoc = buildCreativeDocument(sanitization.sanitizedHtml); + if (owner.generation) { + window.tsjs?.recordAdTrace?.({ + kind: 'pb_render_served', + slotId, + generation: owner.generation, + auctionTraceId: trace?.auctionTraceId, + bidTraceId: trace?.bidTraceId, + reason: 'direct_iframe_created', + }); + } log.info('renderCreativeInline: rendered', { slotId, @@ -150,6 +315,7 @@ function renderCreativeInline({ originalLength: sanitization.originalLength, }); } catch (err) { + recordDirectRejection(owner, 'render_failed'); log.warn('renderCreativeInline: failed', { slotId, seat, creativeId, err }); } } diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 7b81e78a6..0ba40fd5d 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -80,6 +80,126 @@ export interface ApsPrebidRendererEntry { markRendered(): void; } +export type AuctionTraceSource = 'initial_navigation' | 'spa_navigation' | 'auction_api'; +export type AuctionTraceOutcome = 'completed' | 'no_bid' | 'skipped' | 'failed' | 'abandoned'; + +/** Privacy-safe summary emitted only for configured tester traffic. */ +export interface AuctionTraceSummary { + version: 1; + auctionTraceId: string; + source: AuctionTraceSource; + outcome: AuctionTraceOutcome; +} + +/** Privacy-safe trace for one final Trusted Server winning bid. */ +export interface TrustedServerBidTrace { + version: 1; + auctionTraceId: string; + bidTraceId: string; + source: AuctionTraceSource; + slotId: string; + provider: string; + bidder: string; +} + +export type AdTraceConfidence = 'definitive' | 'strong' | 'probable' | 'none'; +export type AdTraceStageName = 'trustedServer' | 'prebid' | 'gam' | 'creative'; +export interface AdTraceStage { + outcome: string; + confidence: AdTraceConfidence; + reason: string; +} + +export type AdTraceEventKind = + | 'ts_auction_observed' + | 'ts_winner_observed' + | 'prebid_auction_init' + | 'prebid_bid_response' + | 'prebid_targeting_selected' + | 'prebid_bid_won' + | 'prebid_auction_end' + | 'prebid_render_succeeded' + | 'prebid_render_failed' + | 'gpt_targeting_applied' + | 'gpt_request_started' + | 'gpt_slot_requested' + | 'gpt_slot_response_received' + | 'gpt_slot_render_ended' + | 'gpt_slot_onload' + | 'aps_display_bids_set' + | 'pb_render_requested' + | 'pb_render_rejected' + | 'pb_render_served' + | 'direct_render_rejected' + | 'creative_load_acknowledged' + | 'generation_superseded'; + +/** Sanitized observation accepted by the optional recorder. */ +export interface AdTraceObservation { + kind: AdTraceEventKind; + slotId?: string; + generation?: number; + auctionTraceId?: string; + bidTraceId?: string; + provider?: string; + bidder?: string; + outcome?: string; + confidence?: AdTraceConfidence; + reason?: string; + isEmpty?: boolean; + isBackfill?: boolean; +} + +export interface AdTraceEvent extends AdTraceObservation { + sequence: number; + timestamp: number; +} + +export interface GenerationTraceSnapshot { + generation: number; + stages: Record; +} + +export interface SlotTraceSnapshot { + slotId: string; + latestGeneration: number; + generations: GenerationTraceSnapshot[]; + /** Convenience view of only the latest retained generation. */ + stages: Record; +} + +export type RenderTraceOutcome = 'confirmed' | 'served' | 'gam_only' | 'empty' | 'unresolved'; +export type RenderTraceVisibility = 'visible' | 'hidden' | 'disconnected' | 'unknown'; + +export interface RenderTraceSnapshot { + sequence: number; + slotId: string; + generation: number; + auctionTraceId?: string; + bidTraceId?: string; + source: 'gpt' | 'pb_render' | 'direct_auction'; + outcome: RenderTraceOutcome; + confidence: AdTraceConfidence; + visibility: RenderTraceVisibility; + createdAt: number; + updatedAt: number; +} + +export interface AdTraceExport { + version: 1; + slots: SlotTraceSnapshot[]; + events: AdTraceEvent[]; + renders: RenderTraceSnapshot[]; + metadata: { droppedEvents: number; evictedSlots: number }; +} + +export interface AdTraceApi { + getSlot(slotId: string): SlotTraceSnapshot | undefined; + getEvents(): readonly AdTraceEvent[]; + getRenderTimeline(): readonly RenderTraceSnapshot[]; + export(): AdTraceExport; +} + /** Bid targeting data from the server-side auction, injected into `window.tsjs.bids`. */ export interface AuctionBidData { hb_pb?: string; @@ -95,6 +215,8 @@ export interface AuctionBidData { burl?: string; /** Typed winning-bid renderer capability. */ renderer?: AuctionBidRenderer; + /** Tester-gated trace; absent for ordinary traffic and malformed input. */ + trace?: TrustedServerBidTrace; /** * Sanitized winning creative markup for local rendering through the pbRender * bridge. Present whenever the winning bid carried a creative that passed the @@ -145,6 +267,80 @@ export interface TsjsApi { * `hb_adid`. The Universal Creative bridge consumes each entry at most once. */ apsPrebidRenderers?: Record; + /** Tester-gated terminal auction summary. */ + auctionTrace?: AuctionTraceSummary; + /** Tester-only immutable diagnostic API. */ + adTrace?: AdTraceApi; + /** Private recorder installed only by the optional ad_trace module. */ + recordAdTrace?: (observation: AdTraceObservation) => void; + /** Private generation allocator installed only by the optional module. */ + nextAdTraceGeneration?: (slotId: string) => number; + /** Private overlay subscription installed only by the optional module. */ + subscribeAdTrace?: (listener: () => void) => () => void; + /** Bind one generation to the exact DOM element captured at its request boundary. */ + bindAdTraceElement?: (slotId: string, generation: number, element: HTMLElement) => void; + /** Resolve only that exact captured element; never searches replacement DOM. */ + getAdTraceElement?: (slotId: string, generation: number) => HTMLElement | undefined; + /** Private live visibility updater used only by the active overlay. */ + updateAdTraceVisibility?: ( + slotId: string, + generation: number, + visibility: RenderTraceVisibility + ) => void; + /** Private request-scoped Prebid correlation ledger; never exported. */ + prebidCorrelation?: Array<{ + auctionId: string; + slotId: string; + requestId: string; + bidder?: string; + adId?: string; + traceToken?: string; + serverTrace?: TrustedServerBidTrace; + events?: AdTraceEventKind[]; + }>; + /** Exact selected participants retained briefly for post-request terminal events. */ + prebidSelectedParticipants?: Array<{ + auctionId: string; + slotId: string; + requestId: string; + adId?: string; + traceToken?: string; + bidder?: string; + generation: number; + selectedAt: number; + }>; + /** Request-scoped root summaries retained until the GPT request boundary. */ + prebidServerSummaries?: Array<{ + auctionId: string; + slotId: string; + summary: AuctionTraceSummary; + }>; + /** Completed Prebid auctions used to identify request-scoped no-bid selections. */ + prebidCompletedAuctions?: Array<{ auctionId: string; slotIds: string[] }>; + /** Private bootstrap queue used until the GPT module installs its capture hook. */ + pendingAdTraceRequests?: Array<{ + slot: unknown; + trigger: string; + snapshot?: { + slotId?: string; + bidder?: string; + adId?: string; + traceToken?: string; + bid?: AuctionBidData; + }; + }>; + /** Private request-boundary hook shared with bootstrap and slim Prebid. */ + captureAdTraceRequest?: ( + slot: unknown, + trigger: string, + snapshot?: { + slotId?: string; + bidder?: string; + adId?: string; + traceToken?: string; + bid?: AuctionBidData; + } + ) => number; /** Initialises GPT slots with server-side bid targeting and calls refresh(). */ adInit?: () => void; /** GPT slot objects TS defined — used to destroy stale slots on SPA navigation. */ @@ -153,12 +349,6 @@ export interface TsjsApi { servicesEnabled?: boolean; /** Maps actualDivId → slotId for slotRenderEnded billing lookup. */ divToSlotId?: Record; - /** - * Win/billing beacons already fired, keyed by `slotId|bidIdentity|kind|url`. - * Used by the GPT render bridge so a bid's nurl/burl fire at most once even - * across repeated Prebid Universal Creative requests for the same adId. - */ - firedBeacons?: Record; /** Slot-level GPT targeting keys TS applied on the previous route. */ prevSlotTargetingKeys?: Record; /** diff --git a/crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts b/crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts new file mode 100644 index 000000000..cf6d6d33c --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts @@ -0,0 +1,98 @@ +import { createAdTraceStore, isBoundedTraceLabel, isCanonicalTraceUuid } from '../../core/ad_trace'; +import type { AdTraceApi, AuctionBidData, AuctionTraceSummary, TsjsApi } from '../../core/types'; + +import { installAdTraceOverlay } from './overlay'; + +const TRACE_SOURCES = new Set(['initial_navigation', 'spa_navigation', 'auction_api']); +const TRACE_OUTCOMES = new Set(['completed', 'no_bid', 'skipped', 'failed', 'abandoned']); + +function validSummary(value: AuctionTraceSummary | undefined): value is AuctionTraceSummary { + return ( + value?.version === 1 && + isCanonicalTraceUuid(value.auctionTraceId) && + TRACE_SOURCES.has(value.source) && + TRACE_OUTCOMES.has(value.outcome) + ); +} + +function validBid(value: AuctionBidData | undefined, slotId: string): boolean { + const trace = value?.trace; + return !!( + trace?.version === 1 && + trace.slotId === slotId && + isCanonicalTraceUuid(trace.auctionTraceId) && + isCanonicalTraceUuid(trace.bidTraceId) && + isBoundedTraceLabel(trace.provider) && + isBoundedTraceLabel(trace.bidder) + ); +} + +function consumeActiveBootstrap(): boolean { + if (window.__tsjs_adTraceActive !== true) return false; + delete window.__tsjs_adTraceActive; + return true; +} + +/** Install the session-scoped recorder, immutable API, and overlay once. */ +export function installAdTrace(): boolean { + if (typeof window === 'undefined') return false; + if (window.tsjs?.adTrace) return true; + if (!consumeActiveBootstrap()) return false; + const ts = (window.tsjs ??= {} as TsjsApi); + + const store = createAdTraceStore(); + const api: AdTraceApi = Object.freeze({ + getSlot: store.getSlot, + getEvents: store.getEvents, + getRenderTimeline: store.getRenderTimeline, + export: store.export, + }); + ts.adTrace = api; + ts.recordAdTrace = store.record; + ts.nextAdTraceGeneration = store.nextGeneration; + ts.subscribeAdTrace = store.subscribe; + ts.bindAdTraceElement = store.bindElement; + ts.getAdTraceElement = store.getBoundElement; + ts.updateAdTraceVisibility = store.updateVisibility; + if (!ts.captureAdTraceRequest) { + ts.captureAdTraceRequest = (slot, trigger, snapshot) => { + const pending = (ts.pendingAdTraceRequests ??= []); + if (pending.length < 64) pending.push({ slot, trigger, snapshot }); + return 0; + }; + } + + const summary = validSummary(ts.auctionTrace) ? ts.auctionTrace : undefined; + for (const slot of ts.adSlots ?? []) { + const bid = ts.bids?.[slot.id]; + if (validBid(bid, slot.id) && bid?.trace) { + store.record({ + kind: 'ts_winner_observed', + slotId: slot.id, + auctionTraceId: bid.trace.auctionTraceId, + bidTraceId: bid.trace.bidTraceId, + provider: bid.trace.provider, + bidder: bid.trace.bidder, + }); + } else if (summary) { + store.record({ + kind: 'ts_auction_observed', + slotId: slot.id, + auctionTraceId: summary.auctionTraceId, + outcome: + summary.outcome === 'completed' || summary.outcome === 'no_bid' + ? 'no_bid' + : summary.outcome === 'skipped' + ? 'skipped' + : 'unresolved', + confidence: 'definitive', + reason: 'terminal_summary', + }); + } + } + + installAdTraceOverlay(api, store.subscribe); + return true; +} + +if (typeof window !== 'undefined') installAdTrace(); diff --git a/crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts b/crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts new file mode 100644 index 000000000..fd08963d0 --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts @@ -0,0 +1,231 @@ +import type { + AdTraceApi, + RenderTraceSnapshot, + RenderTraceVisibility, + SlotTraceSnapshot, +} from '../../core/types'; + +const HOST_ID = 'ts-ad-trace-overlay'; +const TRACE_ATTRIBUTES = [ + 'data-ts-trace-seq', + 'data-ts-trace-generation', + 'data-ts-auction-trace-id', + 'data-ts-bid-trace-id', + 'data-ts-trace-outcome', + 'data-ts-trace-visibility', +] as const; + +function stageLine(label: string, stage: { outcome: string; confidence: string }): string { + return `${label}: ${stage.outcome} · ${stage.confidence}`; +} + +function badgeText(slot: SlotTraceSnapshot, render?: RenderTraceSnapshot): string { + return [ + render ? `#${render.sequence}: ${render.outcome} · ${render.visibility}` : undefined, + stageLine('TS winner', slot.stages.trustedServer), + stageLine('Prebid winner', slot.stages.prebid), + stageLine('GAM result', slot.stages.gam), + stageLine('Creative', slot.stages.creative), + ] + .filter(Boolean) + .join('\n'); +} + +function removeTraceAttributes(element: HTMLElement): void { + for (const attribute of TRACE_ATTRIBUTES) element.removeAttribute(attribute); +} + +function effectiveVisibility(element: HTMLElement, rect: DOMRect): RenderTraceVisibility { + if (!element.isConnected) return 'disconnected'; + if (rect.width <= 0 || rect.height <= 0) return 'hidden'; + let current: HTMLElement | null = element; + while (current) { + const style = getComputedStyle(current); + if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') { + return 'hidden'; + } + current = current.parentElement; + } + return 'visible'; +} + +function stampRender(element: HTMLElement, render: RenderTraceSnapshot): void { + removeTraceAttributes(element); + element.setAttribute('data-ts-trace-seq', String(render.sequence)); + element.setAttribute('data-ts-trace-generation', String(render.generation)); + element.setAttribute('data-ts-trace-outcome', render.outcome); + element.setAttribute('data-ts-trace-visibility', render.visibility); + if (render.auctionTraceId) + element.setAttribute('data-ts-auction-trace-id', render.auctionTraceId); + if (render.bidTraceId) element.setAttribute('data-ts-bid-trace-id', render.bidTraceId); +} + +/** Install one read-only Shadow DOM trace console. */ +export function installAdTraceOverlay( + api: AdTraceApi, + subscribe: (fn: () => void) => () => void +): void { + if (document.getElementById(HOST_ID)) return; + const host = document.createElement('div'); + host.id = HOST_ID; + const root = host.attachShadow({ mode: 'closed' }); + const style = document.createElement('style'); + style.textContent = ` + :host { all: initial; } + .badge { position: fixed; z-index: 2147483647; max-width: 300px; padding: 6px 8px; + border: 1px solid #72e0a6; border-radius: 4px; background: rgba(10,18,16,.94); + color: #eefbf4; font: 11px/1.35 ui-monospace, monospace; white-space: pre; cursor: pointer; } + .badge.probable { border-color: #67a8ff; } + .panel { position: fixed; right: 12px; bottom: 12px; z-index: 2147483647; width: 460px; + max-height: 60vh; overflow: auto; padding: 10px; background: #0a1210; color: #eefbf4; + border: 1px solid #72e0a6; font: 11px/1.4 ui-monospace, monospace; } + .controls { display: flex; gap: 6px; position: sticky; top: 0; background: #0a1210; } + .warning { color: #ffd479; margin: 6px 0; } + .row { border-top: 1px solid #29443a; padding: 6px 0; } + .row strong { color: #72e0a6; } + button { margin-bottom: 6px; } pre { white-space: pre-wrap; }`; + root.appendChild(style); + const badgeLayer = document.createElement('div'); + const panel = document.createElement('div'); + panel.className = 'panel'; + const controls = document.createElement('div'); + controls.className = 'controls'; + const collapseButton = document.createElement('button'); + collapseButton.textContent = 'Collapse'; + const exportButton = document.createElement('button'); + exportButton.textContent = 'Export trace'; + const closeButton = document.createElement('button'); + closeButton.textContent = 'Close'; + const warning = document.createElement('div'); + warning.className = 'warning'; + warning.textContent = 'A non-empty GAM response alone is not proof of a Trusted Server creative.'; + const rows = document.createElement('div'); + const details = document.createElement('pre'); + details.hidden = true; + controls.append(collapseButton, exportButton, closeButton); + panel.append(controls, warning, rows, details); + root.append(badgeLayer, panel); + document.documentElement.appendChild(host); + let cleanup = (): void => {}; + + exportButton.addEventListener('click', () => { + const blob = new Blob([JSON.stringify(api.export(), null, 2)], { type: 'application/json' }); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = 'trusted-server-ad-trace.json'; + link.click(); + URL.revokeObjectURL(url); + }); + collapseButton.addEventListener('click', () => { + rows.hidden = !rows.hidden; + warning.hidden = rows.hidden; + collapseButton.textContent = rows.hidden ? 'Expand' : 'Collapse'; + }); + closeButton.addEventListener('click', () => { + cleanup(); + host.remove(); + }); + + let observedElements = new Set(); + const resizeObserver = + typeof ResizeObserver === 'undefined' ? undefined : new ResizeObserver(() => schedule()); + + const render = (): void => { + badgeLayer.replaceChildren(); + rows.replaceChildren(); + const exported = api.export(); + const slotById = new Map(exported.slots.map((slot) => [slot.slotId, slot])); + const latestBySlot = new Map(); + for (const item of exported.renders) latestBySlot.set(item.slotId, item); + const nextObserved = new Set(); + + for (const item of [...exported.renders].reverse()) { + const row = document.createElement('div'); + row.className = 'row'; + const title = document.createElement('strong'); + title.textContent = `#${item.sequence} ${item.slotId} · ${item.source}`; + const summary = document.createElement('div'); + summary.textContent = `${item.outcome} · ${item.confidence} · ${item.visibility}`; + row.append(title, summary); + row.addEventListener('click', () => { + details.hidden = false; + details.textContent = JSON.stringify( + { render: item, stages: slotById.get(item.slotId)?.stages }, + null, + 2 + ); + }); + rows.appendChild(row); + } + + for (const [slotId, slot] of slotById) { + const item = latestBySlot.get(slotId); + const element = item ? window.tsjs?.getAdTraceElement?.(slotId, item.generation) : undefined; + if (!element || !item) continue; + const rect = element.getBoundingClientRect(); + const visibility = effectiveVisibility(element, rect); + window.tsjs?.updateAdTraceVisibility?.(slotId, item.generation, visibility); + const effectiveItem = visibility === item.visibility ? item : { ...item, visibility }; + if (visibility === 'disconnected') { + resizeObserver?.unobserve(element); + removeTraceAttributes(element); + continue; + } + nextObserved.add(element); + if (!observedElements.has(element)) resizeObserver?.observe(element); + stampRender(element, effectiveItem); + const badge = document.createElement('div'); + badge.className = `badge ${item.outcome === 'confirmed' ? '' : 'probable'}`; + badge.textContent = badgeText(slot, effectiveItem); + badge.style.left = `${Math.max(0, rect.left)}px`; + badge.style.top = `${Math.max(0, rect.top)}px`; + badge.addEventListener('click', () => { + panel.hidden = false; + details.hidden = false; + details.textContent = JSON.stringify( + { render: effectiveItem, stages: slot.stages }, + null, + 2 + ); + }); + badgeLayer.appendChild(badge); + } + for (const element of observedElements) { + if (!nextObserved.has(element)) { + resizeObserver?.unobserve(element); + removeTraceAttributes(element); + } + } + observedElements = nextObserved; + }; + + let framePending = false; + const schedule = (): void => { + if (framePending) return; + framePending = true; + requestAnimationFrame(() => { + framePending = false; + if (host.isConnected) render(); + }); + }; + const unsubscribe = subscribe(schedule); + let cleaned = false; + cleanup = (): void => { + if (cleaned) return; + cleaned = true; + unsubscribe(); + resizeObserver?.disconnect(); + for (const element of observedElements) removeTraceAttributes(element); + window.removeEventListener('scroll', schedule); + window.removeEventListener('resize', schedule); + lifecycleObserver.disconnect(); + }; + const lifecycleObserver = new MutationObserver(() => { + if (!host.isConnected) cleanup(); + }); + lifecycleObserver.observe(document.documentElement, { childList: true, subtree: true }); + window.addEventListener('scroll', schedule, { passive: true }); + window.addEventListener('resize', schedule, { passive: true }); + render(); +} diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 8c0dcacba..8ab89ed6e 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -1,5 +1,5 @@ import { log } from '../../core/log'; -import type { AuctionSlot, AuctionBidData, TsjsApi } from '../../core/types'; +import type { AuctionSlot, AuctionBidData, AuctionTraceSummary, TsjsApi } from '../../core/types'; import { APS_UNIVERSAL_CREATIVE_RENDERER, APS_UNIVERSAL_CREATIVE_RENDERER_VERSION, @@ -40,7 +40,11 @@ const TS_BID_TARGETING_KEYS = [ 'hb_cache_host', 'hb_cache_path', ] as const; -const TS_BASE_TARGETING_KEYS = [...TS_BID_TARGETING_KEYS, TS_INITIAL_TARGETING_KEY] as const; +const TS_BASE_TARGETING_KEYS = [ + ...TS_BID_TARGETING_KEYS, + TS_INITIAL_TARGETING_KEY, + 'ts_trace', +] as const; // ------------------------------------------------------------------ // googletag type stubs (minimal surface needed by the shim) @@ -56,8 +60,189 @@ interface GoogleTagSlot { } interface SlotRenderEndedEvent { - isEmpty: boolean; + isEmpty?: boolean; + isBackfill?: boolean; + slot: GoogleTagSlot; +} + +interface GptSlotEvent { slot: GoogleTagSlot; + isEmpty?: boolean; + isBackfill?: boolean; +} + +interface RenderCandidate { + slotId: string; + generation: number; + slot: GoogleTagSlot; + divId: string; + /** Renderable only when this record's own hb_adid matches the request snapshot. */ + bid?: Readonly; + adId?: string; + traceToken?: string; + createdAt: number; + terminal: boolean; + consumed: boolean; + superseded: boolean; +} + +interface ExpectedRender { + candidate: RenderCandidate; + source: MessageEventSource; + expiresAt: number; + consumed: boolean; +} + +interface AdTraceRequestBoundarySnapshot { + slotId?: string; + bidder?: string; + adId?: string; + traceToken?: string; + bid?: AuctionBidData; +} + +const requestCandidates = new Map(); +const expectedRenders = new Map(); +const fallbackGenerations = new Map(); + +const MAX_EXPECTED_RENDERS = 200; +const MAX_FALLBACK_GENERATIONS = 200; +const MAX_ACTIVE_CACHE_RENDERS = 64; +const MAX_PRIVATE_REQUEST_OWNERS = 64; +let privateNavigationGeneration = 0; + +interface PrivateRequestOwner { + slotId: string; + adId?: string; + bid?: Readonly; + generation?: number; + element: HTMLElement | null; + navigationGeneration: number; + expiresAt: number; + served: boolean; +} + +const latestPrivateRequestBySlot = new Map(); +const staleTsAdIdBits = new Uint32Array(64); + +function staleAdIdHashes(value: string): [number, number] { + let first = 2166136261; + let second = 5381; + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + first = Math.imul(first ^ code, 16777619) >>> 0; + second = (Math.imul(second, 33) ^ code) >>> 0; + } + return [first % 2048, second % 2048]; +} + +function rememberStaleAdIdBits(adId: string): void { + for (const hash of staleAdIdHashes(adId)) { + staleTsAdIdBits[hash >>> 5] |= 1 << (hash & 31); + } +} + +function staleAdIdBitsContain(adId: string): boolean { + return staleAdIdHashes(adId).every( + (hash) => (staleTsAdIdBits[hash >>> 5] & (1 << (hash & 31))) !== 0 + ); +} + +interface ActiveCacheRender { + controller: AbortController; + slotId: string; + adId: string; + source: MessageEventSource | null; + generation?: number; + candidate?: RenderCandidate; + cacheHost: string; + cachePath: string; + traceToken?: string; + navigationGeneration: number; + expiresAt: number; + expiryTimer?: ReturnType; +} + +const activeCacheRenders = new Set(); +const latestCacheRenderBySlot = new Map(); + +function rememberStaleTsOwner(owner: PrivateRequestOwner): void { + if (!owner.bid || !owner.adId) return; + rememberStaleAdIdBits(owner.adId); +} + +function retireActiveCacheRender(render: ActiveCacheRender): void { + if (render.expiryTimer) clearTimeout(render.expiryTimer); + render.controller.abort(); + activeCacheRenders.delete(render); + if (latestCacheRenderBySlot.get(render.slotId) === render) { + latestCacheRenderBySlot.delete(render.slotId); + } +} + +function invalidatePrivateRequestOwners(slotId?: string): void { + const entries = slotId + ? [[slotId, latestPrivateRequestBySlot.get(slotId)] as const] + : [...latestPrivateRequestBySlot.entries()]; + for (const [key, owner] of entries) { + if (!owner) continue; + rememberStaleTsOwner(owner); + latestPrivateRequestBySlot.delete(key); + } +} + +function abortActiveCacheRenders(slotId?: string): void { + privateNavigationGeneration += slotId ? 0 : 1; + for (const render of [...activeCacheRenders]) { + if (!slotId || render.slotId === slotId) retireActiveCacheRender(render); + } + invalidatePrivateRequestOwners(slotId); +} + +function claimPrivateRequestOwner( + slotId: string, + adId: string | undefined, + bid: Readonly | undefined, + element: HTMLElement | null +): PrivateRequestOwner { + for (const render of [...activeCacheRenders]) { + if (render.slotId === slotId) retireActiveCacheRender(render); + } + const previous = latestPrivateRequestBySlot.get(slotId); + if (previous) rememberStaleTsOwner(previous); + const owner: PrivateRequestOwner = { + slotId, + adId, + bid, + element, + navigationGeneration: privateNavigationGeneration, + expiresAt: monotonicNow() + 30_000, + served: false, + }; + latestPrivateRequestBySlot.delete(slotId); + latestPrivateRequestBySlot.set(slotId, owner); + while (latestPrivateRequestBySlot.size > MAX_PRIVATE_REQUEST_OWNERS) { + const oldest = latestPrivateRequestBySlot.keys().next().value as string | undefined; + if (!oldest) break; + const evicted = latestPrivateRequestBySlot.get(oldest); + if (evicted) rememberStaleTsOwner(evicted); + latestPrivateRequestBySlot.delete(oldest); + for (const render of [...activeCacheRenders]) { + if (render.slotId === oldest) retireActiveCacheRender(render); + } + } + return owner; +} + +function isKnownStaleTsAdId(adId: string): boolean { + // The fixed-size bitset intentionally never forgets within the page session: + // false positives fail closed, while bounded-map eviction cannot create a + // false negative that lets a stale TS Universal Creative fall through. + return staleAdIdBitsContain(adId); +} + +function monotonicNow(): number { + return typeof performance === 'undefined' ? Date.now() : performance.now(); } function findSlotElementByDivId(divId: string): HTMLElement | null { @@ -126,7 +311,7 @@ interface GoogleTagPubAdsService { setTargeting(key: string, value: string | string[]): GoogleTagPubAdsService; getTargeting(key: string): string[]; enableSingleRequest(): void; - addEventListener(event: string, fn: (e: SlotRenderEndedEvent) => void): void; + addEventListener(event: string, fn: (e: GptSlotEvent) => void): void; refresh(slots?: GoogleTagSlot[]): void; getSlots?(): GoogleTagSlot[]; disableInitialLoad?(): void; @@ -156,6 +341,39 @@ type GptWindow = Window & { __tsjs_slim_prebid_url?: string; }; +const cacheInvalidationHookedTags = new WeakSet(); +const cacheInvalidationHookedSlots = new WeakSet(); + +function installSlotCacheInvalidationHook(slot: GoogleTagSlot): void { + if (cacheInvalidationHookedSlots.has(slot) || typeof slot.clearTargeting !== 'function') return; + const original = slot.clearTargeting.bind(slot); + slot.clearTargeting = (key?: string) => { + const slotId = slotIdForGptSlot(slot); + if (slotId) abortActiveCacheRenders(slotId); + return original(key); + }; + cacheInvalidationHookedSlots.add(slot); +} + +function installGoogleTagCacheInvalidationHooks(g: Partial): void { + if (cacheInvalidationHookedTags.has(g)) return; + if (typeof g.destroySlots === 'function') { + const original = g.destroySlots.bind(g); + g.destroySlots = (slots?: GoogleTagSlot[]) => { + if (slots) { + slots.forEach((slot) => { + const slotId = slotIdForGptSlot(slot); + if (slotId) abortActiveCacheRenders(slotId); + }); + } else { + abortActiveCacheRenders(); + } + return original(slots); + }; + } + cacheInvalidationHookedTags.add(g); +} + // ------------------------------------------------------------------ // Shim implementation // ------------------------------------------------------------------ @@ -377,24 +595,40 @@ function injectAdmIntoSlot(divId: string, adm: string): void { } } -function fireWinBillingBeacons(slotId: string, bid: AuctionBidData): void { - if (!slotId || (!bid.nurl && !bid.burl)) return; +const MAX_BILLING_DEDUPE_KEYS = 512; +const BILLING_DEDUPE_TTL_MS = 30 * 60_000; +const firedBillingKeys = new Map(); - const fired = (window.tsjs!.firedBeacons ??= {}); +function billingEntries(slotId: string, bid: AuctionBidData): Array<[string, string]> { const bidIdentity = bid.hb_adid ?? bid.nurl ?? bid.burl ?? ''; - const urls = [ - ['nurl', bid.nurl], - ['burl', bid.burl], - ] as const; - - for (const [kind, url] of urls) { - if (!url) continue; + return ( + [ + ['nurl', bid.nurl], + ['burl', bid.burl], + ] as const + ).flatMap(([kind, url]) => + url ? [[`${slotId}|${bidIdentity}|${kind}|${url}`, url] as [string, string]] : [] + ); +} - const beaconKey = `${slotId}|${bidIdentity}|${kind}|${url}`; - if (fired[beaconKey]) continue; +function billingCapacityAvailable(slotId: string, bid: AuctionBidData): boolean { + const now = monotonicNow(); + for (const [key, expiresAt] of firedBillingKeys) { + if (expiresAt <= now) firedBillingKeys.delete(key); + } + const additional = billingEntries(slotId, bid).filter( + ([key]) => !firedBillingKeys.has(key) + ).length; + return firedBillingKeys.size + additional <= MAX_BILLING_DEDUPE_KEYS; +} +function fireWinBillingBeacons(slotId: string, bid: AuctionBidData): void { + if (!slotId) return; + const now = monotonicNow(); + for (const [key, url] of billingEntries(slotId, bid)) { + if (firedBillingKeys.has(key)) continue; if (queueWinBillingBeacon(url)) { - fired[beaconKey] = true; + firedBillingKeys.set(key, now + BILLING_DEDUPE_TTL_MS); } } } @@ -494,8 +728,363 @@ function installInitialLoadDetector(ts: TsjsApi): void { }); } +function slotIdForGptSlot(slot: GoogleTagSlot): string | undefined { + const divId = slot.getSlotElementId?.() ?? ''; + return ( + window.tsjs?.divToSlotId?.[divId] ?? + window.tsjs?.adSlots?.find((item) => { + return ( + divId === item.div_id || + divId === `${item.div_id}-container` || + divId.startsWith(item.div_id) + ); + })?.id + ); +} + +function firstSlotTarget(slot: GoogleTagSlot, key: string): string | undefined { + return slot.getTargeting?.(key)?.find((value) => value.length > 0); +} + +function supersedeCandidate(candidate: RenderCandidate, reason: string): void { + if (candidate.superseded) return; + candidate.superseded = true; + for (const render of [...activeCacheRenders]) { + if (render.candidate === candidate) retireActiveCacheRender(render); + } + window.tsjs?.recordAdTrace?.({ + kind: 'generation_superseded', + slotId: candidate.slotId, + generation: candidate.generation, + bidTraceId: candidate.traceToken, + reason, + }); +} + +export function supersedeAdTraceSlot(slot: GoogleTagSlot, reason: string): void { + const slotId = slotIdForGptSlot(slot); + if (slotId) { + abortActiveCacheRenders(slotId); + if (window.tsjs?.prebidSelectedParticipants) { + window.tsjs.prebidSelectedParticipants = window.tsjs.prebidSelectedParticipants.filter( + (entry) => entry.slotId !== slotId + ); + } + } + for (const candidates of requestCandidates.values()) { + candidates + .filter((candidate) => candidate.slot === slot && !candidate.superseded) + .forEach((candidate) => supersedeCandidate(candidate, reason)); + } +} + +/** Capture immutable attribution immediately before one concrete GPT request. */ +export function captureAdTraceRequest( + slot: GoogleTagSlot, + trigger: string, + snapshot?: AdTraceRequestBoundarySnapshot +): number { + const ts = window.tsjs; + const hasBoundarySnapshot = snapshot !== undefined; + const slotId = hasBoundarySnapshot ? snapshot.slotId : slotIdForGptSlot(slot); + if (!slotId) return 0; + installSlotCacheInvalidationHook(slot); + + // Private service ownership is captured for every GPT request, even when the + // diagnostic recorder is disabled. It must precede all asynchronous render + // work so a later request or navigation can invalidate the exact owner. + const bidder = hasBoundarySnapshot ? snapshot.bidder : firstSlotTarget(slot, 'hb_bidder'); + const adId = hasBoundarySnapshot ? snapshot.adId : firstSlotTarget(slot, 'hb_adid'); + const rawTraceToken = hasBoundarySnapshot + ? snapshot.traceToken + : firstSlotTarget(slot, 'ts_trace'); + const traceToken = + rawTraceToken && TRACE_TOKEN_RE.test(rawTraceToken) ? rawTraceToken : undefined; + const liveBid = hasBoundarySnapshot ? snapshot.bid : ts?.bids?.[slotId]; + const renderBidMatches = + !!liveBid && + !!adId && + liveBid.hb_adid === adId && + (!traceToken || liveBid.trace?.bidTraceId === traceToken); + const divId = slot.getSlotElementId?.() ?? ''; + const privateBid = renderBidMatches ? Object.freeze({ ...liveBid }) : undefined; + const privateOwner = claimPrivateRequestOwner( + slotId, + adId, + privateBid, + divId ? findSlotElementByDivId(divId) : null + ); + + if (!ts?.recordAdTrace) return 0; + (requestCandidates.get(slotId) ?? []) + .filter((candidate) => !candidate.superseded && !candidate.consumed) + .forEach((candidate) => supersedeCandidate(candidate, 'request_replaced')); + const generation = + ts.nextAdTraceGeneration?.(slotId) ?? (fallbackGenerations.get(slotId) ?? 0) + 1; + privateOwner.generation = generation; + fallbackGenerations.delete(slotId); + fallbackGenerations.set(slotId, generation); + while (fallbackGenerations.size > MAX_FALLBACK_GENERATIONS) { + const oldest = fallbackGenerations.keys().next().value as string | undefined; + if (!oldest) break; + fallbackGenerations.delete(oldest); + } + + // Diagnostic attribution reads the same immutable request-boundary values as + // the private owner, but remains optional and independently gated. + const ledger = ts.prebidCorrelation ?? []; + const selectedMatches = traceToken + ? ledger.filter((entry) => entry.slotId === slotId && entry.traceToken === traceToken) + : adId + ? ledger.filter((entry) => entry.slotId === slotId && entry.adId === adId) + : []; + const selectedParticipant = selectedMatches.length === 1 ? selectedMatches[0] : undefined; + const completedAuction = [...(ts.prebidCompletedAuctions ?? [])] + .reverse() + .find((entry) => entry.slotIds.includes(slotId)); + const auctionId = + selectedParticipant?.auctionId ?? (!adId ? completedAuction?.auctionId : undefined); + const participants = auctionId + ? ledger.filter((entry) => entry.slotId === slotId && entry.auctionId === auctionId) + : []; + const hasTracedTsParticipant = participants.some((entry) => !!entry.traceToken); + const tracedServerParticipant = participants.find((entry) => entry.serverTrace); + const serverSummary = auctionId + ? (ts.prebidServerSummaries ?? []).find( + (entry) => entry.auctionId === auctionId && entry.slotId === slotId + )?.summary + : undefined; + if (selectedParticipant) { + const selected = (ts.prebidSelectedParticipants ??= []).filter( + (entry) => monotonicNow() - entry.selectedAt <= 30_000 + ); + selected.push({ + auctionId: selectedParticipant.auctionId, + slotId, + requestId: selectedParticipant.requestId, + adId: selectedParticipant.adId, + traceToken: selectedParticipant.traceToken, + bidder: selectedParticipant.bidder, + generation, + selectedAt: monotonicNow(), + }); + while (selected.length > 128) selected.shift(); + ts.prebidSelectedParticipants = selected; + } + if (auctionId) { + ts.prebidCorrelation = ledger.filter( + (entry) => !(entry.slotId === slotId && entry.auctionId === auctionId) + ); + ts.prebidCompletedAuctions = (ts.prebidCompletedAuctions ?? []).filter( + (entry) => entry.auctionId !== auctionId + ); + ts.prebidServerSummaries = (ts.prebidServerSummaries ?? []).filter( + (entry) => !(entry.auctionId === auctionId && entry.slotId === slotId) + ); + } + + const candidate: RenderCandidate = { + slotId, + generation, + slot, + divId, + ...(privateBid ? { bid: privateBid } : {}), + adId, + traceToken, + createdAt: monotonicNow(), + terminal: false, + consumed: false, + superseded: false, + }; + const capturedElement = candidate.divId ? findSlotElementByDivId(candidate.divId) : null; + if (capturedElement) ts.bindAdTraceElement?.(slotId, generation, capturedElement); + if (!requestCandidates.has(slotId) && requestCandidates.size >= 64) { + const oldestSlotId = requestCandidates.keys().next().value as string | undefined; + if (oldestSlotId) { + requestCandidates + .get(oldestSlotId) + ?.forEach((item) => supersedeCandidate(item, 'slot_evicted')); + requestCandidates.delete(oldestSlotId); + } + } + const candidates = requestCandidates.get(slotId) ?? []; + candidates + .filter((item) => !item.superseded && monotonicNow() - item.createdAt > 30_000) + .forEach((item) => supersedeCandidate(item, 'generation_expired')); + candidates.push(candidate); + if (candidates.length > 8) { + const evicted = candidates.shift(); + if (evicted) supersedeCandidate(evicted, 'generation_evicted'); + } + requestCandidates.set(slotId, candidates); + + const serverTrace = tracedServerParticipant?.serverTrace; + if (serverTrace) { + ts.recordAdTrace({ + kind: 'ts_winner_observed', + slotId, + generation, + auctionTraceId: serverTrace.auctionTraceId, + bidTraceId: serverTrace.bidTraceId, + provider: serverTrace.provider, + bidder: serverTrace.bidder, + }); + } else if (serverSummary) { + ts.recordAdTrace({ + kind: 'ts_auction_observed', + slotId, + generation, + auctionTraceId: serverSummary.auctionTraceId, + outcome: serverSummary.outcome === 'completed' ? 'no_bid' : serverSummary.outcome, + confidence: 'definitive', + reason: 'terminal_summary', + }); + } + + let outcome = 'no_bid'; + let reason = 'no_selected_targeting'; + let confidence: 'definitive' | 'none' = 'definitive'; + if (selectedMatches.length > 1) { + outcome = 'unresolved'; + reason = 'ambiguous_prebid_request'; + confidence = 'none'; + } else if (selectedParticipant) { + if (traceToken && selectedParticipant.traceToken === traceToken) outcome = 'won'; + else if (!traceToken) outcome = hasTracedTsParticipant ? 'lost' : 'client_bid_won'; + else outcome = hasTracedTsParticipant ? 'lost' : 'unresolved'; + reason = 'selected_targeting'; + } else if (completedAuction && !bidder && !adId && !traceToken) { + outcome = 'no_bid'; + reason = 'prebid_no_bid'; + } else if (bidder || adId || traceToken) { + outcome = traceToken && renderBidMatches ? 'not_run' : 'client_bid_won'; + reason = traceToken && renderBidMatches ? 'direct_gpt_request' : 'unjoined_targeting'; + if (!traceToken && !renderBidMatches) confidence = 'none'; + } + ts.recordAdTrace({ + kind: 'prebid_targeting_selected', + slotId, + generation, + bidTraceId: traceToken, + bidder, + outcome, + confidence, + reason, + }); + for (const kind of selectedParticipant?.events ?? []) { + ts.recordAdTrace({ + kind, + slotId, + generation, + bidTraceId: traceToken, + bidder, + }); + } + if (liveBid?.hb_bidder === 'aps' || liveBid?.hb_bidder === 'amazon-aps') { + ts.recordAdTrace({ + kind: 'aps_display_bids_set', + slotId, + generation, + bidTraceId: traceToken, + }); + } + ts.recordAdTrace({ + kind: 'gpt_request_started', + slotId, + generation, + auctionTraceId: liveBid?.trace?.auctionTraceId ?? ts.auctionTrace?.auctionTraceId, + bidTraceId: traceToken, + provider: liveBid?.trace?.provider, + bidder, + reason: trigger, + }); + return generation; +} + +function candidateForSlot( + slot: GoogleTagSlot, + includeTerminal = false +): RenderCandidate | undefined { + const slotId = slotIdForGptSlot(slot); + if (!slotId) return undefined; + const candidates = (requestCandidates.get(slotId) ?? []).filter( + (candidate) => + candidate.slot === slot && + !candidate.superseded && + (includeTerminal || !candidate.terminal) && + monotonicNow() - candidate.createdAt <= 30_000 + ); + if (candidates.length !== 1) { + if (candidates.length > 1) { + candidates.forEach((candidate) => + window.tsjs?.recordAdTrace?.({ + kind: 'gpt_slot_render_ended', + slotId, + generation: candidate.generation, + bidTraceId: candidate.traceToken, + outcome: 'unresolved', + confidence: 'none', + reason: 'overlapping_request', + }) + ); + } else { + window.tsjs?.recordAdTrace?.({ + kind: 'gpt_slot_response_received', + slotId, + outcome: 'unresolved', + confidence: 'none', + reason: 'missing_generation', + }); + } + return undefined; + } + return candidates[0]; +} + +function installGptEvidenceListeners(service: GoogleTagPubAdsService): void { + if (!window.tsjs?.recordAdTrace) return; + const instrumented = service as GoogleTagPubAdsService & { __tsAdTraceListeners?: boolean }; + if (instrumented.__tsAdTraceListeners) return; + instrumented.__tsAdTraceListeners = true; + const record = + (kind: 'gpt_slot_requested' | 'gpt_slot_response_received' | 'gpt_slot_onload') => + (event: GptSlotEvent): void => { + const candidate = candidateForSlot(event.slot, kind === 'gpt_slot_onload'); + if (!candidate) return; + window.tsjs?.recordAdTrace?.({ + kind, + slotId: candidate.slotId, + generation: candidate.generation, + bidTraceId: candidate.traceToken, + }); + }; + service.addEventListener('slotRequested', record('gpt_slot_requested')); + service.addEventListener('slotResponseReceived', record('gpt_slot_response_received')); + service.addEventListener('slotOnload', record('gpt_slot_onload')); + service.addEventListener('slotRenderEnded', (event: GptSlotEvent) => { + const candidate = candidateForSlot(event.slot); + if (!candidate) return; + candidate.terminal = true; + window.tsjs?.recordAdTrace?.({ + kind: 'gpt_slot_render_ended', + slotId: candidate.slotId, + generation: candidate.generation, + bidTraceId: candidate.traceToken, + isEmpty: event.isEmpty, + isBackfill: event.isBackfill, + }); + }); +} + export function installTsAdInit(): void { const ts = (window.tsjs ??= {} as TsjsApi); + const pendingBootstrapRequests = ts.pendingAdTraceRequests ?? []; + ts.pendingAdTraceRequests = []; + ts.captureAdTraceRequest = (slot, trigger, snapshot) => + captureAdTraceRequest(slot as GoogleTagSlot, trigger, snapshot); + pendingBootstrapRequests.forEach(({ slot, trigger, snapshot }) => + ts.captureAdTraceRequest?.(slot, trigger, snapshot) + ); installInitialLoadDetector(ts); ts.adInit = function () { const slots = ts.adSlots ?? []; @@ -503,18 +1092,51 @@ export function installTsAdInit(): void { // The slotRenderEnded listener below reads ts.bids live so SPA navigation // updates (new ts.bids injected before ) are picked up at render time. const bids = ts.bids ?? {}; + const summary = ts.auctionTrace; + for (const slot of slots) { + const bid = bids[slot.id]; + if (bid?.trace && TRACE_TOKEN_RE.test(bid.trace.bidTraceId)) { + ts.recordAdTrace?.({ + kind: 'ts_winner_observed', + slotId: slot.id, + auctionTraceId: bid.trace.auctionTraceId, + bidTraceId: bid.trace.bidTraceId, + provider: bid.trace.provider, + bidder: bid.trace.bidder, + }); + } else if (summary) { + ts.recordAdTrace?.({ + kind: 'ts_auction_observed', + slotId: slot.id, + auctionTraceId: summary.auctionTraceId, + outcome: + summary.outcome === 'completed' || summary.outcome === 'no_bid' + ? 'no_bid' + : summary.outcome === 'skipped' + ? 'skipped' + : 'unresolved', + confidence: 'definitive', + reason: 'terminal_summary', + }); + } + } const g = (window as GptWindow).googletag; if (!g) return; g.cmd?.push(() => { + installGoogleTagCacheInvalidationHooks(g); // Destroy previously defined TS slots before redefining for the new page. if (ts.prevGptSlots && ts.prevGptSlots.length > 0) { + (ts.prevGptSlots as GoogleTagSlot[]).forEach((slot) => + supersedeAdTraceSlot(slot, 'slot_destroyed') + ); g.destroySlots?.(ts.prevGptSlots as GoogleTagSlot[]); ts.prevGptSlots = []; } // Slots TS defined itself — tracked for SPA destroy. Publisher-owned // slots are reused but never destroyed by TS on navigation. + installGptEvidenceListeners(g.pubads!()); const newSlots: GoogleTagSlot[] = []; // Publisher-owned slots TS reused — refreshed to pick up server-side // targeting. The publisher already display()ed these. @@ -542,6 +1164,7 @@ export function installTsAdInit(): void { (g.pubads!().getSlots?.() ?? []).forEach((gptSlot: GoogleTagSlot) => { const elementId = gptSlot.getSlotElementId(); if (!prevTouchedDivIds.has(elementId)) return; + supersedeAdTraceSlot(gptSlot, 'targeting_cleared'); clearTargetingKeys(gptSlot, [ ...TS_BASE_TARGETING_KEYS, ...(prevSlotTargetingKeys[elementId] ?? []), @@ -578,6 +1201,7 @@ export function installTsAdInit(): void { tsOwned = true; } + installSlotCacheInvalidationHook(gptSlot); const slotDivId2 = gptSlot.getSlotElementId?.() ?? actualDivId; clearTargetingKeys(gptSlot, [ ...TS_BASE_TARGETING_KEYS, @@ -589,7 +1213,18 @@ export function installTsAdInit(): void { TS_BID_TARGETING_KEYS.forEach((key) => { if (bid[key]) gptSlot.setTargeting(key, String(bid[key]!)); }); + if (bid.trace?.bidTraceId && TRACE_TOKEN_RE.test(bid.trace.bidTraceId)) { + gptSlot.setTargeting('ts_trace', bid.trace.bidTraceId); + } gptSlot.setTargeting(TS_INITIAL_TARGETING_KEY, '1'); + ts.recordAdTrace?.({ + kind: 'gpt_targeting_applied', + slotId: slot.id, + auctionTraceId: bid.trace?.auctionTraceId, + bidTraceId: bid.trace?.bidTraceId, + provider: bid.trace?.provider, + bidder: bid.trace?.bidder, + }); // Map both inner div and container div → slot ID so slotRenderEnded // (which reports the GPT slot's div, i.e. slotDivId/container) can look up // the slot, while adm injection (which targets the inner div) also works. @@ -605,8 +1240,20 @@ export function installTsAdInit(): void { slotsToRefresh.push(gptSlot); } - // Trusted Server APS winners carry their own typed renderer and never - // enter the publisher-owned native apstag rendering path. + // Typed Trusted Server APS winners render through their own descriptor. + // Only publisher-native APS bids should enter the apstag handoff. + if ( + bid.renderer === undefined && + (bid.hb_bidder === 'aps' || bid.hb_bidder === 'amazon-aps') + ) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (window as any).apstag?.setDisplayBids?.(); + ts.recordAdTrace?.({ + kind: 'aps_display_bids_set', + slotId: slot.id, + bidTraceId: bid.trace?.bidTraceId, + }); + } }); ts.prevGptSlots = newSlots as unknown[]; @@ -654,7 +1301,11 @@ export function installTsAdInit(): void { // called without a matching display call") and misses its impression. // Must run after enableServices(); on SPA navigation services are already // enabled, so this runs unconditionally for any newly-defined slots. - slotsToDisplay.forEach((divId) => g.display?.(divId)); + slotsToDisplay.forEach((divId) => { + const gptSlot = newSlots.find((slot) => slot.getSlotElementId() === divId); + if (gptSlot && !ts.gptInitialLoadDisabled) captureAdTraceRequest(gptSlot, 'display'); + g.display?.(divId); + }); // Slots needing an explicit ad request via refresh(). Reused // publisher-owned slots always need one to pick up the just-applied @@ -677,6 +1328,7 @@ export function installTsAdInit(): void { // the same slots still go through the wrapper normally. ts.adInitRefreshInProgress = true; try { + slotsNeedingRefresh.forEach((slot) => captureAdTraceRequest(slot, 'refresh')); g.pubads!().refresh(slotsNeedingRefresh); } finally { ts.adInitRefreshInProgress = false; @@ -687,6 +1339,7 @@ export function installTsAdInit(): void { } interface PageBidsResponse { + auctionTrace?: AuctionTraceSummary; slots: AuctionSlot[]; bids: Record; } @@ -767,7 +1420,11 @@ export function installSpaAuctionHook(): void { let lastAppliedPath = `${location.pathname}${location.search}`; async function onNavigate(path: string): Promise { + // Navigation invalidates private render ownership even when the resulting + // route key is unchanged (for example a state-only replaceState call). + abortActiveCacheRenders(); if (path === currentPath) return; + ts.prebidSelectedParticipants = []; currentPath = path; inflight?.abort(); const controller = new AbortController(); @@ -798,6 +1455,7 @@ export function installSpaAuctionHook(): void { await waitForSlotElements(data.slots, controller.signal); if (inflight !== controller) return; ts.adSlots = data.slots; + ts.auctionTrace = data.auctionTrace; ts.bids = data.bids; // This route is now the committed, loaded state — a later failed // navigation rolls back here, and a return trip no-ops correctly. @@ -863,6 +1521,8 @@ export function installSlimPrebidLoader(): void { const TS_DISPLAY_RENDERER = '(function(){window.render=function(d,h,w){' + 'var f=h.mkFrame(w.document,{width:d.width||"100%",height:d.height||"100%"});' + + 'if(typeof d.traceToken==="string"){f.addEventListener("load",function(){' + + 'top.postMessage({type:"ts-creative-load",version:1,traceToken:d.traceToken},"*");},{once:true});}' + 'if(d.adUrl&&!d.ad){f.src=d.adUrl;}else{f.srcdoc=d.ad;}' + 'w.document.body.appendChild(f);};})();'; @@ -939,6 +1599,48 @@ export function parseCachedBid(body: string): CachedBid | undefined { }; } +const TRACE_TOKEN_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + +function pruneExpectedRenders(): void { + const now = monotonicNow(); + for (const [token, entries] of expectedRenders) { + const retained = entries.filter((entry) => !entry.consumed && entry.expiresAt >= now); + if (retained.length > 0) expectedRenders.set(token, retained); + else expectedRenders.delete(token); + } +} + +function armExpectedRender( + candidate: RenderCandidate | undefined, + source: MessageEventSource | null +): string | undefined { + if ( + !candidate?.bid || + candidate.superseded || + !candidate.traceToken || + !TRACE_TOKEN_RE.test(candidate.traceToken) || + !source + ) { + return undefined; + } + pruneExpectedRenders(); + const expectedCount = [...expectedRenders.values()].reduce( + (count, entries) => count + entries.length, + 0 + ); + if (expectedCount >= MAX_EXPECTED_RENDERS) return undefined; + candidate.consumed = true; + const entries = expectedRenders.get(candidate.traceToken) ?? []; + entries.push({ + candidate, + source, + expiresAt: monotonicNow() + 30_000, + consumed: false, + }); + expectedRenders.set(candidate.traceToken, entries); + return candidate.traceToken; +} + /** * Install the TS → pbRender bridge. * @@ -996,6 +1698,43 @@ export function installTsRenderBridge(): void { return; } + if (data['type'] === 'ts-creative-load') { + const token = data['traceToken']; + if (data['version'] !== 1 || typeof token !== 'string' || !TRACE_TOKEN_RE.test(token)) return; + const entries = expectedRenders.get(token) ?? []; + entries + .filter((entry) => !entry.consumed && entry.expiresAt < monotonicNow()) + .forEach((entry) => supersedeCandidate(entry.candidate, 'ack_expired')); + const matches = entries.filter( + (entry) => + !entry.consumed && + !entry.candidate.superseded && + entry.expiresAt >= monotonicNow() && + entry.source === e.source && + (requestCandidates.get(entry.candidate.slotId) ?? []).includes(entry.candidate) + ); + if (matches.length !== 1) { + const candidate = entries[0]?.candidate; + window.tsjs?.recordAdTrace?.({ + kind: 'pb_render_rejected', + slotId: candidate?.slotId, + generation: candidate?.generation, + bidTraceId: TRACE_TOKEN_RE.test(token) ? token : undefined, + reason: matches.length > 1 ? 'ambiguous_generation' : 'invalid_acknowledgement', + }); + return; + } + const expected = matches[0]; + expected.consumed = true; + window.tsjs?.recordAdTrace?.({ + kind: 'creative_load_acknowledged', + slotId: expected.candidate.slotId, + generation: expected.candidate.generation, + bidTraceId: token, + }); + return; + } + if (data['message'] !== 'Prebid Request') return; const adId = data['adId'] as string | undefined; if (!adId) return; @@ -1070,18 +1809,57 @@ export function installTsRenderBridge(): void { const sourceSlotId = slotIdForMessageSource(e.source); if (!sourceSlotId) return; - // Resolve the bid by the requesting slot, not by the first bid whose hb_adid - // matches. hb_adid is not unique per bid: absent PBS Cache, it falls back to a - // creative id a bidder may reuse across slots. A first-match-by-adId lookup - // would resolve every duplicate to one slot, so all but that slot render blank. - const bids = window.tsjs?.bids ?? {}; - const slotId = sourceSlotId; - const matchedBid = bids[slotId]; + const allCandidates = requestCandidates.get(sourceSlotId) ?? []; + allCandidates + .filter((candidate) => !candidate.superseded && monotonicNow() - candidate.createdAt > 30_000) + .forEach((candidate) => supersedeCandidate(candidate, 'generation_expired')); + const candidates = allCandidates.filter( + (candidate) => + candidate.adId === adId && + !candidate.consumed && + !candidate.superseded && + monotonicNow() - candidate.createdAt <= 30_000 + ); + const exactCandidate = candidates.length === 1 ? candidates[0] : undefined; + window.tsjs?.recordAdTrace?.({ + kind: candidates.length === 1 ? 'pb_render_requested' : 'pb_render_rejected', + slotId: sourceSlotId, + generation: exactCandidate?.generation, + bidTraceId: exactCandidate?.traceToken, + reason: + candidates.length === 1 + ? 'exact_generation' + : candidates.length > 1 + ? 'ambiguous_generation' + : 'missing_generation', + }); - // Not a TS bid, or the requesting slot's bid does not own this adId — let - // Prebid.js handle it. The adId guard also prevents an iframe under slot A from - // pulling slot B's creative and firing slot B's win/billing beacons. - if (!matchedBid || matchedBid.hb_adid !== adId) return; + const slotId = sourceSlotId; + const requestOwner = latestPrivateRequestBySlot.get(slotId); + const liveBid = window.tsjs?.bids?.[slotId]; + const ownerCurrent = + !!requestOwner?.bid && + requestOwner.adId === adId && + requestOwner.bid.hb_adid === adId && + requestOwner.navigationGeneration === privateNavigationGeneration && + requestOwner.expiresAt >= monotonicNow() && + !requestOwner.served && + !!requestOwner.element?.isConnected && + findSlotElementByDivId(requestOwner.element.id) === requestOwner.element && + slotIdForMessageSource(e.source) === slotId && + liveBid?.hb_adid === requestOwner.bid.hb_adid && + liveBid.hb_cache_host === requestOwner.bid.hb_cache_host && + liveBid.hb_cache_path === requestOwner.bid.hb_cache_path && + liveBid.trace?.bidTraceId === requestOwner.bid.trace?.bidTraceId; + if (!ownerCurrent || !requestOwner?.bid) { + // A once-TS-owned message must not escape to ordinary Prebid after its + // request owner was replaced or invalidated. + if (isKnownStaleTsAdId(adId) || liveBid?.hb_adid === adId) { + e.stopImmediatePropagation(); + } + return; + } + const matchedBid = requestOwner.bid; const slot = window.tsjs?.adSlots?.find((s) => s.id === slotId); // Prefer the winning creative's own dimensions; the first configured slot @@ -1102,6 +1880,7 @@ export function installTsRenderBridge(): void { const rendererKey = `${slotId}|${adId}`; if (renderingKeys.has(rendererKey)) return; renderingKeys.add(rendererKey); + requestOwner.served = true; try { port.postMessage( @@ -1116,8 +1895,16 @@ export function installTsRenderBridge(): void { height: renderer.height, }) ); + window.tsjs?.recordAdTrace?.({ + kind: 'pb_render_served', + slotId, + generation: exactCandidate?.generation, + bidTraceId: exactCandidate?.traceToken, + reason: 'aps_renderer', + }); log.debug(`[tsjs-gpt] pbRender bridge served '${slotId}' through APS renderer`); } catch (err) { + requestOwner.served = false; renderingKeys.delete(rendererKey); log.warn('[tsjs-gpt] pbRender bridge: APS response failed', err); } @@ -1125,6 +1912,9 @@ export function installTsRenderBridge(): void { } if (matchedBid.adm) { + if (!billingCapacityAvailable(slotId, matchedBid)) return; + const traceToken = armExpectedRender(exactCandidate, e.source); + requestOwner.served = true; e.stopImmediatePropagation(); port.postMessage( JSON.stringify({ @@ -1134,9 +1924,16 @@ export function installTsRenderBridge(): void { renderer: TS_DISPLAY_RENDERER, width, height, + ...(traceToken ? { traceToken } : {}), }) ); fireWinBillingBeacons(slotId, matchedBid); + window.tsjs?.recordAdTrace?.({ + kind: 'pb_render_served', + slotId, + generation: exactCandidate?.generation, + bidTraceId: traceToken, + }); log.debug(`[tsjs-gpt] pbRender bridge served '${slotId}' from inline adm`); return; } @@ -1144,31 +1941,121 @@ export function installTsRenderBridge(): void { // No TS render source — let Prebid.js handle it. if (!matchedBid.hb_cache_host || !matchedBid.hb_cache_path) return; - // TS owns this adId — stop Prebid from also processing it. + const capturedSource = e.source; + const capturedElement = requestOwner.element; + const capturedCacheHost = matchedBid.hb_cache_host; + const capturedCachePath = matchedBid.hb_cache_path; + const capturedTraceToken = matchedBid.trace?.bidTraceId; + + const previousOwner = latestCacheRenderBySlot.get(slotId); + if ( + previousOwner && + previousOwner.adId === adId && + previousOwner.source === capturedSource && + previousOwner.generation === requestOwner.generation && + previousOwner.cacheHost === capturedCacheHost && + previousOwner.cachePath === capturedCachePath && + previousOwner.traceToken === capturedTraceToken && + !previousOwner.controller.signal.aborted && + previousOwner.expiresAt >= monotonicNow() + ) { + // A duplicate message for the exact accepted owner must not start a + // second fetch or escape to the ordinary Prebid renderer. + e.stopImmediatePropagation(); + return; + } + if (previousOwner) retireActiveCacheRender(previousOwner); + + // Capacity overflow must not evict a different live billing owner. Leave + // the message untouched so the ordinary Prebid path can process it. + if (activeCacheRenders.size >= MAX_ACTIVE_CACHE_RENDERS) return; + + const controller = new AbortController(); + const activeRender: ActiveCacheRender = { + controller, + slotId, + adId, + source: capturedSource, + generation: requestOwner.generation, + ...(exactCandidate?.generation === requestOwner.generation + ? { candidate: exactCandidate } + : {}), + cacheHost: capturedCacheHost, + cachePath: capturedCachePath, + traceToken: capturedTraceToken, + navigationGeneration: privateNavigationGeneration, + expiresAt: requestOwner.expiresAt, + }; + + activeCacheRenders.add(activeRender); + latestCacheRenderBySlot.set(slotId, activeRender); + activeRender.expiryTimer = setTimeout( + () => retireActiveCacheRender(activeRender), + Math.max(0, activeRender.expiresAt - monotonicNow()) + ); + // TS owns this accepted render — stop Prebid from also processing it. e.stopImmediatePropagation(); - // Skip a concurrent re-render of the same slot's adId so its win/billing - // beacons fire at most once even before the first cache fetch resolves. - const renderingKey = `${slotId}|${adId}`; - if (renderingKeys.has(renderingKey)) return; - renderingKeys.add(renderingKey); + const stillCurrent = (): boolean => { + const liveBid = window.tsjs?.bids?.[slotId]; + const candidateCurrent = + !exactCandidate || + (!exactCandidate.superseded && + (requestCandidates.get(slotId) ?? []).includes(exactCandidate)); + return ( + !controller.signal.aborted && + latestCacheRenderBySlot.get(slotId) === activeRender && + latestPrivateRequestBySlot.get(slotId) === requestOwner && + requestOwner.navigationGeneration === privateNavigationGeneration && + requestOwner.expiresAt >= monotonicNow() && + !requestOwner.served && + activeRender.navigationGeneration === privateNavigationGeneration && + activeRender.expiresAt >= monotonicNow() && + candidateCurrent && + !!capturedElement?.isConnected && + findSlotElementByDivId(capturedElement.id) === capturedElement && + slotIdForMessageSource(capturedSource) === slotId && + liveBid?.hb_adid === adId && + liveBid.hb_cache_host === capturedCacheHost && + liveBid.hb_cache_path === capturedCachePath && + liveBid.trace?.bidTraceId === capturedTraceToken + ); + }; - const cacheUrl = `https://${matchedBid.hb_cache_host}${matchedBid.hb_cache_path}?uuid=${encodeURIComponent(adId)}`; + const cacheUrl = `https://${capturedCacheHost}${capturedCachePath}?uuid=${encodeURIComponent(adId)}`; - fetch(cacheUrl, { mode: 'cors' }) + fetch(cacheUrl, { mode: 'cors', signal: controller.signal }) .then((res) => (res.ok ? res.text() : Promise.reject(res.status))) .then((body) => { // PBS Cache returns the cached bid as a JSON object; decode its creative // and render metadata the same way the Prebid Universal Creative does. const cached = parseCachedBid(body); if (!cached) { - // No renderable creative in the cache payload — decline rather than - // ship a serialized bid document to PUC. Beacons stay unfired. log.warn( `[tsjs-gpt] pbRender bridge: PBS Cache response for '${slotId}' had no renderable adm` ); return; } + if (!stillCurrent()) { + window.tsjs?.recordAdTrace?.({ + kind: 'pb_render_rejected', + slotId, + generation: exactCandidate?.generation, + bidTraceId: exactCandidate?.traceToken, + reason: 'stale_cache_completion', + }); + return; + } + if (!billingCapacityAvailable(slotId, matchedBid)) { + window.tsjs?.recordAdTrace?.({ + kind: 'pb_render_rejected', + slotId, + generation: exactCandidate?.generation, + bidTraceId: exactCandidate?.traceToken, + reason: 'billing_capacity', + }); + return; + } // Resolve the auction-price macro from the cached clearing price, and // size from the cached bid's own dimensions, falling back to the slot // format only when the cache omits them. @@ -1176,6 +2063,8 @@ export function installTsRenderBridge(): void { cached.price !== undefined ? expandAuctionPriceMacro(cached.adm, cached.price) : cached.adm; + const traceToken = armExpectedRender(exactCandidate, capturedSource); + requestOwner.served = true; port.postMessage( JSON.stringify({ message: 'Prebid Response', @@ -1184,16 +2073,28 @@ export function installTsRenderBridge(): void { renderer: TS_DISPLAY_RENDERER, width: cached.width ?? width, height: cached.height ?? height, + ...(traceToken ? { traceToken } : {}), }) ); fireWinBillingBeacons(slotId, matchedBid); + window.tsjs?.recordAdTrace?.({ + kind: 'pb_render_served', + slotId, + generation: exactCandidate?.generation, + bidTraceId: traceToken, + }); log.debug(`[tsjs-gpt] pbRender bridge served '${slotId}' from PBS Cache`); }) .catch((err) => { + if (err instanceof DOMException && err.name === 'AbortError') return; log.warn(`[tsjs-gpt] pbRender bridge: PBS Cache fetch failed for '${slotId}'`, err); }) .finally(() => { - renderingKeys.delete(renderingKey); + if (activeRender.expiryTimer) clearTimeout(activeRender.expiryTimer); + activeCacheRenders.delete(activeRender); + if (latestCacheRenderBySlot.get(slotId) === activeRender) { + latestCacheRenderBySlot.delete(slotId); + } }); }); } diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 735334776..63b9fa7ea 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -28,10 +28,10 @@ import 'prebid.js/modules/userId.js'; import './_adapters.generated'; import { log } from '../../core/log'; -import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; +import { buildAdRequest, parseAuctionResponse, parseAuctionTraceSummary } from '../../core/auction'; import { registerApsPrebidRenderer } from '../aps/render'; import type { AuctionBid, AuctionEid } from '../../core/auction'; -import type { AuctionSlot } from '../../core/types'; +import type { AdTraceEventKind, AuctionSlot, TrustedServerBidTrace } from '../../core/types'; import { INCLUDED_PREBID_USER_ID_MODULES } from './_user_ids.generated'; import { PREBID_USER_ID_MODULE_REGISTRY } from './user_id_modules'; @@ -52,6 +52,7 @@ const TS_REFRESH_TARGETING_KEYS = [ 'hb_adid', 'hb_cache_host', 'hb_cache_path', + 'ts_trace', ] as const; const PUBLISHER_DELIVERY_CONTEXT_TIMEOUT_MS = 1000; @@ -227,6 +228,12 @@ export function auctionBidsToPrebidBids(auctionBids: AuctionBid[], bidRequests: meta: { advertiserDomains: bid.adomain, }, + ...(bid.trace + ? { + adserverTargeting: { ts_trace: bid.trace.bidTraceId }, + tsTrace: bid.trace, + } + : {}), }; }); } @@ -265,6 +272,7 @@ type TrustedServerBidRequest = { adUnitCode?: string; code?: string; bidId?: string; + auctionId?: string; }; type TrustedServerRequest = { method: 'POST'; @@ -590,6 +598,155 @@ function serverSideBidderParamsForRefresh( return params; } +function installAdTracePrebidObservers(): void { + const ts = window.tsjs; + if (!ts?.recordAdTrace) return; + const instrumented = pbjs as unknown as { + __tsAdTraceObserved?: boolean; + onEvent?: (event: string, handler: (data: Record) => void) => void; + setTargetingForGPTAsync?: (codes?: string[]) => unknown; + }; + if (instrumented.__tsAdTraceObserved) return; + instrumented.__tsAdTraceObserved = true; + + const record = + (kind: AdTraceEventKind) => + (data: Record = {}): void => { + const nestedBid = + data.bid && typeof data.bid === 'object' + ? (data.bid as Record) + : undefined; + const evidence = nestedBid ?? data; + const slotId = + typeof evidence.adUnitCode === 'string' + ? evidence.adUnitCode + : typeof evidence.code === 'string' + ? evidence.code + : undefined; + const bidder = + typeof evidence.bidderCode === 'string' + ? evidence.bidderCode + : typeof evidence.bidder === 'string' + ? evidence.bidder + : undefined; + const auctionId = + typeof evidence.auctionId === 'string' + ? evidence.auctionId + : typeof data.auctionId === 'string' + ? data.auctionId + : ''; + const requestId = + typeof evidence.requestId === 'string' + ? evidence.requestId + : typeof evidence.adId === 'string' + ? evidence.adId + : ''; + const adId = typeof evidence.adId === 'string' ? evidence.adId : requestId || undefined; + const targeting = evidence.adserverTargeting as Record | undefined; + const serverTrace = evidence.tsTrace as TrustedServerBidTrace | undefined; + const traceToken = + typeof targeting?.ts_trace === 'string' + ? targeting.ts_trace + : typeof (evidence.tsTrace as { bidTraceId?: unknown } | undefined)?.bidTraceId === + 'string' + ? ((evidence.tsTrace as { bidTraceId: string }).bidTraceId as string) + : undefined; + const ledger = (ts.prebidCorrelation ??= []); + if (kind === 'prebid_bid_response' && auctionId && slotId && requestId) { + ledger.push({ + auctionId, + slotId, + requestId, + bidder, + adId, + traceToken, + serverTrace, + events: [], + }); + if (ledger.length > 256) ledger.shift(); + } else if (kind === 'prebid_auction_end' && auctionId) { + const adUnits = Array.isArray(data.adUnits) + ? (data.adUnits as Array>) + : []; + const received = Array.isArray(data.bidsReceived) + ? (data.bidsReceived as Array>) + : []; + const slotIds = new Set(); + for (const unit of adUnits) { + if (typeof unit.code === 'string') slotIds.add(unit.code); + } + for (const bid of received) { + if (typeof bid.adUnitCode === 'string') slotIds.add(bid.adUnitCode); + } + for (const entry of ledger) { + if (entry.auctionId === auctionId) slotIds.add(entry.slotId); + } + const completed = (ts.prebidCompletedAuctions ??= []); + completed.push({ auctionId, slotIds: [...slotIds] }); + if (completed.length > 64) completed.shift(); + } else if (kind !== 'prebid_auction_init') { + const selected = (ts.prebidSelectedParticipants ?? []).filter( + (entry) => performance.now() - entry.selectedAt <= 30_000 + ); + ts.prebidSelectedParticipants = selected; + const selectedMatches = + auctionId && slotId && requestId + ? selected.filter( + (entry) => + entry.auctionId === auctionId && + entry.slotId === slotId && + (entry.requestId === requestId || entry.adId === adId) && + (!traceToken || entry.traceToken === traceToken) + ) + : []; + if (selectedMatches.length === 1) { + const selectedEntry = selectedMatches[0]; + ts.recordAdTrace?.({ + kind, + slotId, + generation: selectedEntry.generation, + bidTraceId: selectedEntry.traceToken, + bidder: selectedEntry.bidder ?? bidder, + }); + if (kind === 'prebid_render_succeeded' || kind === 'prebid_render_failed') { + ts.prebidSelectedParticipants = selected.filter((entry) => entry !== selectedEntry); + } + return; + } + + const matches = ledger.filter( + (entry) => + (!auctionId || entry.auctionId === auctionId) && + (!slotId || entry.slotId === slotId) && + (!requestId || entry.requestId === requestId || entry.adId === adId) + ); + if (matches.length === 1) { + const events = (matches[0].events ??= []); + events.push(kind); + while (events.length > 16) events.shift(); + } + } + ts.recordAdTrace?.({ kind, slotId, bidder }); + }; + + instrumented.onEvent?.('auctionInit', record('prebid_auction_init')); + instrumented.onEvent?.('bidResponse', record('prebid_bid_response')); + instrumented.onEvent?.('bidWon', record('prebid_bid_won')); + instrumented.onEvent?.('auctionEnd', record('prebid_auction_end')); + instrumented.onEvent?.('adRenderSucceeded', record('prebid_render_succeeded')); + instrumented.onEvent?.('adRenderFailed', record('prebid_render_failed')); + + // Observe the actual selection call once. The GPT request-boundary hook reads + // the resulting slot targeting synchronously; this wrapper never caches it. + const original = instrumented.setTargetingForGPTAsync?.bind(pbjs); + if (!original) return; + instrumented.setTargetingForGPTAsync = function (codes?: string[]) { + const result = original(codes); + ts.recordAdTrace?.({ kind: 'prebid_targeting_selected', reason: 'targeting_applied' }); + return result; + }; +} + function clearRefreshTargeting(slot: RefreshGptSlot): void { if (typeof slot.clearTargeting !== 'function') return; @@ -780,6 +937,16 @@ export function installPrebidNpm(config?: Partial): typeof pbjs log.debug('[tsjs-prebid] interpretResponse', { hasSeatbid: !!body?.seatbid }); const auctionBids = parseAuctionResponse(body); const bidRequests = request?.tsjsBidRequests ?? request?.bidRequests ?? []; + const summary = parseAuctionTraceSummary(body); + if (summary && window.tsjs?.recordAdTrace) { + const summaries = (window.tsjs.prebidServerSummaries ??= []); + for (const bidRequest of bidRequests) { + const auctionId = bidRequest.auctionId; + const slotId = bidRequest.adUnitCode ?? bidRequest.code; + if (auctionId && slotId) summaries.push({ auctionId, slotId, summary }); + } + while (summaries.length > 64) summaries.shift(); + } return auctionBidsToPrebidBids(auctionBids, bidRequests); }, }); @@ -964,6 +1131,7 @@ export function installPrebidNpm(config?: Partial): typeof pbjs // prebid.js via NPM. pbjs.processQueue(); recordUserIdModuleDiagnostics(); + installAdTracePrebidObservers(); // Validate that every client-side bidder has its adapter registered. // Adapters self-register on import, so a missing adapter means the bidder @@ -1106,6 +1274,9 @@ export function installRefreshHandler(timeoutMs = 1500): void { adUnits, bidsBackHandler: () => { pbjs.setTargetingForGPTAsync?.(refreshAdUnitCodes); + targetSlots.forEach((slot) => + window.tsjs?.captureAdTraceRequest?.(slot, 'prebid_refresh') + ); originalRefresh(targetSlots, opts); }, timeout: timeoutMs, diff --git a/crates/trusted-server-js/lib/test/core/ad_trace.test.ts b/crates/trusted-server-js/lib/test/core/ad_trace.test.ts new file mode 100644 index 000000000..0348e8dd7 --- /dev/null +++ b/crates/trusted-server-js/lib/test/core/ad_trace.test.ts @@ -0,0 +1,332 @@ +import { describe, expect, it } from 'vitest'; +import { + AD_TRACE_MAX_EVENTS, + AD_TRACE_MAX_GENERATIONS, + AD_TRACE_MAX_RENDERS, + AD_TRACE_MAX_SLOTS, + createAdTraceStore, +} from '../../src/core/ad_trace'; + +const BID_TRACE_ID = '550e8400-e29b-41d4-a716-446655440000'; + +describe('ad trace reducer', () => { + it('bounds events, slots, and retained generations', () => { + let now = 0; + const store = createAdTraceStore(() => ++now); + for (let i = 0; i < AD_TRACE_MAX_EVENTS + 1; i++) { + store.record({ kind: 'prebid_auction_init', reason: 'observed' }); + } + for (let i = 0; i < AD_TRACE_MAX_SLOTS + 1; i++) { + store.nextGeneration(`slot-${i}`); + } + for (let i = 0; i < AD_TRACE_MAX_GENERATIONS + 1; i++) { + store.nextGeneration('latest-slot'); + } + + const exported = store.export(); + expect(exported.events).toHaveLength(AD_TRACE_MAX_EVENTS); + expect(exported.metadata.droppedEvents).toBe(1); + expect(exported.slots).toHaveLength(AD_TRACE_MAX_SLOTS); + expect(store.getSlot('latest-slot')?.generations).toHaveLength(AD_TRACE_MAX_GENERATIONS); + expect(exported.metadata.evictedSlots).toBeGreaterThan(0); + }); + + it('keeps the four stages independent and only acknowledges an exact load event', () => { + const store = createAdTraceStore(() => 10); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'ts_winner_observed', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + }); + store.record({ + kind: 'gpt_slot_render_ended', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + isEmpty: false, + }); + + expect(store.getSlot('slot-a')?.stages.gam.outcome).toBe('trusted_server_candidate'); + expect(store.getSlot('slot-a')?.stages.creative.outcome).toBe('not_observed'); + + store.record({ + kind: 'creative_load_acknowledged', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + }); + const slot = store.getSlot('slot-a'); + expect(slot?.stages.gam).toMatchObject({ + outcome: 'trusted_server_won', + confidence: 'definitive', + }); + expect(slot?.stages.creative).toMatchObject({ + outcome: 'load_acknowledged', + confidence: 'definitive', + }); + }); + + it('updates only the acknowledged retained generation, never the latest generation', () => { + const store = createAdTraceStore(() => 1); + const first = store.nextGeneration('slot-a'); + const second = store.nextGeneration('slot-a'); + store.record({ + kind: 'creative_load_acknowledged', + slotId: 'slot-a', + generation: first, + bidTraceId: BID_TRACE_ID, + }); + + const slot = store.getSlot('slot-a'); + expect(slot?.latestGeneration).toBe(second); + expect(slot?.stages.creative.outcome).toBe('not_observed'); + expect(slot?.generations[0].stages.creative.outcome).toBe('load_acknowledged'); + expect(slot?.generations[1].stages.creative.outcome).toBe('not_observed'); + }); + + it('never downgrades a definitive acknowledgement with a later GPT callback', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'creative_load_acknowledged', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + }); + store.record({ + kind: 'gpt_slot_render_ended', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + isEmpty: false, + }); + + expect(store.getSlot('slot-a')?.stages.gam.outcome).toBe('trusted_server_won'); + expect(store.getSlot('slot-a')?.stages.creative.outcome).toBe('load_acknowledged'); + }); + + it('preserves acknowledged terminal history when its generation is later cleaned up', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'creative_load_acknowledged', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + }); + store.record({ + kind: 'generation_superseded', + slotId: 'slot-a', + generation, + reason: 'slot_destroyed', + }); + + expect(store.getSlot('slot-a')?.stages.gam.outcome).toBe('trusted_server_won'); + expect(store.getSlot('slot-a')?.stages.creative.outcome).toBe('load_acknowledged'); + }); + + it('does not rewrite a retained generation when the next auction seeds server evidence', () => { + const store = createAdTraceStore(() => 1); + store.record({ + kind: 'ts_winner_observed', + slotId: 'slot-a', + bidTraceId: BID_TRACE_ID, + }); + const first = store.nextGeneration('slot-a'); + store.record({ + kind: 'ts_auction_observed', + slotId: 'slot-a', + outcome: 'no_bid', + confidence: 'definitive', + reason: 'terminal_summary', + }); + const second = store.nextGeneration('slot-a'); + + const slot = store.getSlot('slot-a'); + expect( + slot?.generations.find((item) => item.generation === first)?.stages.trustedServer.outcome + ).toBe('won'); + expect( + slot?.generations.find((item) => item.generation === second)?.stages.trustedServer.outcome + ).toBe('no_bid'); + }); + + it('classifies overlap, client Prebid, APS, no-bid, and superseded states', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'prebid_targeting_selected', + slotId: 'slot-a', + generation, + outcome: 'client_bid_won', + confidence: 'definitive', + reason: 'selected_targeting', + }); + store.record({ + kind: 'prebid_bid_won', + slotId: 'slot-a', + generation, + }); + store.record({ + kind: 'gpt_slot_render_ended', + slotId: 'slot-a', + generation, + isEmpty: false, + }); + expect(store.getSlot('slot-a')?.stages.gam.outcome).toBe('client_prebid_candidate'); + + store.record({ kind: 'aps_display_bids_set', slotId: 'slot-a', generation }); + expect(store.getSlot('slot-a')?.stages.gam.outcome).toBe('client_prebid_candidate'); + + store.record({ + kind: 'generation_superseded', + slotId: 'slot-a', + generation, + reason: 'slot_destroyed', + }); + expect(store.getSlot('slot-a')?.stages.creative.outcome).toBe('not_observed'); + expect(store.getSlot('slot-a')?.stages.gam.outcome).toBe('client_prebid_candidate'); + }); + + it('does not downgrade definitive stage evidence during service or cleanup', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ kind: 'prebid_render_failed', slotId: 'slot-a', generation }); + store.record({ kind: 'pb_render_served', slotId: 'slot-a', generation }); + store.record({ + kind: 'generation_superseded', + slotId: 'slot-a', + generation, + reason: 'navigation', + }); + + expect(store.getSlot('slot-a')?.stages.creative).toMatchObject({ + outcome: 'render_failed', + confidence: 'definitive', + }); + }); + + it('does not downgrade a definitive empty render outcome', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'gpt_slot_render_ended', + slotId: 'slot-a', + generation, + isEmpty: true, + }); + store.record({ + kind: 'generation_superseded', + slotId: 'slot-a', + generation, + reason: 'navigation', + }); + store.record({ kind: 'pb_render_served', slotId: 'slot-a', generation }); + + expect(store.getRenderTimeline()[0]).toMatchObject({ + outcome: 'empty', + confidence: 'definitive', + }); + }); + + it('enriches one bounded render record and keeps visibility independent', () => { + let now = 0; + const store = createAdTraceStore(() => ++now); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'gpt_slot_render_ended', + slotId: 'slot-a', + generation, + isEmpty: false, + }); + store.record({ + kind: 'pb_render_served', + slotId: 'slot-a', + generation, + reason: 'pb_render_response', + }); + store.record({ + kind: 'creative_load_acknowledged', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + }); + store.updateVisibility('slot-a', generation, 'hidden'); + + const timeline = store.getRenderTimeline(); + expect(timeline).toHaveLength(1); + expect(timeline[0]).toMatchObject({ + sequence: 1, + outcome: 'confirmed', + confidence: 'definitive', + visibility: 'hidden', + }); + store.updateVisibility('slot-a', generation, 'visible'); + expect(store.getRenderTimeline()[0]).toMatchObject({ + sequence: 1, + outcome: 'confirmed', + confidence: 'definitive', + visibility: 'visible', + }); + }); + + it('dispatches a frozen privacy-safe render event', () => { + const store = createAdTraceStore(() => 1); + const observed: unknown[] = []; + const listener = (event: Event) => observed.push((event as CustomEvent).detail); + window.addEventListener('tsjs:adRendered', listener); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'pb_render_served', + slotId: 'slot-a', + generation, + reason: 'pb_render_response', + rawUrl: 'https://private.example', + } as never); + window.removeEventListener('tsjs:adRendered', listener); + + expect(observed).toHaveLength(1); + expect(Object.isFrozen(observed[0])).toBe(true); + expect(JSON.stringify(observed[0])).not.toContain('private.example'); + }); + + it('bounds the render timeline without duplicating impression generations', () => { + const store = createAdTraceStore(() => 1); + for (let i = 0; i < AD_TRACE_MAX_RENDERS + 1; i++) { + const slotId = `render-${i}`; + const generation = store.nextGeneration(slotId); + store.record({ kind: 'gpt_request_started', slotId, generation }); + } + expect(store.getRenderTimeline()).toHaveLength(AD_TRACE_MAX_RENDERS); + expect(store.getRenderTimeline()[0].slotId).toBe('render-1'); + }); + + it('rejects malformed runtime event kinds and confidence values', () => { + const store = createAdTraceStore(() => 1); + store.record({ kind: 'not-a-real-kind', slotId: 'slot-a' } as never); + store.record({ + kind: 'ts_winner_observed', + slotId: 'slot-a', + confidence: 'certain', + } as never); + expect(store.getEvents()).toHaveLength(0); + }); + + it('exports an immutable sanitized clone', () => { + const store = createAdTraceStore(() => 1); + store.record({ + kind: 'pb_render_rejected', + slotId: 'slot-a', + reason: 'missing_generation', + // Ensure unknown private fields cannot enter the public export. + rawUrl: 'https://private.example/path', + } as never); + + const exported = store.export(); + expect(Object.isFrozen(exported)).toBe(true); + expect(JSON.stringify(exported)).not.toContain('private.example'); + expect(() => exported.events.push({} as never)).toThrow(); + }); +}); diff --git a/crates/trusted-server-js/lib/test/core/auction.test.ts b/crates/trusted-server-js/lib/test/core/auction.test.ts index 7e9bb2947..f58d6bea0 100644 --- a/crates/trusted-server-js/lib/test/core/auction.test.ts +++ b/crates/trusted-server-js/lib/test/core/auction.test.ts @@ -1,5 +1,10 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { buildAdRequest, parseAuctionResponse, sendAuction } from '../../src/core/auction'; +import { + buildAdRequest, + parseAuctionResponse, + parseAuctionTraceSummary, + sendAuction, +} from '../../src/core/auction'; import envelope from '../fixtures/aps-renderer-v1.json'; function apsRenderer(creativeId?: string) { @@ -299,6 +304,100 @@ describe('auction/parseAuctionResponse', () => { expect(parseAuctionResponse({ seatbid: [] })).toEqual([]); }); + it('strictly joins valid root and bid traces without changing legacy fields', () => { + const body = { + ext: { + trusted_server: { + trace: { + version: 1, + auction_trace_id: '650e8400-e29b-41d4-a716-446655440000', + source: 'auction_api', + outcome: 'completed', + }, + }, + }, + seatbid: [ + { + seat: 'example-bidder', + bid: [ + { + impid: 'slot-1', + price: 1.5, + ext: { + trusted_server: { + trace: { + version: 1, + bid_trace_id: '550e8400-e29b-41d4-a716-446655440000', + slot_id: 'slot-1', + provider: 'prebid', + bidder: 'example-bidder', + }, + }, + }, + }, + ], + }, + ], + }; + + expect(parseAuctionTraceSummary(body)).toEqual({ + version: 1, + auctionTraceId: '650e8400-e29b-41d4-a716-446655440000', + source: 'auction_api', + outcome: 'completed', + }); + expect(parseAuctionResponse(body)[0].trace).toEqual({ + version: 1, + auctionTraceId: '650e8400-e29b-41d4-a716-446655440000', + bidTraceId: '550e8400-e29b-41d4-a716-446655440000', + source: 'auction_api', + slotId: 'slot-1', + provider: 'prebid', + bidder: 'example-bidder', + }); + }); + + it('ignores malformed, contradictory, mismatched, and oversized trace fields', () => { + const body = { + ext: { + trusted_server: { + trace: { + version: 1, + auction_trace_id: '650e8400-e29b-41d4-a716-446655440000', + source: 'auction_api', + outcome: 'no_bid', + }, + }, + }, + seatbid: [ + { + seat: 'seat', + bid: [ + { + impid: 'slot-1', + price: 1, + ext: { + trusted_server: { + trace: { + version: 1, + bid_trace_id: '550e8400-e29b-41d4-a716-446655440000', + slot_id: 'different-slot', + provider: 'p'.repeat(65), + bidder: 'seat', + }, + }, + }, + }, + ], + }, + ], + }; + expect(parseAuctionTraceSummary(body)?.outcome).toBe('no_bid'); + expect(parseAuctionResponse(body)[0].trace).toBeUndefined(); + body.ext.trusted_server.trace.auction_trace_id = 'not-a-uuid'; + expect(parseAuctionTraceSummary(body)).toBeUndefined(); + }); + it('defaults missing fields gracefully', () => { const body = { seatbid: [{ bid: [{ impid: 'slot-1', price: 1.5 }] }], @@ -353,7 +452,7 @@ describe('auction/sendAuction', () => { ], }; - const bids = await sendAuction('/auction', request); + const result = await sendAuction('/auction', request); expect(globalThis.fetch).toHaveBeenCalledWith( '/auction', @@ -363,18 +462,46 @@ describe('auction/sendAuction', () => { body: JSON.stringify(request), }) ); - expect(bids).toHaveLength(1); - expect(bids[0].price).toBe(2.5); + expect(result.kind).toBe('ok'); + if (result.kind !== 'ok') throw new Error('expected successful auction'); + expect(result.bids).toHaveLength(1); + expect(result.bids[0].price).toBe(2.5); }); - it('returns empty array on network error', async () => { + it('distinguishes a network error from a valid empty auction', async () => { globalThis.fetch = vi.fn().mockRejectedValue(new Error('network error')) as any; - const bids = await sendAuction('/auction', { adUnits: [] }); - expect(bids).toEqual([]); + const result = await sendAuction('/auction', { adUnits: [] }); + expect(result).toEqual({ kind: 'transport_error', reason: 'network' }); + }); + + it('accepts legacy empty but rejects malformed seatbid collections', async () => { + globalThis.fetch = vi + .fn() + .mockResolvedValueOnce({ + ok: true, + status: 200, + headers: { get: () => 'application/json' }, + json: async () => ({}), + }) + .mockResolvedValueOnce({ + ok: true, + status: 200, + headers: { get: () => 'application/json' }, + json: async () => ({ seatbid: {} }), + }) as any; + + await expect(sendAuction('/auction', { adUnits: [] })).resolves.toEqual({ + kind: 'ok', + bids: [], + }); + await expect(sendAuction('/auction', { adUnits: [] })).resolves.toEqual({ + kind: 'invalid_response', + reason: 'invalid_shape', + }); }); - it('returns empty array for non-JSON response', async () => { + it('distinguishes a non-JSON response', async () => { globalThis.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, @@ -382,11 +509,11 @@ describe('auction/sendAuction', () => { json: async () => ({}), }) as any; - const bids = await sendAuction('/auction', { adUnits: [] }); - expect(bids).toEqual([]); + const result = await sendAuction('/auction', { adUnits: [] }); + expect(result).toEqual({ kind: 'invalid_response', reason: 'non_json' }); }); - it('returns empty array for non-OK response', async () => { + it('distinguishes a non-OK response', async () => { globalThis.fetch = vi.fn().mockResolvedValue({ ok: false, status: 500, @@ -394,7 +521,7 @@ describe('auction/sendAuction', () => { json: async () => ({}), }) as any; - const bids = await sendAuction('/auction', { adUnits: [] }); - expect(bids).toEqual([]); + const result = await sendAuction('/auction', { adUnits: [] }); + expect(result).toEqual({ kind: 'transport_error', reason: 'http' }); }); }); diff --git a/crates/trusted-server-js/lib/test/core/request.test.ts b/crates/trusted-server-js/lib/test/core/request.test.ts index 8dffd825b..dc3cf9b68 100644 --- a/crates/trusted-server-js/lib/test/core/request.test.ts +++ b/crates/trusted-server-js/lib/test/core/request.test.ts @@ -11,6 +11,7 @@ describe('request.requestAds', () => { beforeEach(async () => { await vi.resetModules(); document.body.innerHTML = ''; + delete window.tsjs; originalFetch = globalThis.fetch; }); @@ -319,9 +320,8 @@ describe('request.requestAds', () => { expect(JSON.stringify(rejectionCall)).not.toContain('[object Object]'); }); - it('does not blank the slot when a later bid for the same slot is rejected', async () => { - // Regression: multi-bid scenario where a rejected bid must not erase an earlier - // successful render into the same slot. + it('rejects an ambiguous multi-winner response without blanking the slot', async () => { + // A final auction response must contain at most one winner per requested slot. const goodCreative = '
Safe Ad
'; (globalThis as any).fetch = vi.fn().mockResolvedValue({ ok: true, @@ -345,16 +345,14 @@ describe('request.requestAds', () => { const { addAdUnits } = await import('../../src/core/registry'); const { requestAds } = await import('../../src/core/request'); - document.body.innerHTML = '
'; + document.body.innerHTML = '
existing
'; addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); requestAds(); await flushRequestAds(); - // The good creative should have rendered; the bad one should not have blanked it. - const iframe = document.querySelector('#slot1 iframe') as HTMLIFrameElement | null; - expect(iframe).toBeTruthy(); - expect(iframe!.srcdoc).toContain(goodCreative); + expect(document.querySelector('#slot1 iframe')).toBeNull(); + expect(document.querySelector('#slot1')?.textContent).toContain('existing'); }); it('rejects creatives that sanitize to empty markup', async () => { @@ -398,6 +396,134 @@ describe('request.requestAds', () => { ); }); + it('keeps the latest direct owner when overlapping responses resolve out of order', async () => { + const resolves: Array<(response: Response) => void> = []; + (globalThis as any).fetch = vi.fn().mockImplementation( + () => + new Promise((resolve) => { + resolves.push(resolve); + }) + ); + const recordAdTrace = vi.fn(); + window.tsjs = { + recordAdTrace, + nextAdTraceGeneration: vi.fn().mockReturnValueOnce(1).mockReturnValueOnce(2), + } as any; + const { addAdUnits } = await import('../../src/core/registry'); + const { requestAds } = await import('../../src/core/request'); + document.body.innerHTML = '
existing
'; + addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); + + requestAds(); + requestAds(); + expect(resolves).toHaveLength(2); + const response = (creative: string) => + ({ + ok: true, + status: 200, + headers: { get: () => 'application/json' }, + json: async () => ({ + seatbid: [{ seat: 'trusted-server', bid: [{ impid: 'slot1', adm: creative }] }], + }), + }) as Response; + + resolves[1](response('
new owner
')); + await flushRequestAds(); + resolves[0](response('
stale owner
')); + await flushRequestAds(); + + const iframe = document.querySelector('#slot1 iframe') as HTMLIFrameElement; + expect(iframe.srcdoc).toContain('new owner'); + expect(iframe.srcdoc).not.toContain('stale owner'); + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'direct_render_rejected', + generation: 1, + reason: 'direct_owner_replaced', + }) + ); + }); + + it('records an exact direct auction winner, placement, and iframe load', async () => { + const auctionTraceId = '550e8400-e29b-41d4-a716-446655440000'; + const bidTraceId = '123e4567-e89b-42d3-a456-426614174000'; + const recordAdTrace = vi.fn(); + window.tsjs = { + recordAdTrace, + nextAdTraceGeneration: vi.fn().mockReturnValue(1), + } as any; + (globalThis as any).fetch = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + headers: { get: () => 'application/json' }, + json: async () => ({ + ext: { + trusted_server: { + trace: { + version: 1, + auction_trace_id: auctionTraceId, + source: 'auction_api', + outcome: 'completed', + }, + }, + }, + seatbid: [ + { + seat: 'trusted-server', + bid: [ + { + impid: 'slot1', + adm: '
direct
', + ext: { + trusted_server: { + trace: { + version: 1, + auction_trace_id: auctionTraceId, + bid_trace_id: bidTraceId, + source: 'auction_api', + slot_id: 'slot1', + provider: 'prebid', + bidder: 'example', + }, + }, + }, + }, + ], + }, + ], + }), + }); + + const { addAdUnits } = await import('../../src/core/registry'); + const { requestAds } = await import('../../src/core/request'); + document.body.innerHTML = '
'; + addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); + + requestAds(); + await flushRequestAds(); + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'ts_winner_observed', + generation: 1, + auctionTraceId, + bidTraceId, + }) + ); + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'pb_render_served', reason: 'direct_iframe_created' }) + ); + + const iframe = document.querySelector('#slot1 iframe') as HTMLIFrameElement; + iframe.dispatchEvent(new Event('load')); + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'creative_load_acknowledged', + generation: 1, + reason: 'direct_iframe_load', + }) + ); + }); + it('skips iframe insertion when slot is missing', async () => { // mock fetch for unified auction endpoint - returns inline HTML (globalThis as any).fetch = vi.fn().mockResolvedValue({ diff --git a/crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts b/crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts new file mode 100644 index 000000000..a3eaa7d6f --- /dev/null +++ b/crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts @@ -0,0 +1,41 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +describe('ad_trace integration gate', () => { + beforeEach(() => { + vi.resetModules(); + document.getElementById('ts-ad-trace-overlay')?.remove(); + delete window.__tsjs_adTraceActive; + delete window.tsjs; + }); + + afterEach(() => { + document.getElementById('ts-ad-trace-overlay')?.remove(); + delete window.__tsjs_adTraceActive; + delete window.tsjs; + }); + + it('leaves API and private recorders absent without the server bootstrap', async () => { + const { installAdTrace } = await import('../../../src/integrations/ad_trace/index'); + expect(installAdTrace()).toBe(false); + expect(window.tsjs?.adTrace).toBeUndefined(); + expect(window.tsjs?.recordAdTrace).toBeUndefined(); + }); + + it('installs one immutable API and consumes the exact bootstrap', async () => { + window.__tsjs_adTraceActive = true; + const { installAdTrace } = await import('../../../src/integrations/ad_trace/index'); + expect(installAdTrace()).toBe(true); + expect(window.__tsjs_adTraceActive).toBeUndefined(); + expect(Object.isFrozen(window.tsjs?.adTrace)).toBe(true); + expect(typeof window.tsjs?.recordAdTrace).toBe('function'); + expect(document.querySelectorAll('#ts-ad-trace-overlay')).toHaveLength(1); + expect(installAdTrace()).toBe(true); + expect(document.querySelectorAll('#ts-ad-trace-overlay')).toHaveLength(1); + }); + + it('does not accept the legacy tester cookie without bootstrap', async () => { + document.cookie = 'ts-tester=true; Path=/'; + const { installAdTrace } = await import('../../../src/integrations/ad_trace/index'); + expect(installAdTrace()).toBe(false); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts b/crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts new file mode 100644 index 000000000..0a5aaa217 --- /dev/null +++ b/crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { installAdTraceOverlay } from '../../../src/integrations/ad_trace/overlay'; +import type { AdTraceApi } from '../../../src/core/types'; + +function api(): AdTraceApi { + const slot = { + slotId: 'slot-a', + latestGeneration: 1, + generations: [], + stages: { + trustedServer: { outcome: 'won', confidence: 'definitive', reason: 'winner' }, + prebid: { outcome: 'not_run', confidence: 'definitive', reason: 'direct' }, + gam: { outcome: 'trusted_server_candidate', confidence: 'probable', reason: 'render' }, + creative: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + }, + } as const; + const renders = [ + { + sequence: 1, + slotId: 'slot-a', + generation: 1, + source: 'gpt', + outcome: 'gam_only', + confidence: 'probable', + visibility: 'unknown', + createdAt: 1, + updatedAt: 1, + }, + ] as const; + return { + getSlot: () => slot as any, + getEvents: () => [], + getRenderTimeline: () => renders as any, + export: () => ({ + version: 1, + slots: [slot as any], + events: [], + renders: renders as any, + metadata: { droppedEvents: 0, evictedSlots: 0 }, + }), + }; +} + +describe('ad trace overlay lifecycle', () => { + afterEach(() => { + document.getElementById('ts-ad-trace-overlay')?.remove(); + document.getElementById('slot-prefix-rendered')?.remove(); + delete window.tsjs; + vi.restoreAllMocks(); + }); + + it('finds prefix slots, observes resize, and coalesces animation frames', () => { + const element = document.createElement('div'); + element.id = 'slot-prefix-rendered'; + const rect = vi.spyOn(element, 'getBoundingClientRect').mockReturnValue({ + left: 10, + top: 20, + width: 300, + height: 250, + } as DOMRect); + document.body.appendChild(element); + const updateVisibility = vi.fn(); + window.tsjs = { + adSlots: [{ id: 'slot-a', div_id: 'slot-prefix' }], + getAdTraceElement: () => element, + updateAdTraceVisibility: updateVisibility, + } as any; + + const observe = vi.fn(); + vi.stubGlobal( + 'ResizeObserver', + class { + observe = observe; + unobserve = vi.fn(); + disconnect = vi.fn(); + } + ); + const frames: FrameRequestCallback[] = []; + vi.spyOn(window, 'requestAnimationFrame').mockImplementation((callback) => { + frames.push(callback); + return frames.length; + }); + let subscriber: (() => void) | undefined; + installAdTraceOverlay(api(), (listener) => { + subscriber = listener; + return vi.fn(); + }); + + expect(rect).toHaveBeenCalledTimes(1); + expect(observe).toHaveBeenCalledWith(element); + expect(updateVisibility).toHaveBeenCalledWith('slot-a', 1, 'visible'); + expect(element.getAttribute('data-ts-trace-seq')).toBe('1'); + expect(element.getAttribute('data-ts-trace-outcome')).toBe('gam_only'); + window.dispatchEvent(new Event('scroll')); + window.dispatchEvent(new Event('scroll')); + subscriber?.(); + expect(frames).toHaveLength(1); + frames.shift()?.(1); + expect(rect).toHaveBeenCalledTimes(2); + + const replacement = document.createElement('div'); + replacement.id = element.id; + element.replaceWith(replacement); + subscriber?.(); + frames.shift()?.(2); + expect(replacement.hasAttribute('data-ts-trace-seq')).toBe(false); + expect(element.hasAttribute('data-ts-trace-seq')).toBe(false); + expect(updateVisibility).toHaveBeenCalledWith('slot-a', 1, 'disconnected'); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index c790d37a0..7232668ab 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -1074,15 +1074,30 @@ describe('installTsRenderBridge', () => { afterEach(() => { vi.unstubAllGlobals(); document.getElementById('div-header')?.remove(); + document.getElementById('div-sidebar')?.remove(); delete (window as TestWindow).tsjs; }); + function capturePrivateOwner(slotId: string, divId: string): void { + const ts = (window as TestWindow).tsjs!; + const bid = ts.bids?.[slotId]; + ts.captureAdTraceRequest?.( + { + getSlotElementId: () => divId, + getTargeting: () => [], + }, + 'test_request', + { slotId, adId: bid?.hb_adid, bid } + ); + } + function createTrustedSlotIframe(): Window { const slot = document.createElement('div'); slot.id = 'div-header'; const iframe = document.createElement('iframe'); slot.appendChild(iframe); document.body.appendChild(slot); + capturePrivateOwner('homepage_header', slot.id); return iframe.contentWindow!; } @@ -1427,7 +1442,7 @@ describe('installTsRenderBridge', () => { expect(fetchStub).toHaveBeenCalledWith( 'https://openads.example.com/cache?uuid=test-cache-uuid', - { mode: 'cors' } + expect.objectContaining({ mode: 'cors', signal: expect.any(AbortSignal) }) ); expect(stopSpy).toHaveBeenCalled(); expect(portMessages).toHaveLength(1); @@ -1449,6 +1464,8 @@ describe('installTsRenderBridge', () => { }) as unknown as MessageEvent ); await new Promise((resolve) => setTimeout(resolve, 50)); + expect(fetchStub).toHaveBeenCalledTimes(1); + expect(portMessages).toHaveLength(1); expect(beaconSpy).toHaveBeenCalledTimes(2); beaconSpy.mockRestore(); }); @@ -1590,12 +1607,8 @@ describe('installTsRenderBridge', () => { }); it('fetches PBS Cache once when two same-adId messages race before the fetch resolves', async () => { - // Concurrent render double-fire guard: two 'Prebid Request' messages for the - // same adId can arrive before the first cache fetch settles. The in-flight - // `renderingAdIds` gate must collapse them to a single fetch — the persistent - // firedBeacons dedup only engages after a fetch resolves, so it cannot stop - // the second fetch on its own. Deferring the fetch keeps both messages in the - // window where only the in-flight gate can prevent the duplicate. + // Two duplicate requests from the exact same private owner collapse to one + // fetch while it remains current. const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); const mockAd = '
Test Creative
'; let resolveFetch: (value: Response) => void = () => {}; @@ -1642,6 +1655,263 @@ describe('installTsRenderBridge', () => { beaconSpy.mockRestore(); }); + it('supersedes a same-adId cache owner from a different source', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + const resolves: Array<(value: Response) => void> = []; + fetchStub.mockImplementation( + () => + new Promise((resolve) => { + resolves.push(resolve); + }) + ); + const bridgeListener = await captureBridgeListener(); + const oldPort = { postMessage: vi.fn() }; + const newPort = { postMessage: vi.fn() }; + const oldSource = createTrustedSlotIframe(); + const newFrame = document.createElement('iframe'); + document.getElementById('div-header')?.appendChild(newFrame); + const newSource = newFrame.contentWindow!; + + const dispatch = (source: Window, port: { postMessage: ReturnType }): void => { + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [port], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + }; + dispatch(oldSource, oldPort); + dispatch(newSource, newPort); + expect(fetchStub).toHaveBeenCalledTimes(2); + + resolves[0]({ ok: true, text: () => Promise.resolve('
old
') } as Response); + resolves[1]({ ok: true, text: () => Promise.resolve('
new
') } as Response); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(oldPort.postMessage).not.toHaveBeenCalled(); + expect(newPort.postMessage).toHaveBeenCalledTimes(1); + expect(beaconSpy).toHaveBeenCalledTimes(2); + beaconSpy.mockRestore(); + }); + + it('allows concurrent same-adId owners in different slots', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + const ts = (window as TestWindow).tsjs!; + ts.bids!.sidebar = { + ...ts.bids!.homepage_header, + nurl: 'https://ssp.example/sidebar-win', + burl: 'https://ssp.example/sidebar-bill', + }; + ts.adSlots!.push({ + id: 'sidebar', + formats: [[300, 250]], + gam_unit_path: '/a/b/sidebar', + div_id: 'div-sidebar', + targeting: {}, + }); + const resolves: Array<(value: Response) => void> = []; + fetchStub.mockImplementation( + () => + new Promise((resolve) => { + resolves.push(resolve); + }) + ); + const bridgeListener = await captureBridgeListener(); + const headerPort = { postMessage: vi.fn() }; + const sidebarPort = { postMessage: vi.fn() }; + const headerSource = createTrustedSlotIframe(); + const sidebar = document.createElement('div'); + sidebar.id = 'div-sidebar'; + const sidebarFrame = document.createElement('iframe'); + sidebar.appendChild(sidebarFrame); + document.body.appendChild(sidebar); + capturePrivateOwner('sidebar', sidebar.id); + + const dispatch = (source: Window, port: { postMessage: ReturnType }): void => { + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [port], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + }; + dispatch(headerSource, headerPort); + dispatch(sidebarFrame.contentWindow!, sidebarPort); + resolves.forEach((resolve, index) => + resolve({ + ok: true, + text: () => Promise.resolve(`
creative ${index}
`), + } as Response) + ); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(headerPort.postMessage).toHaveBeenCalledTimes(1); + expect(sidebarPort.postMessage).toHaveBeenCalledTimes(1); + expect(beaconSpy).toHaveBeenCalledTimes(4); + beaconSpy.mockRestore(); + }); + + it('blocks a late TS message after navigation before page-bids applies', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + const port = { postMessage: vi.fn() }; + const stop = vi.fn(); + + window.dispatchEvent(new PopStateEvent('popstate')); + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [port], + source, + stopImmediatePropagation: stop, + }) as unknown as MessageEvent + ); + + expect(stop).toHaveBeenCalledOnce(); + expect(port.postMessage).not.toHaveBeenCalled(); + expect(fetchStub).not.toHaveBeenCalled(); + expect(beaconSpy).not.toHaveBeenCalled(); + beaconSpy.mockRestore(); + }); + + it('blocks an old traced message after a newer request capture', async () => { + const ts = (window as TestWindow).tsjs!; + ts.recordAdTrace = vi.fn(); + ts.nextAdTraceGeneration = vi.fn().mockReturnValueOnce(1).mockReturnValueOnce(2); + const bridgeListener = await captureBridgeListener(); + const source = createTrustedSlotIframe(); + ts.bids!.homepage_header = { + ...ts.bids!.homepage_header, + hb_adid: 'new-cache-uuid', + }; + capturePrivateOwner('homepage_header', 'div-header'); + const port = { postMessage: vi.fn() }; + const stop = vi.fn(); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [port], + source, + stopImmediatePropagation: stop, + }) as unknown as MessageEvent + ); + + expect(stop).toHaveBeenCalledOnce(); + expect(port.postMessage).not.toHaveBeenCalled(); + expect(fetchStub).not.toHaveBeenCalled(); + }); + + it('drops a detached stale cache completion without responding or billing', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + let resolveFetch: (value: Response) => void = () => {}; + fetchStub.mockReturnValue( + new Promise((resolve) => { + resolveFetch = resolve; + }) + ); + const bridgeListener = await captureBridgeListener(); + const port = { postMessage: vi.fn() }; + const source = createTrustedSlotIframe(); + + bridgeListener( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'test-cache-uuid' }), + ports: [port], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + document.getElementById('div-header')?.remove(); + resolveFetch({ + ok: true, + text: () => Promise.resolve('
stale
'), + } as Response); + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(port.postMessage).not.toHaveBeenCalled(); + expect(beaconSpy).not.toHaveBeenCalled(); + beaconSpy.mockRestore(); + }); + + it('responds with adm without fetching PBS Cache when debug adm is available', async () => { + const beaconSpy = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + const debugAdm = '
Debug Creative
'; + (window as TestWindow).tsjs = { + bids: { + homepage_header: { + hb_adid: 'debug-adid', + hb_bidder: 'mocktioneer', + hb_pb: '0.20', + nurl: 'https://debug.example/win', + burl: 'https://debug.example/bill', + adm: debugAdm, + }, + }, + adSlots: [ + { + id: 'homepage_header', + formats: [[728, 90]] as [number, number][], + gam_unit_path: '/a/b/c', + div_id: 'div-header', + targeting: {}, + }, + ], + }; + + let bridgeListener: ((e: MessageEvent) => unknown) | undefined; + const origAdd = window.addEventListener.bind(window); + const addSpy = vi + .spyOn(window, 'addEventListener') + .mockImplementation( + (type: string, handler: EventListenerOrEventListenerObject, opts?: unknown) => { + if (type === 'message') bridgeListener = handler as (e: MessageEvent) => unknown; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + origAdd(type, handler as EventListener, opts as any); + } + ); + await import('../../../src/integrations/gpt/index'); + addSpy.mockRestore(); + + expect(bridgeListener, 'bridge listener should be registered').toBeDefined(); + + const stopSpy = vi.fn(); + const portMessages: string[] = []; + const fakePort = { postMessage: (s: string) => portMessages.push(s) }; + const source = createTrustedSlotIframe(); + + bridgeListener!( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'debug-adid' }), + ports: [fakePort], + source, + stopImmediatePropagation: stopSpy, + }) as unknown as MessageEvent + ); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + expect(fetchStub).not.toHaveBeenCalled(); + expect(stopSpy).toHaveBeenCalled(); + expect(portMessages).toHaveLength(1); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const parsed = JSON.parse(portMessages[0]) as Record; + expect(parsed.message).toBe('Prebid Response'); + expect(parsed.adId).toBe('debug-adid'); + expect(parsed.ad).toBe(debugAdm); + expect(parsed.width).toBe(728); + expect(parsed.height).toBe(90); + expect(beaconSpy).toHaveBeenCalledWith('https://debug.example/win'); + expect(beaconSpy).toHaveBeenCalledWith('https://debug.example/bill'); + expect(beaconSpy).toHaveBeenCalledTimes(2); + beaconSpy.mockRestore(); + }); + it('does not let one slot block a PBS Cache render for another slot sharing an adId', async () => { // The in-flight guard must be scoped to the requesting slot, not the shared // adId: two distinct slots sharing one hb_adid must each fetch and render. @@ -1696,6 +1966,8 @@ describe('installTsRenderBridge', () => { }; const sourceA = mkIframe('div-a'); const sourceB = mkIframe('div-b'); + capturePrivateOwner('slot_a', 'div-a'); + capturePrivateOwner('slot_b', 'div-b'); try { for (const source of [sourceA, sourceB]) { @@ -1905,6 +2177,7 @@ describe('installTsRenderBridge', () => { slot.appendChild(iframe); document.body.appendChild(slot); const source = iframe.contentWindow!; + capturePrivateOwner('homepage_in_content', 'div-in-content'); try { bridgeListener( diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_trace.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_trace.test.ts new file mode 100644 index 000000000..50d051eaf --- /dev/null +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_trace.test.ts @@ -0,0 +1,327 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const OLD_TOKEN = '550e8400-e29b-41d4-a716-446655440000'; +const NEW_TOKEN = '650e8400-e29b-41d4-a716-446655440000'; + +function slotWithTargeting(values: Record) { + return { + getSlotElementId: () => 'div-header', + getTargeting: (key: string) => (values[key] ? [values[key]] : []), + }; +} + +function trustedSource(): Window { + const root = document.createElement('div'); + root.id = 'div-header'; + const iframe = document.createElement('iframe'); + root.appendChild(iframe); + document.body.appendChild(root); + return iframe.contentWindow!; +} + +describe('GPT immutable ad trace render attribution', () => { + let bridge: (event: MessageEvent) => void; + let module: typeof import('../../../src/integrations/gpt/index'); + let record: ReturnType; + + beforeEach(async () => { + vi.resetModules(); + record = vi.fn(); + Object.defineProperty(navigator, 'sendBeacon', { + value: vi.fn(), + configurable: true, + writable: true, + }); + let generation = 0; + window.tsjs = { + recordAdTrace: record, + nextAdTraceGeneration: () => ++generation, + divToSlotId: { 'div-header': 'slot-a' }, + adSlots: [ + { + id: 'slot-a', + div_id: 'div-header', + gam_unit_path: '/123/example', + formats: [[300, 250]], + }, + ], + bids: { + 'slot-a': { + hb_adid: 'old-ad-id', + adm: '
Old creative
', + nurl: 'https://billing.example/win', + burl: 'https://billing.example/bill', + trace: { + version: 1, + auctionTraceId: '750e8400-e29b-41d4-a716-446655440000', + bidTraceId: OLD_TOKEN, + source: 'initial_navigation', + slotId: 'slot-a', + provider: 'prebid', + bidder: 'example-bidder', + }, + }, + }, + } as any; + const originalAdd = window.addEventListener.bind(window); + const spy = vi + .spyOn(window, 'addEventListener') + .mockImplementation((type, listener, options) => { + if (type === 'message') bridge = listener as (event: MessageEvent) => void; + originalAdd(type, listener, options); + }); + module = await import('../../../src/integrations/gpt/index'); + spy.mockRestore(); + }); + + afterEach(() => { + document.getElementById('div-header')?.remove(); + delete window.tsjs; + vi.restoreAllMocks(); + }); + + it('preserves authoritative missing values in a queued boundary snapshot', () => { + const source = trustedSource(); + const port = { postMessage: vi.fn() }; + const stop = vi.fn(); + const beacon = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + module.captureAdTraceRequest(slotWithTargeting({ hb_adid: 'old-ad-id' }) as any, 'bootstrap', { + slotId: 'slot-a', + bidder: undefined, + adId: undefined, + traceToken: undefined, + bid: undefined, + }); + + bridge( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'old-ad-id' }), + ports: [port], + source, + stopImmediatePropagation: stop, + }) as unknown as MessageEvent + ); + + expect(stop).toHaveBeenCalledOnce(); + expect(port.postMessage).not.toHaveBeenCalled(); + expect(beacon).not.toHaveBeenCalled(); + }); + + it('never pairs a new client or refreshed TS adId with the stale live bid payload', () => { + const source = trustedSource(); + const port = { postMessage: vi.fn() }; + const beacon = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + + for (const targeting of [ + { hb_adid: 'client-ad-id', hb_bidder: 'client-bidder' }, + { hb_adid: 'new-ts-ad-id', hb_bidder: 'example-bidder', ts_trace: NEW_TOKEN }, + ]) { + window.tsjs!.prebidCorrelation = [ + { + auctionId: 'auction-2', + slotId: 'slot-a', + requestId: 'request-2', + adId: targeting.hb_adid, + bidder: targeting.hb_bidder, + ...(targeting.ts_trace ? { traceToken: targeting.ts_trace } : {}), + ...(!targeting.ts_trace ? { events: ['prebid_bid_won' as const] } : {}), + }, + ]; + module.captureAdTraceRequest(slotWithTargeting(targeting) as any, 'prebid_refresh'); + bridge( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: targeting.hb_adid }), + ports: [port], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + } + + expect(port.postMessage).not.toHaveBeenCalled(); + expect(beacon).not.toHaveBeenCalled(); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'prebid_targeting_selected', outcome: 'client_bid_won' }) + ); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'prebid_bid_won', generation: 1 }) + ); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'prebid_targeting_selected', + outcome: 'won', + bidTraceId: NEW_TOKEN, + }) + ); + + window.tsjs!.prebidCorrelation = [ + { + auctionId: 'auction-3', + slotId: 'slot-a', + requestId: 'client-request', + adId: 'winning-client-ad', + bidder: 'client-bidder', + }, + { + auctionId: 'auction-3', + slotId: 'slot-a', + requestId: 'ts-request', + adId: 'losing-ts-ad', + bidder: 'trustedServer', + traceToken: NEW_TOKEN, + }, + ]; + module.captureAdTraceRequest( + slotWithTargeting({ hb_adid: 'winning-client-ad', hb_bidder: 'client-bidder' }) as any, + 'prebid_refresh' + ); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'prebid_targeting_selected', outcome: 'lost' }) + ); + }); + + it('serves, bills once, and acknowledges only the exact immutable generation/source/token', () => { + const source = trustedSource(); + const foreignSource = window; + const port = { postMessage: vi.fn() }; + const beacon = vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + module.captureAdTraceRequest( + slotWithTargeting({ + hb_adid: 'old-ad-id', + hb_bidder: 'example-bidder', + ts_trace: OLD_TOKEN, + }) as any, + 'display' + ); + + bridge( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'old-ad-id' }), + ports: [port], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + const response = JSON.parse(port.postMessage.mock.calls[0][0]); + expect(response.traceToken).toBe(OLD_TOKEN); + expect(response.ad).toBe('
Old creative
'); + expect(beacon).toHaveBeenCalledTimes(2); + + // A newer generation does not steal or invalidate the retained exact ack. + const nextSlot = slotWithTargeting({ hb_adid: 'client-next', hb_bidder: 'client-bidder' }); + module.captureAdTraceRequest(nextSlot as any, 'prebid_refresh'); + + bridge( + Object.assign(new Event('message'), { + data: { type: 'ts-creative-load', version: 1, traceToken: OLD_TOKEN }, + source: foreignSource, + }) as unknown as MessageEvent + ); + expect(record).not.toHaveBeenCalledWith( + expect.objectContaining({ kind: 'creative_load_acknowledged' }) + ); + bridge( + Object.assign(new Event('message'), { + data: { type: 'ts-creative-load', version: 1, traceToken: NEW_TOKEN }, + source, + }) as unknown as MessageEvent + ); + expect(record).not.toHaveBeenCalledWith( + expect.objectContaining({ kind: 'creative_load_acknowledged' }) + ); + + bridge( + Object.assign(new Event('message'), { + data: { type: 'ts-creative-load', version: 1, traceToken: OLD_TOKEN }, + source, + }) as unknown as MessageEvent + ); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'creative_load_acknowledged', + generation: 1, + bidTraceId: OLD_TOKEN, + }) + ); + expect(beacon).toHaveBeenCalledTimes(2); + + module.supersedeAdTraceSlot(nextSlot as any, 'slot_destroyed'); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'generation_superseded', + generation: 2, + reason: 'slot_destroyed', + }) + ); + }); + + it('rejects acknowledgements after the exact slot generation is superseded', () => { + const source = trustedSource(); + const slot = slotWithTargeting({ + hb_adid: 'old-ad-id', + hb_bidder: 'example-bidder', + ts_trace: OLD_TOKEN, + }); + const port = { postMessage: vi.fn() }; + vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + module.captureAdTraceRequest(slot as any, 'display'); + bridge( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'old-ad-id' }), + ports: [port], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + module.supersedeAdTraceSlot(slot as any, 'slot_destroyed'); + record.mockClear(); + bridge( + Object.assign(new Event('message'), { + data: { type: 'ts-creative-load', version: 1, traceToken: OLD_TOKEN }, + source, + }) as unknown as MessageEvent + ); + expect(record).not.toHaveBeenCalledWith( + expect.objectContaining({ kind: 'creative_load_acknowledged' }) + ); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'pb_render_rejected', reason: 'invalid_acknowledgement' }) + ); + }); + + it('expires pending acknowledgements after thirty seconds', () => { + let now = 0; + vi.spyOn(performance, 'now').mockImplementation(() => now); + const source = trustedSource(); + const slot = slotWithTargeting({ + hb_adid: 'old-ad-id', + hb_bidder: 'example-bidder', + ts_trace: OLD_TOKEN, + }); + const port = { postMessage: vi.fn() }; + vi.spyOn(navigator, 'sendBeacon').mockReturnValue(true); + module.captureAdTraceRequest(slot as any, 'display'); + bridge( + Object.assign(new Event('message'), { + data: JSON.stringify({ message: 'Prebid Request', adId: 'old-ad-id' }), + ports: [port], + source, + stopImmediatePropagation: vi.fn(), + }) as unknown as MessageEvent + ); + now = 30_001; + record.mockClear(); + bridge( + Object.assign(new Event('message'), { + data: { type: 'ts-creative-load', version: 1, traceToken: OLD_TOKEN }, + source, + }) as unknown as MessageEvent + ); + expect(record).not.toHaveBeenCalledWith( + expect.objectContaining({ kind: 'creative_load_acknowledged' }) + ); + expect(record).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'generation_superseded', reason: 'ack_expired' }) + ); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts index 406c6d1f5..2f21da832 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts @@ -240,6 +240,14 @@ describe('GPT – installTsAdInit', () => { ['ts_initial', ['1']], ['pos', ['old-pos']], ]); + const clearTargeting = vi.fn((key?: string) => { + if (key) { + slotTargeting.delete(key); + } else { + slotTargeting.clear(); + } + return gptSlot; + }); const gptSlot: any = { getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), getTargeting: vi.fn((key: string) => slotTargeting.get(key) ?? []), @@ -247,14 +255,7 @@ describe('GPT – installTsAdInit', () => { slotTargeting.set(key, Array.isArray(value) ? value : [value]); return gptSlot; }), - clearTargeting: vi.fn((key?: string) => { - if (key) { - slotTargeting.delete(key); - } else { - slotTargeting.clear(); - } - return gptSlot; - }), + clearTargeting, }; const pubads = { getSlots: vi.fn(() => [gptSlot]), @@ -295,13 +296,13 @@ describe('GPT – installTsAdInit', () => { installTsAdInit(); (window as any).tsjs.adInit(); - expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_pb'); - expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_bidder'); - expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_adid'); - expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_cache_host'); - expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_cache_path'); - expect(gptSlot.clearTargeting).toHaveBeenCalledWith('ts_initial'); - expect(gptSlot.clearTargeting).toHaveBeenCalledWith('pos'); + expect(clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(clearTargeting).toHaveBeenCalledWith('hb_bidder'); + expect(clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_host'); + expect(clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(clearTargeting).toHaveBeenCalledWith('ts_initial'); + expect(clearTargeting).toHaveBeenCalledWith('pos'); expect(slotTargeting.get('hb_pb')).toBeUndefined(); expect(slotTargeting.get('hb_bidder')).toBeUndefined(); expect(slotTargeting.get('hb_adid')).toBeUndefined(); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 00cf99dd0..4434f1256 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -211,6 +211,39 @@ describe('prebid/auctionBidsToPrebidBids', () => { ); }); + it('adds adapter targeting only for a validated Trusted Server trace', () => { + const traced: AuctionBid = { + impid: 'slot-traced', + adm: '
Ad
', + price: 2, + width: 300, + height: 250, + seat: 'example-bidder', + creativeId: 'creative-1', + adomain: [], + trace: { + version: 1, + auctionTraceId: '650e8400-e29b-41d4-a716-446655440000', + bidTraceId: '550e8400-e29b-41d4-a716-446655440000', + source: 'auction_api', + slotId: 'slot-traced', + provider: 'prebid', + bidder: 'example-bidder', + }, + }; + + const [bid] = auctionBidsToPrebidBids( + [traced], + [{ adUnitCode: 'slot-traced', bidId: 'request-1' }] + ); + expect(bid.adserverTargeting).toEqual({ + ts_trace: '550e8400-e29b-41d4-a716-446655440000', + }); + expect(auctionBidsToPrebidBids([{ ...traced, trace: undefined }], [])[0]).not.toHaveProperty( + 'adserverTargeting' + ); + }); + it('falls back to impid when no matching bidRequest found', () => { const auctionBids: AuctionBid[] = [ { @@ -280,8 +313,9 @@ describe('prebid/installPrebidNpm', () => { document.cookie = 'ts-eids=; Path=/; Max-Age=0'; delete (window as any).__tsjs_prebid; delete (window as any).__tsjs_prebid_diagnostics; - delete (window as any).tsjs; delete (mockPbjs as any).__tsApsBidResponseListenerInstalled; + delete (mockPbjs as any).__tsAdTraceObserved; + delete window.tsjs; }); afterEach(() => { @@ -940,6 +974,50 @@ describe('prebid/installPrebidNpm', () => { expect(document.cookie).toBe(''); }); + + it('joins late winner and render events to the retained selected generation', () => { + const recordAdTrace = vi.fn(); + window.tsjs = { + recordAdTrace, + prebidSelectedParticipants: [ + { + auctionId: 'auction-1', + slotId: 'slot-a', + requestId: 'request-1', + adId: 'ad-1', + bidder: 'client-bidder', + generation: 7, + selectedAt: performance.now(), + }, + ], + } as any; + installPrebidNpm(); + const handlers = new Map) => void>( + mockOnEvent.mock.calls.map(([event, handler]) => [event, handler]) + ); + const bid = { + auctionId: 'auction-1', + adUnitCode: 'slot-a', + requestId: 'request-1', + adId: 'ad-1', + bidderCode: 'client-bidder', + }; + + handlers.get('bidWon')?.(bid); + handlers.get('adRenderSucceeded')?.({ bid }); + + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ kind: 'prebid_bid_won', generation: 7, slotId: 'slot-a' }) + ); + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'prebid_render_succeeded', + generation: 7, + slotId: 'slot-a', + }) + ); + expect(window.tsjs.prebidSelectedParticipants).toEqual([]); + }); }); }); diff --git a/docs/guide/configuration.md b/docs/guide/configuration.md index 9e8c70b24..1912de6c7 100644 --- a/docs/guide/configuration.md +++ b/docs/guide/configuration.md @@ -998,6 +998,23 @@ apply when the integration section exists in `trusted-server.toml`. | --------- | ------- | ------------------------------ | | `enabled` | Boolean | Enable/disable the integration | +### Ad Trace Integration + +**Section**: `[integrations.ad_trace]` + +| Field | Type | Default | Description | +| --------- | ------- | ------- | ------------------------------------------------ | +| `enabled` | Boolean | `false` | Include tester-only auction trace browser support | + +Browser-visible auction IDs, bid IDs, targeting, API state, and the console require this setting plus an activated browser session. Visit a publisher page with the exact query `?ts_console=true` or `?ts_console=1`; Trusted Server enables the first response and sets a host-only session cookie automatically. Use `?ts_console=false` or `?ts_console=0` to clear the session. The reserved query is removed from downstream requests and cleaned from eligible HTML URLs. Active trace responses are private and non-storeable. + +The integration is disabled by default. The query is a self-service diagnostic toggle, not authorization, and does nothing without the explicit configuration gate. The console never exposes the internal auction request ID, identity data, consent strings, page URLs, partner notification URLs, cache coordinates, raw targeting, or creative markup. A creative marked `confirmed` means its exact Trusted Server renderer iframe load was acknowledged; it does not claim viewability or arbitrary advertiser JavaScript completion. + +```toml +[integrations.ad_trace] +enabled = false +``` + ### Prebid Integration **Section**: `[integrations.prebid]` diff --git a/scripts/generate-integration-viceroy-configs.sh b/scripts/generate-integration-viceroy-configs.sh index 761d06926..97ee870a0 100755 --- a/scripts/generate-integration-viceroy-configs.sh +++ b/scripts/generate-integration-viceroy-configs.sh @@ -13,6 +13,7 @@ ARTIFACTS_DIR="${ARTIFACTS_DIR:-$REPO_ROOT/target/integration-test-artifacts}" CONFIG_DIR="$ARTIFACTS_DIR/configs" TEMPLATE_PATH="crates/trusted-server-integration-tests/fixtures/configs/viceroy-template.toml" APP_CONFIG_PATH="crates/trusted-server-integration-tests/fixtures/configs/trusted-server.integration.toml" +AD_TRACE_APP_CONFIG_PATH="crates/trusted-server-integration-tests/fixtures/configs/trusted-server.ad-trace.integration.toml" INTEGRATION_TARGET_DIR="crates/trusted-server-integration-tests/target" ORIGIN_URL="http://127.0.0.1:$ORIGIN_PORT" HOST_TARGET="$(rustc -vV | sed -n 's/^host: //p')" @@ -41,3 +42,9 @@ fi --app-config "$APP_CONFIG_PATH" \ --output "$CONFIG_DIR/viceroy.toml" \ --origin-url "$ORIGIN_URL" + +"$GENERATOR_BIN" \ + --template "$TEMPLATE_PATH" \ + --app-config "$AD_TRACE_APP_CONFIG_PATH" \ + --output "$CONFIG_DIR/viceroy-ad-trace.toml" \ + --origin-url "$ORIGIN_URL" diff --git a/scripts/integration-tests-browser.sh b/scripts/integration-tests-browser.sh index 714de510b..08dc5b5bc 100755 --- a/scripts/integration-tests-browser.sh +++ b/scripts/integration-tests-browser.sh @@ -38,6 +38,19 @@ TRUSTED_SERVER__PROXY__CERTIFICATE_CHECK=false \ echo "==> Generating Viceroy configs..." INTEGRATION_ORIGIN_PORT="$ORIGIN_PORT" ./scripts/generate-integration-viceroy-configs.sh GENERATED_VICEROY_CONFIG_PATH="$REPO_ROOT/target/integration-test-artifacts/configs/viceroy.toml" +GENERATED_AD_TRACE_CONFIG_PATH="$REPO_ROOT/target/integration-test-artifacts/configs/viceroy-ad-trace.toml" + +# Build the actual external Prebid bundle consumed by the isolated ad-trace +# fixture. The browser routes its first-party managed URL to this local asset; +# no public ad network is contacted. +echo "==> Building deterministic external Prebid fixture bundle..." +rm -rf "$REPO_ROOT/target/integration-test-artifacts/prebid" +mkdir -p "$REPO_ROOT/target/integration-test-artifacts/prebid" +npm ci --prefix crates/trusted-server-js/lib +npm run --prefix crates/trusted-server-js/lib build:prebid-external -- \ + --adapters=rubicon \ + --user-id-modules=sharedIdSystem \ + --out "$REPO_ROOT/target/integration-test-artifacts/prebid" # --- Build Docker images --- echo "==> Building WordPress test container..." @@ -50,6 +63,12 @@ docker build \ -t test-nextjs:latest \ crates/trusted-server-integration-tests/fixtures/frameworks/nextjs/ +echo "==> Building ad-trace test container..." +docker build \ + -f crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/Dockerfile \ + -t test-ad-trace:latest \ + . + # --- Install Playwright --- echo "==> Installing Playwright dependencies..." cd "$REPO_ROOT/$BROWSER_DIR" @@ -80,15 +99,22 @@ stop_matching_containers() { } cleanup() { + stop_matching_containers test-ad-trace:latest stop_matching_containers test-nextjs:latest stop_matching_containers test-wordpress:latest } trap cleanup EXIT # --- Run tests for each framework --- -for framework in nextjs wordpress; do +for framework in nextjs wordpress ad-trace; do echo "==> Running Playwright tests for $framework..." - TEST_FRAMEWORK="$framework" npx playwright test "$@" + if [ "$framework" = "ad-trace" ]; then + TEST_FRAMEWORK="$framework" VICEROY_CONFIG_PATH="$GENERATED_AD_TRACE_CONFIG_PATH" \ + npx playwright test "$@" + else + TEST_FRAMEWORK="$framework" VICEROY_CONFIG_PATH="$GENERATED_VICEROY_CONFIG_PATH" \ + npx playwright test "$@" + fi done echo "==> All browser tests passed." diff --git a/tinybird/datasources/auction_events_raw.datasource b/tinybird/datasources/auction_events_raw.datasource index d62f8ae5d..592158713 100644 --- a/tinybird/datasources/auction_events_raw.datasource +++ b/tinybird/datasources/auction_events_raw.datasource @@ -32,6 +32,7 @@ SCHEMA > `price_cpm` Nullable(Float64), `currency` LowCardinality(Nullable(String)), `is_win` Nullable(UInt8), + `bid_trace_id` Nullable(UUID), `ad_domain` Nullable(String), `ad_id` Nullable(String), `event_date` Date DEFAULT toDate(event_ts) diff --git a/tinybird/fixtures/auction_events_raw.ndjson b/tinybird/fixtures/auction_events_raw.ndjson index 078d0c533..10626e3ad 100644 --- a/tinybird/fixtures/auction_events_raw.ndjson +++ b/tinybird/fixtures/auction_events_raw.ndjson @@ -1,7 +1,7 @@ {"event_ts":"2026-06-23 12:00:00.000","event_kind":"summary","auction_id":"550e8400-e29b-41d4-a716-446655440000","auction_source":"auction_api","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":"completed","terminal_reason":null,"slot_count":2,"total_time_ms":120,"winning_bid_count":1,"provider":null,"provider_role":null,"status":null,"provider_response_time_ms":null,"provider_bid_count":null,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} {"event_ts":"2026-06-23 12:00:00.000","event_kind":"provider_call","auction_id":"550e8400-e29b-41d4-a716-446655440000","auction_source":"auction_api","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"prebid","provider_role":"bidder","status":"success","provider_response_time_ms":80,"provider_bid_count":2,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} {"event_ts":"2026-06-23 12:00:00.000","event_kind":"provider_call","auction_id":"550e8400-e29b-41d4-a716-446655440000","auction_source":"auction_api","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"aps","provider_role":"bidder","status":"nobid","provider_response_time_ms":95,"provider_bid_count":0,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} -{"event_ts":"2026-06-23 12:00:00.000","event_kind":"bid","auction_id":"550e8400-e29b-41d4-a716-446655440000","auction_source":"auction_api","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"prebid","provider_role":null,"status":null,"provider_response_time_ms":null,"provider_bid_count":null,"slot_id":"slot-1","slot_w":300,"slot_h":250,"media_type":"banner","seat":"kargo","price_cpm":1.25,"currency":"USD","is_win":1,"ad_domain":"advertiser.example","ad_id":"ad-1"} +{"event_ts":"2026-06-23 12:00:00.000","event_kind":"bid","auction_id":"550e8400-e29b-41d4-a716-446655440000","auction_source":"auction_api","publisher_domain":"test-publisher.example","page_path":"/article/:id","country":"US","region":"CA","is_mobile":0,"is_known_browser":1,"gdpr_applies":0,"consent_present":0,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"prebid","provider_role":null,"status":null,"provider_response_time_ms":null,"provider_bid_count":null,"slot_id":"slot-1","slot_w":300,"slot_h":250,"media_type":"banner","seat":"kargo","price_cpm":1.25,"currency":"USD","is_win":1,"bid_trace_id":"950e8400-e29b-41d4-a716-446655440000","ad_domain":"advertiser.example","ad_id":"ad-1"} {"event_ts":"2026-06-23 12:01:00.000","event_kind":"summary","auction_id":"650e8400-e29b-41d4-a716-446655440000","auction_source":"initial_navigation","publisher_domain":"test-publisher.example","page_path":"/sports","country":"US","region":"CA","is_mobile":1,"is_known_browser":1,"gdpr_applies":0,"consent_present":1,"terminal_status":"abandoned","terminal_reason":"pass_through_response","slot_count":1,"total_time_ms":35,"winning_bid_count":0,"provider":null,"provider_role":null,"status":null,"provider_response_time_ms":null,"provider_bid_count":null,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} {"event_ts":"2026-06-23 12:01:00.000","event_kind":"provider_call","auction_id":"650e8400-e29b-41d4-a716-446655440000","auction_source":"initial_navigation","publisher_domain":"test-publisher.example","page_path":"/sports","country":"US","region":"CA","is_mobile":1,"is_known_browser":1,"gdpr_applies":0,"consent_present":1,"terminal_status":null,"terminal_reason":null,"slot_count":null,"total_time_ms":null,"winning_bid_count":null,"provider":"prebid","provider_role":"bidder","status":"abandoned","provider_response_time_ms":35,"provider_bid_count":0,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} {"event_ts":"2026-06-23 12:02:00.000","event_kind":"summary","auction_id":"750e8400-e29b-41d4-a716-446655440000","auction_source":"spa_navigation","publisher_domain":"test-publisher.example","page_path":"/privacy","country":"DE","region":null,"is_mobile":2,"is_known_browser":2,"gdpr_applies":1,"consent_present":1,"terminal_status":"skipped","terminal_reason":"consent_denied","slot_count":1,"total_time_ms":0,"winning_bid_count":0,"provider":null,"provider_role":null,"status":null,"provider_response_time_ms":null,"provider_bid_count":null,"slot_id":null,"slot_w":null,"slot_h":null,"media_type":null,"seat":null,"price_cpm":null,"currency":null,"is_win":null,"ad_domain":null,"ad_id":null} diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 7cd16133e..3d76102ca 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -59,6 +59,11 @@ enabled = false rewrite_attributes = ["href", "link", "siteBaseUrl", "siteProductionDomain", "url"] max_combined_payload_bytes = 10485760 +# Session-scoped auction-to-creative trace diagnostics. When enabled, visit a +# publisher page with `?ts_console=1` or `?ts_console=true` to open the console. +[integrations.ad_trace] +enabled = false + [integrations.testlight] enabled = false endpoint = "https://testlight.example.com/openrtb2/auction" From fd1cbd8de7f735f0bd629fe3c07bcf4b1a1f3fc7 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 23 Jul 2026 15:52:27 -0500 Subject: [PATCH 107/198] Clarify auction trace console evidence --- .../tests/ad-trace/auction-trace.spec.ts | 42 +++- .../lib/src/core/ad_trace.ts | 43 +++- .../trusted-server-js/lib/src/core/request.ts | 20 +- .../trusted-server-js/lib/src/core/types.ts | 6 + .../lib/src/integrations/ad_trace/index.ts | 14 +- .../lib/src/integrations/ad_trace/overlay.ts | 101 +++++--- .../src/integrations/ad_trace/presentation.ts | 174 +++++++++++++ .../lib/src/integrations/aps/render.ts | 13 +- .../lib/src/integrations/gpt/index.ts | 31 ++- .../lib/test/core/ad_trace.test.ts | 107 ++++++++ .../lib/test/core/request.test.ts | 66 +++++ .../test/integrations/ad_trace/index.test.ts | 27 ++ .../integrations/ad_trace/overlay.test.ts | 216 ++++++++++++++++ .../ad_trace/presentation.test.ts | 231 ++++++++++++++++++ .../lib/test/integrations/aps/render.test.ts | 9 +- .../lib/test/integrations/gpt/ad_init.test.ts | 121 +++++++++ 16 files changed, 1147 insertions(+), 74 deletions(-) create mode 100644 crates/trusted-server-js/lib/src/integrations/ad_trace/presentation.ts create mode 100644 crates/trusted-server-js/lib/test/integrations/ad_trace/presentation.test.ts diff --git a/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts b/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts index 9c2935d07..075c30b57 100644 --- a/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts +++ b/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts @@ -118,7 +118,9 @@ test.describe("tester-only auction trace contract", () => { waitUntil: "domcontentloaded", }); await expect(page).toHaveURL(runtimeUrl("/")); - expect(activation?.headers()["cache-control"]).toBe("private, no-store"); + expect(activation?.headers()["cache-control"]).toBe( + "private, no-store", + ); await expect .poll(() => page.evaluate( @@ -210,9 +212,13 @@ test.describe("tester-only auction trace contract", () => { const visibleText = tree.nodes .map((node) => node.name?.value || "") .join("\n"); - expect(visibleText).toContain("TS winner: won · definitive"); + expect(visibleText).toContain("Trusted Server selected a bid"); expect(visibleText).toContain( - "Creative: load_acknowledged · definitive", + "GAM selected the Trusted Server creative", + ); + expect(visibleText).toContain("Trusted Server creative load confirmed"); + expect(visibleText).not.toMatch( + /definitive|strong|probable|not_run|gam_only|TS winner|Prebid winner|#\d/, ); }); @@ -224,12 +230,14 @@ test.describe("tester-only auction trace contract", () => { const direct = document.createElement("div"); direct.id = "direct-api-slot"; document.body.appendChild(direct); - const ts = (window as Window & { - tsjs: { - addAdUnits(unit: unknown): void; - requestAds(): void; - }; - }).tsjs; + const ts = ( + window as Window & { + tsjs: { + addAdUnits(unit: unknown): void; + requestAds(): void; + }; + } + ).tsjs; ts.addAdUnits({ code: "direct-api-slot", mediaTypes: { banner: { sizes: [[300, 250]] } }, @@ -268,7 +276,7 @@ test.describe("tester-only auction trace contract", () => { }); }); - test("actual generated Prebid selects the traced TS bid before a probable GAM result", async ({ + test("actual generated Prebid selects the traced TS bid before an unattributed GAM render", async ({ page, }) => { await openTesterPage(page); @@ -352,6 +360,20 @@ test.describe("tester-only auction trace contract", () => { }) .toEqual({ prebid: "lost", gam: "client_prebid_candidate" }); + const session = await page.context().newCDPSession(page); + await expect + .poll(async () => { + const tree = (await session.send( + "Accessibility.getFullAXTree", + )) as { + nodes: Array<{ name?: { value?: string } }>; + }; + return tree.nodes + .map((node) => node.name?.value || "") + .join("\n"); + }) + .toContain("Prebid selected a client bid"); + await page.evaluate(() => { const win = window as Window & { adTraceFixture: { diff --git a/crates/trusted-server-js/lib/src/core/ad_trace.ts b/crates/trusted-server-js/lib/src/core/ad_trace.ts index aca037f5d..1ea51503f 100644 --- a/crates/trusted-server-js/lib/src/core/ad_trace.ts +++ b/crates/trusted-server-js/lib/src/core/ad_trace.ts @@ -39,7 +39,9 @@ const EVENT_KINDS = new Set([ 'gpt_slot_response_received', 'gpt_slot_render_ended', 'gpt_slot_onload', + 'gpt_impression_viewable', 'aps_display_bids_set', + 'aps_renderer_ready', 'pb_render_requested', 'pb_render_rejected', 'pb_render_served', @@ -135,12 +137,21 @@ function updateStage(target: Record, event: AdTr if (explicit && target.prebid.confidence !== 'definitive') target.prebid = explicit; break; case 'prebid_bid_won': - if (target.prebid.outcome === 'client_bid_won' || target.prebid.outcome === 'lost') { + if ( + target.prebid.outcome === 'won' || + target.prebid.outcome === 'client_bid_won' || + target.prebid.outcome === 'lost' + ) { + // A Prebid win corroborates selection only. It is never creative-load + // evidence, including when the selected bid originated from Trusted Server. target.prebid = { ...target.prebid, reason: 'selected_targeting_with_bid_won', }; - if (target.gam.outcome === 'direct_or_unattributed') { + if ( + (target.prebid.outcome === 'client_bid_won' || target.prebid.outcome === 'lost') && + target.gam.outcome === 'direct_or_unattributed' + ) { target.gam = { outcome: 'client_prebid_candidate', confidence: 'probable', @@ -202,6 +213,15 @@ function updateStage(target: Record, event: AdTr // APS setting display bids is a handoff only. GAM attribution remains // unobserved until a correlated non-empty GPT render arrives. break; + case 'aps_renderer_ready': + if (target.creative.confidence !== 'definitive') { + target.creative = { + outcome: 'aps_renderer_ready', + confidence: 'strong', + reason: event.reason ?? 'aps_renderer_ready', + }; + } + break; case 'gpt_slot_onload': if (target.creative.outcome === 'not_observed') target.creative = { @@ -232,7 +252,8 @@ function updateStage(target: Record, event: AdTr target.creative = { outcome: 'load_acknowledged', confidence: 'definitive', - reason: 'source_validated_load', + reason: + event.reason === 'direct_iframe_load' ? 'direct_iframe_load' : 'source_validated_load', }; if (event.reason !== 'direct_iframe_load') { target.gam = { @@ -268,6 +289,8 @@ function isRenderEvent(kind: AdTraceEventKind): boolean { return ( kind === 'gpt_request_started' || kind === 'gpt_slot_render_ended' || + kind === 'gpt_impression_viewable' || + kind === 'aps_renderer_ready' || kind === 'prebid_render_succeeded' || kind === 'prebid_render_failed' || kind === 'pb_render_requested' || @@ -375,6 +398,8 @@ export function createAdTraceStore( render.source = render.source === 'direct_auction' ? render.source : 'pb_render'; if (event.auctionTraceId) render.auctionTraceId = event.auctionTraceId; if (event.bidTraceId) render.bidTraceId = event.bidTraceId; + if (event.reason) render.reason = event.reason; + if (event.kind === 'gpt_impression_viewable') render.viewability = 'viewable'; render.updatedAt = timestamp; emitRender(render); }; @@ -497,6 +522,18 @@ export function createAdTraceStore( }; } +/** + * Map a public terminal auction outcome to an internal stage outcome. + * + * A completed auction without a final slot winner is a no-bid result. A + * completed auction with a winner is immediately followed by winner evidence, + * but remains distinct here so callers never erase failed or abandoned results. + */ +export function terminalSummaryStageOutcome(outcome: string, hasWinner = false): string { + if (outcome === 'completed') return hasWinner ? 'completed' : 'no_bid'; + return outcome; +} + export function isCanonicalTraceUuid(value: unknown): value is string { return safeUuid(value) !== undefined; } diff --git a/crates/trusted-server-js/lib/src/core/request.ts b/crates/trusted-server-js/lib/src/core/request.ts index b7429d01c..47d31a013 100644 --- a/crates/trusted-server-js/lib/src/core/request.ts +++ b/crates/trusted-server-js/lib/src/core/request.ts @@ -1,6 +1,7 @@ // Request orchestration for tsjs: unified auction endpoint with iframe-based creative rendering. import { renderApsCreative } from '../integrations/aps/render'; +import { terminalSummaryStageOutcome } from './ad_trace'; import { buildAdRequest, sendAuction } from './auction'; import { collectContext } from './context'; import { log } from './log'; @@ -74,7 +75,7 @@ function recordRootSummary( slotId: owner.slotId, generation: owner.generation, auctionTraceId: summary.auctionTraceId, - outcome: summary.outcome === 'completed' && !hasWinner ? 'no_bid' : summary.outcome, + outcome: terminalSummaryStageOutcome(summary.outcome, hasWinner), confidence: 'definitive', reason: 'terminal_summary', }); @@ -170,7 +171,22 @@ export function requestAds( } if (bid.renderer) { if (!ownerIsCurrent(owner)) continue; - if (!renderApsCreative({ slotId, renderer: bid.renderer })) { + const started = renderApsCreative({ + slotId, + renderer: bid.renderer, + onReady: () => { + if (!ownerIsCurrent(owner) || !owner.generation) return; + window.tsjs?.recordAdTrace?.({ + kind: 'aps_renderer_ready', + slotId, + generation: owner.generation, + auctionTraceId: trace?.auctionTraceId, + bidTraceId: trace?.bidTraceId, + reason: 'direct_aps_renderer_ready', + }); + }, + }); + if (!started) { recordDirectRejection(owner, 'aps_render_rejected'); } else if (owner.generation) { window.tsjs?.recordAdTrace?.({ diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 0ba40fd5d..f54e45b44 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -126,7 +126,9 @@ export type AdTraceEventKind = | 'gpt_slot_response_received' | 'gpt_slot_render_ended' | 'gpt_slot_onload' + | 'gpt_impression_viewable' | 'aps_display_bids_set' + | 'aps_renderer_ready' | 'pb_render_requested' | 'pb_render_rejected' | 'pb_render_served' @@ -181,6 +183,10 @@ export interface RenderTraceSnapshot { outcome: RenderTraceOutcome; confidence: AdTraceConfidence; visibility: RenderTraceVisibility; + /** GPT reported this exact retained slot generation viewable. */ + viewability?: 'viewable'; + /** Bounded privacy-safe reason for the latest render evidence. */ + reason?: string; createdAt: number; updatedAt: number; } diff --git a/crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts b/crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts index cf6d6d33c..27bf58418 100644 --- a/crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts @@ -1,4 +1,9 @@ -import { createAdTraceStore, isBoundedTraceLabel, isCanonicalTraceUuid } from '../../core/ad_trace'; +import { + createAdTraceStore, + isBoundedTraceLabel, + isCanonicalTraceUuid, + terminalSummaryStageOutcome, +} from '../../core/ad_trace'; import type { AdTraceApi, AuctionBidData, AuctionTraceSummary, TsjsApi } from '../../core/types'; import { installAdTraceOverlay } from './overlay'; @@ -79,12 +84,7 @@ export function installAdTrace(): boolean { kind: 'ts_auction_observed', slotId: slot.id, auctionTraceId: summary.auctionTraceId, - outcome: - summary.outcome === 'completed' || summary.outcome === 'no_bid' - ? 'no_bid' - : summary.outcome === 'skipped' - ? 'skipped' - : 'unresolved', + outcome: terminalSummaryStageOutcome(summary.outcome), confidence: 'definitive', reason: 'terminal_summary', }); diff --git a/crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts b/crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts index fd08963d0..827f4e397 100644 --- a/crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts +++ b/crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts @@ -1,10 +1,14 @@ import type { AdTraceApi, + AdTraceStage, + AdTraceStageName, RenderTraceSnapshot, RenderTraceVisibility, SlotTraceSnapshot, } from '../../core/types'; +import { presentTraceOverlay } from './presentation'; + const HOST_ID = 'ts-ad-trace-overlay'; const TRACE_ATTRIBUTES = [ 'data-ts-trace-seq', @@ -15,20 +19,29 @@ const TRACE_ATTRIBUTES = [ 'data-ts-trace-visibility', ] as const; -function stageLine(label: string, stage: { outcome: string; confidence: string }): string { - return `${label}: ${stage.outcome} · ${stage.confidence}`; +const EMPTY_STAGES: Record = { + trustedServer: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + prebid: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + gam: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + creative: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, +}; + +function stagesForRender(slot: SlotTraceSnapshot, render: RenderTraceSnapshot) { + return ( + slot.generations.find((generation) => generation.generation === render.generation)?.stages ?? + (slot.latestGeneration === render.generation ? slot.stages : undefined) + ); } -function badgeText(slot: SlotTraceSnapshot, render?: RenderTraceSnapshot): string { - return [ - render ? `#${render.sequence}: ${render.outcome} · ${render.visibility}` : undefined, - stageLine('TS winner', slot.stages.trustedServer), - stageLine('Prebid winner', slot.stages.prebid), - stageLine('GAM result', slot.stages.gam), - stageLine('Creative', slot.stages.creative), - ] - .filter(Boolean) - .join('\n'); +function latestRenderForSlot( + renders: readonly RenderTraceSnapshot[], + slot: SlotTraceSnapshot +): RenderTraceSnapshot | undefined { + for (let index = renders.length - 1; index >= 0; index -= 1) { + const render = renders[index]; + if (render.slotId === slot.slotId && render.generation === slot.latestGeneration) return render; + } + return undefined; } function removeTraceAttributes(element: HTMLElement): void { @@ -75,7 +88,10 @@ export function installAdTraceOverlay( .badge { position: fixed; z-index: 2147483647; max-width: 300px; padding: 6px 8px; border: 1px solid #72e0a6; border-radius: 4px; background: rgba(10,18,16,.94); color: #eefbf4; font: 11px/1.35 ui-monospace, monospace; white-space: pre; cursor: pointer; } - .badge.probable { border-color: #67a8ff; } + .badge.attributed { border-color: #72e0a6; } + .badge.unattributed { border-color: #67a8ff; } + .badge.empty { border-color: #ffd479; } + .badge.failed { border-color: #ff7b72; } .panel { position: fixed; right: 12px; bottom: 12px; z-index: 2147483647; width: 460px; max-height: 60vh; overflow: auto; padding: 10px; background: #0a1210; color: #eefbf4; border: 1px solid #72e0a6; font: 11px/1.4 ui-monospace, monospace; } @@ -136,31 +152,30 @@ export function installAdTraceOverlay( rows.replaceChildren(); const exported = api.export(); const slotById = new Map(exported.slots.map((slot) => [slot.slotId, slot])); - const latestBySlot = new Map(); - for (const item of exported.renders) latestBySlot.set(item.slotId, item); const nextObserved = new Set(); for (const item of [...exported.renders].reverse()) { + const slot = slotById.get(item.slotId); + const stages = slot && stagesForRender(slot, item); + // Render history outlives bounded generation-stage retention. Its own + // factual render outcome remains safe to show when the stages are gone. + const presentation = presentTraceOverlay(stages ?? EMPTY_STAGES, item); const row = document.createElement('div'); - row.className = 'row'; + row.className = `row ${presentation.className}`; const title = document.createElement('strong'); - title.textContent = `#${item.sequence} ${item.slotId} · ${item.source}`; + title.textContent = item.slotId; const summary = document.createElement('div'); - summary.textContent = `${item.outcome} · ${item.confidence} · ${item.visibility}`; + summary.textContent = presentation.primaryStatus ?? 'No trace result observed'; row.append(title, summary); row.addEventListener('click', () => { details.hidden = false; - details.textContent = JSON.stringify( - { render: item, stages: slotById.get(item.slotId)?.stages }, - null, - 2 - ); + details.textContent = JSON.stringify({ render: item, stages }, null, 2); }); rows.appendChild(row); } for (const [slotId, slot] of slotById) { - const item = latestBySlot.get(slotId); + const item = latestRenderForSlot(exported.renders, slot); const element = item ? window.tsjs?.getAdTraceElement?.(slotId, item.generation) : undefined; if (!element || !item) continue; const rect = element.getBoundingClientRect(); @@ -175,21 +190,29 @@ export function installAdTraceOverlay( nextObserved.add(element); if (!observedElements.has(element)) resizeObserver?.observe(element); stampRender(element, effectiveItem); - const badge = document.createElement('div'); - badge.className = `badge ${item.outcome === 'confirmed' ? '' : 'probable'}`; - badge.textContent = badgeText(slot, effectiveItem); - badge.style.left = `${Math.max(0, rect.left)}px`; - badge.style.top = `${Math.max(0, rect.top)}px`; - badge.addEventListener('click', () => { - panel.hidden = false; - details.hidden = false; - details.textContent = JSON.stringify( - { render: effectiveItem, stages: slot.stages }, - null, - 2 - ); - }); - badgeLayer.appendChild(badge); + const presentation = presentTraceOverlay(slot.stages, effectiveItem); + // A visibility calculation alone is not trace evidence. Do not place a + // marker over an ad until the trace has at least one observed fact. + const traceFacts = presentation.facts.filter( + (fact) => !fact.startsWith('Slot element currently ') + ); + if (traceFacts.length > 0) { + const badge = document.createElement('div'); + badge.className = `badge ${presentation.className}`; + badge.textContent = presentation.facts.join('\n'); + badge.style.left = `${Math.max(0, rect.left)}px`; + badge.style.top = `${Math.max(0, rect.top)}px`; + badge.addEventListener('click', () => { + panel.hidden = false; + details.hidden = false; + details.textContent = JSON.stringify( + { render: effectiveItem, stages: slot.stages }, + null, + 2 + ); + }); + badgeLayer.appendChild(badge); + } } for (const element of observedElements) { if (!nextObserved.has(element)) { diff --git a/crates/trusted-server-js/lib/src/integrations/ad_trace/presentation.ts b/crates/trusted-server-js/lib/src/integrations/ad_trace/presentation.ts new file mode 100644 index 000000000..e443e5fa0 --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/ad_trace/presentation.ts @@ -0,0 +1,174 @@ +import type { + AdTraceStage, + AdTraceStageName, + RenderTraceSnapshot, + RenderTraceVisibility, +} from '../../core/types'; + +export type TraceOverlayPresentationClass = 'attributed' | 'unattributed' | 'empty' | 'failed'; + +export interface TraceOverlayPresentation { + /** Facts suitable for operator-facing badges and primary timeline rows. */ + facts: readonly string[]; + /** One concise description of the render result when render evidence exists. */ + renderStatus?: string; + /** Best observed fact for a compact primary timeline row. */ + primaryStatus?: string; + className: TraceOverlayPresentationClass; +} + +type TraceStages = Record; + +const STAGE_ORDER: readonly AdTraceStageName[] = ['trustedServer', 'prebid', 'gam', 'creative']; + +function stageFacts(name: AdTraceStageName, stage: AdTraceStage): readonly string[] { + if (stage.outcome === 'not_observed' || stage.outcome === 'not_run') return []; + + switch (name) { + case 'trustedServer': + switch (stage.outcome) { + case 'won': + return ['Trusted Server selected a bid']; + case 'no_bid': + return ['Trusted Server returned no bid']; + case 'skipped': + return ['Trusted Server auction skipped']; + case 'failed': + case 'abandoned': + return ['Trusted Server auction did not complete']; + default: + return []; + } + case 'prebid': + if (stage.outcome === 'won') { + return stage.reason === 'selected_targeting_with_bid_won' + ? ['Prebid selected the Trusted Server bid', 'Prebid reported the bid won'] + : ['Prebid selected the Trusted Server bid']; + } + if (stage.outcome === 'client_bid_won' || stage.outcome === 'lost') { + return stage.reason === 'selected_targeting_with_bid_won' + ? ['Prebid selected a client bid', 'Prebid reported the bid won'] + : ['Prebid selected a client bid']; + } + return []; + case 'gam': + switch (stage.outcome) { + case 'empty': + return ['GAM returned no ad']; + case 'backfill': + return ['GAM returned backfill']; + case 'trusted_server_won': + return ['GAM selected the Trusted Server creative']; + case 'trusted_server_candidate': + case 'client_prebid_candidate': + case 'direct_or_unattributed': + return ['GAM rendered an ad — source not attributed']; + default: + return []; + } + case 'creative': + switch (stage.outcome) { + case 'gpt_iframe_onload': + return ['GAM creative iframe loaded']; + case 'load_acknowledged': + return [ + stage.reason === 'direct_iframe_load' + ? 'Creative iframe load confirmed' + : 'Trusted Server creative load confirmed', + ]; + case 'prebid_render_succeeded': + return ['Prebid reported render succeeded']; + case 'render_failed': + return ['Prebid reported render failed']; + case 'aps_renderer_ready': + return ['APS renderer reported ready']; + case 'renderer_served': + if (stage.reason === 'direct_aps_renderer') { + return ['APS renderer started creative loading']; + } + if (stage.reason === 'aps_renderer') return ['APS renderer response sent']; + return ['Creative response sent to the renderer']; + case 'rejected': + return ['Trusted Server direct render rejected']; + default: + return []; + } + } +} + +function renderStatus(render?: RenderTraceSnapshot): string | undefined { + if (!render) return undefined; + + switch (render.outcome) { + case 'confirmed': + return render.source === 'direct_auction' + ? 'Creative iframe load confirmed' + : 'Trusted Server creative load confirmed'; + case 'served': + if (render.reason === 'direct_aps_renderer_ready') return 'APS renderer reported ready'; + if (render.reason === 'direct_aps_renderer') return 'APS renderer started creative loading'; + if (render.reason === 'aps_renderer') return 'APS renderer response sent'; + if (render.reason === 'direct_iframe_created') return 'Creative iframe created'; + return 'Creative response sent to the renderer'; + case 'gam_only': + return 'GAM rendered an ad — source not attributed'; + case 'empty': + return 'GAM returned no ad'; + case 'unresolved': + return undefined; + } +} + +function visibilityFact(visibility: RenderTraceVisibility | undefined): string | undefined { + if (visibility === 'visible') return 'Slot element currently visible'; + if (visibility === 'hidden') return 'Slot element currently hidden'; + return undefined; +} + +function presentationClass( + stages: TraceStages, + render?: RenderTraceSnapshot +): TraceOverlayPresentationClass { + if ( + stages.creative.outcome === 'load_acknowledged' || + stages.gam.outcome === 'trusted_server_won' || + render?.outcome === 'confirmed' + ) { + return 'attributed'; + } + if (stages.creative.outcome === 'render_failed') return 'failed'; + if (stages.gam.outcome === 'empty' || render?.outcome === 'empty') return 'empty'; + return 'unattributed'; +} + +/** + * Convert internal trace stages into factual operator-facing language. + * + * Raw outcomes, confidence, reasons, sequence IDs, and generation IDs remain + * available in technical details and exports; they are intentionally excluded + * from this presentation surface. + */ +export function presentTraceOverlay( + stages: TraceStages, + render?: RenderTraceSnapshot +): TraceOverlayPresentation { + const facts = new Set(); + for (const name of STAGE_ORDER) { + for (const fact of stageFacts(name, stages[name])) facts.add(fact); + } + const status = renderStatus(render); + if (status) facts.add(status); + const visibility = visibilityFact(render?.visibility); + if (visibility) facts.add(visibility); + if (render?.viewability === 'viewable') facts.add('Viewable impression observed'); + const factList = [...facts]; + const primaryStatus = + status ?? [...factList].reverse().find((fact) => !fact.startsWith('Slot element currently ')); + + return { + facts: factList, + renderStatus: status, + ...(primaryStatus ? { primaryStatus } : {}), + className: presentationClass(stages, render), + }; +} diff --git a/crates/trusted-server-js/lib/src/integrations/aps/render.ts b/crates/trusted-server-js/lib/src/integrations/aps/render.ts index ff37e2cb4..fed5a063e 100644 --- a/crates/trusted-server-js/lib/src/integrations/aps/render.ts +++ b/crates/trusted-server-js/lib/src/integrations/aps/render.ts @@ -294,10 +294,16 @@ export function apsRendererUrl(pageOrigin = window.location.origin): string | un export interface RenderApsCreativeOptions { slotId: string; renderer: unknown; + /** Observational callback after the renderer's nonce-bound ready message. */ + onReady?: () => void; } /** Render APS through the static endpoint under an outer opaque-origin sandbox. */ -export function renderApsCreative({ slotId, renderer: input }: RenderApsCreativeOptions): boolean { +export function renderApsCreative({ + slotId, + renderer: input, + onReady, +}: RenderApsCreativeOptions): boolean { const renderer = validateApsRenderer(input); const rendererUrl = apsRendererUrl(); const nonce = createNonce(); @@ -346,6 +352,11 @@ export function renderApsCreative({ slotId, renderer: input }: RenderApsCreative if (child !== iframe) child.remove(); } iframe.style.display = ''; + try { + onReady?.(); + } catch { + // Read-only diagnostics must never affect a successfully committed render. + } }; function receive(event: MessageEvent): void { if (event.source !== iframe.contentWindow || !hasExactKeys(event.data, ['message', 'nonce'])) { diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 8ab89ed6e..1f9756601 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -1,3 +1,4 @@ +import { terminalSummaryStageOutcome } from '../../core/ad_trace'; import { log } from '../../core/log'; import type { AuctionSlot, AuctionBidData, AuctionTraceSummary, TsjsApi } from '../../core/types'; import { @@ -935,7 +936,7 @@ export function captureAdTraceRequest( slotId, generation, auctionTraceId: serverSummary.auctionTraceId, - outcome: serverSummary.outcome === 'completed' ? 'no_bid' : serverSummary.outcome, + outcome: terminalSummaryStageOutcome(serverSummary.outcome), confidence: 'definitive', reason: 'terminal_summary', }); @@ -1012,7 +1013,12 @@ function candidateForSlot( candidate.slot === slot && !candidate.superseded && (includeTerminal || !candidate.terminal) && - monotonicNow() - candidate.createdAt <= 30_000 + // Terminal evidence such as iframe load and viewability can arrive well + // after the 30-second render-request window. Retain the exact terminal + // candidate until a replacement, navigation, or bounded eviction supersedes it. + (includeTerminal && candidate.terminal + ? true + : monotonicNow() - candidate.createdAt <= 30_000) ); if (candidates.length !== 1) { if (candidates.length > 1) { @@ -1047,9 +1053,18 @@ function installGptEvidenceListeners(service: GoogleTagPubAdsService): void { if (instrumented.__tsAdTraceListeners) return; instrumented.__tsAdTraceListeners = true; const record = - (kind: 'gpt_slot_requested' | 'gpt_slot_response_received' | 'gpt_slot_onload') => + ( + kind: + | 'gpt_slot_requested' + | 'gpt_slot_response_received' + | 'gpt_slot_onload' + | 'gpt_impression_viewable' + ) => (event: GptSlotEvent): void => { - const candidate = candidateForSlot(event.slot, kind === 'gpt_slot_onload'); + const candidate = candidateForSlot( + event.slot, + kind === 'gpt_slot_onload' || kind === 'gpt_impression_viewable' + ); if (!candidate) return; window.tsjs?.recordAdTrace?.({ kind, @@ -1061,6 +1076,7 @@ function installGptEvidenceListeners(service: GoogleTagPubAdsService): void { service.addEventListener('slotRequested', record('gpt_slot_requested')); service.addEventListener('slotResponseReceived', record('gpt_slot_response_received')); service.addEventListener('slotOnload', record('gpt_slot_onload')); + service.addEventListener('impressionViewable', record('gpt_impression_viewable')); service.addEventListener('slotRenderEnded', (event: GptSlotEvent) => { const candidate = candidateForSlot(event.slot); if (!candidate) return; @@ -1109,12 +1125,7 @@ export function installTsAdInit(): void { kind: 'ts_auction_observed', slotId: slot.id, auctionTraceId: summary.auctionTraceId, - outcome: - summary.outcome === 'completed' || summary.outcome === 'no_bid' - ? 'no_bid' - : summary.outcome === 'skipped' - ? 'skipped' - : 'unresolved', + outcome: terminalSummaryStageOutcome(summary.outcome), confidence: 'definitive', reason: 'terminal_summary', }); diff --git a/crates/trusted-server-js/lib/test/core/ad_trace.test.ts b/crates/trusted-server-js/lib/test/core/ad_trace.test.ts index 0348e8dd7..2a16a58e7 100644 --- a/crates/trusted-server-js/lib/test/core/ad_trace.test.ts +++ b/crates/trusted-server-js/lib/test/core/ad_trace.test.ts @@ -5,6 +5,7 @@ import { AD_TRACE_MAX_RENDERS, AD_TRACE_MAX_SLOTS, createAdTraceStore, + terminalSummaryStageOutcome, } from '../../src/core/ad_trace'; const BID_TRACE_ID = '550e8400-e29b-41d4-a716-446655440000'; @@ -153,6 +154,50 @@ describe('ad trace reducer', () => { ).toBe('no_bid'); }); + it('retains a Trusted Server Prebid selection when bidWon arrives without claiming creative load', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'prebid_targeting_selected', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + outcome: 'won', + confidence: 'definitive', + reason: 'selected_targeting', + }); + store.record({ + kind: 'prebid_bid_won', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + }); + + expect(store.getSlot('slot-a')?.stages.prebid).toMatchObject({ + outcome: 'won', + reason: 'selected_targeting_with_bid_won', + }); + expect(store.getSlot('slot-a')?.stages.creative.outcome).toBe('not_observed'); + }); + + it('preserves the direct iframe acknowledgement boundary without claiming GAM selection', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'creative_load_acknowledged', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + reason: 'direct_iframe_load', + }); + + expect(store.getSlot('slot-a')?.stages.creative).toMatchObject({ + outcome: 'load_acknowledged', + reason: 'direct_iframe_load', + }); + expect(store.getSlot('slot-a')?.stages.gam.outcome).toBe('not_observed'); + }); + it('classifies overlap, client Prebid, APS, no-bid, and superseded states', () => { const store = createAdTraceStore(() => 1); const generation = store.nextGeneration('slot-a'); @@ -272,6 +317,60 @@ describe('ad trace reducer', () => { }); }); + it('keeps GPT viewability separate from element visibility and creative load', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'gpt_slot_render_ended', + slotId: 'slot-a', + generation, + isEmpty: false, + }); + store.updateVisibility('slot-a', generation, 'hidden'); + store.record({ kind: 'gpt_impression_viewable', slotId: 'slot-a', generation }); + + expect(store.getRenderTimeline()[0]).toMatchObject({ + outcome: 'gam_only', + visibility: 'hidden', + viewability: 'viewable', + }); + expect(store.getSlot('slot-a')?.stages.creative.outcome).toBe('not_observed'); + }); + + it('distinguishes APS renderer start from the validated ready boundary', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'pb_render_served', + slotId: 'slot-a', + generation, + reason: 'direct_aps_renderer', + }); + expect(store.getSlot('slot-a')?.stages.creative).toMatchObject({ + outcome: 'renderer_served', + reason: 'direct_aps_renderer', + }); + expect(store.getRenderTimeline()[0]).toMatchObject({ + outcome: 'served', + reason: 'direct_aps_renderer', + }); + + store.record({ + kind: 'aps_renderer_ready', + slotId: 'slot-a', + generation, + reason: 'direct_aps_renderer_ready', + }); + expect(store.getSlot('slot-a')?.stages.creative).toMatchObject({ + outcome: 'aps_renderer_ready', + reason: 'direct_aps_renderer_ready', + }); + expect(store.getRenderTimeline()[0]).toMatchObject({ + outcome: 'served', + reason: 'direct_aps_renderer_ready', + }); + }); + it('dispatches a frozen privacy-safe render event', () => { const store = createAdTraceStore(() => 1); const observed: unknown[] = []; @@ -303,6 +402,14 @@ describe('ad trace reducer', () => { expect(store.getRenderTimeline()[0].slotId).toBe('render-1'); }); + it('preserves failed and abandoned terminal summaries while mapping completed no-winner to no bid', () => { + expect(terminalSummaryStageOutcome('completed')).toBe('no_bid'); + expect(terminalSummaryStageOutcome('completed', true)).toBe('completed'); + expect(terminalSummaryStageOutcome('failed')).toBe('failed'); + expect(terminalSummaryStageOutcome('abandoned')).toBe('abandoned'); + expect(terminalSummaryStageOutcome('skipped')).toBe('skipped'); + }); + it('rejects malformed runtime event kinds and confidence values', () => { const store = createAdTraceStore(() => 1); store.record({ kind: 'not-a-real-kind', slotId: 'slot-a' } as never); diff --git a/crates/trusted-server-js/lib/test/core/request.test.ts b/crates/trusted-server-js/lib/test/core/request.test.ts index dc3cf9b68..2c39edb49 100644 --- a/crates/trusted-server-js/lib/test/core/request.test.ts +++ b/crates/trusted-server-js/lib/test/core/request.test.ts @@ -81,6 +81,11 @@ describe('request.requestAds', () => { width: apsBid.w, height: apsBid.h, }; + const recordAdTrace = vi.fn(); + window.tsjs = { + recordAdTrace, + nextAdTraceGeneration: vi.fn().mockReturnValue(1), + } as any; (globalThis as any).fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, @@ -117,6 +122,16 @@ describe('request.requestAds', () => { expect(iframe!.srcdoc).toBe(''); expect(iframe!.getAttribute('sandbox')).not.toContain('allow-same-origin'); expect(document.querySelector('#slot1 span')).not.toBeNull(); + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'pb_render_served', + generation: 1, + reason: 'direct_aps_renderer', + }) + ); + expect(recordAdTrace).not.toHaveBeenCalledWith( + expect.objectContaining({ kind: 'aps_renderer_ready' }) + ); const postMessage = vi.spyOn(iframe!.contentWindow!, 'postMessage'); iframe!.dispatchEvent(new Event('load')); @@ -131,6 +146,13 @@ describe('request.requestAds', () => { }) ); expect(document.querySelector('#slot1 span')).toBeNull(); + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'aps_renderer_ready', + generation: 1, + reason: 'direct_aps_renderer_ready', + }) + ); }); it('does not mutate the slot for an invalid APS descriptor', async () => { @@ -524,6 +546,50 @@ describe('request.requestAds', () => { ); }); + it.each(['failed', 'abandoned'] as const)( + 'preserves a direct auction %s terminal summary', + async (outcome) => { + const recordAdTrace = vi.fn(); + window.tsjs = { + recordAdTrace, + nextAdTraceGeneration: vi.fn().mockReturnValue(1), + } as any; + (globalThis as any).fetch = vi.fn().mockResolvedValue({ + ok: true, + status: 200, + headers: { get: () => 'application/json' }, + json: async () => ({ + ext: { + trusted_server: { + trace: { + version: 1, + auction_trace_id: '550e8400-e29b-41d4-a716-446655440000', + source: 'auction_api', + outcome, + }, + }, + }, + seatbid: [], + }), + }); + const { addAdUnits } = await import('../../src/core/registry'); + const { requestAds } = await import('../../src/core/request'); + document.body.innerHTML = '
'; + addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); + + requestAds(); + await flushRequestAds(); + + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'ts_auction_observed', + outcome, + reason: 'terminal_summary', + }) + ); + } + ); + it('skips iframe insertion when slot is missing', async () => { // mock fetch for unified auction endpoint - returns inline HTML (globalThis as any).fetch = vi.fn().mockResolvedValue({ diff --git a/crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts b/crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts index a3eaa7d6f..f24a513b7 100644 --- a/crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts @@ -33,6 +33,33 @@ describe('ad_trace integration gate', () => { expect(document.querySelectorAll('#ts-ad-trace-overlay')).toHaveLength(1); }); + it.each(['failed', 'abandoned'] as const)( + 'preserves a %s terminal summary when seeding a slot', + async (outcome) => { + window.__tsjs_adTraceActive = true; + window.tsjs = { + adSlots: [{ id: 'slot-a' }], + auctionTrace: { + version: 1, + auctionTraceId: '550e8400-e29b-41d4-a716-446655440000', + source: 'initial_navigation', + outcome, + }, + } as any; + + const { installAdTrace } = await import('../../../src/integrations/ad_trace/index'); + expect(installAdTrace()).toBe(true); + const generation = window.tsjs?.nextAdTraceGeneration?.('slot-a'); + expect( + window.tsjs?.adTrace?.getSlot('slot-a')?.generations[0]?.stages.trustedServer + ).toMatchObject({ + outcome, + reason: 'terminal_summary', + }); + expect(generation).toBeGreaterThan(0); + } + ); + it('does not accept the legacy tester cookie without bootstrap', async () => { document.cookie = 'ts-tester=true; Path=/'; const { installAdTrace } = await import('../../../src/integrations/ad_trace/index'); diff --git a/crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts b/crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts index 0a5aaa217..6e0770d21 100644 --- a/crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts @@ -67,6 +67,15 @@ describe('ad trace overlay lifecycle', () => { } as any; const observe = vi.fn(); + const attachShadow = HTMLElement.prototype.attachShadow; + let shadow: ShadowRoot | undefined; + vi.spyOn(HTMLElement.prototype, 'attachShadow').mockImplementation(function ( + this: HTMLElement, + init: ShadowRootInit + ) { + shadow = attachShadow.call(this, init); + return shadow; + }); vi.stubGlobal( 'ResizeObserver', class { @@ -89,6 +98,15 @@ describe('ad trace overlay lifecycle', () => { expect(rect).toHaveBeenCalledTimes(1); expect(observe).toHaveBeenCalledWith(element); expect(updateVisibility).toHaveBeenCalledWith('slot-a', 1, 'visible'); + const badge = shadow?.querySelector('.badge'); + const row = shadow?.querySelector('.row'); + expect(badge?.textContent).toBe( + 'Trusted Server selected a bid\nGAM rendered an ad — source not attributed\nSlot element currently visible' + ); + expect(row?.textContent).toContain('GAM rendered an ad — source not attributed'); + expect(`${badge?.textContent}\n${row?.textContent}`).not.toMatch( + /definitive|strong|probable|not_run|gam_only|TS winner|Prebid winner|#1/ + ); expect(element.getAttribute('data-ts-trace-seq')).toBe('1'); expect(element.getAttribute('data-ts-trace-outcome')).toBe('gam_only'); window.dispatchEvent(new Event('scroll')); @@ -107,4 +125,202 @@ describe('ad trace overlay lifecycle', () => { expect(element.hasAttribute('data-ts-trace-seq')).toBe(false); expect(updateVisibility).toHaveBeenCalledWith('slot-a', 1, 'disconnected'); }); + + it('does not add an empty badge when no operator-facing fact was observed', () => { + const element = document.createElement('div'); + document.body.appendChild(element); + window.tsjs = { + getAdTraceElement: () => element, + updateAdTraceVisibility: vi.fn(), + } as any; + const attachShadow = HTMLElement.prototype.attachShadow; + let shadow: ShadowRoot | undefined; + vi.spyOn(HTMLElement.prototype, 'attachShadow').mockImplementation(function ( + this: HTMLElement, + init: ShadowRootInit + ) { + shadow = attachShadow.call(this, init); + return shadow; + }); + vi.stubGlobal( + 'ResizeObserver', + class { + observe() {} + unobserve() {} + disconnect() {} + } + ); + vi.spyOn(element, 'getBoundingClientRect').mockReturnValue({ + left: 0, + top: 0, + width: 300, + height: 250, + } as DOMRect); + const slot = { + slotId: 'slot-a', + latestGeneration: 1, + generations: [], + stages: { + trustedServer: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + prebid: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + gam: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + creative: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + }, + }; + const render = { + sequence: 1, + slotId: 'slot-a', + generation: 1, + source: 'gpt', + outcome: 'unresolved', + confidence: 'none', + visibility: 'unknown', + createdAt: 1, + updatedAt: 1, + }; + installAdTraceOverlay( + { + getSlot: () => slot as any, + getEvents: () => [], + getRenderTimeline: () => [render] as any, + export: () => + ({ + version: 1, + slots: [slot], + events: [], + renders: [render], + metadata: { droppedEvents: 0, evictedSlots: 0 }, + }) as any, + }, + () => vi.fn() + ); + + expect(shadow?.querySelector('.badge')).toBeNull(); + expect(shadow?.querySelector('.row')?.textContent).toContain('No trace result observed'); + }); + + it('uses observed stage evidence when a render row has no render outcome', () => { + const attachShadow = HTMLElement.prototype.attachShadow; + let shadow: ShadowRoot | undefined; + vi.spyOn(HTMLElement.prototype, 'attachShadow').mockImplementation(function ( + this: HTMLElement, + init: ShadowRootInit + ) { + shadow = attachShadow.call(this, init); + return shadow; + }); + vi.stubGlobal( + 'ResizeObserver', + class { + observe() {} + unobserve() {} + disconnect() {} + } + ); + const stages = { + trustedServer: { outcome: 'won', confidence: 'definitive', reason: 'winner' }, + prebid: { outcome: 'won', confidence: 'definitive', reason: 'selected_targeting' }, + gam: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + creative: { outcome: 'render_failed', confidence: 'definitive', reason: 'failure' }, + }; + const slot = { + slotId: 'slot-a', + latestGeneration: 1, + generations: [{ generation: 1, stages }], + stages, + }; + const render = { + sequence: 1, + slotId: 'slot-a', + generation: 1, + source: 'gpt', + outcome: 'unresolved', + confidence: 'none', + visibility: 'unknown', + createdAt: 1, + updatedAt: 1, + }; + installAdTraceOverlay( + { + getSlot: () => slot as any, + getEvents: () => [], + getRenderTimeline: () => [render] as any, + export: () => + ({ + version: 1, + slots: [slot], + events: [], + renders: [render], + metadata: { droppedEvents: 0, evictedSlots: 0 }, + }) as any, + }, + () => vi.fn() + ); + + const row = shadow?.querySelector('.row'); + expect(row?.textContent).toContain('Prebid reported render failed'); + expect(row?.textContent).not.toContain('No trace result observed'); + }); + + it('gives a retained render a factual status after its generation stages were evicted', () => { + const attachShadow = HTMLElement.prototype.attachShadow; + let shadow: ShadowRoot | undefined; + vi.spyOn(HTMLElement.prototype, 'attachShadow').mockImplementation(function ( + this: HTMLElement, + init: ShadowRootInit + ) { + shadow = attachShadow.call(this, init); + return shadow; + }); + vi.stubGlobal( + 'ResizeObserver', + class { + observe() {} + unobserve() {} + disconnect() {} + } + ); + const slot = { + slotId: 'slot-a', + latestGeneration: 2, + generations: [], + stages: { + trustedServer: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + prebid: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + gam: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + creative: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + }, + }; + const render = { + sequence: 1, + slotId: 'slot-a', + generation: 1, + source: 'gpt', + outcome: 'gam_only', + confidence: 'probable', + visibility: 'unknown', + createdAt: 1, + updatedAt: 1, + }; + installAdTraceOverlay( + { + getSlot: () => slot as any, + getEvents: () => [], + getRenderTimeline: () => [render] as any, + export: () => + ({ + version: 1, + slots: [slot], + events: [], + renders: [render], + metadata: { droppedEvents: 0, evictedSlots: 0 }, + }) as any, + }, + () => vi.fn() + ); + + const row = shadow?.querySelector('.row'); + expect(row?.textContent).toContain('GAM rendered an ad — source not attributed'); + expect(row?.textContent).not.toMatch(/probable|gam_only|#1/); + }); }); diff --git a/crates/trusted-server-js/lib/test/integrations/ad_trace/presentation.test.ts b/crates/trusted-server-js/lib/test/integrations/ad_trace/presentation.test.ts new file mode 100644 index 000000000..d8008d1cb --- /dev/null +++ b/crates/trusted-server-js/lib/test/integrations/ad_trace/presentation.test.ts @@ -0,0 +1,231 @@ +import { describe, expect, it } from 'vitest'; + +import { presentTraceOverlay } from '../../../src/integrations/ad_trace/presentation'; +import type { AdTraceStage, AdTraceStageName, RenderTraceSnapshot } from '../../../src/core/types'; + +function stage(outcome = 'not_observed', reason = 'none'): AdTraceStage { + return { outcome, confidence: 'none', reason }; +} + +function stages(overrides: Partial> = {}) { + return { + trustedServer: stage(), + prebid: stage(), + gam: stage(), + creative: stage(), + ...overrides, + }; +} + +function render( + outcome: RenderTraceSnapshot['outcome'], + overrides: Partial = {} +): RenderTraceSnapshot { + return { + sequence: 4, + slotId: 'slot-a', + generation: 2, + source: 'gpt', + outcome, + confidence: 'probable', + visibility: 'unknown', + createdAt: 1, + updatedAt: 1, + ...overrides, + }; +} + +describe('presentTraceOverlay', () => { + it.each([ + [ + 'server winner', + stages({ trustedServer: stage('won') }), + undefined, + ['Trusted Server selected a bid'], + ], + [ + 'server no bid', + stages({ trustedServer: stage('no_bid') }), + undefined, + ['Trusted Server returned no bid'], + ], + [ + 'server skip', + stages({ trustedServer: stage('skipped') }), + undefined, + ['Trusted Server auction skipped'], + ], + [ + 'server failure', + stages({ trustedServer: stage('failed') }), + undefined, + ['Trusted Server auction did not complete'], + ], + [ + 'server abandonment', + stages({ trustedServer: stage('abandoned') }), + undefined, + ['Trusted Server auction did not complete'], + ], + [ + 'traced Prebid selection', + stages({ prebid: stage('won') }), + undefined, + ['Prebid selected the Trusted Server bid'], + ], + [ + 'client Prebid selection', + stages({ prebid: stage('client_bid_won') }), + undefined, + ['Prebid selected a client bid'], + ], + [ + 'client Prebid selection recorded as lost server targeting', + stages({ prebid: stage('lost') }), + undefined, + ['Prebid selected a client bid'], + ], + [ + 'reported Prebid win', + stages({ prebid: stage('won', 'selected_targeting_with_bid_won') }), + undefined, + ['Prebid selected the Trusted Server bid', 'Prebid reported the bid won'], + ], + ['GAM empty', stages({ gam: stage('empty') }), undefined, ['GAM returned no ad']], + ['GAM backfill', stages({ gam: stage('backfill') }), undefined, ['GAM returned backfill']], + [ + 'unattributed GAM render', + stages({ gam: stage('direct_or_unattributed') }), + undefined, + ['GAM rendered an ad — source not attributed'], + ], + [ + 'selected Trusted Server GAM creative', + stages({ gam: stage('trusted_server_won') }), + undefined, + ['GAM selected the Trusted Server creative'], + ], + [ + 'GAM iframe load', + stages({ creative: stage('gpt_iframe_onload') }), + undefined, + ['GAM creative iframe loaded'], + ], + [ + 'creative acknowledgement', + stages({ creative: stage('load_acknowledged') }), + undefined, + ['Trusted Server creative load confirmed'], + ], + [ + 'direct iframe acknowledgement', + stages({ creative: stage('load_acknowledged', 'direct_iframe_load') }), + render('confirmed', { source: 'direct_auction' }), + ['Creative iframe load confirmed'], + ], + [ + 'Prebid render success', + stages({ creative: stage('prebid_render_succeeded') }), + undefined, + ['Prebid reported render succeeded'], + ], + [ + 'Prebid render failure', + stages({ creative: stage('render_failed') }), + undefined, + ['Prebid reported render failed'], + ], + [ + 'direct APS renderer started', + stages({ creative: stage('renderer_served', 'direct_aps_renderer') }), + render('served', { source: 'direct_auction', reason: 'direct_aps_renderer' }), + ['APS renderer started creative loading'], + ], + [ + 'direct APS renderer ready', + stages({ creative: stage('aps_renderer_ready', 'direct_aps_renderer_ready') }), + render('served', { source: 'direct_auction', reason: 'direct_aps_renderer_ready' }), + ['APS renderer reported ready'], + ], + [ + 'direct render rejection', + stages({ creative: stage('rejected') }), + undefined, + ['Trusted Server direct render rejected'], + ], + [ + 'current visibility', + stages(), + render('unresolved', { visibility: 'visible' }), + ['Slot element currently visible'], + ], + [ + 'viewable impression independent of live visibility', + stages({ creative: stage('gpt_iframe_onload') }), + render('gam_only', { visibility: 'hidden', viewability: 'viewable' }), + [ + 'GAM creative iframe loaded', + 'GAM rendered an ad — source not attributed', + 'Slot element currently hidden', + 'Viewable impression observed', + ], + ], + ])('%s uses factual operator language', (_name, input, snapshot, expected) => { + expect(presentTraceOverlay(input, snapshot).facts).toEqual(expected); + }); + + it('hides unobserved and inapplicable stages without leaking internal vocabulary', () => { + const presentation = presentTraceOverlay( + stages({ + trustedServer: stage('unresolved'), + prebid: stage('not_run', 'direct'), + gam: stage('not_observed'), + creative: stage('not_observed'), + }), + render('unresolved', { visibility: 'unknown' }) + ); + + expect(presentation.facts).toEqual([]); + expect(presentation.renderStatus).toBeUndefined(); + expect(JSON.stringify(presentation)).not.toMatch( + /definitive|strong|probable|not_run|not_observed|unresolved|gam_only|client_bid_won/ + ); + }); + + it.each([ + ['attributed', stages({ creative: stage('load_acknowledged') }), undefined], + ['empty', stages({ gam: stage('empty') }), undefined], + ['failed', stages({ creative: stage('render_failed') }), undefined], + ['unattributed', stages({ gam: stage('trusted_server_candidate') }), render('gam_only')], + ] as const)('uses an evidence-based %s presentation class', (expected, input, snapshot) => { + expect(presentTraceOverlay(input, snapshot).className).toBe(expected); + }); + + it.each([ + [ + 'confirmed Trusted Server creative', + render('confirmed'), + 'Trusted Server creative load confirmed', + ], + [ + 'confirmed direct creative', + render('confirmed', { source: 'direct_auction' }), + 'Creative iframe load confirmed', + ], + ['served renderer', render('served'), 'Creative response sent to the renderer'], + ['unattributed GAM render', render('gam_only'), 'GAM rendered an ad — source not attributed'], + ['empty GAM response', render('empty'), 'GAM returned no ad'], + ] as const)('renders %s as a concise factual row status', (_name, snapshot, expected) => { + expect(presentTraceOverlay(stages(), snapshot).renderStatus).toBe(expected); + }); + + it('falls back to the strongest observed stage fact for a primary row', () => { + const presentation = presentTraceOverlay( + stages({ creative: stage('render_failed') }), + render('unresolved') + ); + + expect(presentation.renderStatus).toBeUndefined(); + expect(presentation.primaryStatus).toBe('Prebid reported render failed'); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts index e7978e1cb..b7cf2e2f0 100644 --- a/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/aps/render.test.ts @@ -182,8 +182,11 @@ describe('direct APS rendering', () => { document.body.innerHTML = ''; }); - it('loads the static route with a fragment-bound 128-bit nonce and opaque sandbox', () => { - expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor() })).toBe(true); + it('loads the static route with a fragment-bound 128-bit nonce and reports only validated readiness', () => { + const onReady = vi.fn(); + expect(renderApsCreative({ slotId: 'fictional-slot', renderer: descriptor(), onReady })).toBe( + true + ); const slot = document.getElementById('fictional-slot')!; const iframe = slot.querySelector('iframe')!; @@ -219,6 +222,7 @@ describe('direct APS rendering', () => { }) ); expect(slot.querySelector('span')).not.toBeNull(); + expect(onReady).not.toHaveBeenCalled(); window.dispatchEvent( new MessageEvent('message', { @@ -228,6 +232,7 @@ describe('direct APS rendering', () => { ); expect(slot.querySelector('span')).toBeNull(); expect(iframe.style.display).toBe(''); + expect(onReady).toHaveBeenCalledTimes(1); }); it('leaves existing slot content intact when validation or loading fails', () => { diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 7232668ab..222ba95e6 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -155,6 +155,127 @@ describe('installTsAdInit', () => { fetchSpy.mockRestore(); }); + it.each(['failed', 'abandoned'] as const)( + 'preserves a %s terminal summary when adInit has no traced bid', + async (outcome) => { + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([mockSlot]), + addEventListener: vi.fn(), + refresh: vi.fn(), + }; + const recordAdTrace = vi.fn(); + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + auctionTrace: { + version: 1, + auctionTraceId: '550e8400-e29b-41d4-a716-446655440000', + source: 'initial_navigation', + outcome, + }, + recordAdTrace, + } as any; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'ts_auction_observed', + outcome, + reason: 'terminal_summary', + }) + ); + } + ); + + it('records late GPT viewability on the exact terminal request generation', async () => { + let now = 0; + vi.spyOn(performance, 'now').mockImplementation(() => now); + const listeners: Record void>> = {}; + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + clearTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn((key: string) => { + if (key === 'hb_adid') return ['client-ad-id']; + if (key === 'hb_bidder') return ['client-bidder']; + return []; + }), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([mockSlot]), + addEventListener: vi.fn((event: string, fn: (value: SlotRenderEvent) => void) => { + (listeners[event] ??= []).push(fn); + }), + refresh: vi.fn(), + }; + const recordAdTrace = vi.fn(); + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + recordAdTrace, + nextAdTraceGeneration: vi.fn().mockReturnValue(1), + } as any; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + now = 1; + listeners.slotRenderEnded?.forEach((listener) => listener({ isEmpty: false, slot: mockSlot })); + now = 60_001; + listeners.impressionViewable?.forEach((listener) => + listener({ isEmpty: false, slot: mockSlot }) + ); + + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'gpt_impression_viewable', + slotId: 'atf_sidebar_ad', + generation: 1, + }) + ); + }); + it('displays TS-defined slots and does not include them in refresh', async () => { const mockSlot = { addService: vi.fn().mockReturnThis(), From ea6aadfb41ea1aa86505bf15414feb4b883de32e Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 23 Jul 2026 15:52:27 -0500 Subject: [PATCH 108/198] Clarify auction trace console evidence --- .../tests/ad-trace/auction-trace.spec.ts | 42 +++- .../lib/src/core/ad_trace.ts | 43 +++- .../trusted-server-js/lib/src/core/request.ts | 3 +- .../trusted-server-js/lib/src/core/types.ts | 6 + .../lib/src/integrations/ad_trace/index.ts | 14 +- .../lib/src/integrations/ad_trace/overlay.ts | 101 +++++--- .../src/integrations/ad_trace/presentation.ts | 174 +++++++++++++ .../lib/src/integrations/gpt/index.ts | 31 ++- .../lib/test/core/ad_trace.test.ts | 107 ++++++++ .../test/integrations/ad_trace/index.test.ts | 27 ++ .../integrations/ad_trace/overlay.test.ts | 216 ++++++++++++++++ .../ad_trace/presentation.test.ts | 231 ++++++++++++++++++ .../lib/test/integrations/gpt/ad_init.test.ts | 121 +++++++++ 13 files changed, 1046 insertions(+), 70 deletions(-) create mode 100644 crates/trusted-server-js/lib/src/integrations/ad_trace/presentation.ts create mode 100644 crates/trusted-server-js/lib/test/integrations/ad_trace/presentation.test.ts diff --git a/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts b/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts index 9c2935d07..075c30b57 100644 --- a/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts +++ b/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts @@ -118,7 +118,9 @@ test.describe("tester-only auction trace contract", () => { waitUntil: "domcontentloaded", }); await expect(page).toHaveURL(runtimeUrl("/")); - expect(activation?.headers()["cache-control"]).toBe("private, no-store"); + expect(activation?.headers()["cache-control"]).toBe( + "private, no-store", + ); await expect .poll(() => page.evaluate( @@ -210,9 +212,13 @@ test.describe("tester-only auction trace contract", () => { const visibleText = tree.nodes .map((node) => node.name?.value || "") .join("\n"); - expect(visibleText).toContain("TS winner: won · definitive"); + expect(visibleText).toContain("Trusted Server selected a bid"); expect(visibleText).toContain( - "Creative: load_acknowledged · definitive", + "GAM selected the Trusted Server creative", + ); + expect(visibleText).toContain("Trusted Server creative load confirmed"); + expect(visibleText).not.toMatch( + /definitive|strong|probable|not_run|gam_only|TS winner|Prebid winner|#\d/, ); }); @@ -224,12 +230,14 @@ test.describe("tester-only auction trace contract", () => { const direct = document.createElement("div"); direct.id = "direct-api-slot"; document.body.appendChild(direct); - const ts = (window as Window & { - tsjs: { - addAdUnits(unit: unknown): void; - requestAds(): void; - }; - }).tsjs; + const ts = ( + window as Window & { + tsjs: { + addAdUnits(unit: unknown): void; + requestAds(): void; + }; + } + ).tsjs; ts.addAdUnits({ code: "direct-api-slot", mediaTypes: { banner: { sizes: [[300, 250]] } }, @@ -268,7 +276,7 @@ test.describe("tester-only auction trace contract", () => { }); }); - test("actual generated Prebid selects the traced TS bid before a probable GAM result", async ({ + test("actual generated Prebid selects the traced TS bid before an unattributed GAM render", async ({ page, }) => { await openTesterPage(page); @@ -352,6 +360,20 @@ test.describe("tester-only auction trace contract", () => { }) .toEqual({ prebid: "lost", gam: "client_prebid_candidate" }); + const session = await page.context().newCDPSession(page); + await expect + .poll(async () => { + const tree = (await session.send( + "Accessibility.getFullAXTree", + )) as { + nodes: Array<{ name?: { value?: string } }>; + }; + return tree.nodes + .map((node) => node.name?.value || "") + .join("\n"); + }) + .toContain("Prebid selected a client bid"); + await page.evaluate(() => { const win = window as Window & { adTraceFixture: { diff --git a/crates/trusted-server-js/lib/src/core/ad_trace.ts b/crates/trusted-server-js/lib/src/core/ad_trace.ts index aca037f5d..1ea51503f 100644 --- a/crates/trusted-server-js/lib/src/core/ad_trace.ts +++ b/crates/trusted-server-js/lib/src/core/ad_trace.ts @@ -39,7 +39,9 @@ const EVENT_KINDS = new Set([ 'gpt_slot_response_received', 'gpt_slot_render_ended', 'gpt_slot_onload', + 'gpt_impression_viewable', 'aps_display_bids_set', + 'aps_renderer_ready', 'pb_render_requested', 'pb_render_rejected', 'pb_render_served', @@ -135,12 +137,21 @@ function updateStage(target: Record, event: AdTr if (explicit && target.prebid.confidence !== 'definitive') target.prebid = explicit; break; case 'prebid_bid_won': - if (target.prebid.outcome === 'client_bid_won' || target.prebid.outcome === 'lost') { + if ( + target.prebid.outcome === 'won' || + target.prebid.outcome === 'client_bid_won' || + target.prebid.outcome === 'lost' + ) { + // A Prebid win corroborates selection only. It is never creative-load + // evidence, including when the selected bid originated from Trusted Server. target.prebid = { ...target.prebid, reason: 'selected_targeting_with_bid_won', }; - if (target.gam.outcome === 'direct_or_unattributed') { + if ( + (target.prebid.outcome === 'client_bid_won' || target.prebid.outcome === 'lost') && + target.gam.outcome === 'direct_or_unattributed' + ) { target.gam = { outcome: 'client_prebid_candidate', confidence: 'probable', @@ -202,6 +213,15 @@ function updateStage(target: Record, event: AdTr // APS setting display bids is a handoff only. GAM attribution remains // unobserved until a correlated non-empty GPT render arrives. break; + case 'aps_renderer_ready': + if (target.creative.confidence !== 'definitive') { + target.creative = { + outcome: 'aps_renderer_ready', + confidence: 'strong', + reason: event.reason ?? 'aps_renderer_ready', + }; + } + break; case 'gpt_slot_onload': if (target.creative.outcome === 'not_observed') target.creative = { @@ -232,7 +252,8 @@ function updateStage(target: Record, event: AdTr target.creative = { outcome: 'load_acknowledged', confidence: 'definitive', - reason: 'source_validated_load', + reason: + event.reason === 'direct_iframe_load' ? 'direct_iframe_load' : 'source_validated_load', }; if (event.reason !== 'direct_iframe_load') { target.gam = { @@ -268,6 +289,8 @@ function isRenderEvent(kind: AdTraceEventKind): boolean { return ( kind === 'gpt_request_started' || kind === 'gpt_slot_render_ended' || + kind === 'gpt_impression_viewable' || + kind === 'aps_renderer_ready' || kind === 'prebid_render_succeeded' || kind === 'prebid_render_failed' || kind === 'pb_render_requested' || @@ -375,6 +398,8 @@ export function createAdTraceStore( render.source = render.source === 'direct_auction' ? render.source : 'pb_render'; if (event.auctionTraceId) render.auctionTraceId = event.auctionTraceId; if (event.bidTraceId) render.bidTraceId = event.bidTraceId; + if (event.reason) render.reason = event.reason; + if (event.kind === 'gpt_impression_viewable') render.viewability = 'viewable'; render.updatedAt = timestamp; emitRender(render); }; @@ -497,6 +522,18 @@ export function createAdTraceStore( }; } +/** + * Map a public terminal auction outcome to an internal stage outcome. + * + * A completed auction without a final slot winner is a no-bid result. A + * completed auction with a winner is immediately followed by winner evidence, + * but remains distinct here so callers never erase failed or abandoned results. + */ +export function terminalSummaryStageOutcome(outcome: string, hasWinner = false): string { + if (outcome === 'completed') return hasWinner ? 'completed' : 'no_bid'; + return outcome; +} + export function isCanonicalTraceUuid(value: unknown): value is string { return safeUuid(value) !== undefined; } diff --git a/crates/trusted-server-js/lib/src/core/request.ts b/crates/trusted-server-js/lib/src/core/request.ts index 40b41d524..a109b03fd 100644 --- a/crates/trusted-server-js/lib/src/core/request.ts +++ b/crates/trusted-server-js/lib/src/core/request.ts @@ -3,6 +3,7 @@ import { log } from './log'; import { collectContext } from './context'; import { getAllUnits, firstSize } from './registry'; import { createAdIframe, findSlot, buildCreativeDocument, sanitizeCreativeHtml } from './render'; +import { terminalSummaryStageOutcome } from './ad_trace'; import { buildAdRequest, sendAuction } from './auction'; import type { AuctionTraceSummary, TrustedServerBidTrace } from './types'; @@ -72,7 +73,7 @@ function recordRootSummary( slotId: owner.slotId, generation: owner.generation, auctionTraceId: summary.auctionTraceId, - outcome: summary.outcome === 'completed' && !hasWinner ? 'no_bid' : summary.outcome, + outcome: terminalSummaryStageOutcome(summary.outcome, hasWinner), confidence: 'definitive', reason: 'terminal_summary', }); diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 615f174ef..14a9da358 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -94,7 +94,9 @@ export type AdTraceEventKind = | 'gpt_slot_response_received' | 'gpt_slot_render_ended' | 'gpt_slot_onload' + | 'gpt_impression_viewable' | 'aps_display_bids_set' + | 'aps_renderer_ready' | 'pb_render_requested' | 'pb_render_rejected' | 'pb_render_served' @@ -149,6 +151,10 @@ export interface RenderTraceSnapshot { outcome: RenderTraceOutcome; confidence: AdTraceConfidence; visibility: RenderTraceVisibility; + /** GPT reported this exact retained slot generation viewable. */ + viewability?: 'viewable'; + /** Bounded privacy-safe reason for the latest render evidence. */ + reason?: string; createdAt: number; updatedAt: number; } diff --git a/crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts b/crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts index cf6d6d33c..27bf58418 100644 --- a/crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/ad_trace/index.ts @@ -1,4 +1,9 @@ -import { createAdTraceStore, isBoundedTraceLabel, isCanonicalTraceUuid } from '../../core/ad_trace'; +import { + createAdTraceStore, + isBoundedTraceLabel, + isCanonicalTraceUuid, + terminalSummaryStageOutcome, +} from '../../core/ad_trace'; import type { AdTraceApi, AuctionBidData, AuctionTraceSummary, TsjsApi } from '../../core/types'; import { installAdTraceOverlay } from './overlay'; @@ -79,12 +84,7 @@ export function installAdTrace(): boolean { kind: 'ts_auction_observed', slotId: slot.id, auctionTraceId: summary.auctionTraceId, - outcome: - summary.outcome === 'completed' || summary.outcome === 'no_bid' - ? 'no_bid' - : summary.outcome === 'skipped' - ? 'skipped' - : 'unresolved', + outcome: terminalSummaryStageOutcome(summary.outcome), confidence: 'definitive', reason: 'terminal_summary', }); diff --git a/crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts b/crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts index fd08963d0..827f4e397 100644 --- a/crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts +++ b/crates/trusted-server-js/lib/src/integrations/ad_trace/overlay.ts @@ -1,10 +1,14 @@ import type { AdTraceApi, + AdTraceStage, + AdTraceStageName, RenderTraceSnapshot, RenderTraceVisibility, SlotTraceSnapshot, } from '../../core/types'; +import { presentTraceOverlay } from './presentation'; + const HOST_ID = 'ts-ad-trace-overlay'; const TRACE_ATTRIBUTES = [ 'data-ts-trace-seq', @@ -15,20 +19,29 @@ const TRACE_ATTRIBUTES = [ 'data-ts-trace-visibility', ] as const; -function stageLine(label: string, stage: { outcome: string; confidence: string }): string { - return `${label}: ${stage.outcome} · ${stage.confidence}`; +const EMPTY_STAGES: Record = { + trustedServer: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + prebid: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + gam: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + creative: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, +}; + +function stagesForRender(slot: SlotTraceSnapshot, render: RenderTraceSnapshot) { + return ( + slot.generations.find((generation) => generation.generation === render.generation)?.stages ?? + (slot.latestGeneration === render.generation ? slot.stages : undefined) + ); } -function badgeText(slot: SlotTraceSnapshot, render?: RenderTraceSnapshot): string { - return [ - render ? `#${render.sequence}: ${render.outcome} · ${render.visibility}` : undefined, - stageLine('TS winner', slot.stages.trustedServer), - stageLine('Prebid winner', slot.stages.prebid), - stageLine('GAM result', slot.stages.gam), - stageLine('Creative', slot.stages.creative), - ] - .filter(Boolean) - .join('\n'); +function latestRenderForSlot( + renders: readonly RenderTraceSnapshot[], + slot: SlotTraceSnapshot +): RenderTraceSnapshot | undefined { + for (let index = renders.length - 1; index >= 0; index -= 1) { + const render = renders[index]; + if (render.slotId === slot.slotId && render.generation === slot.latestGeneration) return render; + } + return undefined; } function removeTraceAttributes(element: HTMLElement): void { @@ -75,7 +88,10 @@ export function installAdTraceOverlay( .badge { position: fixed; z-index: 2147483647; max-width: 300px; padding: 6px 8px; border: 1px solid #72e0a6; border-radius: 4px; background: rgba(10,18,16,.94); color: #eefbf4; font: 11px/1.35 ui-monospace, monospace; white-space: pre; cursor: pointer; } - .badge.probable { border-color: #67a8ff; } + .badge.attributed { border-color: #72e0a6; } + .badge.unattributed { border-color: #67a8ff; } + .badge.empty { border-color: #ffd479; } + .badge.failed { border-color: #ff7b72; } .panel { position: fixed; right: 12px; bottom: 12px; z-index: 2147483647; width: 460px; max-height: 60vh; overflow: auto; padding: 10px; background: #0a1210; color: #eefbf4; border: 1px solid #72e0a6; font: 11px/1.4 ui-monospace, monospace; } @@ -136,31 +152,30 @@ export function installAdTraceOverlay( rows.replaceChildren(); const exported = api.export(); const slotById = new Map(exported.slots.map((slot) => [slot.slotId, slot])); - const latestBySlot = new Map(); - for (const item of exported.renders) latestBySlot.set(item.slotId, item); const nextObserved = new Set(); for (const item of [...exported.renders].reverse()) { + const slot = slotById.get(item.slotId); + const stages = slot && stagesForRender(slot, item); + // Render history outlives bounded generation-stage retention. Its own + // factual render outcome remains safe to show when the stages are gone. + const presentation = presentTraceOverlay(stages ?? EMPTY_STAGES, item); const row = document.createElement('div'); - row.className = 'row'; + row.className = `row ${presentation.className}`; const title = document.createElement('strong'); - title.textContent = `#${item.sequence} ${item.slotId} · ${item.source}`; + title.textContent = item.slotId; const summary = document.createElement('div'); - summary.textContent = `${item.outcome} · ${item.confidence} · ${item.visibility}`; + summary.textContent = presentation.primaryStatus ?? 'No trace result observed'; row.append(title, summary); row.addEventListener('click', () => { details.hidden = false; - details.textContent = JSON.stringify( - { render: item, stages: slotById.get(item.slotId)?.stages }, - null, - 2 - ); + details.textContent = JSON.stringify({ render: item, stages }, null, 2); }); rows.appendChild(row); } for (const [slotId, slot] of slotById) { - const item = latestBySlot.get(slotId); + const item = latestRenderForSlot(exported.renders, slot); const element = item ? window.tsjs?.getAdTraceElement?.(slotId, item.generation) : undefined; if (!element || !item) continue; const rect = element.getBoundingClientRect(); @@ -175,21 +190,29 @@ export function installAdTraceOverlay( nextObserved.add(element); if (!observedElements.has(element)) resizeObserver?.observe(element); stampRender(element, effectiveItem); - const badge = document.createElement('div'); - badge.className = `badge ${item.outcome === 'confirmed' ? '' : 'probable'}`; - badge.textContent = badgeText(slot, effectiveItem); - badge.style.left = `${Math.max(0, rect.left)}px`; - badge.style.top = `${Math.max(0, rect.top)}px`; - badge.addEventListener('click', () => { - panel.hidden = false; - details.hidden = false; - details.textContent = JSON.stringify( - { render: effectiveItem, stages: slot.stages }, - null, - 2 - ); - }); - badgeLayer.appendChild(badge); + const presentation = presentTraceOverlay(slot.stages, effectiveItem); + // A visibility calculation alone is not trace evidence. Do not place a + // marker over an ad until the trace has at least one observed fact. + const traceFacts = presentation.facts.filter( + (fact) => !fact.startsWith('Slot element currently ') + ); + if (traceFacts.length > 0) { + const badge = document.createElement('div'); + badge.className = `badge ${presentation.className}`; + badge.textContent = presentation.facts.join('\n'); + badge.style.left = `${Math.max(0, rect.left)}px`; + badge.style.top = `${Math.max(0, rect.top)}px`; + badge.addEventListener('click', () => { + panel.hidden = false; + details.hidden = false; + details.textContent = JSON.stringify( + { render: effectiveItem, stages: slot.stages }, + null, + 2 + ); + }); + badgeLayer.appendChild(badge); + } } for (const element of observedElements) { if (!nextObserved.has(element)) { diff --git a/crates/trusted-server-js/lib/src/integrations/ad_trace/presentation.ts b/crates/trusted-server-js/lib/src/integrations/ad_trace/presentation.ts new file mode 100644 index 000000000..e443e5fa0 --- /dev/null +++ b/crates/trusted-server-js/lib/src/integrations/ad_trace/presentation.ts @@ -0,0 +1,174 @@ +import type { + AdTraceStage, + AdTraceStageName, + RenderTraceSnapshot, + RenderTraceVisibility, +} from '../../core/types'; + +export type TraceOverlayPresentationClass = 'attributed' | 'unattributed' | 'empty' | 'failed'; + +export interface TraceOverlayPresentation { + /** Facts suitable for operator-facing badges and primary timeline rows. */ + facts: readonly string[]; + /** One concise description of the render result when render evidence exists. */ + renderStatus?: string; + /** Best observed fact for a compact primary timeline row. */ + primaryStatus?: string; + className: TraceOverlayPresentationClass; +} + +type TraceStages = Record; + +const STAGE_ORDER: readonly AdTraceStageName[] = ['trustedServer', 'prebid', 'gam', 'creative']; + +function stageFacts(name: AdTraceStageName, stage: AdTraceStage): readonly string[] { + if (stage.outcome === 'not_observed' || stage.outcome === 'not_run') return []; + + switch (name) { + case 'trustedServer': + switch (stage.outcome) { + case 'won': + return ['Trusted Server selected a bid']; + case 'no_bid': + return ['Trusted Server returned no bid']; + case 'skipped': + return ['Trusted Server auction skipped']; + case 'failed': + case 'abandoned': + return ['Trusted Server auction did not complete']; + default: + return []; + } + case 'prebid': + if (stage.outcome === 'won') { + return stage.reason === 'selected_targeting_with_bid_won' + ? ['Prebid selected the Trusted Server bid', 'Prebid reported the bid won'] + : ['Prebid selected the Trusted Server bid']; + } + if (stage.outcome === 'client_bid_won' || stage.outcome === 'lost') { + return stage.reason === 'selected_targeting_with_bid_won' + ? ['Prebid selected a client bid', 'Prebid reported the bid won'] + : ['Prebid selected a client bid']; + } + return []; + case 'gam': + switch (stage.outcome) { + case 'empty': + return ['GAM returned no ad']; + case 'backfill': + return ['GAM returned backfill']; + case 'trusted_server_won': + return ['GAM selected the Trusted Server creative']; + case 'trusted_server_candidate': + case 'client_prebid_candidate': + case 'direct_or_unattributed': + return ['GAM rendered an ad — source not attributed']; + default: + return []; + } + case 'creative': + switch (stage.outcome) { + case 'gpt_iframe_onload': + return ['GAM creative iframe loaded']; + case 'load_acknowledged': + return [ + stage.reason === 'direct_iframe_load' + ? 'Creative iframe load confirmed' + : 'Trusted Server creative load confirmed', + ]; + case 'prebid_render_succeeded': + return ['Prebid reported render succeeded']; + case 'render_failed': + return ['Prebid reported render failed']; + case 'aps_renderer_ready': + return ['APS renderer reported ready']; + case 'renderer_served': + if (stage.reason === 'direct_aps_renderer') { + return ['APS renderer started creative loading']; + } + if (stage.reason === 'aps_renderer') return ['APS renderer response sent']; + return ['Creative response sent to the renderer']; + case 'rejected': + return ['Trusted Server direct render rejected']; + default: + return []; + } + } +} + +function renderStatus(render?: RenderTraceSnapshot): string | undefined { + if (!render) return undefined; + + switch (render.outcome) { + case 'confirmed': + return render.source === 'direct_auction' + ? 'Creative iframe load confirmed' + : 'Trusted Server creative load confirmed'; + case 'served': + if (render.reason === 'direct_aps_renderer_ready') return 'APS renderer reported ready'; + if (render.reason === 'direct_aps_renderer') return 'APS renderer started creative loading'; + if (render.reason === 'aps_renderer') return 'APS renderer response sent'; + if (render.reason === 'direct_iframe_created') return 'Creative iframe created'; + return 'Creative response sent to the renderer'; + case 'gam_only': + return 'GAM rendered an ad — source not attributed'; + case 'empty': + return 'GAM returned no ad'; + case 'unresolved': + return undefined; + } +} + +function visibilityFact(visibility: RenderTraceVisibility | undefined): string | undefined { + if (visibility === 'visible') return 'Slot element currently visible'; + if (visibility === 'hidden') return 'Slot element currently hidden'; + return undefined; +} + +function presentationClass( + stages: TraceStages, + render?: RenderTraceSnapshot +): TraceOverlayPresentationClass { + if ( + stages.creative.outcome === 'load_acknowledged' || + stages.gam.outcome === 'trusted_server_won' || + render?.outcome === 'confirmed' + ) { + return 'attributed'; + } + if (stages.creative.outcome === 'render_failed') return 'failed'; + if (stages.gam.outcome === 'empty' || render?.outcome === 'empty') return 'empty'; + return 'unattributed'; +} + +/** + * Convert internal trace stages into factual operator-facing language. + * + * Raw outcomes, confidence, reasons, sequence IDs, and generation IDs remain + * available in technical details and exports; they are intentionally excluded + * from this presentation surface. + */ +export function presentTraceOverlay( + stages: TraceStages, + render?: RenderTraceSnapshot +): TraceOverlayPresentation { + const facts = new Set(); + for (const name of STAGE_ORDER) { + for (const fact of stageFacts(name, stages[name])) facts.add(fact); + } + const status = renderStatus(render); + if (status) facts.add(status); + const visibility = visibilityFact(render?.visibility); + if (visibility) facts.add(visibility); + if (render?.viewability === 'viewable') facts.add('Viewable impression observed'); + const factList = [...facts]; + const primaryStatus = + status ?? [...factList].reverse().find((fact) => !fact.startsWith('Slot element currently ')); + + return { + facts: factList, + renderStatus: status, + ...(primaryStatus ? { primaryStatus } : {}), + className: presentationClass(stages, render), + }; +} diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 63aca2833..dbe55d233 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -1,3 +1,4 @@ +import { terminalSummaryStageOutcome } from '../../core/ad_trace'; import { log } from '../../core/log'; import type { AuctionSlot, AuctionBidData, AuctionTraceSummary, TsjsApi } from '../../core/types'; @@ -886,7 +887,7 @@ export function captureAdTraceRequest( slotId, generation, auctionTraceId: serverSummary.auctionTraceId, - outcome: serverSummary.outcome === 'completed' ? 'no_bid' : serverSummary.outcome, + outcome: terminalSummaryStageOutcome(serverSummary.outcome), confidence: 'definitive', reason: 'terminal_summary', }); @@ -963,7 +964,12 @@ function candidateForSlot( candidate.slot === slot && !candidate.superseded && (includeTerminal || !candidate.terminal) && - monotonicNow() - candidate.createdAt <= 30_000 + // Terminal evidence such as iframe load and viewability can arrive well + // after the 30-second render-request window. Retain the exact terminal + // candidate until a replacement, navigation, or bounded eviction supersedes it. + (includeTerminal && candidate.terminal + ? true + : monotonicNow() - candidate.createdAt <= 30_000) ); if (candidates.length !== 1) { if (candidates.length > 1) { @@ -998,9 +1004,18 @@ function installGptEvidenceListeners(service: GoogleTagPubAdsService): void { if (instrumented.__tsAdTraceListeners) return; instrumented.__tsAdTraceListeners = true; const record = - (kind: 'gpt_slot_requested' | 'gpt_slot_response_received' | 'gpt_slot_onload') => + ( + kind: + | 'gpt_slot_requested' + | 'gpt_slot_response_received' + | 'gpt_slot_onload' + | 'gpt_impression_viewable' + ) => (event: GptSlotEvent): void => { - const candidate = candidateForSlot(event.slot, kind === 'gpt_slot_onload'); + const candidate = candidateForSlot( + event.slot, + kind === 'gpt_slot_onload' || kind === 'gpt_impression_viewable' + ); if (!candidate) return; window.tsjs?.recordAdTrace?.({ kind, @@ -1012,6 +1027,7 @@ function installGptEvidenceListeners(service: GoogleTagPubAdsService): void { service.addEventListener('slotRequested', record('gpt_slot_requested')); service.addEventListener('slotResponseReceived', record('gpt_slot_response_received')); service.addEventListener('slotOnload', record('gpt_slot_onload')); + service.addEventListener('impressionViewable', record('gpt_impression_viewable')); service.addEventListener('slotRenderEnded', (event: GptSlotEvent) => { const candidate = candidateForSlot(event.slot); if (!candidate) return; @@ -1060,12 +1076,7 @@ export function installTsAdInit(): void { kind: 'ts_auction_observed', slotId: slot.id, auctionTraceId: summary.auctionTraceId, - outcome: - summary.outcome === 'completed' || summary.outcome === 'no_bid' - ? 'no_bid' - : summary.outcome === 'skipped' - ? 'skipped' - : 'unresolved', + outcome: terminalSummaryStageOutcome(summary.outcome), confidence: 'definitive', reason: 'terminal_summary', }); diff --git a/crates/trusted-server-js/lib/test/core/ad_trace.test.ts b/crates/trusted-server-js/lib/test/core/ad_trace.test.ts index 0348e8dd7..2a16a58e7 100644 --- a/crates/trusted-server-js/lib/test/core/ad_trace.test.ts +++ b/crates/trusted-server-js/lib/test/core/ad_trace.test.ts @@ -5,6 +5,7 @@ import { AD_TRACE_MAX_RENDERS, AD_TRACE_MAX_SLOTS, createAdTraceStore, + terminalSummaryStageOutcome, } from '../../src/core/ad_trace'; const BID_TRACE_ID = '550e8400-e29b-41d4-a716-446655440000'; @@ -153,6 +154,50 @@ describe('ad trace reducer', () => { ).toBe('no_bid'); }); + it('retains a Trusted Server Prebid selection when bidWon arrives without claiming creative load', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'prebid_targeting_selected', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + outcome: 'won', + confidence: 'definitive', + reason: 'selected_targeting', + }); + store.record({ + kind: 'prebid_bid_won', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + }); + + expect(store.getSlot('slot-a')?.stages.prebid).toMatchObject({ + outcome: 'won', + reason: 'selected_targeting_with_bid_won', + }); + expect(store.getSlot('slot-a')?.stages.creative.outcome).toBe('not_observed'); + }); + + it('preserves the direct iframe acknowledgement boundary without claiming GAM selection', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'creative_load_acknowledged', + slotId: 'slot-a', + generation, + bidTraceId: BID_TRACE_ID, + reason: 'direct_iframe_load', + }); + + expect(store.getSlot('slot-a')?.stages.creative).toMatchObject({ + outcome: 'load_acknowledged', + reason: 'direct_iframe_load', + }); + expect(store.getSlot('slot-a')?.stages.gam.outcome).toBe('not_observed'); + }); + it('classifies overlap, client Prebid, APS, no-bid, and superseded states', () => { const store = createAdTraceStore(() => 1); const generation = store.nextGeneration('slot-a'); @@ -272,6 +317,60 @@ describe('ad trace reducer', () => { }); }); + it('keeps GPT viewability separate from element visibility and creative load', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'gpt_slot_render_ended', + slotId: 'slot-a', + generation, + isEmpty: false, + }); + store.updateVisibility('slot-a', generation, 'hidden'); + store.record({ kind: 'gpt_impression_viewable', slotId: 'slot-a', generation }); + + expect(store.getRenderTimeline()[0]).toMatchObject({ + outcome: 'gam_only', + visibility: 'hidden', + viewability: 'viewable', + }); + expect(store.getSlot('slot-a')?.stages.creative.outcome).toBe('not_observed'); + }); + + it('distinguishes APS renderer start from the validated ready boundary', () => { + const store = createAdTraceStore(() => 1); + const generation = store.nextGeneration('slot-a'); + store.record({ + kind: 'pb_render_served', + slotId: 'slot-a', + generation, + reason: 'direct_aps_renderer', + }); + expect(store.getSlot('slot-a')?.stages.creative).toMatchObject({ + outcome: 'renderer_served', + reason: 'direct_aps_renderer', + }); + expect(store.getRenderTimeline()[0]).toMatchObject({ + outcome: 'served', + reason: 'direct_aps_renderer', + }); + + store.record({ + kind: 'aps_renderer_ready', + slotId: 'slot-a', + generation, + reason: 'direct_aps_renderer_ready', + }); + expect(store.getSlot('slot-a')?.stages.creative).toMatchObject({ + outcome: 'aps_renderer_ready', + reason: 'direct_aps_renderer_ready', + }); + expect(store.getRenderTimeline()[0]).toMatchObject({ + outcome: 'served', + reason: 'direct_aps_renderer_ready', + }); + }); + it('dispatches a frozen privacy-safe render event', () => { const store = createAdTraceStore(() => 1); const observed: unknown[] = []; @@ -303,6 +402,14 @@ describe('ad trace reducer', () => { expect(store.getRenderTimeline()[0].slotId).toBe('render-1'); }); + it('preserves failed and abandoned terminal summaries while mapping completed no-winner to no bid', () => { + expect(terminalSummaryStageOutcome('completed')).toBe('no_bid'); + expect(terminalSummaryStageOutcome('completed', true)).toBe('completed'); + expect(terminalSummaryStageOutcome('failed')).toBe('failed'); + expect(terminalSummaryStageOutcome('abandoned')).toBe('abandoned'); + expect(terminalSummaryStageOutcome('skipped')).toBe('skipped'); + }); + it('rejects malformed runtime event kinds and confidence values', () => { const store = createAdTraceStore(() => 1); store.record({ kind: 'not-a-real-kind', slotId: 'slot-a' } as never); diff --git a/crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts b/crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts index a3eaa7d6f..f24a513b7 100644 --- a/crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/ad_trace/index.test.ts @@ -33,6 +33,33 @@ describe('ad_trace integration gate', () => { expect(document.querySelectorAll('#ts-ad-trace-overlay')).toHaveLength(1); }); + it.each(['failed', 'abandoned'] as const)( + 'preserves a %s terminal summary when seeding a slot', + async (outcome) => { + window.__tsjs_adTraceActive = true; + window.tsjs = { + adSlots: [{ id: 'slot-a' }], + auctionTrace: { + version: 1, + auctionTraceId: '550e8400-e29b-41d4-a716-446655440000', + source: 'initial_navigation', + outcome, + }, + } as any; + + const { installAdTrace } = await import('../../../src/integrations/ad_trace/index'); + expect(installAdTrace()).toBe(true); + const generation = window.tsjs?.nextAdTraceGeneration?.('slot-a'); + expect( + window.tsjs?.adTrace?.getSlot('slot-a')?.generations[0]?.stages.trustedServer + ).toMatchObject({ + outcome, + reason: 'terminal_summary', + }); + expect(generation).toBeGreaterThan(0); + } + ); + it('does not accept the legacy tester cookie without bootstrap', async () => { document.cookie = 'ts-tester=true; Path=/'; const { installAdTrace } = await import('../../../src/integrations/ad_trace/index'); diff --git a/crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts b/crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts index 0a5aaa217..6e0770d21 100644 --- a/crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/ad_trace/overlay.test.ts @@ -67,6 +67,15 @@ describe('ad trace overlay lifecycle', () => { } as any; const observe = vi.fn(); + const attachShadow = HTMLElement.prototype.attachShadow; + let shadow: ShadowRoot | undefined; + vi.spyOn(HTMLElement.prototype, 'attachShadow').mockImplementation(function ( + this: HTMLElement, + init: ShadowRootInit + ) { + shadow = attachShadow.call(this, init); + return shadow; + }); vi.stubGlobal( 'ResizeObserver', class { @@ -89,6 +98,15 @@ describe('ad trace overlay lifecycle', () => { expect(rect).toHaveBeenCalledTimes(1); expect(observe).toHaveBeenCalledWith(element); expect(updateVisibility).toHaveBeenCalledWith('slot-a', 1, 'visible'); + const badge = shadow?.querySelector('.badge'); + const row = shadow?.querySelector('.row'); + expect(badge?.textContent).toBe( + 'Trusted Server selected a bid\nGAM rendered an ad — source not attributed\nSlot element currently visible' + ); + expect(row?.textContent).toContain('GAM rendered an ad — source not attributed'); + expect(`${badge?.textContent}\n${row?.textContent}`).not.toMatch( + /definitive|strong|probable|not_run|gam_only|TS winner|Prebid winner|#1/ + ); expect(element.getAttribute('data-ts-trace-seq')).toBe('1'); expect(element.getAttribute('data-ts-trace-outcome')).toBe('gam_only'); window.dispatchEvent(new Event('scroll')); @@ -107,4 +125,202 @@ describe('ad trace overlay lifecycle', () => { expect(element.hasAttribute('data-ts-trace-seq')).toBe(false); expect(updateVisibility).toHaveBeenCalledWith('slot-a', 1, 'disconnected'); }); + + it('does not add an empty badge when no operator-facing fact was observed', () => { + const element = document.createElement('div'); + document.body.appendChild(element); + window.tsjs = { + getAdTraceElement: () => element, + updateAdTraceVisibility: vi.fn(), + } as any; + const attachShadow = HTMLElement.prototype.attachShadow; + let shadow: ShadowRoot | undefined; + vi.spyOn(HTMLElement.prototype, 'attachShadow').mockImplementation(function ( + this: HTMLElement, + init: ShadowRootInit + ) { + shadow = attachShadow.call(this, init); + return shadow; + }); + vi.stubGlobal( + 'ResizeObserver', + class { + observe() {} + unobserve() {} + disconnect() {} + } + ); + vi.spyOn(element, 'getBoundingClientRect').mockReturnValue({ + left: 0, + top: 0, + width: 300, + height: 250, + } as DOMRect); + const slot = { + slotId: 'slot-a', + latestGeneration: 1, + generations: [], + stages: { + trustedServer: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + prebid: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + gam: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + creative: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + }, + }; + const render = { + sequence: 1, + slotId: 'slot-a', + generation: 1, + source: 'gpt', + outcome: 'unresolved', + confidence: 'none', + visibility: 'unknown', + createdAt: 1, + updatedAt: 1, + }; + installAdTraceOverlay( + { + getSlot: () => slot as any, + getEvents: () => [], + getRenderTimeline: () => [render] as any, + export: () => + ({ + version: 1, + slots: [slot], + events: [], + renders: [render], + metadata: { droppedEvents: 0, evictedSlots: 0 }, + }) as any, + }, + () => vi.fn() + ); + + expect(shadow?.querySelector('.badge')).toBeNull(); + expect(shadow?.querySelector('.row')?.textContent).toContain('No trace result observed'); + }); + + it('uses observed stage evidence when a render row has no render outcome', () => { + const attachShadow = HTMLElement.prototype.attachShadow; + let shadow: ShadowRoot | undefined; + vi.spyOn(HTMLElement.prototype, 'attachShadow').mockImplementation(function ( + this: HTMLElement, + init: ShadowRootInit + ) { + shadow = attachShadow.call(this, init); + return shadow; + }); + vi.stubGlobal( + 'ResizeObserver', + class { + observe() {} + unobserve() {} + disconnect() {} + } + ); + const stages = { + trustedServer: { outcome: 'won', confidence: 'definitive', reason: 'winner' }, + prebid: { outcome: 'won', confidence: 'definitive', reason: 'selected_targeting' }, + gam: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + creative: { outcome: 'render_failed', confidence: 'definitive', reason: 'failure' }, + }; + const slot = { + slotId: 'slot-a', + latestGeneration: 1, + generations: [{ generation: 1, stages }], + stages, + }; + const render = { + sequence: 1, + slotId: 'slot-a', + generation: 1, + source: 'gpt', + outcome: 'unresolved', + confidence: 'none', + visibility: 'unknown', + createdAt: 1, + updatedAt: 1, + }; + installAdTraceOverlay( + { + getSlot: () => slot as any, + getEvents: () => [], + getRenderTimeline: () => [render] as any, + export: () => + ({ + version: 1, + slots: [slot], + events: [], + renders: [render], + metadata: { droppedEvents: 0, evictedSlots: 0 }, + }) as any, + }, + () => vi.fn() + ); + + const row = shadow?.querySelector('.row'); + expect(row?.textContent).toContain('Prebid reported render failed'); + expect(row?.textContent).not.toContain('No trace result observed'); + }); + + it('gives a retained render a factual status after its generation stages were evicted', () => { + const attachShadow = HTMLElement.prototype.attachShadow; + let shadow: ShadowRoot | undefined; + vi.spyOn(HTMLElement.prototype, 'attachShadow').mockImplementation(function ( + this: HTMLElement, + init: ShadowRootInit + ) { + shadow = attachShadow.call(this, init); + return shadow; + }); + vi.stubGlobal( + 'ResizeObserver', + class { + observe() {} + unobserve() {} + disconnect() {} + } + ); + const slot = { + slotId: 'slot-a', + latestGeneration: 2, + generations: [], + stages: { + trustedServer: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + prebid: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + gam: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + creative: { outcome: 'not_observed', confidence: 'none', reason: 'none' }, + }, + }; + const render = { + sequence: 1, + slotId: 'slot-a', + generation: 1, + source: 'gpt', + outcome: 'gam_only', + confidence: 'probable', + visibility: 'unknown', + createdAt: 1, + updatedAt: 1, + }; + installAdTraceOverlay( + { + getSlot: () => slot as any, + getEvents: () => [], + getRenderTimeline: () => [render] as any, + export: () => + ({ + version: 1, + slots: [slot], + events: [], + renders: [render], + metadata: { droppedEvents: 0, evictedSlots: 0 }, + }) as any, + }, + () => vi.fn() + ); + + const row = shadow?.querySelector('.row'); + expect(row?.textContent).toContain('GAM rendered an ad — source not attributed'); + expect(row?.textContent).not.toMatch(/probable|gam_only|#1/); + }); }); diff --git a/crates/trusted-server-js/lib/test/integrations/ad_trace/presentation.test.ts b/crates/trusted-server-js/lib/test/integrations/ad_trace/presentation.test.ts new file mode 100644 index 000000000..d8008d1cb --- /dev/null +++ b/crates/trusted-server-js/lib/test/integrations/ad_trace/presentation.test.ts @@ -0,0 +1,231 @@ +import { describe, expect, it } from 'vitest'; + +import { presentTraceOverlay } from '../../../src/integrations/ad_trace/presentation'; +import type { AdTraceStage, AdTraceStageName, RenderTraceSnapshot } from '../../../src/core/types'; + +function stage(outcome = 'not_observed', reason = 'none'): AdTraceStage { + return { outcome, confidence: 'none', reason }; +} + +function stages(overrides: Partial> = {}) { + return { + trustedServer: stage(), + prebid: stage(), + gam: stage(), + creative: stage(), + ...overrides, + }; +} + +function render( + outcome: RenderTraceSnapshot['outcome'], + overrides: Partial = {} +): RenderTraceSnapshot { + return { + sequence: 4, + slotId: 'slot-a', + generation: 2, + source: 'gpt', + outcome, + confidence: 'probable', + visibility: 'unknown', + createdAt: 1, + updatedAt: 1, + ...overrides, + }; +} + +describe('presentTraceOverlay', () => { + it.each([ + [ + 'server winner', + stages({ trustedServer: stage('won') }), + undefined, + ['Trusted Server selected a bid'], + ], + [ + 'server no bid', + stages({ trustedServer: stage('no_bid') }), + undefined, + ['Trusted Server returned no bid'], + ], + [ + 'server skip', + stages({ trustedServer: stage('skipped') }), + undefined, + ['Trusted Server auction skipped'], + ], + [ + 'server failure', + stages({ trustedServer: stage('failed') }), + undefined, + ['Trusted Server auction did not complete'], + ], + [ + 'server abandonment', + stages({ trustedServer: stage('abandoned') }), + undefined, + ['Trusted Server auction did not complete'], + ], + [ + 'traced Prebid selection', + stages({ prebid: stage('won') }), + undefined, + ['Prebid selected the Trusted Server bid'], + ], + [ + 'client Prebid selection', + stages({ prebid: stage('client_bid_won') }), + undefined, + ['Prebid selected a client bid'], + ], + [ + 'client Prebid selection recorded as lost server targeting', + stages({ prebid: stage('lost') }), + undefined, + ['Prebid selected a client bid'], + ], + [ + 'reported Prebid win', + stages({ prebid: stage('won', 'selected_targeting_with_bid_won') }), + undefined, + ['Prebid selected the Trusted Server bid', 'Prebid reported the bid won'], + ], + ['GAM empty', stages({ gam: stage('empty') }), undefined, ['GAM returned no ad']], + ['GAM backfill', stages({ gam: stage('backfill') }), undefined, ['GAM returned backfill']], + [ + 'unattributed GAM render', + stages({ gam: stage('direct_or_unattributed') }), + undefined, + ['GAM rendered an ad — source not attributed'], + ], + [ + 'selected Trusted Server GAM creative', + stages({ gam: stage('trusted_server_won') }), + undefined, + ['GAM selected the Trusted Server creative'], + ], + [ + 'GAM iframe load', + stages({ creative: stage('gpt_iframe_onload') }), + undefined, + ['GAM creative iframe loaded'], + ], + [ + 'creative acknowledgement', + stages({ creative: stage('load_acknowledged') }), + undefined, + ['Trusted Server creative load confirmed'], + ], + [ + 'direct iframe acknowledgement', + stages({ creative: stage('load_acknowledged', 'direct_iframe_load') }), + render('confirmed', { source: 'direct_auction' }), + ['Creative iframe load confirmed'], + ], + [ + 'Prebid render success', + stages({ creative: stage('prebid_render_succeeded') }), + undefined, + ['Prebid reported render succeeded'], + ], + [ + 'Prebid render failure', + stages({ creative: stage('render_failed') }), + undefined, + ['Prebid reported render failed'], + ], + [ + 'direct APS renderer started', + stages({ creative: stage('renderer_served', 'direct_aps_renderer') }), + render('served', { source: 'direct_auction', reason: 'direct_aps_renderer' }), + ['APS renderer started creative loading'], + ], + [ + 'direct APS renderer ready', + stages({ creative: stage('aps_renderer_ready', 'direct_aps_renderer_ready') }), + render('served', { source: 'direct_auction', reason: 'direct_aps_renderer_ready' }), + ['APS renderer reported ready'], + ], + [ + 'direct render rejection', + stages({ creative: stage('rejected') }), + undefined, + ['Trusted Server direct render rejected'], + ], + [ + 'current visibility', + stages(), + render('unresolved', { visibility: 'visible' }), + ['Slot element currently visible'], + ], + [ + 'viewable impression independent of live visibility', + stages({ creative: stage('gpt_iframe_onload') }), + render('gam_only', { visibility: 'hidden', viewability: 'viewable' }), + [ + 'GAM creative iframe loaded', + 'GAM rendered an ad — source not attributed', + 'Slot element currently hidden', + 'Viewable impression observed', + ], + ], + ])('%s uses factual operator language', (_name, input, snapshot, expected) => { + expect(presentTraceOverlay(input, snapshot).facts).toEqual(expected); + }); + + it('hides unobserved and inapplicable stages without leaking internal vocabulary', () => { + const presentation = presentTraceOverlay( + stages({ + trustedServer: stage('unresolved'), + prebid: stage('not_run', 'direct'), + gam: stage('not_observed'), + creative: stage('not_observed'), + }), + render('unresolved', { visibility: 'unknown' }) + ); + + expect(presentation.facts).toEqual([]); + expect(presentation.renderStatus).toBeUndefined(); + expect(JSON.stringify(presentation)).not.toMatch( + /definitive|strong|probable|not_run|not_observed|unresolved|gam_only|client_bid_won/ + ); + }); + + it.each([ + ['attributed', stages({ creative: stage('load_acknowledged') }), undefined], + ['empty', stages({ gam: stage('empty') }), undefined], + ['failed', stages({ creative: stage('render_failed') }), undefined], + ['unattributed', stages({ gam: stage('trusted_server_candidate') }), render('gam_only')], + ] as const)('uses an evidence-based %s presentation class', (expected, input, snapshot) => { + expect(presentTraceOverlay(input, snapshot).className).toBe(expected); + }); + + it.each([ + [ + 'confirmed Trusted Server creative', + render('confirmed'), + 'Trusted Server creative load confirmed', + ], + [ + 'confirmed direct creative', + render('confirmed', { source: 'direct_auction' }), + 'Creative iframe load confirmed', + ], + ['served renderer', render('served'), 'Creative response sent to the renderer'], + ['unattributed GAM render', render('gam_only'), 'GAM rendered an ad — source not attributed'], + ['empty GAM response', render('empty'), 'GAM returned no ad'], + ] as const)('renders %s as a concise factual row status', (_name, snapshot, expected) => { + expect(presentTraceOverlay(stages(), snapshot).renderStatus).toBe(expected); + }); + + it('falls back to the strongest observed stage fact for a primary row', () => { + const presentation = presentTraceOverlay( + stages({ creative: stage('render_failed') }), + render('unresolved') + ); + + expect(presentation.renderStatus).toBeUndefined(); + expect(presentation.primaryStatus).toBe('Prebid reported render failed'); + }); +}); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 24f900123..ec3d7c681 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -138,6 +138,127 @@ describe('installTsAdInit', () => { fetchSpy.mockRestore(); }); + it.each(['failed', 'abandoned'] as const)( + 'preserves a %s terminal summary when adInit has no traced bid', + async (outcome) => { + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([mockSlot]), + addEventListener: vi.fn(), + refresh: vi.fn(), + }; + const recordAdTrace = vi.fn(); + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + auctionTrace: { + version: 1, + auctionTraceId: '550e8400-e29b-41d4-a716-446655440000', + source: 'initial_navigation', + outcome, + }, + recordAdTrace, + } as any; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'ts_auction_observed', + outcome, + reason: 'terminal_summary', + }) + ); + } + ); + + it('records late GPT viewability on the exact terminal request generation', async () => { + let now = 0; + vi.spyOn(performance, 'now').mockImplementation(() => now); + const listeners: Record void>> = {}; + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + clearTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn((key: string) => { + if (key === 'hb_adid') return ['client-ad-id']; + if (key === 'hb_bidder') return ['client-bidder']; + return []; + }), + }; + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([mockSlot]), + addEventListener: vi.fn((event: string, fn: (value: SlotRenderEvent) => void) => { + (listeners[event] ??= []).push(fn); + }), + refresh: vi.fn(), + }; + const recordAdTrace = vi.fn(); + (window as TestWindow).googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + recordAdTrace, + nextAdTraceGeneration: vi.fn().mockReturnValue(1), + } as any; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + now = 1; + listeners.slotRenderEnded?.forEach((listener) => listener({ isEmpty: false, slot: mockSlot })); + now = 60_001; + listeners.impressionViewable?.forEach((listener) => + listener({ isEmpty: false, slot: mockSlot }) + ); + + expect(recordAdTrace).toHaveBeenCalledWith( + expect.objectContaining({ + kind: 'gpt_impression_viewable', + slotId: 'atf_sidebar_ad', + generation: 1, + }) + ); + }); + it('displays TS-defined slots and does not include them in refresh', async () => { const mockSlot = { addService: vi.fn().mockReturnThis(), From 5094f6fda724fd6e5ee2d8f496a2d09d0b663748 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 23 Jul 2026 17:06:45 -0500 Subject: [PATCH 109/198] Trace publisher-owned GPT slot lifecycles --- .../tests/ad-trace/auction-trace.spec.ts | 266 +++++++++ .../frameworks/ad-trace/public/index.php | 77 ++- .../frameworks/ad-trace/public/router.php | 2 +- .../lib/src/core/ad_trace.ts | 4 +- .../src/integrations/ad_trace/presentation.ts | 6 +- .../lib/src/integrations/gpt/index.ts | 551 +++++++++++++++--- .../lib/test/core/ad_trace.test.ts | 45 ++ .../ad_trace/presentation.test.ts | 20 + .../lib/test/integrations/gpt/ad_init.test.ts | 438 ++++++++++++++ 9 files changed, 1306 insertions(+), 103 deletions(-) diff --git a/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts b/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts index 075c30b57..56db80bc9 100644 --- a/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts +++ b/crates/trusted-server-integration-tests/browser/tests/ad-trace/auction-trace.spec.ts @@ -110,6 +110,75 @@ test.describe("tester-only auction trace contract", () => { ).toBe("undefined"); }); + test("publisher-only pages install universal GPT tracing without adInit", async ({ + page, + }) => { + await serveBuiltPrebid(page); + await page.goto(runtimeUrl("/publisher-only?ts_console=1"), { + waitUntil: "domcontentloaded", + }); + await expect(page).toHaveURL(runtimeUrl("/publisher-only")); + await expect + .poll(() => + page.evaluate( + () => + typeof ( + window as Window & { + tsjs?: { adTrace?: unknown }; + } + ).tsjs?.adTrace, + ), + ) + .toBe("object"); + + await page.evaluate(() => { + ( + window as Window & { + adTraceFixture: { + requestPublisherLazy(flags: { + isBackfill?: boolean; + }): void; + }; + } + ).adTraceFixture.requestPublisherLazy({ isBackfill: true }); + }); + await expect + .poll(() => + page.evaluate(() => { + const slots = ( + window as Window & { + tsjs: { + adTrace: { + export(): { + slots: Array<{ + slotId: string; + stages: { + trustedServer: { + outcome: string; + }; + gam: { outcome: string }; + }; + }>; + }; + }; + }; + } + ).tsjs.adTrace.export().slots; + const slot = slots.find((item) => + item.slotId.startsWith("gpt_slot_"), + ); + return slot + ? { + trustedServer: + slot.stages.trustedServer.outcome, + gam: slot.stages.gam.outcome, + } + : undefined; + }), + ) + .toEqual({ trustedServer: "not_observed", gam: "backfill" }); + }); + test("console session supports true, persists privately, and can be disabled", async ({ page, }) => { @@ -222,6 +291,203 @@ test.describe("tester-only auction trace contract", () => { ); }); + test("publisher-owned lazy GPT slots receive factual overlays without TS ownership", async ({ + page, + }) => { + await openTesterPage(page); + await page.locator("#publisher-lazy-slot").scrollIntoViewIfNeeded(); + await page.evaluate(() => { + const fixture = ( + window as Window & { + adTraceFixture: { + requestPublisherLazy(flags: { + isBackfill?: boolean; + isEmpty?: boolean; + }): void; + }; + } + ).adTraceFixture; + fixture.requestPublisherLazy({ isBackfill: true }); + }); + + await expect + .poll(() => + page.evaluate(() => { + const result = ( + window as Window & { + tsjs: { + adTrace: { + export(): { + slots: Array<{ + slotId: string; + latestGeneration: number; + stages: Record< + string, + { outcome: string } + >; + }>; + }; + }; + }; + } + ).tsjs.adTrace.export(); + const slot = result.slots.find((item) => + item.slotId.startsWith("gpt_slot_"), + ); + return slot + ? { + slotId: slot.slotId, + generation: slot.latestGeneration, + trustedServer: + slot.stages.trustedServer.outcome, + prebid: slot.stages.prebid.outcome, + gam: slot.stages.gam.outcome, + creative: slot.stages.creative.outcome, + } + : undefined; + }), + ) + .toEqual( + expect.objectContaining({ + slotId: expect.stringMatching(/^gpt_slot_\d+$/), + trustedServer: "not_observed", + prebid: "not_observed", + gam: "backfill", + creative: "gpt_iframe_onload", + }), + ); + + await page.evaluate(() => { + ( + window as Window & { + adTraceFixture: { markPublisherLazyViewable(): void }; + } + ).adTraceFixture.markPublisherLazyViewable(); + }); + await expect + .poll(() => + page.evaluate(() => { + const result = ( + window as Window & { + tsjs: { + adTrace: { + export(): { + renders: Array<{ + slotId: string; + viewability?: string; + }>; + }; + }; + }; + } + ).tsjs.adTrace.export(); + return result.renders.find((item) => + item.slotId.startsWith("gpt_slot_"), + )?.viewability; + }), + ) + .toBe("viewable"); + + await expect(page.locator("#publisher-lazy-slot")).toHaveAttribute( + "data-ts-trace-outcome", + "gam_only", + ); + const genericExport = await page.evaluate(() => + JSON.stringify( + ( + window as Window & { + tsjs: { adTrace: { export(): unknown } }; + } + ).tsjs.adTrace.export(), + ), + ); + expect(genericExport).not.toContain("publisher-lazy-slot"); + expect(genericExport).not.toContain("/123456789/example/publisher-lazy"); + const session = await page.context().newCDPSession(page); + await expect + .poll(async () => { + const tree = (await session.send( + "Accessibility.getFullAXTree", + )) as { + nodes: Array<{ name?: { value?: string } }>; + }; + return tree.nodes + .map((node) => node.name?.value || "") + .join("\n"); + }) + .toContain("GAM returned backfill"); + + const firstGeneration = await page.evaluate(() => { + const result = ( + window as Window & { + tsjs: { + adTrace: { + export(): { + slots: Array<{ + slotId: string; + latestGeneration: number; + }>; + }; + }; + }; + } + ).tsjs.adTrace.export(); + return result.slots.find((item) => + item.slotId.startsWith("gpt_slot_"), + )?.latestGeneration; + }); + await page.evaluate(() => { + ( + window as Window & { + adTraceFixture: { + requestPublisherLazy(flags: { + isEmpty?: boolean; + }): void; + }; + } + ).adTraceFixture.requestPublisherLazy({ isEmpty: true }); + }); + await expect + .poll(() => + page.evaluate((previousGeneration) => { + const result = ( + window as Window & { + tsjs: { + adTrace: { + export(): { + slots: Array<{ + slotId: string; + latestGeneration: number; + stages: Record< + string, + { outcome: string } + >; + }>; + }; + }; + }; + } + ).tsjs.adTrace.export(); + const slot = result.slots.find((item) => + item.slotId.startsWith("gpt_slot_"), + ); + return slot + ? { + generationAdvanced: + slot.latestGeneration > previousGeneration, + gam: slot.stages.gam.outcome, + creative: slot.stages.creative.outcome, + } + : undefined; + }, firstGeneration ?? 0), + ) + .toEqual({ + generationAdvanced: true, + gam: "empty", + creative: "not_observed", + }); + }); + test("direct auction API render reaches an exact iframe-load acknowledgement", async ({ page, }) => { diff --git a/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/index.php b/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/index.php index e7bfca062..89e2f0076 100644 --- a/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/index.php +++ b/crates/trusted-server-integration-tests/fixtures/frameworks/ad-trace/public/index.php @@ -6,8 +6,9 @@ Trusted Server ad trace fixture +``` + +For `[]`, omit the property with `skip_serializing_if`, matching the existing +`clientSideBidders` convention. A browser that receives no property treats it as an +empty list. This makes upgrade and rollback backwards compatible: old configuration +has no behavior change; an older external bundle safely ignores the extra injected +property; and a newer bundle with old configuration has no exclusions. + +## 5. Matching and refresh behavior + +### 5.1 Match predicate + +Extend `RefreshGptSlot` with: + +```ts +getAdUnitPath?: () => string; +``` + +At each publisher refresh, derive a `Set` from +`getInjectedConfig()?.excludedGamAdUnitPathSuffixes ?? []`. A slot is excluded only +when all of the following hold: + +1. The normalized set is non-empty. +2. `slot.getAdUnitPath` is a function. +3. Calling it returns a string. +4. The returned GAM path `endsWith()` at least one configured suffix, using exact, + case-sensitive JavaScript string comparison. + +Do not derive paths from the element ID or injected `adSlots` metadata. Do not use +`getSizes()` as a fallback. A missing getter, a non-string return value, an empty +path, or a getter that throws is **fail-open**: the slot remains auction-eligible. +The implementation catches only the getter failure around that call; it neither +suppresses the GPT refresh nor broadens an exclusion because telemetry is absent. + +A matching path is excluded only from the synthetic refresh auction. It is not +removed from GPT's target list. + +### 5.2 Required algorithm + +Keep the existing `adInitRefreshInProgress` check as the first branch, before slot +resolution, targeting cleanup, and path inspection: + +```text +if adInitRefreshInProgress: + originalRefresh(slots, options) + return + +targetSlots = explicit slots, or pubads.getSlots() for bare refresh +if targetSlots is empty: + originalRefresh(slots, options) + return + +clear TS/Prebid refresh-targeting keys from every target slot +auctionSlots = targetSlots excluding suffix-matched slots + +if auctionSlots is empty: + originalRefresh(targetSlots, options) + return + +adUnits = synthetic refresh ad units for auctionSlots only +pbjs.requestBids({ adUnits, timeout, bidsBackHandler }) +bidsBackHandler: + pbjs.setTargetingForGPTAsync(auction-slot codes only) + originalRefresh(targetSlots, options) +``` + +Build candidate codes, recover publisher bidder params, and recover client-side bids +only for `auctionSlots`; excluded slots must not be represented in `adUnits` at all. +The existing scoped targeting behavior therefore continues to affect only eligible +slots. + +### 5.3 Refresh sequences + +| Call and slot set | Prebid behavior | GPT behavior | +| -------------------------------------------------------- | ----------------------------------------------------------------------------------- | --------------------------------------------------------------------------------- | +| `refresh([normal], options)` | Auction `normal`, target its synthetic code after bids return. | Refresh `[normal]` with the same options after the callback. | +| `refresh([excluded], options)` | Clear TS/Prebid keys; do not call `requestBids()` or `setTargetingForGPTAsync()`. | Immediately refresh `[excluded]` with the same options. | +| Bare `refresh(options)`; all slots excluded | Resolve `pubads.getSlots()`, clear their TS/Prebid keys, then make no Prebid calls. | Immediately refresh the resolved complete slot list with the same options. | +| Bare `refresh(options)`; mixed normal and excluded slots | Clear every target slot; auction and target only normal slots. | After the callback, refresh the complete resolved list, including excluded slots. | +| Any refresh while `adInitRefreshInProgress` is true | No cleanup, match, auction, or targeting. | Directly pass through the original `slots` and options unchanged. | +| Missing/throwing `getAdUnitPath()` | Treat the slot as normal and auction it. | Existing post-auction refresh behavior. | + +Passing the resolved list in the all-excluded and mixed cases is deliberate: it is +the same concrete list used for cleanup and makes the final GAM refresh list +explicit. The original options object is passed through unchanged. + +### 5.4 Targeting and initial-load invariants + +The cleanup step remains before filtering and is limited to the existing +`TS_REFRESH_TARGETING_KEYS`. It removes stale Trusted Server/Prebid winner data +from excluded slots so GAM cannot serve using an obsolete header-bid winner, while +preserving GAM path metadata and every unrelated publisher targeting key. + +`adInitRefreshInProgress` continues to bypass cleanup and auctioning directly. This +preserves `disableInitialLoad()` and the initial Trusted Server targeting handoff: +that one internal refresh must deliver already-applied targeting to GAM instead of +being converted into a client-side refresh auction. + +## 6. Implementation areas + +| File | Planned change | +| ----------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `crates/trusted-server-core/src/integrations/prebid.rs` | Add config field, validation/canonicalization, head-injected camel-case array, and Rust tests. | +| `crates/trusted-server-js/lib/src/integrations/prebid/index.ts` | Add injected config/type support, guarded path matcher, and filter `targetSlots` into `auctionSlots` after cleanup. | +| `crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts` | Add explicit/global/mixed/fail-open refresh tests. | +| `trusted-server.example.toml` | Add a commented fictional configuration example. | +| `docs/guide/integrations/prebid.md` | Document the field, exact matcher semantics, and GAM-preservation caveat. | +| `docs/superpowers/specs/2026-07-24-prebid-refresh-gam-path-opt-out-design.md` | Update only if implementation exposes a necessary design correction. | +| `docs/superpowers/plans/2026-07-24-prebid-refresh-gam-path-opt-out.md` | Mark implementation evidence/status only if project practice requires it. | + +No generated `dist` file, minified external bundle, or publisher source file is a +source-of-truth edit target. + +## 7. Test matrix + +### Rust configuration and injection + +- Default/omitted field yields `Vec::new()` and omits + `excludedGamAdUnitPathSuffixes` from injected JSON. +- A valid array parses, normalizes exact duplicates to one entry in declaration + order, and injects the expected camel-case array. +- Empty, whitespace-padded, whitespace-only, missing-leading-slash, and `/` values + fail enabled Prebid configuration validation with field-specific errors. +- Existing Prebid config/head-injector tests continue to pass with the new empty + field initialized in helper literals. + +### Browser refresh wrapper + +- Normal explicit slot with a nonmatching path still calls `requestBids()`, creates + its ad unit, scopes targeting to its code, and refreshes it after the callback. +- Explicit matching slot clears each existing TS/Prebid key, does not call + `requestBids()` or `setTargetingForGPTAsync()`, and immediately calls original + GPT refresh with the exact slot array and options. +- Global all-excluded slots resolve through `getSlots()`, clear every target, make + no Prebid calls, and refresh the complete resolved list with options. +- Global mixed slots clear both categories, auction only eligible slots, scope + targeting to eligible synthetic codes, and refresh the complete list after bids + return. +- Missing `getAdUnitPath`, non-string path, and a throwing getter each fail open to + the normal auction path without throwing from the wrapper. +- Case mismatch and a trailing-slash mismatch do not exclude, proving literal + case-sensitive suffix behavior. +- Existing `adInitRefreshInProgress` test still proves direct pass-through without + cleanup or auction; existing normal refresh and client-side-bid recovery tests + remain green. + +## 8. External bundle and browser verification + +`crates/trusted-server-js/lib/build-prebid-external.mjs` is the supported source +build path for the immutable external Prebid bundle; `build-all.mjs` intentionally +does not build Prebid. Implementers must change TypeScript source and regenerate a +new external bundle through the supported `ts prebid bundle` workflow (or its +underlying supported generator), not edit a generated/minified asset. + +Roll out the application/config and the regenerated external bundle together: + +1. Build and test the source change. +2. Generate and upload the new external bundle. +3. Update the operator bundle URL/hash/SRI metadata as required by the existing + bundle workflow and deploy the Trusted Server application/config containing the + suffix list. +4. Verify the first-party bundle URL resolves to the new bytes and the injected + `window.__tsjs_prebid.excludedGamAdUnitPathSuffixes` has the expected values. +5. In browser instrumentation, verify a matching slot calls GPT refresh without a + corresponding Trusted Server refresh `/auction` request, while a normal display + slot in the same global refresh still produces `/auction` and receives refreshed + Prebid targeting. +6. Verify GAM records the excluded slot's request/impression. Use a controlled + staging page or harness rather than relying on the unstable production host. + +A config-only deployment with an old cached external bundle cannot apply the browser +filter; a bundle-only deployment without the injected configuration remains a no-op. + +## 9. Operational caveats and risks + +- The exclusion is limited to Trusted Server's wrapper around GPT refresh. It does + not block a publisher's unrelated direct `pbjs.requestBids()`, APS calls, direct + `/auction` use, or any other auction wrapper. +- The feature relies on GPT's supported `getAdUnitPath()` API. A missing or throwing + getter deliberately fails open, which may continue auctioning a tracking slot + rather than risk silently suppressing display inventory. +- Literal suffix matching can be over-broad if an operator chooses a generic suffix + such as `/only`; use a unique terminal GAM path segment and validate on a staging + page. `/` is rejected, but other overly broad valid values remain an operator + responsibility. +- Excluded slots have only Trusted Server/Prebid targeting cleared; unrelated GAM + targeting and GAM request behavior are intentionally untouched. +- Browser code and injected config must reach the same deployed page. Cache/version + rollout mistakes are the primary operational risk. diff --git a/trusted-server.example.toml b/trusted-server.example.toml index 69df213e3..1e27e2544 100644 --- a/trusted-server.example.toml +++ b/trusted-server.example.toml @@ -45,6 +45,9 @@ timeout_ms = 1000 bidders = [] debug = false client_side_bidders = [] +# Keep selected GAM inventory out of Trusted Server's Prebid refresh auctions. +# Matching slots still refresh through GAM. +# excluded_gam_ad_unit_path_suffixes = ["/trackingonly"] # Runtime bundle metadata. Set these after running `ts prebid bundle` and uploading the bundle. # external_bundle_url = "https://assets.example.com/prebid/trusted-prebid-.js" # external_bundle_sha256 = "" From 76c56263fa780dbd7b6d0f1fd9e32099516e9f6e Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 24 Jul 2026 13:09:53 -0500 Subject: [PATCH 119/198] Prevent duplicate GPT slot requests --- .../src/integrations/gpt.rs | 29 +++ .../src/integrations/gpt_bootstrap.js | 160 +++++++++++-- .../trusted-server-js/lib/src/core/types.ts | 18 ++ .../lib/src/integrations/gpt/index.ts | 164 ++++++++++++- .../lib/test/integrations/gpt/ad_init.test.ts | 174 +++++++++++++- ...-24-prevent-duplicate-gpt-slot-requests.md | 219 ++++++++++++++++++ ...vent-duplicate-gpt-slot-requests-design.md | 165 +++++++++++++ 7 files changed, 900 insertions(+), 29 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-24-prevent-duplicate-gpt-slot-requests.md create mode 100644 docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index e21058a21..f53a21cf0 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -1222,6 +1222,35 @@ mod tests { ); } + #[test] + fn head_inserts_bootstrap_installs_inner_div_slot_handoff() { + let integration = GptIntegration::new(test_config()); + let doc_state = IntegrationDocumentState::default(); + let ctx = IntegrationHtmlContext { + request_host: "edge.example.com", + request_scheme: "https", + origin_host: "example.com", + document_state: &doc_state, + }; + let combined = integration.head_inserts(&ctx).join(""); + assert!( + combined.contains("gptSlotHandoffs"), + "bootstrap should keep late publisher slot handoff state on window.tsjs" + ); + assert!( + combined.contains("__tsSlotHandoffPatched"), + "bootstrap should install idempotent GPT handoff wrappers" + ); + assert!( + combined.contains("return googletag.defineSlot") && combined.contains("actualDivId"), + "bootstrap should define the TS fallback on the actual inner div" + ); + assert!( + !combined.contains("actualDivId + \"-container\""), + "bootstrap must not define a competing outer-container GPT slot" + ); + } + #[test] fn head_inserts_bootstrap_guards_enable_services_with_idempotency_flag() { let config = test_config(); diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index cc4c5c00c..0c2697357 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -42,6 +42,127 @@ pubads.__tsInitialLoadHooked = true; }); + function findSlotByElementId(pubads, elementId) { + var slots = pubads.getSlots ? pubads.getSlots() : []; + return ( + slots.find(function (slot) { + return slot.getSlotElementId() === elementId; + }) || null + ); + } + + function runHandoffInternal(callback) { + var wasInternal = ts.gptSlotHandoffInternal; + ts.gptSlotHandoffInternal = true; + try { + return callback(); + } finally { + ts.gptSlotHandoffInternal = wasInternal; + } + } + + // TS cannot wait an arbitrary amount of time for a framework to define a + // slot: publishers that never define one would render blank. Instead, TS + // defines its fallback on the actual inner div and aliases only a later + // publisher defineSlot() for that exact div to the same GPT slot. + function installSlotHandoff() { + window.googletag.cmd.push(function () { + var tag = window.googletag; + var pubads = tag.pubads && tag.pubads(); + if (!tag.defineSlot || !tag.display || !pubads) return; + + if (!tag.defineSlot.__tsSlotHandoffPatched) { + var originalDefineSlot = tag.defineSlot.bind(tag); + var patchedDefineSlot = function (adUnitPath, formats, elementId) { + var handoff = ts.gptSlotHandoffs && ts.gptSlotHandoffs[elementId]; + if (!ts.gptSlotHandoffInternal && handoff) { + var existingSlot = findSlotByElementId(pubads, elementId); + if (existingSlot) { + if (!handoff.publisherClaimed) { + handoff.publisherClaimed = true; + handoff.suppressPublisherDisplay = true; + handoff.suppressPublisherRefresh = + ts.gptInitialLoadDisabled === true; + ts.prevGptSlots = (ts.prevGptSlots || []).filter( + function (ownedSlot) { + return ownedSlot !== existingSlot; + }, + ); + if ( + handoff.gamUnitPath !== adUnitPath || + JSON.stringify(handoff.formats) !== JSON.stringify(formats) + ) { + ts.log && + ts.log.warn && + ts.log.warn( + "GPT slot handoff: publisher definition differs from TS configuration", + elementId, + ); + } + } + return existingSlot; + } + } + return originalDefineSlot(adUnitPath, formats, elementId); + }; + patchedDefineSlot.__tsSlotHandoffPatched = true; + tag.defineSlot = patchedDefineSlot; + } + + if (!tag.display.__tsSlotHandoffPatched) { + var originalDisplay = tag.display.bind(tag); + var patchedDisplay = function (elementId) { + var handoff = ts.gptSlotHandoffs && ts.gptSlotHandoffs[elementId]; + if ( + !ts.gptSlotHandoffInternal && + handoff && + handoff.suppressPublisherDisplay + ) { + handoff.suppressPublisherDisplay = false; + return; + } + originalDisplay(elementId); + }; + patchedDisplay.__tsSlotHandoffPatched = true; + tag.display = patchedDisplay; + } + + if (!pubads.refresh.__tsSlotHandoffPatched) { + var originalRefresh = pubads.refresh.bind(pubads); + var patchedRefresh = function (requestedSlots) { + if (ts.gptSlotHandoffInternal) { + originalRefresh(requestedSlots); + return; + } + var slots = + requestedSlots || (pubads.getSlots ? pubads.getSlots() : null); + if (!slots) { + originalRefresh(requestedSlots); + return; + } + var suppressed = false; + var remainingSlots = slots.filter(function (slot) { + var handoff = + ts.gptSlotHandoffs && ts.gptSlotHandoffs[slot.getSlotElementId()]; + if (!handoff || !handoff.suppressPublisherRefresh) return true; + handoff.suppressPublisherRefresh = false; + suppressed = true; + return false; + }); + if (!suppressed) { + originalRefresh(requestedSlots); + } else if (remainingSlots.length > 0) { + originalRefresh(remainingSlots); + } + }; + patchedRefresh.__tsSlotHandoffPatched = true; + pubads.refresh = patchedRefresh; + } + }); + } + + installSlotHandoff(); + ts.adInit = function () { var slots = ts.adSlots || []; var bids = ts.bids || {}; @@ -88,15 +209,26 @@ }) || null; var tsOwned = false; if (!s) { - // Use outer container div for TS's slot when publisher hasn't defined - // theirs yet — keeps both slots on separate divs so publisher's - // later defineSlot on the inner div doesn't conflict. - var containerEl = document.getElementById(actualDivId + "-container"); - var slotDivId = containerEl ? containerEl.id : actualDivId; - s = googletag.defineSlot(slot.gam_unit_path, slot.formats, slotDivId); + // Define TS's fallback on the publisher's actual div. The scoped + // handoff wrapper returns this slot if the publisher defines it later. + s = runHandoffInternal(function () { + return googletag.defineSlot( + slot.gam_unit_path, + slot.formats, + actualDivId, + ); + }); if (!s) return; s.addService(googletag.pubads()); tsOwned = true; + ts.gptSlotHandoffs = ts.gptSlotHandoffs || {}; + ts.gptSlotHandoffs[actualDivId] = { + gamUnitPath: slot.gam_unit_path, + formats: slot.formats, + publisherClaimed: false, + suppressPublisherDisplay: false, + suppressPublisherRefresh: false, + }; } Object.entries(slot.targeting || {}).forEach(function (e) { @@ -113,11 +245,9 @@ }); // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts s.setTargeting("ts_initial", "1"); - // Map both the inner div and the GPT slot's element ID (the - // "-container" div when TS defined the slot there) into divToSlotId. - // This bootstrap fires no beacons and registers no slotRenderEnded - // listener; the map is consumed by the bundle's render bridge (index.ts) - // once it loads, which reports the GPT slot element ID. + // Map the resolved inner div to the slot ID. This bootstrap fires no + // beacons and registers no slotRenderEnded listener; the map is consumed + // by the bundle's render bridge (index.ts) once it loads. divToSlotId[actualDivId] = slot.id; var slotElementId = s.getSlotElementId(); if (slotElementId && slotElementId !== actualDivId) { @@ -143,7 +273,9 @@ // impression. Runs after enableServices(); on SPA navigation services are // already enabled, so this runs unconditionally for new slots. slotsToDisplay.forEach(function (divId) { - googletag.display(divId); + runHandoffInternal(function () { + googletag.display(divId); + }); }); // Reused publisher-owned slots always need a refresh to pick up the // server-side targeting. TS-defined slots are fetched by display() above @@ -161,7 +293,9 @@ // bundle's adInit() in crates/trusted-server-js/lib/src/integrations/gpt/index.ts. ts.adInitRefreshInProgress = true; try { - googletag.pubads().refresh(slotsNeedingRefresh); + runHandoffInternal(function () { + googletag.pubads().refresh(slotsNeedingRefresh); + }); } finally { ts.adInitRefreshInProgress = false; } diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 360e2aa49..cd25133d1 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -63,6 +63,20 @@ export interface AuctionBidData { debug_bid?: AuctionDebugBidData; } +/** + * Lifecycle state for a GPT slot TS created before its publisher declares it. + * + * Stored on `window.tsjs` so the head bootstrap and the full TSJS bundle share + * one handoff protocol. + */ +export interface GptSlotHandoff { + gamUnitPath: string; + formats: Array<[number, number]>; + publisherClaimed: boolean; + suppressPublisherDisplay: boolean; + suppressPublisherRefresh: boolean; +} + export interface TsjsApi { version: string; que: Array<() => void>; @@ -121,6 +135,10 @@ export interface TsjsApi { * defined slots so they are not left blank. */ gptInitialLoadDisabled?: boolean; + /** Late publisher claims for TS-created GPT slots, keyed by actual div ID. */ + gptSlotHandoffs?: Record; + /** True only while TS calls a GPT function that the handoff wrappers observe. */ + gptSlotHandoffInternal?: boolean; /** Guards SPA pushState hook installation. */ spaHookInstalled?: boolean; } diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index ca4689684..8853997c3 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -1,5 +1,5 @@ import { log } from '../../core/log'; -import type { AuctionSlot, AuctionBidData, TsjsApi } from '../../core/types'; +import type { AuctionSlot, AuctionBidData, GptSlotHandoff, TsjsApi } from '../../core/types'; import { installGptGuard } from './script_guard'; @@ -445,9 +445,143 @@ function installInitialLoadDetector(ts: TsjsApi): void { }); } +interface HandoffPatchedFunction { + __tsSlotHandoffPatched?: boolean; +} + +function findGptSlotByElementId( + pubads: GoogleTagPubAdsService, + elementId: string +): GoogleTagSlot | undefined { + return pubads.getSlots?.().find((slot) => slot.getSlotElementId() === elementId); +} + +function handoffForSlot(ts: TsjsApi, slot: GoogleTagSlot): GptSlotHandoff | undefined { + return ts.gptSlotHandoffs?.[slot.getSlotElementId()]; +} + +function withGptSlotHandoffInternal(ts: TsjsApi, callback: () => T): T { + const wasInternal = ts.gptSlotHandoffInternal; + ts.gptSlotHandoffInternal = true; + try { + return callback(); + } finally { + ts.gptSlotHandoffInternal = wasInternal; + } +} + +/** + * Reuse a TS-created inner-div slot when its publisher defines that div later. + * + * TS cannot wait an arbitrary amount of time for framework hydration: doing so + * would leave placements blank when no publisher slot is ever defined. Instead, + * TS creates its fallback on the publisher's actual div and aliases only a later + * `defineSlot()` for that exact div. The first duplicate publisher request is + * suppressed because TS has already issued the initial request with TS targeting. + */ +function installLatePublisherSlotHandoff(ts: TsjsApi): void { + const win = window as GptWindow; + const cmd = win.googletag?.cmd; + if (!cmd) return; + + cmd.push(() => { + const g = win.googletag; + const pubads = g?.pubads?.(); + if (!g?.defineSlot || !g.display || !pubads) return; + + const defineSlot = g.defineSlot; + if (!(defineSlot as HandoffPatchedFunction).__tsSlotHandoffPatched) { + const originalDefineSlot = defineSlot.bind(g); + const patchedDefineSlot = ( + adUnitPath: string, + formats: Array, + elementId: string + ): GoogleTagSlot | null => { + const handoff = ts.gptSlotHandoffs?.[elementId]; + if (!ts.gptSlotHandoffInternal && handoff) { + const existingSlot = findGptSlotByElementId(pubads, elementId); + if (existingSlot) { + if (!handoff.publisherClaimed) { + handoff.publisherClaimed = true; + handoff.suppressPublisherDisplay = true; + handoff.suppressPublisherRefresh = ts.gptInitialLoadDisabled === true; + ts.prevGptSlots = (ts.prevGptSlots ?? []).filter( + (ownedSlot) => ownedSlot !== existingSlot + ); + if ( + handoff.gamUnitPath !== adUnitPath || + JSON.stringify(handoff.formats) !== JSON.stringify(formats) + ) { + log.warn('GPT slot handoff: publisher definition differs from TS configuration', { + elementId, + tsGamUnitPath: handoff.gamUnitPath, + publisherGamUnitPath: adUnitPath, + }); + } + } + return existingSlot; + } + } + return originalDefineSlot(adUnitPath, formats, elementId); + }; + (patchedDefineSlot as HandoffPatchedFunction).__tsSlotHandoffPatched = true; + g.defineSlot = patchedDefineSlot; + } + + const display = g.display; + if (!(display as HandoffPatchedFunction).__tsSlotHandoffPatched) { + const originalDisplay = display.bind(g); + const patchedDisplay = (elementId: string): void => { + const handoff = ts.gptSlotHandoffs?.[elementId]; + if (!ts.gptSlotHandoffInternal && handoff?.suppressPublisherDisplay) { + handoff.suppressPublisherDisplay = false; + return; + } + originalDisplay(elementId); + }; + (patchedDisplay as HandoffPatchedFunction).__tsSlotHandoffPatched = true; + g.display = patchedDisplay; + } + + const refresh = pubads.refresh; + if (!(refresh as HandoffPatchedFunction).__tsSlotHandoffPatched) { + const originalRefresh = refresh.bind(pubads); + const patchedRefresh = (requestedSlots?: GoogleTagSlot[]): void => { + if (ts.gptSlotHandoffInternal) { + originalRefresh(requestedSlots); + return; + } + + const slots = requestedSlots ?? pubads.getSlots?.(); + if (!slots) { + originalRefresh(requestedSlots); + return; + } + + let suppressed = false; + const remainingSlots = slots.filter((slot) => { + const handoff = handoffForSlot(ts, slot); + if (!handoff?.suppressPublisherRefresh) return true; + handoff.suppressPublisherRefresh = false; + suppressed = true; + return false; + }); + if (!suppressed) { + originalRefresh(requestedSlots); + } else if (remainingSlots.length > 0) { + originalRefresh(remainingSlots); + } + }; + (patchedRefresh as HandoffPatchedFunction).__tsSlotHandoffPatched = true; + pubads.refresh = patchedRefresh; + } + }); +} + export function installTsAdInit(): void { const ts = (window.tsjs ??= {} as TsjsApi); installInitialLoadDetector(ts); + installLatePublisherSlotHandoff(ts); ts.adInit = function () { const slots = ts.adSlots ?? []; // Snapshot bids at adInit() call time — correct for targeting setup. @@ -517,16 +651,23 @@ export function installTsAdInit(): void { if (existingSlot) { gptSlot = existingSlot; } else { - // Use outer container div for TS's slot when publisher hasn't defined - // theirs yet — keeps both slots on separate divs so publisher's - // later defineSlot on the inner div doesn't conflict. - const containerEl = document.getElementById(`${actualDivId}-container`); - const slotDivId = containerEl?.id ?? actualDivId; - const defined = g.defineSlot?.(slot.gam_unit_path, slot.formats, slotDivId); + // Define TS's fallback on the publisher's actual div. A late publisher + // defineSlot() for this div is handed the same slot by the scoped GPT + // wrapper, preventing a competing container-slot request. + const defined = withGptSlotHandoffInternal(ts, () => + g.defineSlot?.(slot.gam_unit_path, slot.formats, actualDivId) + ); if (!defined) return; defined.addService(g.pubads!()); gptSlot = defined; tsOwned = true; + (ts.gptSlotHandoffs ??= {})[actualDivId] = { + gamUnitPath: slot.gam_unit_path, + formats: slot.formats, + publisherClaimed: false, + suppressPublisherDisplay: false, + suppressPublisherRefresh: false, + }; } const slotDivId2 = gptSlot.getSlotElementId?.() ?? actualDivId; @@ -541,9 +682,8 @@ export function installTsAdInit(): void { if (bid[key]) gptSlot.setTargeting(key, String(bid[key]!)); }); gptSlot.setTargeting(TS_INITIAL_TARGETING_KEY, '1'); - // Map both inner div and container div → slot ID so slotRenderEnded - // (which reports the GPT slot's div, i.e. slotDivId/container) can look up - // the slot, while adm injection (which targets the inner div) also works. + // Map the resolved inner div to the slot ID so slotRenderEnded and ADM + // injection address the same, single GPT slot. divToSlotId[actualDivId] = slot.id; if (slotDivId2 !== actualDivId) divToSlotId[slotDivId2] = slot.id; const slotTargetingKeys = Object.keys(slot.targeting ?? {}); @@ -607,7 +747,7 @@ export function installTsAdInit(): void { // called without a matching display call") and misses its impression. // Must run after enableServices(); on SPA navigation services are already // enabled, so this runs unconditionally for any newly-defined slots. - slotsToDisplay.forEach((divId) => g.display?.(divId)); + slotsToDisplay.forEach((divId) => withGptSlotHandoffInternal(ts, () => g.display?.(divId))); // Slots needing an explicit ad request via refresh(). Reused // publisher-owned slots always need one to pick up the just-applied @@ -630,7 +770,7 @@ export function installTsAdInit(): void { // the same slots still go through the wrapper normally. ts.adInitRefreshInProgress = true; try { - g.pubads!().refresh(slotsNeedingRefresh); + withGptSlotHandoffInternal(ts, () => g.pubads!().refresh(slotsNeedingRefresh)); } finally { ts.adInitRefreshInProgress = false; } diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index 4a6368768..f99d90fa7 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -145,12 +145,13 @@ describe('installTsAdInit', () => { getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), getTargeting: vi.fn().mockReturnValue([]), }; + const nativeRefresh = vi.fn(); const mockPubads = { enableSingleRequest: vi.fn(), // Publisher has not defined this slot, so TS defines (owns) it. getSlots: vi.fn().mockReturnValue([]), addEventListener: vi.fn(), - refresh: vi.fn(), + refresh: nativeRefresh, }; const defineSlotMock = vi.fn().mockReturnValue(mockSlot); const displayMock = vi.fn(); @@ -184,7 +185,171 @@ describe('installTsAdInit', () => { expect(displayMock).toHaveBeenCalledWith('div-atf-sidebar'); // TS-owned slots are displayed, not refreshed (refresh() no-ops for a slot // that was never displayed). - expect(mockPubads.refresh).not.toHaveBeenCalled(); + expect(nativeRefresh).not.toHaveBeenCalled(); + }); + + it('hands a late publisher definition the TS inner-div slot without a second request', async () => { + type FakeSlot = { + addService(service: unknown): FakeSlot; + setTargeting(key: string, value: string | string[]): FakeSlot; + getSlotElementId(): string; + getTargeting(key?: string): string[]; + }; + const slots = new Map(); + const requests: string[] = []; + const makeSlot = (elementId: string): FakeSlot => ({ + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue(elementId), + getTargeting: vi.fn().mockReturnValue([]), + }); + const pubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn(() => Array.from(slots.values())), + addEventListener: vi.fn(), + refresh: vi.fn((requestedSlots?: FakeSlot[]) => { + (requestedSlots ?? Array.from(slots.values())).forEach((slot) => + requests.push(slot.getSlotElementId()) + ); + }), + }; + const nativeDefineSlot = vi.fn( + (_adUnitPath: string, _formats: number[][], elementId: string) => { + const slot = makeSlot(elementId); + slots.set(elementId, slot); + return slot; + } + ); + const nativeDisplay = vi.fn((elementId: string) => requests.push(elementId)); + const destroySlots = vi.fn(); + const googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: nativeDefineSlot, + display: nativeDisplay, + pubads: vi.fn().mockReturnValue(pubads), + destroySlots, + enableServices: vi.fn(), + }; + (window as TestWindow).googletag = googletag; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + }; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + (window as TestWindow).tsjs!.adInit!(); + + const publisherDefineSlot = googletag.defineSlot as unknown as ( + adUnitPath: string, + formats: number[][], + elementId: string + ) => FakeSlot; + const publisherDisplay = googletag.display as unknown as (elementId: string) => void; + const publisherSlot = publisherDefineSlot('/123/atf', [[300, 250]], 'div-atf-sidebar'); + publisherSlot.addService(pubads); + publisherDisplay('div-atf-sidebar'); + + expect(nativeDefineSlot).toHaveBeenCalledTimes(1); + expect(nativeDisplay).toHaveBeenCalledTimes(1); + expect(requests).toEqual(['div-atf-sidebar']); + expect((window as TestWindow).tsjs!.prevGptSlots).toEqual([]); + + (window as TestWindow).tsjs!.adSlots = []; + (window as TestWindow).tsjs!.adInit!(); + expect(destroySlots).not.toHaveBeenCalled(); + }); + + it('suppresses only the claimed slot from the first disabled-load publisher refresh', async () => { + type FakeSlot = { + addService(service: unknown): FakeSlot; + setTargeting(key: string, value: string | string[]): FakeSlot; + getSlotElementId(): string; + getTargeting(key?: string): string[]; + }; + const slots = new Map(); + const requests: string[] = []; + let initialLoadDisabled = false; + const makeSlot = (elementId: string): FakeSlot => ({ + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue(elementId), + getTargeting: vi.fn().mockReturnValue([]), + }); + const pubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn(() => Array.from(slots.values())), + addEventListener: vi.fn(), + refresh: vi.fn((requestedSlots?: FakeSlot[]) => { + (requestedSlots ?? Array.from(slots.values())).forEach((slot) => + requests.push(slot.getSlotElementId()) + ); + }), + disableInitialLoad: vi.fn(() => { + initialLoadDisabled = true; + }), + }; + const nativeDefineSlot = vi.fn( + (_adUnitPath: string, _formats: number[][], elementId: string) => { + const slot = makeSlot(elementId); + slots.set(elementId, slot); + return slot; + } + ); + const nativeDisplay = vi.fn((elementId: string) => { + if (!initialLoadDisabled) requests.push(elementId); + }); + const googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: nativeDefineSlot, + display: nativeDisplay, + pubads: vi.fn().mockReturnValue(pubads), + enableServices: vi.fn(), + }; + (window as TestWindow).googletag = googletag; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: {}, + }; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + pubads.disableInitialLoad(); + (window as TestWindow).tsjs!.adInit!(); + + const publisherDefineSlot = googletag.defineSlot as unknown as ( + adUnitPath: string, + formats: number[][], + elementId: string + ) => FakeSlot; + const publisherDisplay = googletag.display as unknown as (elementId: string) => void; + const publisherRefresh = pubads.refresh as unknown as () => void; + const publisherSlot = publisherDefineSlot('/123/atf', [[300, 250]], 'div-atf-sidebar'); + publisherSlot.addService(pubads); + publisherDisplay('div-atf-sidebar'); + slots.set('div-unrelated', makeSlot('div-unrelated')); + publisherRefresh(); + + expect(nativeDefineSlot).toHaveBeenCalledTimes(1); + expect(nativeDisplay).toHaveBeenCalledTimes(1); + expect(requests.filter((elementId) => elementId === 'div-atf-sidebar')).toHaveLength(1); + expect(requests).toContain('div-unrelated'); }); it('refreshes TS-defined slots when the publisher disabled GPT initial load', async () => { @@ -197,12 +362,13 @@ describe('installTsAdInit', () => { getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), getTargeting: vi.fn().mockReturnValue([]), }; + const nativeRefresh = vi.fn(); const mockPubads = { enableSingleRequest: vi.fn(), // Publisher has not defined this slot, so TS defines (owns) it. getSlots: vi.fn().mockReturnValue([]), addEventListener: vi.fn(), - refresh: vi.fn(), + refresh: nativeRefresh, disableInitialLoad: vi.fn(), }; const displayMock = vi.fn(); @@ -240,7 +406,7 @@ describe('installTsAdInit', () => { // The slot is still registered via display(), and additionally refreshed so // it actually requests an ad under disableInitialLoad(). expect(displayMock).toHaveBeenCalledWith('div-atf-sidebar'); - expect(mockPubads.refresh).toHaveBeenCalledWith([mockSlot]); + expect(nativeRefresh).toHaveBeenCalledWith([mockSlot]); }); it('sets adInitRefreshInProgress only for the duration of the internal refresh', async () => { diff --git a/docs/superpowers/plans/2026-07-24-prevent-duplicate-gpt-slot-requests.md b/docs/superpowers/plans/2026-07-24-prevent-duplicate-gpt-slot-requests.md new file mode 100644 index 000000000..4699e3c42 --- /dev/null +++ b/docs/superpowers/plans/2026-07-24-prevent-duplicate-gpt-slot-requests.md @@ -0,0 +1,219 @@ +# Prevent Duplicate GPT Slot Requests — Implementation Plan + +> **Status:** Implemented locally; production-like browser validation remains pending. +> +> **Spec:** `docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md` + +**Goal:** Ensure one GPT slot and one initial request per configured placement when +TS `adInit()` runs before a publisher later defines the placement's inner GPT div. + +**Architecture:** TS creates its fallback on the resolved inner div and records a +handoff claim. Narrow, idempotent wrappers around GPT's `defineSlot`, `display`, and +`pubads().refresh` alias a matching late publisher definition to that slot and +suppress only the duplicate initial publisher request. A successful handoff transfers +SPA-destruction ownership to the publisher. The head bootstrap and full TSJS bundle +share this runtime protocol through `window.tsjs`. + +**Primary files:** + +- `crates/trusted-server-js/lib/src/core/types.ts` +- `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` +- `crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts` +- `crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts` +- `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` +- `crates/trusted-server-core/src/integrations/gpt.rs` + +## Preconditions + +- [ ] Confirm with the issue owner that the intended late-owner behavior is slot + handoff (publisher receives the existing inner-div slot), not a hydration-delay + policy. +- [ ] Capture representative publisher call sequences for normal initial load and + `disableInitialLoad()` before changing wrappers. The expected sequence is + `defineSlot` → `addService` → `display`; initial-load-disabled pages additionally + call `refresh`. +- [ ] Establish an automated fake-GPT request counter: calling native `display` with + initial load enabled, or native `refresh` with initial load disabled, records a + request. Assertions must use this counter rather than only `getSlots()`. + +## Task 1: Add the shared handoff state and typed GPT wrapper surface + +**Files:** + +- Modify `crates/trusted-server-js/lib/src/core/types.ts` +- Modify `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` + +- [ ] Add a `TsjsApi` property for a div-ID-keyed handoff registry. Each entry must + retain serializable lifecycle flags: TS-created, ownership-transferred, initial + request made, and one-shot publisher display/refresh suppression state. +- [ ] Add only the minimal optional/internal type surface needed for idempotence + markers on GPT functions and `pubads`. Do not weaken the public GPT types with + `any`. +- [ ] Add helper functions in `index.ts` to: + - find a live GPT slot by exact element ID; + - register and retrieve a claim; + - remove a transferred slot from `ts.prevGptSlots`; + - run an internal TS GPT call behind a short-lived guard; + - filter a requested refresh list (including no-argument/global refresh) by the + entries whose one-shot publisher refresh must be suppressed. +- [ ] Keep the registry on `window.tsjs`, not in module scope, so the bootstrap state + survives bundle loading. + +**Focused checks:** + +```bash +cd crates/trusted-server-js/lib +npx vitest run test/integrations/gpt/ad_init.test.ts test/integrations/gpt/index.test.ts +``` + +## Task 2: Install scoped idempotent handoff wrappers + +**File:** `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` + +- [ ] From the GPT command queue, install wrappers once GPT exposes the real methods. + Mark the wrapped functions/service so a later `installTsAdInit()` call or the + bootstrap-to-bundle handoff cannot stack wrappers. +- [ ] `defineSlot` wrapper: + - pass through TS-internal calls and IDs absent from the registry; + - for a late publisher call on a claimed inner div, find and return the existing + slot without calling native `defineSlot`; + - mark ownership transferred and remove that slot from `prevGptSlots` before + returning it; + - log, but do not create a second slot, if publisher arguments differ from the TS + configuration. +- [ ] `display` wrapper: consume the one permitted publisher post-handoff display + call without invoking native `display`; pass every other call through unchanged. +- [ ] `refresh` wrapper: when initial load was disabled, consume the one permitted + post-handoff refresh for each claimed slot. If called with no slot list, expand + `getSlots()`, filter only the claimed slots, and forward the remaining slots + explicitly. Preserve all unrelated refreshes. +- [ ] Ensure wrapper installation precedes the fallback definition path and does not + change existing publisher-owned-slot behavior. + +**Focused checks:** + +```bash +cd crates/trusted-server-js/lib +npx vitest run test/integrations/gpt/ad_init.test.ts +``` + +## Task 3: Change fallback creation to the actual inner div + +**File:** `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` + +- [ ] Delete the `${actualDivId}-container` fallback selection. When no existing + publisher slot is found, call `defineSlot` with `actualDivId`. +- [ ] Register the handoff claim immediately after successful TS definition. +- [ ] Keep `display()` for TS-created slots; with initial load disabled, retain the + single TS `refresh()` that makes the required initial request. +- [ ] Simplify `divToSlotId` and `prevSlotTargetingKeys` to the actual inner div; + remove only mappings that existed exclusively for the container fallback. +- [ ] On SPA navigation, destroy only claims that remain TS-owned. A transferred + claim must participate in stale-targeting cleanup but never be passed to + `destroySlots()`. +- [ ] Retain exact match then prefix-based dynamic-ID lookup; do not interpolate + publisher-provided IDs into CSS selectors. + +## Task 4: Add request-level regression coverage for the full bundle + +**Files:** + +- Modify `crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts` +- Modify `crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts` if the + shared wrapper setup belongs there + +- [ ] Introduce a reusable fake GPT fixture that models slots by element ID and + records native `defineSlot`, `display`, `refresh`, and request events. Its + `getSlots()` result must update when a slot is defined so the test cannot pass by + asserting a stale static array. +- [ ] Add a failing regression test for the critical sequence: + 1. TS finds the inner div and runs `adInit()` before publisher setup; + 2. TS defines/displays the inner div and makes one request; + 3. publisher calls `defineSlot(innerDiv).addService(...); display(innerDiv)`; + 4. assert native `defineSlot` was called once, there is one slot, and there is one + request. +- [ ] Add the same sequence with `disableInitialLoad()`: TS display plus its refresh + makes one request; the publisher's first refresh cannot make a second request. +- [ ] Add a no-argument publisher refresh test containing an unrelated slot. Assert + the claimed slot is suppressed once and the unrelated slot is refreshed. +- [ ] Add an already publisher-owned test proving TS does not install a claim, applies + targeting, and refreshes that slot. +- [ ] Add a no-publisher test proving TS still creates, displays, and requests its + inner-div slot exactly once. +- [ ] Add a SPA handoff test: after late publisher claim, the next `adInit()` does not + destroy the transferred slot, clears old TS keys, and reapplies current-route + targeting. +- [ ] Retain or extend the dynamic prefix-ID test to prove a resolved runtime ID is + the handoff key. + +## Task 5: Mirror the runtime protocol in the head bootstrap + +**Files:** + +- Modify `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` +- Modify `crates/trusted-server-core/src/integrations/gpt.rs` + +- [ ] Port the same actual-inner-div fallback, registry names, lifecycle flags, and + idempotence markers to the plain-JavaScript bootstrap. +- [ ] Use the existing bootstrap `window.tsjs` properties exactly so `index.ts` can + adopt the initial claim after the bundle loads. +- [ ] Ensure its internal definition/display/refresh calls use the same guards as the + bundle; bootstrap must not transfer or suppress its own operations. +- [ ] Extend the `gpt.rs` head-insert tests to assert that the bootstrap contains the + inner-div handoff protocol and no longer contains the container fallback. +- [ ] Add an executable bootstrap behavior test if practical by evaluating the + injected script against the same fake GPT fixture. If the test setup cannot execute + the included asset without duplication, record that limitation and keep the Rust + source-contract assertion plus identical bundle lifecycle tests as the minimum + coverage. + +## Task 6: Validate, inspect, and ship + +- [ ] Run focused request-level tests: + + ```bash + cd crates/trusted-server-js/lib + npx vitest run test/integrations/gpt/ad_init.test.ts test/integrations/gpt/index.test.ts + ``` + +- [ ] Run all TSJS tests and formatting: + + ```bash + cd crates/trusted-server-js/lib + npx vitest run + npm run format + ``` + +- [ ] Run the target-matched Rust tests that cover the embedded bootstrap, followed by + project formatting and linting: + + ```bash + cargo test-axum + cargo fmt --all -- --check + cargo clippy-fastly && cargo clippy-axum && cargo clippy-cloudflare + ``` + +- [ ] Before PR handoff, run the full required CI gates from `CLAUDE.md`, including + Fastly, Axum, Cloudflare, Spin, integration parity, JS build/tests/format, and docs + format. +- [ ] Review the diff specifically for bootstrap/bundle protocol drift and for any + use of container IDs in GPT slot creation. +- [ ] In a controlled production-like browser capture, verify one initial request for + each affected visible placement and independently verify an unrelated placement + remains requestable. +- [ ] Update issue #944 with the ownership-handoff decision, test evidence, and + browser-capture result. + +## Stop conditions + +Stop and return to design review instead of adding heuristics if any of these occur: + +- A publisher relies on a late `defineSlot` with materially different path or size + arguments and cannot accept the existing TS slot. +- The publisher's first initial-load-disabled refresh cannot be identified without + suppressing unrelated legitimate refreshes. +- A cross-bundle bootstrap handoff requires module-local identity that cannot be + represented safely through `window.tsjs`. +- Browser validation shows a second request despite native `defineSlot`/`display`/ + `refresh` suppression; capture the GPT event ordering before choosing another + strategy. diff --git a/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md b/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md new file mode 100644 index 000000000..770718199 --- /dev/null +++ b/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md @@ -0,0 +1,165 @@ +# Prevent Duplicate GPT Slot Requests — Design Specification + +## Problem + +When `tsjs.adInit()` executes before a publisher's framework later calls +`googletag.defineSlot()` for the same placement, TS currently defines and displays a +slot on the outer `-container` element. The publisher subsequently defines and +displays an inner-div slot. These are distinct GPT slots, so they make separate GAM +requests for one visible placement. + +The affected paths are deliberately duplicated today: + +- `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` is the full bundle + implementation used after the TSJS bundle loads. +- `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` is the head-injected + implementation that can make the initial request before the bundle loads. + +A fix must keep both implementations in sync. + +## Goals + +1. A configured placement has at most one initial GPT slot and ad request when TS + runs before a publisher defines its inner div. +2. Apply TS targeting and the `ts_initial=1` marker before that single initial + request. +3. Continue reusing a slot that the publisher has already defined. +4. Keep the TS-only fallback: if the publisher never defines the placement, TS still + displays it and makes exactly one initial request. +5. Preserve `disableInitialLoad()`, SPA targeting cleanup, and the rule that TS does + not destroy genuinely publisher-owned slots. +6. Keep dynamic div-ID prefix resolution intact. + +## Non-goals + +- Deduplicating by GAM ad-unit path. Multiple visible placements may validly share a + path. +- Changing publisher GAM configuration, line items, or refresh policy. +- Delaying the initial TS request while waiting an arbitrary amount of time for + framework hydration. A time-based grace period cannot distinguish a slow + publisher-owned slot from a placement that the publisher will never define. +- General interception of unrelated GPT slots. + +## Decision: one inner-div slot with late-definition handoff + +TS will define its fallback slot on the **actual inner div**, never on its outer +`-container` element. It will record a narrowly scoped handoff claim keyed by that +inner div ID. A `googletag.defineSlot` wrapper then recognizes a later publisher +request for that exact div and returns the existing TS slot rather than invoking +GPT's native `defineSlot` again. + +GPT requires a one-to-one slot-to-div relationship and documents that a slot should +be displayed only once. Sharing the initial inner-div slot therefore avoids both the +competing container slot and an invalid duplicate definition. + +### Lifecycle + +1. **Already publisher-owned** — `getSlots()` finds a slot for the resolved inner + div. TS applies targeting, records it as publisher-owned, and refreshes it as it + does today. +2. **No slot yet** — TS defines a slot on the resolved inner div, applies targeting, + enables services when needed, and displays it. When initial load is disabled, TS + performs its existing one explicit refresh. TS records this slot as TS-owned and + handoff-eligible. +3. **Publisher defines later** — the scoped `defineSlot` wrapper sees the recorded + inner-div claim, returns the existing slot, and transfers ownership: it removes + the slot from TS's future `destroySlots()` set. The publisher's setup continues + against that same slot. +4. **Publisher's first request call** — the wrapper suppresses the duplicate + publisher `display()` call. With `disableInitialLoad()`, it instead suppresses + only the publisher's first refresh for the transferred slot, because TS has + already issued the required initial refresh. For a no-argument/global refresh, + the wrapper must expand `getSlots()`, remove only the one-shot suppressed slots, + and forward the remaining slots explicitly so unrelated slots still refresh. +5. **Later refreshes and SPA navigation** — after the one-shot suppression is + consumed, publisher refreshes are untouched. On navigation, TS clears its + targeting from the shared slot and may reuse it for the next route; it must not + destroy a slot after ownership has transferred. + +The wrapper is not a global deduplicator. It only handles IDs present in TS's +handoff registry and must preserve native `defineSlot`, `display`, and `refresh` +behavior for every other placement. + +## Implementation shape + +### Shared runtime state + +Add a small, serializable `window.tsjs` registry that both initial implementations +can read after the bundle replaces the bootstrap implementation. It is keyed by the +resolved actual div ID and records at least: + +- whether TS created the slot and whether ownership has transferred; +- whether one publisher `display()` or initial-load-disabled `refresh()` remains to + suppress. + +Do not rely only on module-local state: the bootstrap can define the initial slot +before `index.ts` is loaded. Look up the live slot by element ID through +`pubads().getSlots()` when a wrapper needs it. + +Install idempotent markers on the wrapped GPT functions/services so the bootstrap and +bundle do not stack wrappers. Each wrapper must retain and call the original bound +function for non-claimed slots. Internal TS calls need a short-lived guard so the +wrappers do not mistake TS's own `defineSlot`, `display`, or `refresh` for a +publisher handoff. + +### Full bundle + +In `crates/trusted-server-js/lib/src/integrations/gpt/index.ts`: + +- Replace the container fallback with `actualDivId`. +- Add the typed handoff-registry state to `TsjsApi` in + `crates/trusted-server-js/lib/src/core/types.ts`. +- Install the idempotent `defineSlot`, `display`, and `pubads().refresh` handoff + wrappers from the GPT command queue before `adInit()` can create a fallback slot. +- When a late publisher definition is aliased to the existing slot, remove it from + `prevGptSlots` and mark it transferred before returning it. +- Keep targeting cleanup keyed by the real inner div. Remove the old dual + inner/container mappings because the slot element ID is now the inner div. + +### Head bootstrap + +Mirror the same ownership registry and wrappers in +`crates/trusted-server-core/src/integrations/gpt_bootstrap.js`. The bootstrap must +leave the registry and idempotence markers in `window.tsjs` so the full bundle adopts +rather than re-wraps or reclaims the initial slot. + +This duplication is intentional for now: the head bootstrap is needed to apply +server-side targeting before the normal bundle becomes available. The regression +suite must exercise both implementations' observable contract. + +## Compatibility rules and risks + +| Risk | Mitigation | +| -------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Publisher passes a different ad-unit path or sizes in its late `defineSlot` call | Return the existing claimed slot but log a diagnostic. Do not define a second slot. Treat the TS configuration and publisher configuration mismatch as an integration error to resolve separately. | +| Publisher invokes global `refresh()` after `disableInitialLoad()` | Filter the one-shot claimed slot from the expanded slot list and refresh all remaining slots. A no-argument refresh must not be silently dropped. | +| Publisher calls a legitimate refresh without an initial display | The one-shot suppression is consumed only immediately after a successful late handoff. Document and test the standard publisher sequence (`defineSlot` → `addService` → `display`, with `refresh` when initial load is disabled). Escalate unusual publisher lifecycle requirements rather than adding a time heuristic. | +| Publisher-owned slot is destroyed on SPA navigation | Transfer ownership synchronously in the `defineSlot` wrapper and remove the slot from `prevGptSlots`. | +| Bootstrap and bundle diverge | Give both paths the same black-box regression cases; retain a Rust source-contract assertion for bootstrap-specific sentinels. | +| A framework creates the inner element only after `adInit()` | TS still skips an absent element, as it does today; when the publisher owns that later-created slot it will not be duplicated. Supplying TS targeting to such a slot is a separate readiness problem, not part of this duplicate-request fix. | + +## Acceptance criteria + +- A late `defineSlot(innerDiv)` aliases the already-created inner-div TS slot; native + `defineSlot` is not called a second time for that placement. +- Request instrumentation records one initial request for the placement in normal and + initial-load-disabled modes. +- The late publisher `display()` (and its first initial-load-disabled refresh) cannot + create a second request, while unrelated slots retain their normal calls. +- Existing publisher slots are still reused and receive TS targeting. +- A slot that no publisher claims is displayed and requested once by TS. +- A transferred slot is absent from TS's SPA `destroySlots()` argument; targeting is + still cleared and reapplied correctly on the next route. +- Dynamic resolved div IDs work without constructing a CSS selector from the ID. +- Bootstrap and bundle paths pass the same ownership/request assertions. + +## Validation + +1. Add focused Vitest lifecycle tests with a fake GPT that records native + `defineSlot`, `display`, `refresh`, and synthetic request events. +2. Run the focused GPT test files, then the full TSJS Vitest suite and formatter. +3. Run the target-matched Rust test suite so the included bootstrap and its source + assertions compile and pass. +4. In a controlled browser capture, verify that one configured header and one + configured fixed placement each produce one initial slot request, while a distinct + in-content placement remains independently requestable. From b65e1aedd33440a281cf28443c0ffd6e46b9de02 Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 24 Jul 2026 14:14:54 -0500 Subject: [PATCH 120/198] Gate publisher GPT requests until targeting is ready --- .../src/integrations/gpt.rs | 7 +- .../src/integrations/gpt_bootstrap.js | 108 +++++++++++--- .../trusted-server-js/lib/src/core/types.ts | 11 ++ .../lib/src/integrations/gpt/index.ts | 133 ++++++++++++----- .../lib/test/integrations/gpt/ad_init.test.ts | 134 +++++++++++++++++- ...-24-prevent-duplicate-gpt-slot-requests.md | 50 ++++--- ...vent-duplicate-gpt-slot-requests-design.md | 60 +++++--- 7 files changed, 398 insertions(+), 105 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index f53a21cf0..20e4a9492 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -474,7 +474,8 @@ impl IntegrationHeadInjector for GptIntegration { /// ## Scroll / refresh handoff contract (Phase 1) /// /// `tsjs.adInit` handles **initial render only**: it wires server-side bid - /// targeting into GPT slots and refreshes them. Win/billing beacons fire + /// targeting into GPT slots and replays only publisher requests held until + /// that targeting was available. Win/billing beacons fire /// only from the TS render bridge in the JS bundle, where a matching /// Prebid Universal Creative request proves the TS creative rendered. /// It does **not** trigger refresh auctions or handle GPT slot refresh events. @@ -1241,6 +1242,10 @@ mod tests { combined.contains("__tsSlotHandoffPatched"), "bootstrap should install idempotent GPT handoff wrappers" ); + assert!( + combined.contains("gptInitialRequestGate") && combined.contains("pendingDisplays"), + "bootstrap should hold configured publisher requests until initial targeting is applied" + ); assert!( combined.contains("return googletag.defineSlot") && combined.contains("actualDivId"), "bootstrap should define the TS fallback on the actual inner div" diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 0c2697357..257da5908 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -10,8 +10,8 @@ // - Both implementations must set `window.tsjs.servicesEnabled = true` // after calling `enableSingleRequest()`/`enableServices()` so a // subsequent call becomes a no-op. -// - `refresh()` is called only for the slots defined in this pass, -// never the global slot list. +// - `refresh()` is called only for TS-defined slots in this pass and +// publisher requests the initial gate held, never the global slot list. // // Only installed if `window.tsjs.adInit` isn't already defined. (function () { @@ -51,6 +51,45 @@ ); } + function configuredSlotForElementId(elementId) { + return (ts.adSlots || []).find(function (slot) { + return ( + slot.div_id && + (elementId === slot.div_id || elementId.startsWith(slot.div_id)) && + !elementId.endsWith("-container") + ); + }); + } + + function initialRequestGate() { + if (!ts.gptInitialRequestGate) { + ts.gptInitialRequestGate = { + pendingDisplays: {}, + pendingRefreshes: {}, + released: false, + }; + } + return ts.gptInitialRequestGate; + } + + function takeInitialPublisherRequests(pubads) { + var gate = initialRequestGate(); + if (gate.released) return { displayIds: [], refreshSlots: [] }; + + gate.released = true; + var displayIds = Object.keys(gate.pendingDisplays); + var refreshIds = Object.keys(gate.pendingRefreshes); + gate.pendingDisplays = {}; + gate.pendingRefreshes = {}; + var slots = pubads.getSlots ? pubads.getSlots() : []; + return { + displayIds: displayIds, + refreshSlots: slots.filter(function (slot) { + return refreshIds.includes(slot.getSlotElementId()); + }), + }; + } + function runHandoffInternal(callback) { var wasInternal = ts.gptSlotHandoffInternal; ts.gptSlotHandoffInternal = true; @@ -121,6 +160,15 @@ handoff.suppressPublisherDisplay = false; return; } + var gate = initialRequestGate(); + if ( + !ts.gptSlotHandoffInternal && + !gate.released && + configuredSlotForElementId(elementId) + ) { + gate.pendingDisplays[elementId] = true; + return; + } originalDisplay(elementId); }; patchedDisplay.__tsSlotHandoffPatched = true; @@ -141,13 +189,22 @@ return; } var suppressed = false; + var gate = initialRequestGate(); var remainingSlots = slots.filter(function (slot) { var handoff = ts.gptSlotHandoffs && ts.gptSlotHandoffs[slot.getSlotElementId()]; - if (!handoff || !handoff.suppressPublisherRefresh) return true; - handoff.suppressPublisherRefresh = false; - suppressed = true; - return false; + if (handoff && handoff.suppressPublisherRefresh) { + handoff.suppressPublisherRefresh = false; + suppressed = true; + return false; + } + var elementId = slot.getSlotElementId(); + if (!gate.released && configuredSlotForElementId(elementId)) { + gate.pendingRefreshes[elementId] = true; + suppressed = true; + return false; + } + return true; }); if (!suppressed) { originalRefresh(requestedSlots); @@ -172,13 +229,15 @@ // Slots TS defined itself — tracked for SPA destroy. Publisher-owned // slots are reused but never destroyed by TS on navigation. var newSlots = []; - // Publisher-owned slots TS reused — refreshed to pick up server-side - // targeting. The publisher already display()ed these. + // Publisher-owned slots can be refreshed on SPA navigation. On initial + // load their first request is held until the targeting below is applied. var slotsToRefresh = []; + var isInitialAdInit = !ts.gptInitialAdInitCompleted; // Element IDs of slots TS defined itself. GPT requires display() to // register/render a freshly-defined slot; refresh() alone no-ops for a // slot that was never displayed, so these are display()ed instead. var slotsToDisplay = []; + var hasAppliedTargeting = false; slots.forEach(function (slot) { // Resolve actual div ID: exact match first, then safe prefix scan. // div_id in config may be a stable prefix (e.g. "ad-header-0-") when @@ -245,6 +304,7 @@ }); // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts s.setTargeting("ts_initial", "1"); + hasAppliedTargeting = true; // Map the resolved inner div to the slot ID. This bootstrap fires no // beacons and registers no slotRenderEnded listener; the map is consumed // by the bundle's render bridge (index.ts) once it loads. @@ -257,34 +317,36 @@ newSlots.push(s); var displayId = s.getSlotElementId() || actualDivId; slotsToDisplay.push(displayId); - } else { + } else if (!isInitialAdInit) { slotsToRefresh.push(s); } }); ts.prevGptSlots = newSlots; ts.divToSlotId = divToSlotId; - if (!ts.servicesEnabled) { + var heldPublisherRequests = isInitialAdInit + ? takeInitialPublisherRequests(googletag.pubads()) + : { displayIds: [], refreshSlots: [] }; + ts.gptInitialAdInitCompleted = true; + if (!ts.servicesEnabled && (hasAppliedTargeting || heldPublisherRequests.displayIds.length > 0 || heldPublisherRequests.refreshSlots.length > 0)) { googletag.pubads().enableSingleRequest(); googletag.enableServices(); ts.servicesEnabled = true; } - // Register and render TS-defined slots. GPT requires display() for a - // freshly-defined slot; without it the slot no-ops and misses its - // impression. Runs after enableServices(); on SPA navigation services are - // already enabled, so this runs unconditionally for new slots. - slotsToDisplay.forEach(function (divId) { + // Register/render TS-defined slots and replay publisher displays held + // before server-side bids were available. The replay is the publisher's + // one initial request, not a later TS refresh. + heldPublisherRequests.displayIds.concat(slotsToDisplay).forEach(function (divId) { runHandoffInternal(function () { googletag.display(divId); }); }); - // Reused publisher-owned slots always need a refresh to pick up the - // server-side targeting. TS-defined slots are fetched by display() above - // unless the publisher disabled initial load, in which case display() only - // registers them and refresh() must request the ad — otherwise they render - // blank. Only add them in that case to avoid double-requesting. - var slotsNeedingRefresh = ts.gptInitialLoadDisabled - ? slotsToRefresh.concat(newSlots) - : slotsToRefresh; + // Replay held publisher refreshes after targeting. On SPA navigation TS + // refreshes reused publisher slots as before; TS-defined slots need a + // refresh only when initial load was disabled. + var slotsNeedingRefresh = heldPublisherRequests.refreshSlots.concat( + slotsToRefresh, + ts.gptInitialLoadDisabled ? newSlots : [], + ); if (slotsNeedingRefresh.length > 0) { // One-shot bypass: this internal refresh delivers the just-applied // server-side targeting to GAM. If slim-Prebid has already wrapped diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index cd25133d1..2ced11086 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -77,6 +77,13 @@ export interface GptSlotHandoff { suppressPublisherRefresh: boolean; } +/** Publisher requests held until initial TS targeting has been applied. */ +export interface GptInitialRequestGate { + pendingDisplays: Record; + pendingRefreshes: Record; + released: boolean; +} + export interface TsjsApi { version: string; que: Array<() => void>; @@ -137,6 +144,10 @@ export interface TsjsApi { gptInitialLoadDisabled?: boolean; /** Late publisher claims for TS-created GPT slots, keyed by actual div ID. */ gptSlotHandoffs?: Record; + /** Publisher initial requests held until TS has applied server-side targeting. */ + gptInitialRequestGate?: GptInitialRequestGate; + /** True after the first page-load `adInit()` has handled publisher slots. */ + gptInitialAdInitCompleted?: boolean; /** True only while TS calls a GPT function that the handoff wrappers observe. */ gptSlotHandoffInternal?: boolean; /** Guards SPA pushState hook installation. */ diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index 8853997c3..effae7ada 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -1,5 +1,11 @@ import { log } from '../../core/log'; -import type { AuctionSlot, AuctionBidData, GptSlotHandoff, TsjsApi } from '../../core/types'; +import type { + AuctionSlot, + AuctionBidData, + GptInitialRequestGate, + GptSlotHandoff, + TsjsApi, +} from '../../core/types'; import { installGptGuard } from './script_guard'; @@ -460,6 +466,41 @@ function handoffForSlot(ts: TsjsApi, slot: GoogleTagSlot): GptSlotHandoff | unde return ts.gptSlotHandoffs?.[slot.getSlotElementId()]; } +function configuredSlotForElementId(ts: TsjsApi, elementId: string): AuctionSlot | undefined { + return ts.adSlots?.find( + (slot) => + !!slot.div_id && + (elementId === slot.div_id || elementId.startsWith(slot.div_id)) && + !elementId.endsWith('-container') + ); +} + +function initialRequestGate(ts: TsjsApi): GptInitialRequestGate { + return (ts.gptInitialRequestGate ??= { + pendingDisplays: {}, + pendingRefreshes: {}, + released: false, + }); +} + +function takeInitialPublisherRequests( + ts: TsjsApi, + pubads: GoogleTagPubAdsService +): { displayIds: string[]; refreshSlots: GoogleTagSlot[] } { + const gate = initialRequestGate(ts); + if (gate.released) return { displayIds: [], refreshSlots: [] }; + + gate.released = true; + const displayIds = Object.keys(gate.pendingDisplays); + const refreshIds = new Set(Object.keys(gate.pendingRefreshes)); + gate.pendingDisplays = {}; + gate.pendingRefreshes = {}; + const refreshSlots = (pubads.getSlots?.() ?? []).filter((slot) => + refreshIds.has(slot.getSlotElementId()) + ); + return { displayIds, refreshSlots }; +} + function withGptSlotHandoffInternal(ts: TsjsApi, callback: () => T): T { const wasInternal = ts.gptSlotHandoffInternal; ts.gptSlotHandoffInternal = true; @@ -537,6 +578,15 @@ function installLatePublisherSlotHandoff(ts: TsjsApi): void { handoff.suppressPublisherDisplay = false; return; } + const gate = initialRequestGate(ts); + if ( + !ts.gptSlotHandoffInternal && + !gate.released && + configuredSlotForElementId(ts, elementId) + ) { + gate.pendingDisplays[elementId] = true; + return; + } originalDisplay(elementId); }; (patchedDisplay as HandoffPatchedFunction).__tsSlotHandoffPatched = true; @@ -559,12 +609,21 @@ function installLatePublisherSlotHandoff(ts: TsjsApi): void { } let suppressed = false; + const gate = initialRequestGate(ts); const remainingSlots = slots.filter((slot) => { const handoff = handoffForSlot(ts, slot); - if (!handoff?.suppressPublisherRefresh) return true; - handoff.suppressPublisherRefresh = false; - suppressed = true; - return false; + if (handoff?.suppressPublisherRefresh) { + handoff.suppressPublisherRefresh = false; + suppressed = true; + return false; + } + const elementId = slot.getSlotElementId(); + if (!gate.released && configuredSlotForElementId(ts, elementId)) { + gate.pendingRefreshes[elementId] = true; + suppressed = true; + return false; + } + return true; }); if (!suppressed) { originalRefresh(requestedSlots); @@ -601,14 +660,17 @@ export function installTsAdInit(): void { // Slots TS defined itself — tracked for SPA destroy. Publisher-owned // slots are reused but never destroyed by TS on navigation. const newSlots: GoogleTagSlot[] = []; - // Publisher-owned slots TS reused — refreshed to pick up server-side - // targeting. The publisher already display()ed these. + // Publisher-owned slots can be refreshed on SPA navigation. On initial + // load their first request is held by the head-installed gate and replayed + // only after the targeting below has been applied. const slotsToRefresh: GoogleTagSlot[] = []; + const isInitialAdInit = !ts.gptInitialAdInitCompleted; // Element IDs of slots TS defined itself this call. GPT requires a // display() call to register/render a freshly-defined slot; refresh() // alone no-ops for a slot that was never displayed, so these are // display()ed instead of refreshed. const slotsToDisplay: string[] = []; + let hasAppliedTargeting = false; const divToSlotId: Record = {}; const prevSlotTargetingKeys = ts.prevSlotTargetingKeys ?? {}; const nextSlotTargetingKeys: Record = {}; @@ -682,6 +744,7 @@ export function installTsAdInit(): void { if (bid[key]) gptSlot.setTargeting(key, String(bid[key]!)); }); gptSlot.setTargeting(TS_INITIAL_TARGETING_KEY, '1'); + hasAppliedTargeting = true; // Map the resolved inner div to the slot ID so slotRenderEnded and ADM // injection address the same, single GPT slot. divToSlotId[actualDivId] = slot.id; @@ -692,7 +755,7 @@ export function installTsAdInit(): void { if (tsOwned) { newSlots.push(gptSlot); slotsToDisplay.push(slotDivId2); - } else { + } else if (!isInitialAdInit) { slotsToRefresh.push(gptSlot); } @@ -709,11 +772,20 @@ export function installTsAdInit(): void { // Replace (not merge) so destroyed slots from previous navigation don't linger. ts.divToSlotId = divToSlotId; ts.prevSlotTargetingKeys = nextSlotTargetingKeys; - - // Whether this call produced any TS slot to render. A gated page-bids - // response (auction kill switch or consent denial) returns no slots, so - // the loops above leave these empty. - const hasRenderableWork = slotsToDisplay.length > 0 || slotsToRefresh.length > 0; + const heldPublisherRequests = isInitialAdInit + ? takeInitialPublisherRequests(ts, g.pubads!()) + : { displayIds: [], refreshSlots: [] }; + ts.gptInitialAdInitCompleted = true; + + // Whether this call produced a request to make. A gated page-bids response + // (auction kill switch or consent denial) returns no slots, so the loops + // above leave these empty. + const hasRenderableWork = + slotsToDisplay.length > 0 || + slotsToRefresh.length > 0 || + heldPublisherRequests.displayIds.length > 0 || + heldPublisherRequests.refreshSlots.length > 0 || + hasAppliedTargeting; // enableSingleRequest and enableServices must only be called once per page // load. Skip activating GPT services when TS has nothing to display or @@ -742,25 +814,22 @@ export function installTsAdInit(): void { }); } - // Register and render TS-defined slots. GPT requires display() for a - // freshly-defined slot — without it the slot no-ops ("defineSlot was - // called without a matching display call") and misses its impression. - // Must run after enableServices(); on SPA navigation services are already - // enabled, so this runs unconditionally for any newly-defined slots. - slotsToDisplay.forEach((divId) => withGptSlotHandoffInternal(ts, () => g.display?.(divId))); - - // Slots needing an explicit ad request via refresh(). Reused - // publisher-owned slots always need one to pick up the just-applied - // server-side targeting. TS-defined slots are normally fetched by the - // display() above — but when the publisher called - // pubads().disableInitialLoad(), display() only registers the slot and the - // ad request must come from refresh(). Without this, a TS-owned - // first-impression slot renders blank on initial-load-disabled pages. Only - // add them in that case; otherwise display() + refresh() would - // double-request the impression. - const slotsNeedingRefresh = ts.gptInitialLoadDisabled - ? slotsToRefresh.concat(newSlots) - : slotsToRefresh; + // Register/render TS-defined slots and replay publisher displays held + // before the server-side bids were available. The gate is released only + // after targeting has been applied, so this remains the publisher's one + // initial request rather than a later TS refresh. + heldPublisherRequests.displayIds + .concat(slotsToDisplay) + .forEach((divId) => withGptSlotHandoffInternal(ts, () => g.display?.(divId))); + + // Slots needing an explicit ad request via refresh(). Publisher refreshes + // held on the initial page load are replayed after targeting. On SPA + // navigation TS refreshes reused publisher slots as before. TS-defined + // slots need a refresh only when the publisher disabled initial load. + const slotsNeedingRefresh = heldPublisherRequests.refreshSlots.concat( + slotsToRefresh, + ts.gptInitialLoadDisabled ? newSlots : [] + ); if (slotsNeedingRefresh.length > 0) { // One-shot bypass: this internal refresh delivers the just-applied diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index f99d90fa7..bdfc7123b 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -77,7 +77,7 @@ describe('installTsAdInit', () => { document.getElementById("ad'prefix-real")?.remove(); }); - it('reads window.tsjs.bids synchronously and applies bid targeting before refresh', async () => { + it('reads window.tsjs.bids synchronously without re-requesting an existing publisher slot', async () => { const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), @@ -133,11 +133,130 @@ describe('installTsAdInit', () => { expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_cache_host', 'cache.example.com'); expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_cache_path', '/pbc/v1/cache'); expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); - expect(mockPubads.refresh).toHaveBeenCalled(); + expect(mockPubads.refresh).not.toHaveBeenCalled(); fetchSpy.mockRestore(); }); + it('holds and replays a publisher display once after applying initial targeting', async () => { + const requests: string[] = []; + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const nativeDisplay = vi.fn((elementId: string) => requests.push(elementId)); + const nativeRefresh = vi.fn(); + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([mockSlot]), + addEventListener: vi.fn(), + refresh: nativeRefresh, + }; + const googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + display: nativeDisplay, + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).googletag = googletag; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: { pos: 'atf' }, + }, + ], + bids: { atf_sidebar_ad: { hb_pb: '1.00' } }, + }; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + + googletag.display('div-atf-sidebar'); + expect(nativeDisplay).not.toHaveBeenCalled(); + + (window as TestWindow).tsjs!.adInit!(); + + expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00'); + expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); + expect(nativeDisplay).toHaveBeenCalledTimes(1); + expect(requests).toEqual(['div-atf-sidebar']); + expect(nativeRefresh).not.toHaveBeenCalled(); + }); + + it('holds and replays a disabled-load publisher refresh once after targeting', async () => { + const requests: string[] = []; + let initialLoadDisabled = false; + const mockSlot = { + addService: vi.fn().mockReturnThis(), + setTargeting: vi.fn().mockReturnThis(), + getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), + getTargeting: vi.fn().mockReturnValue([]), + }; + const unrelatedSlot = { + getSlotElementId: vi.fn().mockReturnValue('div-unrelated'), + }; + const nativeDisplay = vi.fn((elementId: string) => { + if (!initialLoadDisabled) requests.push(elementId); + }); + const nativeRefresh = vi.fn((slots?: Array) => { + (slots ?? [mockSlot, unrelatedSlot]).forEach((slot) => + requests.push(slot.getSlotElementId()) + ); + }); + const mockPubads = { + enableSingleRequest: vi.fn(), + getSlots: vi.fn().mockReturnValue([mockSlot, unrelatedSlot]), + addEventListener: vi.fn(), + refresh: nativeRefresh, + disableInitialLoad: vi.fn(() => { + initialLoadDisabled = true; + }), + }; + const googletag = { + cmd: { push: vi.fn((fn: () => void) => fn()) }, + defineSlot: vi.fn().mockReturnValue(mockSlot), + display: nativeDisplay, + pubads: vi.fn().mockReturnValue(mockPubads), + enableServices: vi.fn(), + }; + (window as TestWindow).googletag = googletag; + (window as TestWindow).tsjs = { + adSlots: [ + { + id: 'atf_sidebar_ad', + gam_unit_path: '/123/atf', + div_id: 'div-atf-sidebar', + formats: [[300, 250]], + targeting: {}, + }, + ], + bids: { atf_sidebar_ad: { hb_pb: '1.00' } }, + }; + + const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); + installTsAdInit(); + mockPubads.disableInitialLoad(); + googletag.display('div-atf-sidebar'); + mockPubads.refresh(); + expect(nativeDisplay).not.toHaveBeenCalled(); + expect(nativeRefresh).toHaveBeenCalledWith([unrelatedSlot]); + + (window as TestWindow).tsjs!.adInit!(); + + expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00'); + expect(nativeDisplay).toHaveBeenCalledTimes(1); + expect(nativeRefresh).toHaveBeenCalledTimes(2); + expect(nativeRefresh).toHaveBeenLastCalledWith([mockSlot]); + expect(requests).toEqual(['div-unrelated', 'div-atf-sidebar']); + }); + it('displays TS-defined slots and does not include them in refresh', async () => { const mockSlot = { addService: vi.fn().mockReturnThis(), @@ -444,6 +563,9 @@ describe('installTsAdInit', () => { }, ], bids: {}, + // This models a route update: existing publisher slots are refreshed on + // SPA navigation, while initial-load publisher slots are not re-requested. + gptInitialAdInitCompleted: true, // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; @@ -589,7 +711,7 @@ describe('installTsAdInit', () => { expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_bidder', 'mocktioneer'); expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_adid', 'debug-uuid'); expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); - expect(mockPubads.refresh).toHaveBeenCalledWith([mockSlot]); + expect(mockPubads.refresh).not.toHaveBeenCalled(); }); it('does not fire win/billing beacons from slotRenderEnded targeting alone', async () => { @@ -914,7 +1036,7 @@ describe('installTsAdInit', () => { delete (window as TestWindow).apstag; }); - it('calls refresh even when tsjs.bids is empty (graceful fallback)', async () => { + it('does not re-request an existing publisher slot when tsjs.bids is empty', async () => { const emptyTestSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), @@ -953,7 +1075,7 @@ describe('installTsAdInit', () => { installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); - expect(mockPubads.refresh).toHaveBeenCalled(); + expect(mockPubads.refresh).not.toHaveBeenCalled(); }); it('resolves dynamic div prefixes without interpolating div_id into a CSS selector', async () => { @@ -996,7 +1118,7 @@ describe('installTsAdInit', () => { installTsAdInit(); expect(() => (window as TestWindow).tsjs!.adInit!()).not.toThrow(); - expect(mockPubads.refresh).toHaveBeenCalledWith([dynamicSlot]); + expect(mockPubads.refresh).not.toHaveBeenCalled(); }); }); diff --git a/docs/superpowers/plans/2026-07-24-prevent-duplicate-gpt-slot-requests.md b/docs/superpowers/plans/2026-07-24-prevent-duplicate-gpt-slot-requests.md index 4699e3c42..24879c8e4 100644 --- a/docs/superpowers/plans/2026-07-24-prevent-duplicate-gpt-slot-requests.md +++ b/docs/superpowers/plans/2026-07-24-prevent-duplicate-gpt-slot-requests.md @@ -1,6 +1,7 @@ # Prevent Duplicate GPT Slot Requests — Implementation Plan -> **Status:** Implemented locally; production-like browser validation remains pending. +> **Status:** Revised after production-like validation found a second request for +> publisher-owned slots when hydration-safe scheduling defers `adInit()`. > > **Spec:** `docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md` @@ -8,10 +9,11 @@ TS `adInit()` runs before a publisher later defines the placement's inner GPT div. **Architecture:** TS creates its fallback on the resolved inner div and records a -handoff claim. Narrow, idempotent wrappers around GPT's `defineSlot`, `display`, and -`pubads().refresh` alias a matching late publisher definition to that slot and -suppress only the duplicate initial publisher request. A successful handoff transfers -SPA-destruction ownership to the publisher. The head bootstrap and full TSJS bundle +handoff claim. Narrow, idempotent GPT wrappers also gate a configured publisher +slot's first `display`/`refresh` while the server auction result is unavailable. At +`adInit()`, TS applies targeting to that same publisher slot and replays the held +native request once; it does not issue a second TS refresh. Late-definition handoff +and SPA ownership transfer remain unchanged. The head bootstrap and full TSJS bundle share this runtime protocol through `window.tsjs`. **Primary files:** @@ -43,9 +45,9 @@ share this runtime protocol through `window.tsjs`. - Modify `crates/trusted-server-js/lib/src/core/types.ts` - Modify `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` -- [ ] Add a `TsjsApi` property for a div-ID-keyed handoff registry. Each entry must - retain serializable lifecycle flags: TS-created, ownership-transferred, initial - request made, and one-shot publisher display/refresh suppression state. +- [ ] Add `TsjsApi` state for both the div-ID-keyed late-handoff registry and an + initial publisher-request gate. The gate records held display/refresh IDs and + a released marker so it applies only once per page load. - [ ] Add only the minimal optional/internal type surface needed for idempotence markers on GPT functions and `pubads`. Do not weaken the public GPT types with `any`. @@ -81,14 +83,14 @@ npx vitest run test/integrations/gpt/ad_init.test.ts test/integrations/gpt/index returning it; - log, but do not create a second slot, if publisher arguments differ from the TS configuration. -- [ ] `display` wrapper: consume the one permitted publisher post-handoff display - call without invoking native `display`; pass every other call through unchanged. -- [ ] `refresh` wrapper: when initial load was disabled, consume the one permitted - post-handoff refresh for each claimed slot. If called with no slot list, expand - `getSlots()`, filter only the claimed slots, and forward the remaining slots - explicitly. Preserve all unrelated refreshes. -- [ ] Ensure wrapper installation precedes the fallback definition path and does not - change existing publisher-owned-slot behavior. +- [ ] `display` wrapper: consume the one permitted post-handoff display; before the + first `adInit()`, also hold a configured publisher slot's native display. +- [ ] `refresh` wrapper: consume one permitted post-handoff disabled-load refresh; + before the first `adInit()`, hold configured publisher refreshes and forward + all unrelated slots explicitly, including a no-argument/global refresh. +- [ ] At initial `adInit()`, apply targeting then replay held native calls; never + refresh an existing publisher-owned slot that has already requested. +- [ ] Ensure wrapper installation precedes publisher setup and fallback creation. **Focused checks:** @@ -136,8 +138,9 @@ npx vitest run test/integrations/gpt/ad_init.test.ts makes one request; the publisher's first refresh cannot make a second request. - [ ] Add a no-argument publisher refresh test containing an unrelated slot. Assert the claimed slot is suppressed once and the unrelated slot is refreshed. -- [ ] Add an already publisher-owned test proving TS does not install a claim, applies - targeting, and refreshes that slot. +- [ ] Add publisher-owned tests proving TS holds normal and disabled-load initial + requests, applies targeting, and replays exactly one native request. Also prove + an already-requested publisher slot is not refreshed again. - [ ] Add a no-publisher test proving TS still creates, displays, and requests its inner-div slot exactly once. - [ ] Add a SPA handoff test: after late publisher claim, the next `adInit()` does not @@ -153,8 +156,8 @@ npx vitest run test/integrations/gpt/ad_init.test.ts - Modify `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` - Modify `crates/trusted-server-core/src/integrations/gpt.rs` -- [ ] Port the same actual-inner-div fallback, registry names, lifecycle flags, and - idempotence markers to the plain-JavaScript bootstrap. +- [ ] Port the same initial-request gate, actual-inner-div fallback, registry names, + lifecycle flags, and idempotence markers to the plain-JavaScript bootstrap. - [ ] Use the existing bootstrap `window.tsjs` properties exactly so `index.ts` can adopt the initial claim after the bundle loads. - [ ] Ensure its internal definition/display/refresh calls use the same guards as the @@ -198,9 +201,10 @@ npx vitest run test/integrations/gpt/ad_init.test.ts format. - [ ] Review the diff specifically for bootstrap/bundle protocol drift and for any use of container IDs in GPT slot creation. -- [ ] In a controlled production-like browser capture, verify one initial request for - each affected visible placement and independently verify an unrelated placement - remains requestable. +- [ ] In a controlled production-like browser capture with the hydration-safe + deferred `adInit()` path, verify one targeted initial request for each affected + visible placement and independently verify an unrelated placement remains + requestable. - [ ] Update issue #944 with the ownership-handoff decision, test evidence, and browser-capture result. diff --git a/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md b/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md index 770718199..c94e25b2c 100644 --- a/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md +++ b/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md @@ -8,6 +8,12 @@ slot on the outer `-container` element. The publisher subsequently defines and displays an inner-div slot. These are distinct GPT slots, so they make separate GAM requests for one visible placement. +A production deployment also exposed the inverse ordering: the hydration-safe +body bootstrap delays `adInit()` until after `window.load`, so publisher code can +already have defined **and requested** its inner-div slot. In that ordering, +reusing the slot and refreshing it applies targeting too late and creates a second +SRA request. + The affected paths are deliberately duplicated today: - `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` is the full bundle @@ -40,7 +46,7 @@ A fix must keep both implementations in sync. publisher-owned slot from a placement that the publisher will never define. - General interception of unrelated GPT slots. -## Decision: one inner-div slot with late-definition handoff +## Decision: inner-div fallback, late-definition handoff, and an initial request gate TS will define its fallback slot on the **actual inner div**, never on its outer `-container` element. It will record a narrowly scoped handoff claim keyed by that @@ -54,31 +60,36 @@ competing container slot and an invalid duplicate definition. ### Lifecycle -1. **Already publisher-owned** — `getSlots()` finds a slot for the resolved inner - div. TS applies targeting, records it as publisher-owned, and refreshes it as it - does today. -2. **No slot yet** — TS defines a slot on the resolved inner div, applies targeting, +1. **Publisher-owned before bids are available** — a scoped head-installed gate + holds the configured placement's first publisher `display()` or `refresh()`. + At `adInit()`, TS finds the publisher slot, applies targeting, and replays that + held native call exactly once. It never adds a second TS refresh. +2. **Already-requested publisher-owned slot** — if a configured publisher request + was not observed by the gate, TS applies targeting for later lifecycle work but + does not re-request the already-served initial impression. +3. **No slot yet** — TS defines a slot on the resolved inner div, applies targeting, enables services when needed, and displays it. When initial load is disabled, TS performs its existing one explicit refresh. TS records this slot as TS-owned and handoff-eligible. -3. **Publisher defines later** — the scoped `defineSlot` wrapper sees the recorded +4. **Publisher defines later** — the scoped `defineSlot` wrapper sees the recorded inner-div claim, returns the existing slot, and transfers ownership: it removes the slot from TS's future `destroySlots()` set. The publisher's setup continues against that same slot. -4. **Publisher's first request call** — the wrapper suppresses the duplicate +5. **Publisher's first request call after a late handoff** — the wrapper suppresses the duplicate publisher `display()` call. With `disableInitialLoad()`, it instead suppresses only the publisher's first refresh for the transferred slot, because TS has already issued the required initial refresh. For a no-argument/global refresh, the wrapper must expand `getSlots()`, remove only the one-shot suppressed slots, and forward the remaining slots explicitly so unrelated slots still refresh. -5. **Later refreshes and SPA navigation** — after the one-shot suppression is +6. **Later refreshes and SPA navigation** — after the one-shot suppression is consumed, publisher refreshes are untouched. On navigation, TS clears its targeting from the shared slot and may reuse it for the next route; it must not destroy a slot after ownership has transferred. -The wrapper is not a global deduplicator. It only handles IDs present in TS's -handoff registry and must preserve native `defineSlot`, `display`, and `refresh` -behavior for every other placement. +The wrappers are not global deduplicators. The initial request gate only holds the +first `display`/`refresh` for a configured placement until initial TS targeting is +available; handoff suppression only handles IDs present in TS's handoff registry. +All unrelated GPT calls retain native behavior. ## Implementation shape @@ -89,8 +100,10 @@ can read after the bundle replaces the bootstrap implementation. It is keyed by resolved actual div ID and records at least: - whether TS created the slot and whether ownership has transferred; -- whether one publisher `display()` or initial-load-disabled `refresh()` remains to - suppress. +- whether one post-handoff publisher `display()` or initial-load-disabled `refresh()` + remains to suppress; +- configured publisher displays and refreshes held before initial targeting, plus a + released marker so the gate applies only once per page load. Do not rely only on module-local state: the bootstrap can define the initial slot before `index.ts` is loaded. Look up the live slot by element ID through @@ -109,8 +122,12 @@ In `crates/trusted-server-js/lib/src/integrations/gpt/index.ts`: - Replace the container fallback with `actualDivId`. - Add the typed handoff-registry state to `TsjsApi` in `crates/trusted-server-js/lib/src/core/types.ts`. -- Install the idempotent `defineSlot`, `display`, and `pubads().refresh` handoff - wrappers from the GPT command queue before `adInit()` can create a fallback slot. +- Install idempotent `defineSlot`, `display`, and `pubads().refresh` wrappers from + the GPT command queue before publisher setup. The latter two also hold the first + configured publisher request until `adInit()` has applied initial targeting. +- Replay held initial publisher displays/refreshes after targeting rather than + refreshing an existing publisher-owned slot. Retain reused-slot refreshes only for + later SPA navigations. - When a late publisher definition is aliased to the existing slot, remove it from `prevGptSlots` and mark it transferred before returning it. - Keep targeting cleanup keyed by the real inner div. Remove the old dual @@ -132,7 +149,7 @@ suite must exercise both implementations' observable contract. | Risk | Mitigation | | -------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | Publisher passes a different ad-unit path or sizes in its late `defineSlot` call | Return the existing claimed slot but log a diagnostic. Do not define a second slot. Treat the TS configuration and publisher configuration mismatch as an integration error to resolve separately. | -| Publisher invokes global `refresh()` after `disableInitialLoad()` | Filter the one-shot claimed slot from the expanded slot list and refresh all remaining slots. A no-argument refresh must not be silently dropped. | +| Publisher invokes global `refresh()` before bids after `disableInitialLoad()` | Filter only configured held slots from the expanded list, forward unrelated slots immediately, then replay the held slots once after targeting. A no-argument refresh must not be silently dropped. | | Publisher calls a legitimate refresh without an initial display | The one-shot suppression is consumed only immediately after a successful late handoff. Document and test the standard publisher sequence (`defineSlot` → `addService` → `display`, with `refresh` when initial load is disabled). Escalate unusual publisher lifecycle requirements rather than adding a time heuristic. | | Publisher-owned slot is destroyed on SPA navigation | Transfer ownership synchronously in the `defineSlot` wrapper and remove the slot from `prevGptSlots`. | | Bootstrap and bundle diverge | Give both paths the same black-box regression cases; retain a Rust source-contract assertion for bootstrap-specific sentinels. | @@ -146,7 +163,9 @@ suite must exercise both implementations' observable contract. initial-load-disabled modes. - The late publisher `display()` (and its first initial-load-disabled refresh) cannot create a second request, while unrelated slots retain their normal calls. -- Existing publisher slots are still reused and receive TS targeting. +- A configured publisher slot whose first request occurs before the deferred + `adInit()` is held, receives TS targeting, and makes exactly one replayed native + request. An already-requested publisher slot is never re-requested by TS. - A slot that no publisher claims is displayed and requested once by TS. - A transferred slot is absent from TS's SPA `destroySlots()` argument; targeting is still cleared and reapplied correctly on the next route. @@ -160,6 +179,7 @@ suite must exercise both implementations' observable contract. 2. Run the focused GPT test files, then the full TSJS Vitest suite and formatter. 3. Run the target-matched Rust test suite so the included bootstrap and its source assertions compile and pass. -4. In a controlled browser capture, verify that one configured header and one - configured fixed placement each produce one initial slot request, while a distinct - in-content placement remains independently requestable. +4. In a controlled browser capture with deferred `adInit()`, verify that one + configured header and one configured fixed placement each produce one initial + slot request with TS targeting, while a distinct in-content placement remains + independently requestable. From f3c1e6bcbefcfc20144d874945d5277587612de1 Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 24 Jul 2026 16:17:31 -0500 Subject: [PATCH 121/198] Revert "Gate publisher GPT requests until targeting is ready" This reverts commit b65e1aedd33440a281cf28443c0ffd6e46b9de02. --- .../src/integrations/gpt.rs | 7 +- .../src/integrations/gpt_bootstrap.js | 108 +++----------- .../trusted-server-js/lib/src/core/types.ts | 11 -- .../lib/src/integrations/gpt/index.ts | 133 +++++------------ .../lib/test/integrations/gpt/ad_init.test.ts | 134 +----------------- ...-24-prevent-duplicate-gpt-slot-requests.md | 50 +++---- ...vent-duplicate-gpt-slot-requests-design.md | 60 +++----- 7 files changed, 105 insertions(+), 398 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index 20e4a9492..f53a21cf0 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -474,8 +474,7 @@ impl IntegrationHeadInjector for GptIntegration { /// ## Scroll / refresh handoff contract (Phase 1) /// /// `tsjs.adInit` handles **initial render only**: it wires server-side bid - /// targeting into GPT slots and replays only publisher requests held until - /// that targeting was available. Win/billing beacons fire + /// targeting into GPT slots and refreshes them. Win/billing beacons fire /// only from the TS render bridge in the JS bundle, where a matching /// Prebid Universal Creative request proves the TS creative rendered. /// It does **not** trigger refresh auctions or handle GPT slot refresh events. @@ -1242,10 +1241,6 @@ mod tests { combined.contains("__tsSlotHandoffPatched"), "bootstrap should install idempotent GPT handoff wrappers" ); - assert!( - combined.contains("gptInitialRequestGate") && combined.contains("pendingDisplays"), - "bootstrap should hold configured publisher requests until initial targeting is applied" - ); assert!( combined.contains("return googletag.defineSlot") && combined.contains("actualDivId"), "bootstrap should define the TS fallback on the actual inner div" diff --git a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js index 257da5908..0c2697357 100644 --- a/crates/trusted-server-core/src/integrations/gpt_bootstrap.js +++ b/crates/trusted-server-core/src/integrations/gpt_bootstrap.js @@ -10,8 +10,8 @@ // - Both implementations must set `window.tsjs.servicesEnabled = true` // after calling `enableSingleRequest()`/`enableServices()` so a // subsequent call becomes a no-op. -// - `refresh()` is called only for TS-defined slots in this pass and -// publisher requests the initial gate held, never the global slot list. +// - `refresh()` is called only for the slots defined in this pass, +// never the global slot list. // // Only installed if `window.tsjs.adInit` isn't already defined. (function () { @@ -51,45 +51,6 @@ ); } - function configuredSlotForElementId(elementId) { - return (ts.adSlots || []).find(function (slot) { - return ( - slot.div_id && - (elementId === slot.div_id || elementId.startsWith(slot.div_id)) && - !elementId.endsWith("-container") - ); - }); - } - - function initialRequestGate() { - if (!ts.gptInitialRequestGate) { - ts.gptInitialRequestGate = { - pendingDisplays: {}, - pendingRefreshes: {}, - released: false, - }; - } - return ts.gptInitialRequestGate; - } - - function takeInitialPublisherRequests(pubads) { - var gate = initialRequestGate(); - if (gate.released) return { displayIds: [], refreshSlots: [] }; - - gate.released = true; - var displayIds = Object.keys(gate.pendingDisplays); - var refreshIds = Object.keys(gate.pendingRefreshes); - gate.pendingDisplays = {}; - gate.pendingRefreshes = {}; - var slots = pubads.getSlots ? pubads.getSlots() : []; - return { - displayIds: displayIds, - refreshSlots: slots.filter(function (slot) { - return refreshIds.includes(slot.getSlotElementId()); - }), - }; - } - function runHandoffInternal(callback) { var wasInternal = ts.gptSlotHandoffInternal; ts.gptSlotHandoffInternal = true; @@ -160,15 +121,6 @@ handoff.suppressPublisherDisplay = false; return; } - var gate = initialRequestGate(); - if ( - !ts.gptSlotHandoffInternal && - !gate.released && - configuredSlotForElementId(elementId) - ) { - gate.pendingDisplays[elementId] = true; - return; - } originalDisplay(elementId); }; patchedDisplay.__tsSlotHandoffPatched = true; @@ -189,22 +141,13 @@ return; } var suppressed = false; - var gate = initialRequestGate(); var remainingSlots = slots.filter(function (slot) { var handoff = ts.gptSlotHandoffs && ts.gptSlotHandoffs[slot.getSlotElementId()]; - if (handoff && handoff.suppressPublisherRefresh) { - handoff.suppressPublisherRefresh = false; - suppressed = true; - return false; - } - var elementId = slot.getSlotElementId(); - if (!gate.released && configuredSlotForElementId(elementId)) { - gate.pendingRefreshes[elementId] = true; - suppressed = true; - return false; - } - return true; + if (!handoff || !handoff.suppressPublisherRefresh) return true; + handoff.suppressPublisherRefresh = false; + suppressed = true; + return false; }); if (!suppressed) { originalRefresh(requestedSlots); @@ -229,15 +172,13 @@ // Slots TS defined itself — tracked for SPA destroy. Publisher-owned // slots are reused but never destroyed by TS on navigation. var newSlots = []; - // Publisher-owned slots can be refreshed on SPA navigation. On initial - // load their first request is held until the targeting below is applied. + // Publisher-owned slots TS reused — refreshed to pick up server-side + // targeting. The publisher already display()ed these. var slotsToRefresh = []; - var isInitialAdInit = !ts.gptInitialAdInitCompleted; // Element IDs of slots TS defined itself. GPT requires display() to // register/render a freshly-defined slot; refresh() alone no-ops for a // slot that was never displayed, so these are display()ed instead. var slotsToDisplay = []; - var hasAppliedTargeting = false; slots.forEach(function (slot) { // Resolve actual div ID: exact match first, then safe prefix scan. // div_id in config may be a stable prefix (e.g. "ad-header-0-") when @@ -304,7 +245,6 @@ }); // Keep in sync with TS_INITIAL_TARGETING_KEY in index.ts s.setTargeting("ts_initial", "1"); - hasAppliedTargeting = true; // Map the resolved inner div to the slot ID. This bootstrap fires no // beacons and registers no slotRenderEnded listener; the map is consumed // by the bundle's render bridge (index.ts) once it loads. @@ -317,36 +257,34 @@ newSlots.push(s); var displayId = s.getSlotElementId() || actualDivId; slotsToDisplay.push(displayId); - } else if (!isInitialAdInit) { + } else { slotsToRefresh.push(s); } }); ts.prevGptSlots = newSlots; ts.divToSlotId = divToSlotId; - var heldPublisherRequests = isInitialAdInit - ? takeInitialPublisherRequests(googletag.pubads()) - : { displayIds: [], refreshSlots: [] }; - ts.gptInitialAdInitCompleted = true; - if (!ts.servicesEnabled && (hasAppliedTargeting || heldPublisherRequests.displayIds.length > 0 || heldPublisherRequests.refreshSlots.length > 0)) { + if (!ts.servicesEnabled) { googletag.pubads().enableSingleRequest(); googletag.enableServices(); ts.servicesEnabled = true; } - // Register/render TS-defined slots and replay publisher displays held - // before server-side bids were available. The replay is the publisher's - // one initial request, not a later TS refresh. - heldPublisherRequests.displayIds.concat(slotsToDisplay).forEach(function (divId) { + // Register and render TS-defined slots. GPT requires display() for a + // freshly-defined slot; without it the slot no-ops and misses its + // impression. Runs after enableServices(); on SPA navigation services are + // already enabled, so this runs unconditionally for new slots. + slotsToDisplay.forEach(function (divId) { runHandoffInternal(function () { googletag.display(divId); }); }); - // Replay held publisher refreshes after targeting. On SPA navigation TS - // refreshes reused publisher slots as before; TS-defined slots need a - // refresh only when initial load was disabled. - var slotsNeedingRefresh = heldPublisherRequests.refreshSlots.concat( - slotsToRefresh, - ts.gptInitialLoadDisabled ? newSlots : [], - ); + // Reused publisher-owned slots always need a refresh to pick up the + // server-side targeting. TS-defined slots are fetched by display() above + // unless the publisher disabled initial load, in which case display() only + // registers them and refresh() must request the ad — otherwise they render + // blank. Only add them in that case to avoid double-requesting. + var slotsNeedingRefresh = ts.gptInitialLoadDisabled + ? slotsToRefresh.concat(newSlots) + : slotsToRefresh; if (slotsNeedingRefresh.length > 0) { // One-shot bypass: this internal refresh delivers the just-applied // server-side targeting to GAM. If slim-Prebid has already wrapped diff --git a/crates/trusted-server-js/lib/src/core/types.ts b/crates/trusted-server-js/lib/src/core/types.ts index 2ced11086..cd25133d1 100644 --- a/crates/trusted-server-js/lib/src/core/types.ts +++ b/crates/trusted-server-js/lib/src/core/types.ts @@ -77,13 +77,6 @@ export interface GptSlotHandoff { suppressPublisherRefresh: boolean; } -/** Publisher requests held until initial TS targeting has been applied. */ -export interface GptInitialRequestGate { - pendingDisplays: Record; - pendingRefreshes: Record; - released: boolean; -} - export interface TsjsApi { version: string; que: Array<() => void>; @@ -144,10 +137,6 @@ export interface TsjsApi { gptInitialLoadDisabled?: boolean; /** Late publisher claims for TS-created GPT slots, keyed by actual div ID. */ gptSlotHandoffs?: Record; - /** Publisher initial requests held until TS has applied server-side targeting. */ - gptInitialRequestGate?: GptInitialRequestGate; - /** True after the first page-load `adInit()` has handled publisher slots. */ - gptInitialAdInitCompleted?: boolean; /** True only while TS calls a GPT function that the handoff wrappers observe. */ gptSlotHandoffInternal?: boolean; /** Guards SPA pushState hook installation. */ diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index effae7ada..8853997c3 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -1,11 +1,5 @@ import { log } from '../../core/log'; -import type { - AuctionSlot, - AuctionBidData, - GptInitialRequestGate, - GptSlotHandoff, - TsjsApi, -} from '../../core/types'; +import type { AuctionSlot, AuctionBidData, GptSlotHandoff, TsjsApi } from '../../core/types'; import { installGptGuard } from './script_guard'; @@ -466,41 +460,6 @@ function handoffForSlot(ts: TsjsApi, slot: GoogleTagSlot): GptSlotHandoff | unde return ts.gptSlotHandoffs?.[slot.getSlotElementId()]; } -function configuredSlotForElementId(ts: TsjsApi, elementId: string): AuctionSlot | undefined { - return ts.adSlots?.find( - (slot) => - !!slot.div_id && - (elementId === slot.div_id || elementId.startsWith(slot.div_id)) && - !elementId.endsWith('-container') - ); -} - -function initialRequestGate(ts: TsjsApi): GptInitialRequestGate { - return (ts.gptInitialRequestGate ??= { - pendingDisplays: {}, - pendingRefreshes: {}, - released: false, - }); -} - -function takeInitialPublisherRequests( - ts: TsjsApi, - pubads: GoogleTagPubAdsService -): { displayIds: string[]; refreshSlots: GoogleTagSlot[] } { - const gate = initialRequestGate(ts); - if (gate.released) return { displayIds: [], refreshSlots: [] }; - - gate.released = true; - const displayIds = Object.keys(gate.pendingDisplays); - const refreshIds = new Set(Object.keys(gate.pendingRefreshes)); - gate.pendingDisplays = {}; - gate.pendingRefreshes = {}; - const refreshSlots = (pubads.getSlots?.() ?? []).filter((slot) => - refreshIds.has(slot.getSlotElementId()) - ); - return { displayIds, refreshSlots }; -} - function withGptSlotHandoffInternal(ts: TsjsApi, callback: () => T): T { const wasInternal = ts.gptSlotHandoffInternal; ts.gptSlotHandoffInternal = true; @@ -578,15 +537,6 @@ function installLatePublisherSlotHandoff(ts: TsjsApi): void { handoff.suppressPublisherDisplay = false; return; } - const gate = initialRequestGate(ts); - if ( - !ts.gptSlotHandoffInternal && - !gate.released && - configuredSlotForElementId(ts, elementId) - ) { - gate.pendingDisplays[elementId] = true; - return; - } originalDisplay(elementId); }; (patchedDisplay as HandoffPatchedFunction).__tsSlotHandoffPatched = true; @@ -609,21 +559,12 @@ function installLatePublisherSlotHandoff(ts: TsjsApi): void { } let suppressed = false; - const gate = initialRequestGate(ts); const remainingSlots = slots.filter((slot) => { const handoff = handoffForSlot(ts, slot); - if (handoff?.suppressPublisherRefresh) { - handoff.suppressPublisherRefresh = false; - suppressed = true; - return false; - } - const elementId = slot.getSlotElementId(); - if (!gate.released && configuredSlotForElementId(ts, elementId)) { - gate.pendingRefreshes[elementId] = true; - suppressed = true; - return false; - } - return true; + if (!handoff?.suppressPublisherRefresh) return true; + handoff.suppressPublisherRefresh = false; + suppressed = true; + return false; }); if (!suppressed) { originalRefresh(requestedSlots); @@ -660,17 +601,14 @@ export function installTsAdInit(): void { // Slots TS defined itself — tracked for SPA destroy. Publisher-owned // slots are reused but never destroyed by TS on navigation. const newSlots: GoogleTagSlot[] = []; - // Publisher-owned slots can be refreshed on SPA navigation. On initial - // load their first request is held by the head-installed gate and replayed - // only after the targeting below has been applied. + // Publisher-owned slots TS reused — refreshed to pick up server-side + // targeting. The publisher already display()ed these. const slotsToRefresh: GoogleTagSlot[] = []; - const isInitialAdInit = !ts.gptInitialAdInitCompleted; // Element IDs of slots TS defined itself this call. GPT requires a // display() call to register/render a freshly-defined slot; refresh() // alone no-ops for a slot that was never displayed, so these are // display()ed instead of refreshed. const slotsToDisplay: string[] = []; - let hasAppliedTargeting = false; const divToSlotId: Record = {}; const prevSlotTargetingKeys = ts.prevSlotTargetingKeys ?? {}; const nextSlotTargetingKeys: Record = {}; @@ -744,7 +682,6 @@ export function installTsAdInit(): void { if (bid[key]) gptSlot.setTargeting(key, String(bid[key]!)); }); gptSlot.setTargeting(TS_INITIAL_TARGETING_KEY, '1'); - hasAppliedTargeting = true; // Map the resolved inner div to the slot ID so slotRenderEnded and ADM // injection address the same, single GPT slot. divToSlotId[actualDivId] = slot.id; @@ -755,7 +692,7 @@ export function installTsAdInit(): void { if (tsOwned) { newSlots.push(gptSlot); slotsToDisplay.push(slotDivId2); - } else if (!isInitialAdInit) { + } else { slotsToRefresh.push(gptSlot); } @@ -772,20 +709,11 @@ export function installTsAdInit(): void { // Replace (not merge) so destroyed slots from previous navigation don't linger. ts.divToSlotId = divToSlotId; ts.prevSlotTargetingKeys = nextSlotTargetingKeys; - const heldPublisherRequests = isInitialAdInit - ? takeInitialPublisherRequests(ts, g.pubads!()) - : { displayIds: [], refreshSlots: [] }; - ts.gptInitialAdInitCompleted = true; - - // Whether this call produced a request to make. A gated page-bids response - // (auction kill switch or consent denial) returns no slots, so the loops - // above leave these empty. - const hasRenderableWork = - slotsToDisplay.length > 0 || - slotsToRefresh.length > 0 || - heldPublisherRequests.displayIds.length > 0 || - heldPublisherRequests.refreshSlots.length > 0 || - hasAppliedTargeting; + + // Whether this call produced any TS slot to render. A gated page-bids + // response (auction kill switch or consent denial) returns no slots, so + // the loops above leave these empty. + const hasRenderableWork = slotsToDisplay.length > 0 || slotsToRefresh.length > 0; // enableSingleRequest and enableServices must only be called once per page // load. Skip activating GPT services when TS has nothing to display or @@ -814,22 +742,25 @@ export function installTsAdInit(): void { }); } - // Register/render TS-defined slots and replay publisher displays held - // before the server-side bids were available. The gate is released only - // after targeting has been applied, so this remains the publisher's one - // initial request rather than a later TS refresh. - heldPublisherRequests.displayIds - .concat(slotsToDisplay) - .forEach((divId) => withGptSlotHandoffInternal(ts, () => g.display?.(divId))); - - // Slots needing an explicit ad request via refresh(). Publisher refreshes - // held on the initial page load are replayed after targeting. On SPA - // navigation TS refreshes reused publisher slots as before. TS-defined - // slots need a refresh only when the publisher disabled initial load. - const slotsNeedingRefresh = heldPublisherRequests.refreshSlots.concat( - slotsToRefresh, - ts.gptInitialLoadDisabled ? newSlots : [] - ); + // Register and render TS-defined slots. GPT requires display() for a + // freshly-defined slot — without it the slot no-ops ("defineSlot was + // called without a matching display call") and misses its impression. + // Must run after enableServices(); on SPA navigation services are already + // enabled, so this runs unconditionally for any newly-defined slots. + slotsToDisplay.forEach((divId) => withGptSlotHandoffInternal(ts, () => g.display?.(divId))); + + // Slots needing an explicit ad request via refresh(). Reused + // publisher-owned slots always need one to pick up the just-applied + // server-side targeting. TS-defined slots are normally fetched by the + // display() above — but when the publisher called + // pubads().disableInitialLoad(), display() only registers the slot and the + // ad request must come from refresh(). Without this, a TS-owned + // first-impression slot renders blank on initial-load-disabled pages. Only + // add them in that case; otherwise display() + refresh() would + // double-request the impression. + const slotsNeedingRefresh = ts.gptInitialLoadDisabled + ? slotsToRefresh.concat(newSlots) + : slotsToRefresh; if (slotsNeedingRefresh.length > 0) { // One-shot bypass: this internal refresh delivers the just-applied diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index bdfc7123b..f99d90fa7 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -77,7 +77,7 @@ describe('installTsAdInit', () => { document.getElementById("ad'prefix-real")?.remove(); }); - it('reads window.tsjs.bids synchronously without re-requesting an existing publisher slot', async () => { + it('reads window.tsjs.bids synchronously and applies bid targeting before refresh', async () => { const mockSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), @@ -133,130 +133,11 @@ describe('installTsAdInit', () => { expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_cache_host', 'cache.example.com'); expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_cache_path', '/pbc/v1/cache'); expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); - expect(mockPubads.refresh).not.toHaveBeenCalled(); + expect(mockPubads.refresh).toHaveBeenCalled(); fetchSpy.mockRestore(); }); - it('holds and replays a publisher display once after applying initial targeting', async () => { - const requests: string[] = []; - const mockSlot = { - addService: vi.fn().mockReturnThis(), - setTargeting: vi.fn().mockReturnThis(), - getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), - getTargeting: vi.fn().mockReturnValue([]), - }; - const nativeDisplay = vi.fn((elementId: string) => requests.push(elementId)); - const nativeRefresh = vi.fn(); - const mockPubads = { - enableSingleRequest: vi.fn(), - getSlots: vi.fn().mockReturnValue([mockSlot]), - addEventListener: vi.fn(), - refresh: nativeRefresh, - }; - const googletag = { - cmd: { push: vi.fn((fn: () => void) => fn()) }, - defineSlot: vi.fn().mockReturnValue(mockSlot), - display: nativeDisplay, - pubads: vi.fn().mockReturnValue(mockPubads), - enableServices: vi.fn(), - }; - (window as TestWindow).googletag = googletag; - (window as TestWindow).tsjs = { - adSlots: [ - { - id: 'atf_sidebar_ad', - gam_unit_path: '/123/atf', - div_id: 'div-atf-sidebar', - formats: [[300, 250]], - targeting: { pos: 'atf' }, - }, - ], - bids: { atf_sidebar_ad: { hb_pb: '1.00' } }, - }; - - const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); - installTsAdInit(); - - googletag.display('div-atf-sidebar'); - expect(nativeDisplay).not.toHaveBeenCalled(); - - (window as TestWindow).tsjs!.adInit!(); - - expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00'); - expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); - expect(nativeDisplay).toHaveBeenCalledTimes(1); - expect(requests).toEqual(['div-atf-sidebar']); - expect(nativeRefresh).not.toHaveBeenCalled(); - }); - - it('holds and replays a disabled-load publisher refresh once after targeting', async () => { - const requests: string[] = []; - let initialLoadDisabled = false; - const mockSlot = { - addService: vi.fn().mockReturnThis(), - setTargeting: vi.fn().mockReturnThis(), - getSlotElementId: vi.fn().mockReturnValue('div-atf-sidebar'), - getTargeting: vi.fn().mockReturnValue([]), - }; - const unrelatedSlot = { - getSlotElementId: vi.fn().mockReturnValue('div-unrelated'), - }; - const nativeDisplay = vi.fn((elementId: string) => { - if (!initialLoadDisabled) requests.push(elementId); - }); - const nativeRefresh = vi.fn((slots?: Array) => { - (slots ?? [mockSlot, unrelatedSlot]).forEach((slot) => - requests.push(slot.getSlotElementId()) - ); - }); - const mockPubads = { - enableSingleRequest: vi.fn(), - getSlots: vi.fn().mockReturnValue([mockSlot, unrelatedSlot]), - addEventListener: vi.fn(), - refresh: nativeRefresh, - disableInitialLoad: vi.fn(() => { - initialLoadDisabled = true; - }), - }; - const googletag = { - cmd: { push: vi.fn((fn: () => void) => fn()) }, - defineSlot: vi.fn().mockReturnValue(mockSlot), - display: nativeDisplay, - pubads: vi.fn().mockReturnValue(mockPubads), - enableServices: vi.fn(), - }; - (window as TestWindow).googletag = googletag; - (window as TestWindow).tsjs = { - adSlots: [ - { - id: 'atf_sidebar_ad', - gam_unit_path: '/123/atf', - div_id: 'div-atf-sidebar', - formats: [[300, 250]], - targeting: {}, - }, - ], - bids: { atf_sidebar_ad: { hb_pb: '1.00' } }, - }; - - const { installTsAdInit } = await import('../../../src/integrations/gpt/index'); - installTsAdInit(); - mockPubads.disableInitialLoad(); - googletag.display('div-atf-sidebar'); - mockPubads.refresh(); - expect(nativeDisplay).not.toHaveBeenCalled(); - expect(nativeRefresh).toHaveBeenCalledWith([unrelatedSlot]); - - (window as TestWindow).tsjs!.adInit!(); - - expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_pb', '1.00'); - expect(nativeDisplay).toHaveBeenCalledTimes(1); - expect(nativeRefresh).toHaveBeenCalledTimes(2); - expect(nativeRefresh).toHaveBeenLastCalledWith([mockSlot]); - expect(requests).toEqual(['div-unrelated', 'div-atf-sidebar']); - }); - it('displays TS-defined slots and does not include them in refresh', async () => { const mockSlot = { addService: vi.fn().mockReturnThis(), @@ -563,9 +444,6 @@ describe('installTsAdInit', () => { }, ], bids: {}, - // This models a route update: existing publisher slots are refreshed on - // SPA navigation, while initial-load publisher slots are not re-requested. - gptInitialAdInitCompleted: true, // eslint-disable-next-line @typescript-eslint/no-explicit-any } as any; @@ -711,7 +589,7 @@ describe('installTsAdInit', () => { expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_bidder', 'mocktioneer'); expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_adid', 'debug-uuid'); expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); - expect(mockPubads.refresh).not.toHaveBeenCalled(); + expect(mockPubads.refresh).toHaveBeenCalledWith([mockSlot]); }); it('does not fire win/billing beacons from slotRenderEnded targeting alone', async () => { @@ -1036,7 +914,7 @@ describe('installTsAdInit', () => { delete (window as TestWindow).apstag; }); - it('does not re-request an existing publisher slot when tsjs.bids is empty', async () => { + it('calls refresh even when tsjs.bids is empty (graceful fallback)', async () => { const emptyTestSlot = { addService: vi.fn().mockReturnThis(), setTargeting: vi.fn().mockReturnThis(), @@ -1075,7 +953,7 @@ describe('installTsAdInit', () => { installTsAdInit(); (window as TestWindow).tsjs!.adInit!(); - expect(mockPubads.refresh).not.toHaveBeenCalled(); + expect(mockPubads.refresh).toHaveBeenCalled(); }); it('resolves dynamic div prefixes without interpolating div_id into a CSS selector', async () => { @@ -1118,7 +996,7 @@ describe('installTsAdInit', () => { installTsAdInit(); expect(() => (window as TestWindow).tsjs!.adInit!()).not.toThrow(); - expect(mockPubads.refresh).not.toHaveBeenCalled(); + expect(mockPubads.refresh).toHaveBeenCalledWith([dynamicSlot]); }); }); diff --git a/docs/superpowers/plans/2026-07-24-prevent-duplicate-gpt-slot-requests.md b/docs/superpowers/plans/2026-07-24-prevent-duplicate-gpt-slot-requests.md index 24879c8e4..4699e3c42 100644 --- a/docs/superpowers/plans/2026-07-24-prevent-duplicate-gpt-slot-requests.md +++ b/docs/superpowers/plans/2026-07-24-prevent-duplicate-gpt-slot-requests.md @@ -1,7 +1,6 @@ # Prevent Duplicate GPT Slot Requests — Implementation Plan -> **Status:** Revised after production-like validation found a second request for -> publisher-owned slots when hydration-safe scheduling defers `adInit()`. +> **Status:** Implemented locally; production-like browser validation remains pending. > > **Spec:** `docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md` @@ -9,11 +8,10 @@ TS `adInit()` runs before a publisher later defines the placement's inner GPT div. **Architecture:** TS creates its fallback on the resolved inner div and records a -handoff claim. Narrow, idempotent GPT wrappers also gate a configured publisher -slot's first `display`/`refresh` while the server auction result is unavailable. At -`adInit()`, TS applies targeting to that same publisher slot and replays the held -native request once; it does not issue a second TS refresh. Late-definition handoff -and SPA ownership transfer remain unchanged. The head bootstrap and full TSJS bundle +handoff claim. Narrow, idempotent wrappers around GPT's `defineSlot`, `display`, and +`pubads().refresh` alias a matching late publisher definition to that slot and +suppress only the duplicate initial publisher request. A successful handoff transfers +SPA-destruction ownership to the publisher. The head bootstrap and full TSJS bundle share this runtime protocol through `window.tsjs`. **Primary files:** @@ -45,9 +43,9 @@ share this runtime protocol through `window.tsjs`. - Modify `crates/trusted-server-js/lib/src/core/types.ts` - Modify `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` -- [ ] Add `TsjsApi` state for both the div-ID-keyed late-handoff registry and an - initial publisher-request gate. The gate records held display/refresh IDs and - a released marker so it applies only once per page load. +- [ ] Add a `TsjsApi` property for a div-ID-keyed handoff registry. Each entry must + retain serializable lifecycle flags: TS-created, ownership-transferred, initial + request made, and one-shot publisher display/refresh suppression state. - [ ] Add only the minimal optional/internal type surface needed for idempotence markers on GPT functions and `pubads`. Do not weaken the public GPT types with `any`. @@ -83,14 +81,14 @@ npx vitest run test/integrations/gpt/ad_init.test.ts test/integrations/gpt/index returning it; - log, but do not create a second slot, if publisher arguments differ from the TS configuration. -- [ ] `display` wrapper: consume the one permitted post-handoff display; before the - first `adInit()`, also hold a configured publisher slot's native display. -- [ ] `refresh` wrapper: consume one permitted post-handoff disabled-load refresh; - before the first `adInit()`, hold configured publisher refreshes and forward - all unrelated slots explicitly, including a no-argument/global refresh. -- [ ] At initial `adInit()`, apply targeting then replay held native calls; never - refresh an existing publisher-owned slot that has already requested. -- [ ] Ensure wrapper installation precedes publisher setup and fallback creation. +- [ ] `display` wrapper: consume the one permitted publisher post-handoff display + call without invoking native `display`; pass every other call through unchanged. +- [ ] `refresh` wrapper: when initial load was disabled, consume the one permitted + post-handoff refresh for each claimed slot. If called with no slot list, expand + `getSlots()`, filter only the claimed slots, and forward the remaining slots + explicitly. Preserve all unrelated refreshes. +- [ ] Ensure wrapper installation precedes the fallback definition path and does not + change existing publisher-owned-slot behavior. **Focused checks:** @@ -138,9 +136,8 @@ npx vitest run test/integrations/gpt/ad_init.test.ts makes one request; the publisher's first refresh cannot make a second request. - [ ] Add a no-argument publisher refresh test containing an unrelated slot. Assert the claimed slot is suppressed once and the unrelated slot is refreshed. -- [ ] Add publisher-owned tests proving TS holds normal and disabled-load initial - requests, applies targeting, and replays exactly one native request. Also prove - an already-requested publisher slot is not refreshed again. +- [ ] Add an already publisher-owned test proving TS does not install a claim, applies + targeting, and refreshes that slot. - [ ] Add a no-publisher test proving TS still creates, displays, and requests its inner-div slot exactly once. - [ ] Add a SPA handoff test: after late publisher claim, the next `adInit()` does not @@ -156,8 +153,8 @@ npx vitest run test/integrations/gpt/ad_init.test.ts - Modify `crates/trusted-server-core/src/integrations/gpt_bootstrap.js` - Modify `crates/trusted-server-core/src/integrations/gpt.rs` -- [ ] Port the same initial-request gate, actual-inner-div fallback, registry names, - lifecycle flags, and idempotence markers to the plain-JavaScript bootstrap. +- [ ] Port the same actual-inner-div fallback, registry names, lifecycle flags, and + idempotence markers to the plain-JavaScript bootstrap. - [ ] Use the existing bootstrap `window.tsjs` properties exactly so `index.ts` can adopt the initial claim after the bundle loads. - [ ] Ensure its internal definition/display/refresh calls use the same guards as the @@ -201,10 +198,9 @@ npx vitest run test/integrations/gpt/ad_init.test.ts format. - [ ] Review the diff specifically for bootstrap/bundle protocol drift and for any use of container IDs in GPT slot creation. -- [ ] In a controlled production-like browser capture with the hydration-safe - deferred `adInit()` path, verify one targeted initial request for each affected - visible placement and independently verify an unrelated placement remains - requestable. +- [ ] In a controlled production-like browser capture, verify one initial request for + each affected visible placement and independently verify an unrelated placement + remains requestable. - [ ] Update issue #944 with the ownership-handoff decision, test evidence, and browser-capture result. diff --git a/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md b/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md index c94e25b2c..770718199 100644 --- a/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md +++ b/docs/superpowers/specs/2026-07-24-prevent-duplicate-gpt-slot-requests-design.md @@ -8,12 +8,6 @@ slot on the outer `-container` element. The publisher subsequently defines and displays an inner-div slot. These are distinct GPT slots, so they make separate GAM requests for one visible placement. -A production deployment also exposed the inverse ordering: the hydration-safe -body bootstrap delays `adInit()` until after `window.load`, so publisher code can -already have defined **and requested** its inner-div slot. In that ordering, -reusing the slot and refreshing it applies targeting too late and creates a second -SRA request. - The affected paths are deliberately duplicated today: - `crates/trusted-server-js/lib/src/integrations/gpt/index.ts` is the full bundle @@ -46,7 +40,7 @@ A fix must keep both implementations in sync. publisher-owned slot from a placement that the publisher will never define. - General interception of unrelated GPT slots. -## Decision: inner-div fallback, late-definition handoff, and an initial request gate +## Decision: one inner-div slot with late-definition handoff TS will define its fallback slot on the **actual inner div**, never on its outer `-container` element. It will record a narrowly scoped handoff claim keyed by that @@ -60,36 +54,31 @@ competing container slot and an invalid duplicate definition. ### Lifecycle -1. **Publisher-owned before bids are available** — a scoped head-installed gate - holds the configured placement's first publisher `display()` or `refresh()`. - At `adInit()`, TS finds the publisher slot, applies targeting, and replays that - held native call exactly once. It never adds a second TS refresh. -2. **Already-requested publisher-owned slot** — if a configured publisher request - was not observed by the gate, TS applies targeting for later lifecycle work but - does not re-request the already-served initial impression. -3. **No slot yet** — TS defines a slot on the resolved inner div, applies targeting, +1. **Already publisher-owned** — `getSlots()` finds a slot for the resolved inner + div. TS applies targeting, records it as publisher-owned, and refreshes it as it + does today. +2. **No slot yet** — TS defines a slot on the resolved inner div, applies targeting, enables services when needed, and displays it. When initial load is disabled, TS performs its existing one explicit refresh. TS records this slot as TS-owned and handoff-eligible. -4. **Publisher defines later** — the scoped `defineSlot` wrapper sees the recorded +3. **Publisher defines later** — the scoped `defineSlot` wrapper sees the recorded inner-div claim, returns the existing slot, and transfers ownership: it removes the slot from TS's future `destroySlots()` set. The publisher's setup continues against that same slot. -5. **Publisher's first request call after a late handoff** — the wrapper suppresses the duplicate +4. **Publisher's first request call** — the wrapper suppresses the duplicate publisher `display()` call. With `disableInitialLoad()`, it instead suppresses only the publisher's first refresh for the transferred slot, because TS has already issued the required initial refresh. For a no-argument/global refresh, the wrapper must expand `getSlots()`, remove only the one-shot suppressed slots, and forward the remaining slots explicitly so unrelated slots still refresh. -6. **Later refreshes and SPA navigation** — after the one-shot suppression is +5. **Later refreshes and SPA navigation** — after the one-shot suppression is consumed, publisher refreshes are untouched. On navigation, TS clears its targeting from the shared slot and may reuse it for the next route; it must not destroy a slot after ownership has transferred. -The wrappers are not global deduplicators. The initial request gate only holds the -first `display`/`refresh` for a configured placement until initial TS targeting is -available; handoff suppression only handles IDs present in TS's handoff registry. -All unrelated GPT calls retain native behavior. +The wrapper is not a global deduplicator. It only handles IDs present in TS's +handoff registry and must preserve native `defineSlot`, `display`, and `refresh` +behavior for every other placement. ## Implementation shape @@ -100,10 +89,8 @@ can read after the bundle replaces the bootstrap implementation. It is keyed by resolved actual div ID and records at least: - whether TS created the slot and whether ownership has transferred; -- whether one post-handoff publisher `display()` or initial-load-disabled `refresh()` - remains to suppress; -- configured publisher displays and refreshes held before initial targeting, plus a - released marker so the gate applies only once per page load. +- whether one publisher `display()` or initial-load-disabled `refresh()` remains to + suppress. Do not rely only on module-local state: the bootstrap can define the initial slot before `index.ts` is loaded. Look up the live slot by element ID through @@ -122,12 +109,8 @@ In `crates/trusted-server-js/lib/src/integrations/gpt/index.ts`: - Replace the container fallback with `actualDivId`. - Add the typed handoff-registry state to `TsjsApi` in `crates/trusted-server-js/lib/src/core/types.ts`. -- Install idempotent `defineSlot`, `display`, and `pubads().refresh` wrappers from - the GPT command queue before publisher setup. The latter two also hold the first - configured publisher request until `adInit()` has applied initial targeting. -- Replay held initial publisher displays/refreshes after targeting rather than - refreshing an existing publisher-owned slot. Retain reused-slot refreshes only for - later SPA navigations. +- Install the idempotent `defineSlot`, `display`, and `pubads().refresh` handoff + wrappers from the GPT command queue before `adInit()` can create a fallback slot. - When a late publisher definition is aliased to the existing slot, remove it from `prevGptSlots` and mark it transferred before returning it. - Keep targeting cleanup keyed by the real inner div. Remove the old dual @@ -149,7 +132,7 @@ suite must exercise both implementations' observable contract. | Risk | Mitigation | | -------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | Publisher passes a different ad-unit path or sizes in its late `defineSlot` call | Return the existing claimed slot but log a diagnostic. Do not define a second slot. Treat the TS configuration and publisher configuration mismatch as an integration error to resolve separately. | -| Publisher invokes global `refresh()` before bids after `disableInitialLoad()` | Filter only configured held slots from the expanded list, forward unrelated slots immediately, then replay the held slots once after targeting. A no-argument refresh must not be silently dropped. | +| Publisher invokes global `refresh()` after `disableInitialLoad()` | Filter the one-shot claimed slot from the expanded slot list and refresh all remaining slots. A no-argument refresh must not be silently dropped. | | Publisher calls a legitimate refresh without an initial display | The one-shot suppression is consumed only immediately after a successful late handoff. Document and test the standard publisher sequence (`defineSlot` → `addService` → `display`, with `refresh` when initial load is disabled). Escalate unusual publisher lifecycle requirements rather than adding a time heuristic. | | Publisher-owned slot is destroyed on SPA navigation | Transfer ownership synchronously in the `defineSlot` wrapper and remove the slot from `prevGptSlots`. | | Bootstrap and bundle diverge | Give both paths the same black-box regression cases; retain a Rust source-contract assertion for bootstrap-specific sentinels. | @@ -163,9 +146,7 @@ suite must exercise both implementations' observable contract. initial-load-disabled modes. - The late publisher `display()` (and its first initial-load-disabled refresh) cannot create a second request, while unrelated slots retain their normal calls. -- A configured publisher slot whose first request occurs before the deferred - `adInit()` is held, receives TS targeting, and makes exactly one replayed native - request. An already-requested publisher slot is never re-requested by TS. +- Existing publisher slots are still reused and receive TS targeting. - A slot that no publisher claims is displayed and requested once by TS. - A transferred slot is absent from TS's SPA `destroySlots()` argument; targeting is still cleared and reapplied correctly on the next route. @@ -179,7 +160,6 @@ suite must exercise both implementations' observable contract. 2. Run the focused GPT test files, then the full TSJS Vitest suite and formatter. 3. Run the target-matched Rust test suite so the included bootstrap and its source assertions compile and pass. -4. In a controlled browser capture with deferred `adInit()`, verify that one - configured header and one configured fixed placement each produce one initial - slot request with TS targeting, while a distinct in-content placement remains - independently requestable. +4. In a controlled browser capture, verify that one configured header and one + configured fixed placement each produce one initial slot request, while a distinct + in-content placement remains independently requestable. From 001ad385c5c67b18b4de963bf1b233c57e793370 Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Fri, 24 Jul 2026 15:04:53 -0700 Subject: [PATCH 122/198] Decouple the prebid tsjs shim from the bundled Prebid.js The external bundle is now pure Prebid.js (core, consent and user ID modules, client-side bid adapters) and stamps a manifest on window.__tsjs_prebid_bundle. The shim ships as a server-served deferred tsjs module that installs the trustedServer adapter onto the window.pbjs global via public APIs only, so shim fixes deploy with the server instead of requiring an external bundle re-upload. --- .../src/integrations/prebid.rs | 6 +- .../src/integrations/registry.rs | 22 ++-- crates/trusted-server-core/src/publisher.rs | 6 +- crates/trusted-server-core/src/tsjs.rs | 11 +- crates/trusted-server-js/lib/build-all.mjs | 11 +- .../lib/build-prebid-external.mjs | 40 ++++++- .../lib/src/integrations/prebid/index.ts | 110 ++++++++++++------ .../test/integrations/prebid/index.test.ts | 98 +++++++++------- docs/guide/integrations/prebid.md | 12 +- 9 files changed, 208 insertions(+), 108 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index a1b40281d..c15248fb0 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -921,7 +921,7 @@ pub fn register( .with_proxy(integration.clone()) .with_attribute_rewriter(integration.clone()) .with_head_injector(integration) - .without_js() + .with_deferred_js() .build(), )) } @@ -2937,8 +2937,8 @@ passphrase = "test-secret-key-32-bytes-minimum" "External prebid bundle route should be injected" ); assert!( - !processed.contains("tsjs-prebid.min.js"), - "Embedded deferred prebid bundle should not be injected" + processed.contains("tsjs-prebid.min.js"), + "Deferred tsjs prebid shim should be injected" ); } diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 6f1b4dcfd..23e83d1de 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1949,7 +1949,7 @@ mod tests { } #[test] - fn js_module_ids_exclude_prebid_and_include_core_js_only_modules() { + fn js_module_ids_defer_prebid_and_include_core_js_only_modules() { let settings = crate::test_support::tests::create_test_settings(); let mut settings_with_prebid = settings; settings_with_prebid @@ -1975,8 +1975,8 @@ mod tests { let deferred = registry.js_module_ids_deferred(); assert!( - !all.contains(&"prebid"), - "should not include prebid in embedded TSJS module IDs" + all.contains(&"prebid"), + "should include the prebid shim in embedded TSJS module IDs" ); assert!( immediate.contains(&"creative"), @@ -1991,8 +1991,8 @@ mod tests { "should not include prebid in immediate IDs" ); assert!( - !deferred.contains(&"prebid"), - "should not include prebid in deferred IDs" + deferred.contains(&"prebid"), + "should serve the prebid shim as a deferred module" ); } @@ -2077,7 +2077,7 @@ mod tests { } #[test] - fn js_module_ids_exclude_prebid_when_external_bundle_is_configured() { + fn js_module_ids_defer_prebid_shim_when_external_bundle_is_configured() { let mut settings = crate::test_support::tests::create_test_settings(); settings .integrations @@ -2094,16 +2094,16 @@ mod tests { let registry = IntegrationRegistry::new(&settings).expect("should create registry"); assert!( - !registry.js_module_ids().contains(&"prebid"), - "external bundle mode should not include prebid in embedded TSJS modules" + registry.js_module_ids().contains(&"prebid"), + "external bundle mode should include the prebid shim in embedded TSJS modules" ); assert!( !registry.js_module_ids_immediate().contains(&"prebid"), - "external bundle mode should not include prebid in immediate TSJS modules" + "the prebid shim should not load in the immediate TSJS bundle" ); assert!( - !registry.js_module_ids_deferred().contains(&"prebid"), - "external bundle mode should not include prebid in deferred TSJS modules" + registry.js_module_ids_deferred().contains(&"prebid"), + "the prebid shim should load as a deferred TSJS module" ); assert!( registry.has_route(&Method::GET, "/integrations/prebid/bundle.js"), diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4d6e9d2ee..32c4b37ab 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -3796,7 +3796,7 @@ mod tests { } #[test] - fn tsjs_dynamic_does_not_serve_embedded_prebid() { + fn tsjs_dynamic_serves_prebid_shim_when_enabled() { let settings = create_test_settings(); let registry = IntegrationRegistry::new(&settings).expect("should create integration registry"); @@ -3808,8 +3808,8 @@ mod tests { let response = handle_tsjs_dynamic(&req, ®istry).expect("should handle tsjs request"); assert_eq!( response.status(), - StatusCode::NOT_FOUND, - "should not serve embedded prebid module" + StatusCode::OK, + "should serve the deferred prebid shim module when prebid is enabled" ); } diff --git a/crates/trusted-server-core/src/tsjs.rs b/crates/trusted-server-core/src/tsjs.rs index 45aee02eb..a7b4cc2ef 100644 --- a/crates/trusted-server-core/src/tsjs.rs +++ b/crates/trusted-server-core/src/tsjs.rs @@ -192,12 +192,13 @@ mod tests { } #[test] - fn tsjs_deferred_script_src_uses_empty_hash_for_external_or_unknown_module() { - assert_eq!( - tsjs_deferred_script_src("prebid"), - "/static/tsjs=tsjs-prebid.min.js?v=", - "prebid now ships as an external bundle and has no local hash" + fn tsjs_deferred_script_src_hashes_prebid_shim_and_empties_unknown_module() { + let prebid_src = tsjs_deferred_script_src("prebid"); + assert!( + prebid_src.starts_with("/static/tsjs=tsjs-prebid.min.js?v="), + "prebid shim should be served from the deferred tsjs route" ); + assert_sha256_hex_hash(hash_query_value(&prebid_src)); assert_eq!( tsjs_deferred_script_src("unknown-module"), "/static/tsjs=tsjs-unknown-module.min.js?v=", diff --git a/crates/trusted-server-js/lib/build-all.mjs b/crates/trusted-server-js/lib/build-all.mjs index df261bd4f..2bfee01b1 100644 --- a/crates/trusted-server-js/lib/build-all.mjs +++ b/crates/trusted-server-js/lib/build-all.mjs @@ -8,9 +8,10 @@ * tsjs-core.js — core API (always included) * tsjs-.js — one per discovered integration * - * Prebid is intentionally excluded from this embedded build. Use - * build-prebid-external.mjs to generate publisher-specific Prebid bundles - * outside the Cargo build. + * The prebid integration builds here as the tsjs shim only — Prebid.js itself + * is never bundled into tsjs. Use build-prebid-external.mjs to generate the + * pure Prebid.js external bundle (core + adapters + user ID modules) that the + * shim requires at runtime via integrations.prebid.external_bundle_url. */ import fs from 'node:fs'; @@ -34,9 +35,7 @@ const integrationModules = fs.existsSync(integrationsDir) .filter((name) => { const fullPath = path.join(integrationsDir, name); return ( - name !== 'prebid' && - fs.statSync(fullPath).isDirectory() && - fs.existsSync(path.join(fullPath, 'index.ts')) + fs.statSync(fullPath).isDirectory() && fs.existsSync(path.join(fullPath, 'index.ts')) ); }) .sort() diff --git a/crates/trusted-server-js/lib/build-prebid-external.mjs b/crates/trusted-server-js/lib/build-prebid-external.mjs index 4e89723ed..8c343065c 100644 --- a/crates/trusted-server-js/lib/build-prebid-external.mjs +++ b/crates/trusted-server-js/lib/build-prebid-external.mjs @@ -182,9 +182,39 @@ function createTemporaryModulePaths() { temporaryDir, adaptersFile: path.join(temporaryDir, '_adapters.generated.ts'), userIdsFile: path.join(temporaryDir, '_user_ids.generated.ts'), + entryFile: path.join(temporaryDir, '_external_entry.generated.ts'), }; } +function generateExternalEntry(entryFile, adapters) { + const content = [ + '// Auto-generated by build-prebid-external.mjs.', + '//', + '// Pure Prebid.js external bundle: core, consent modules, user ID modules,', + '// and client-side bid adapters. The Trusted Server prebid shim', + '// (tsjs-prebid, served by the server) installs the trustedServer adapter', + '// onto the `window.pbjs` global this bundle populates and drives queue', + '// processing — this bundle intentionally does NOT call processQueue().', + "import 'prebid.js';", + "import 'prebid.js/modules/consentManagementTcf.js';", + "import 'prebid.js/modules/consentManagementGpp.js';", + "import 'prebid.js/modules/consentManagementUsp.js';", + "import 'prebid.js/modules/userId.js';", + "import './_adapters.generated';", + "import { INCLUDED_PREBID_USER_ID_MODULES } from './_user_ids.generated';", + '', + '// Manifest consumed by the tsjs prebid shim to validate that every', + '// configured client_side_bidder has its adapter compiled in.', + '(window as unknown as Record).__tsjs_prebid_bundle = Object.freeze({', + ` adapters: ${JSON.stringify(adapters)},`, + ' userIdModules: INCLUDED_PREBID_USER_ID_MODULES,', + '});', + '', + ].join('\n'); + + fs.writeFileSync(entryFile, content); +} + export function deriveBundleMetadata(bundleBytes) { const sha256 = crypto.createHash('sha256').update(bundleBytes).digest('hex'); const sri = `sha384-${crypto.createHash('sha384').update(bundleBytes).digest('base64')}`; @@ -224,6 +254,13 @@ async function buildExternalBundle(outDir, generatedModules) { 'node_modules/prebid.js/dist/src/src/adapterManager.js' ), }, + { + find: 'prebid.js/src/adRendering.js', + replacement: path.resolve( + __dirname, + 'node_modules/prebid.js/dist/src/src/adRendering.js' + ), + }, ], }, build: { @@ -233,7 +270,7 @@ async function buildExternalBundle(outDir, generatedModules) { sourcemap: false, minify: 'esbuild', rollupOptions: { - input: path.join(prebidDir, 'index.ts'), + input: generatedModules.entryFile, output: { format: 'iife', dir: outDir, @@ -270,6 +307,7 @@ export async function main(argv = process.argv.slice(2)) { try { const adapters = generateAdapterImports(args.adapters, generatedModules.adaptersFile); const userIdModules = generateUserIdImports(args.userIdModules, generatedModules.userIdsFile); + generateExternalEntry(generatedModules.entryFile, adapters); const bundle = await buildExternalBundle(args.outDir, generatedModules); const manifest = { prebidVersion: prebidPackageVersion(), diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 342e4038d..825dfa628 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -11,29 +11,55 @@ // The shim on requestBids injects "trustedServer" into every ad unit so all // bids flow through the orchestrator. -import pbjs from 'prebid.js'; -import adapterManager from 'prebid.js/src/adapterManager.js'; -import 'prebid.js/modules/consentManagementTcf.js'; -import 'prebid.js/modules/consentManagementGpp.js'; -import 'prebid.js/modules/consentManagementUsp.js'; -import 'prebid.js/modules/userId.js'; - -// Client-side bid adapters — self-register with prebid.js on import. -// The external bundle generator aliases these placeholder modules to temporary -// modules built from its --adapters and --user-id-modules options. When a bidder -// is listed in `client_side_bidders` in trusted-server.toml, the requestBids -// shim leaves its bids untouched and the corresponding adapter handles them -// natively in the browser. -import './_adapters.generated'; - import { log } from '../../core/log'; import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; import type { AuctionBid, AuctionEid } from '../../core/auction'; import type { AuctionSlot } from '../../core/types'; +import type _pbjsDefault from 'prebid.js'; -import { INCLUDED_PREBID_USER_ID_MODULES } from './_user_ids.generated'; import { PREBID_USER_ID_MODULE_REGISTRY } from './user_id_modules'; +/** + * Prebid.js public API surface (type-only; erased at build time). + * + * `getUserIdsAsEids` is added by the userId module at runtime, which the base + * package typing does not model. + */ +type PbjsGlobal = typeof _pbjsDefault & { + getUserIdsAsEids?: () => unknown[]; +}; + +// Prebid.js itself is NOT bundled into this module. It is served as the +// external bundle configured via `integrations.prebid.external_bundle_url` +// (required whenever the prebid integration is enabled) and owns the +// `window.pbjs` global. The Rust head injector emits a stub +// (`window.pbjs = window.pbjs || {que:[],cmd:[]}`) before any script runs and +// Prebid.js installs its API onto that same object, so capturing the reference +// at module scope is safe regardless of evaluation order. +const pbjs: PbjsGlobal = ( + typeof window !== 'undefined' + ? // eslint-disable-next-line @typescript-eslint/no-explicit-any + ((window as any).pbjs ??= { que: [], cmd: [] }) + : { que: [], cmd: [] } +) as PbjsGlobal; + +/** + * Manifest stamped on `window.__tsjs_prebid_bundle` by the external Prebid.js + * bundle (see build-prebid-external.mjs): which client-side bid adapters and + * user ID modules were compiled into it. + */ +interface ExternalPrebidBundleManifest { + adapters?: string[]; + userIdModules?: string[]; +} + +function getExternalBundleManifest(): ExternalPrebidBundleManifest | undefined { + if (typeof window === 'undefined') { + return undefined; + } + return (window as { __tsjs_prebid_bundle?: ExternalPrebidBundleManifest }).__tsjs_prebid_bundle; +} + const ADAPTER_CODE = 'trustedServer'; // OpenRTB permits vendor-specific agent types; PAIR uses 571187. // Keep this range aligned with the signed 32-bit Rust/OpenRTB representation. @@ -139,10 +165,11 @@ function readConfiguredUserIdNames(): string[] { } function recordUserIdModuleDiagnostics(): PrebidUserIdDiagnostics { + const includedUserIdModules = getExternalBundleManifest()?.userIdModules ?? []; const configuredUserIdNames = [...new Set(readConfiguredUserIdNames())].sort(); const coveredConfigNames = new Set( PREBID_USER_ID_MODULE_REGISTRY.filter((entry) => - INCLUDED_PREBID_USER_ID_MODULES.includes(entry.moduleName) + includedUserIdModules.includes(entry.moduleName) ).flatMap((entry) => entry.configNames) ); const missingConfiguredUserIdNames = configuredUserIdNames.filter( @@ -150,7 +177,7 @@ function recordUserIdModuleDiagnostics(): PrebidUserIdDiagnostics { ); const diagnostics: PrebidUserIdDiagnostics = { - includedModules: [...INCLUDED_PREBID_USER_ID_MODULES], + includedModules: [...includedUserIdModules], configuredUserIdNames, missingConfiguredUserIdNames, }; @@ -502,6 +529,18 @@ function collectAuctionEids(): AuctionEid[] | undefined { * 2. `config` argument — explicit overrides from the publisher's JS */ export function installPrebidNpm(config?: Partial): typeof pbjs { + // The prebid integration requires the external Prebid.js bundle + // (integrations.prebid.external_bundle_url). When it failed to load (network + // error, SRI mismatch) window.pbjs is still the head-injected stub with no + // API — installing the adapter is impossible, so bail out loudly. + if (typeof (pbjs as { registerBidAdapter?: unknown }).registerBidAdapter !== 'function') { + log.error( + '[tsjs-prebid] window.pbjs has no Prebid.js API — the external Prebid bundle ' + + 'failed to load. Prebid integration disabled.' + ); + return pbjs; + } + const injected = getInjectedConfig(); const merged: PrebidNpmConfig = { endpoint: config?.endpoint, @@ -661,7 +700,7 @@ export function installPrebidNpm(config?: Partial): typeof pbjs opts.bidsBackHandler = function (...args: unknown[]) { syncPrebidEidsCookie(); if (typeof originalBidsBack === 'function') { - originalBidsBack.apply(this, args); + (originalBidsBack as (...handlerArgs: unknown[]) => void).apply(this, args); } }; @@ -682,24 +721,27 @@ export function installPrebidNpm(config?: Partial): typeof pbjs pbjs.processQueue(); recordUserIdModuleDiagnostics(); - // Validate that every client-side bidder has its adapter registered. - // Adapters self-register on import, so a missing adapter means the bidder - // was listed in client_side_bidders but not included in the generated - // external Prebid bundle. Without the adapter the bidder is silently dropped - // from both server-side and client-side auctions. - for (const bidder of clientSideBidders) { - try { - if (!adapterManager.getBidAdapter(bidder)) { + // Validate that every client-side bidder has its adapter compiled into the + // external Prebid.js bundle. The bundle stamps its adapter list on + // window.__tsjs_prebid_bundle; a missing adapter means the bidder was listed + // in client_side_bidders but not included in the generated bundle, so it is + // silently dropped from both server-side and client-side auctions. + const bundledAdapters = getExternalBundleManifest()?.adapters; + if (bundledAdapters === undefined) { + if (clientSideBidders.size > 0) { + log.warn( + '[tsjs-prebid] external Prebid bundle did not stamp an adapter manifest; ' + + 'cannot verify client_side_bidders adapters' + ); + } + } else { + for (const bidder of clientSideBidders) { + if (!bundledAdapters.includes(bidder)) { log.error( - `[tsjs-prebid] client-side bidder "${bidder}" has no adapter loaded. ` + - `Add it to build-prebid-external.mjs --adapters.` + `[tsjs-prebid] client-side bidder "${bidder}" has no adapter in the external ` + + `Prebid bundle. Add it to build-prebid-external.mjs --adapters.` ); } - } catch { - log.error( - `[tsjs-prebid] client-side bidder "${bidder}" has no adapter loaded. ` + - `Add it to build-prebid-external.mjs --adapters.` - ); } } diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 726f40b49..8827ddfea 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -1,6 +1,19 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -// Define mocks using vi.hoisted so they're available inside vi.mock factories +/** + * Default external-bundle manifest for tests. Mirrors what the real external + * Prebid.js bundle stamps on `window.__tsjs_prebid_bundle` (see + * build-prebid-external.mjs). Individual tests override and restore it. + */ +const DEFAULT_BUNDLE_MANIFEST = { + adapters: ['rubicon', 'openx', 'exampleBrowser', 'appnexus'], + userIdModules: ['sharedIdSystem'], +}; + +// Define mocks using vi.hoisted so they exist before the module under test is +// imported. The shim reads Prebid.js from the `window.pbjs` global (owned by +// the external bundle in production), so tests install the mock there instead +// of mocking module imports. const { mockSetConfig, mockProcessQueue, @@ -9,14 +22,11 @@ const { mockGetUserIdsAsEids, mockGetConfig, mockPbjs, - mockGetBidAdapter, - mockAdapterManager, } = vi.hoisted(() => { const mockSetConfig = vi.fn(); const mockProcessQueue = vi.fn(); const mockRequestBids = vi.fn(); const mockRegisterBidAdapter = vi.fn(); - const mockGetBidAdapter = vi.fn(); const mockGetUserIdsAsEids = vi.fn( () => [] as Array<{ source: string; uids?: Array<{ id: string; atype?: number }> }> ); @@ -29,10 +39,20 @@ const { getUserIdsAsEids: mockGetUserIdsAsEids, getConfig: mockGetConfig, adUnits: [] as any[], + que: [] as Array<() => void>, + cmd: [] as Array<() => void>, }; - const mockAdapterManager = { - getBidAdapter: mockGetBidAdapter, + + // Install the mock global BEFORE the shim module evaluates — the shim + // captures `window.pbjs` at module scope. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const w = globalThis.window as any; + w.pbjs = mockPbjs; + w.__tsjs_prebid_bundle = { + adapters: ['rubicon', 'openx', 'exampleBrowser', 'appnexus'], + userIdModules: ['sharedIdSystem'], }; + return { mockSetConfig, mockProcessQueue, @@ -41,28 +61,9 @@ const { mockGetUserIdsAsEids, mockGetConfig, mockPbjs, - mockGetBidAdapter, - mockAdapterManager, }; }); -// Mock prebid.js before importing the module under test. -// The real prebid.js cannot run in jsdom, so we provide a minimal stub. -vi.mock('prebid.js', () => ({ default: mockPbjs })); -vi.mock('prebid.js/src/adapterManager.js', () => ({ default: mockAdapterManager })); - -// Side-effect imports are no-ops in tests -vi.mock('prebid.js/modules/consentManagementTcf.js', () => ({})); -vi.mock('prebid.js/modules/consentManagementGpp.js', () => ({})); -vi.mock('prebid.js/modules/consentManagementUsp.js', () => ({})); -vi.mock('prebid.js/modules/userId.js', () => ({})); - -// Mock the build-generated imports in tests. -vi.mock('../../../src/integrations/prebid/_adapters.generated', () => ({})); -vi.mock('../../../src/integrations/prebid/_user_ids.generated', () => ({ - INCLUDED_PREBID_USER_ID_MODULES: ['sharedIdSystem'], -})); - import { collectBidders, getInjectedConfig, @@ -1410,8 +1411,8 @@ describe('prebid/client-side bidders', () => { mockPbjs.adUnits = []; mockGetUserIdsAsEids.mockReset(); mockGetUserIdsAsEids.mockReturnValue([]); - // By default, pretend all adapters are registered - mockGetBidAdapter.mockReturnValue({}); + // By default the manifest declares all adapters compiled in. + (window as any).__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; delete (window as any).__tsjs_prebid; }); @@ -1558,21 +1559,18 @@ describe('prebid/client-side bidders', () => { expect(tsBid.params.bidderParams).toEqual({}); }); - it('logs error when a client-side bidder has no adapter loaded', () => { - // rubicon is registered, but openx is not - mockGetBidAdapter.mockImplementation((bidder: string) => - bidder === 'rubicon' ? {} : undefined - ); + it('logs error when a client-side bidder has no adapter in the external bundle', () => { + // rubicon is compiled into the external bundle, but openx is not + (window as any).__tsjs_prebid_bundle = { + ...DEFAULT_BUNDLE_MANIFEST, + adapters: ['rubicon'], + }; (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon', 'openx'] }; const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); installPrebidNpm(); - // Should have been called to check both bidders - expect(mockGetBidAdapter).toHaveBeenCalledWith('rubicon'); - expect(mockGetBidAdapter).toHaveBeenCalledWith('openx'); - // Should log an error for the missing adapter. // log.error() uses styled console output: console.error('%c[tsjs]%c ...:', style, reset, ...args) // so the actual message is the 4th argument. @@ -1580,22 +1578,40 @@ describe('prebid/client-side bidders', () => { const hasOpenxError = errorCalls.some((args) => args.some( (a) => - typeof a === 'string' && a.includes('client-side bidder "openx" has no adapter loaded') + typeof a === 'string' && + a.includes('client-side bidder "openx" has no adapter in the external Prebid bundle') ) ); expect(hasOpenxError).toBe(true); - // Should NOT log an error for the registered adapter + // Should NOT log an error for the compiled-in adapter const hasRubiconError = errorCalls.some((args) => args.some((a) => typeof a === 'string' && a.includes('client-side bidder "rubicon"')) ); expect(hasRubiconError).toBe(false); errorSpy.mockRestore(); + (window as any).__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; + }); + + it('warns when the external bundle stamped no adapter manifest', () => { + delete (window as any).__tsjs_prebid_bundle; + (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + + const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); + + installPrebidNpm(); + + const hasManifestWarn = warnSpy.mock.calls.some((args) => + args.some((a) => typeof a === 'string' && a.includes('did not stamp an adapter manifest')) + ); + expect(hasManifestWarn).toBe(true); + + warnSpy.mockRestore(); + (window as any).__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; }); it('does not log errors when all client-side bidders have adapters', () => { - mockGetBidAdapter.mockReturnValue({}); (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); @@ -1603,7 +1619,9 @@ describe('prebid/client-side bidders', () => { installPrebidNpm(); const hasAdapterError = errorSpy.mock.calls.some((args) => - args.some((a) => typeof a === 'string' && a.includes('has no adapter loaded')) + args.some( + (a) => typeof a === 'string' && a.includes('has no adapter in the external Prebid bundle') + ) ); expect(hasAdapterError).toBe(false); diff --git a/docs/guide/integrations/prebid.md b/docs/guide/integrations/prebid.md index 4bc73c0ce..e496fdd3c 100644 --- a/docs/guide/integrations/prebid.md +++ b/docs/guide/integrations/prebid.md @@ -374,11 +374,13 @@ available modules and default preset are checked in at `--user-id-modules` to `build-prebid-external.mjs` when a publisher needs a specific subset; omit it to use the default preset. -This is deliberate: Trusted Server injects a generated Prebid.js bundle so we -can install the `trustedServer` adapter and route auctions through `/auction`, -but publishers often need different User ID submodules. Moving that selection to -the external bundle keeps publisher-specific Prebid choices out of the Trusted -Server WASM artifact while preserving a manifest and bundle hash for auditing. +This is deliberate: the external bundle is pure Prebid.js (core, consent and +User ID modules, and client-side bid adapters) while the server-served TSJS +prebid shim installs the `trustedServer` adapter onto `window.pbjs` and routes +auctions through `/auction` — but publishers often need different User ID +submodules. Moving that selection to the external bundle keeps +publisher-specific Prebid choices out of the Trusted Server WASM artifact while +preserving a manifest and bundle hash for auditing. The current preset includes common ID modules such as Yahoo ConnectID, Criteo, LiveIntent, SharedID, UID2, ID5, LiveRamp IdentityLink, PubProvidedID, and From f63f31aa74b4f7da32ac59f344f23a3008ec95b3 Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Sat, 25 Jul 2026 09:49:23 -0700 Subject: [PATCH 123/198] Order the prebid.js type import before relative imports --- crates/trusted-server-js/lib/src/integrations/prebid/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 825dfa628..6e97a1aa8 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -11,11 +11,12 @@ // The shim on requestBids injects "trustedServer" into every ad unit so all // bids flow through the orchestrator. +import type _pbjsDefault from 'prebid.js'; + import { log } from '../../core/log'; import { buildAdRequest, parseAuctionResponse } from '../../core/auction'; import type { AuctionBid, AuctionEid } from '../../core/auction'; import type { AuctionSlot } from '../../core/types'; -import type _pbjsDefault from 'prebid.js'; import { PREBID_USER_ID_MODULE_REGISTRY } from './user_id_modules'; From 2ddd19311397e6f5c49e6f2f3a52598028741e61 Mon Sep 17 00:00:00 2001 From: Aram Grigoryan <132480+aram356@users.noreply.github.com> Date: Sat, 25 Jul 2026 17:28:46 -0700 Subject: [PATCH 124/198] Lint test files and replace explicit any casts with typed helpers Widen the lint script to cover test/**, and make every test file pass it with real types: typed window views, TestBid/TestAdUnit shapes, adapter spec and requestBids parameter types, typeof-fetch casts for fetch mocks, and signature-free spies instead of unused typed parameters. --- crates/trusted-server-js/lib/package.json | 4 +- .../lib/test/core/auction.test.ts | 11 +- .../lib/test/core/config.test.ts | 2 +- .../lib/test/core/index.test.ts | 20 +- .../lib/test/core/registry.test.ts | 2 +- .../lib/test/core/request.test.ts | 74 +++-- .../test/integrations/creative/click.test.ts | 2 +- .../integrations/creative/proxy_sign.test.ts | 2 +- .../datadome/script_guard.test.ts | 1 + .../test/integrations/didomi/index.test.ts | 6 +- .../lib/test/integrations/gpt/index.test.ts | 30 +- .../integrations/lockr/script_guard.test.ts | 1 + .../test/integrations/prebid/index.test.ts | 280 +++++++++++------- .../lib/test/shared/beacon_guard.test.ts | 7 +- 14 files changed, 279 insertions(+), 163 deletions(-) diff --git a/crates/trusted-server-js/lib/package.json b/crates/trusted-server-js/lib/package.json index 2ffed57e7..47f4e29cf 100644 --- a/crates/trusted-server-js/lib/package.json +++ b/crates/trusted-server-js/lib/package.json @@ -10,8 +10,8 @@ "dev": "vite build --watch", "test": "vitest run", "test:watch": "vitest", - "lint": "eslint \"src/**/*.{ts,tsx}\"", - "lint:fix": "eslint --fix \"src/**/*.{ts,tsx}\"", + "lint": "eslint \"src/**/*.{ts,tsx}\" \"test/**/*.{ts,tsx}\"", + "lint:fix": "eslint --fix \"src/**/*.{ts,tsx}\" \"test/**/*.{ts,tsx}\"", "format": "prettier --check \"**/*.{ts,tsx,js,json,css,md}\"", "format:write": "prettier --write \"**/*.{ts,tsx,js,json,css,md}\"" }, diff --git a/crates/trusted-server-js/lib/test/core/auction.test.ts b/crates/trusted-server-js/lib/test/core/auction.test.ts index 31e020eff..50f078d0b 100644 --- a/crates/trusted-server-js/lib/test/core/auction.test.ts +++ b/crates/trusted-server-js/lib/test/core/auction.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; + import { buildAdRequest, parseAuctionResponse, sendAuction } from '../../src/core/auction'; describe('auction/buildAdRequest', () => { @@ -247,7 +248,7 @@ describe('auction/sendAuction', () => { ], }), }; - globalThis.fetch = vi.fn().mockResolvedValue(mockResponse) as any; + globalThis.fetch = vi.fn().mockResolvedValue(mockResponse) as unknown as typeof fetch; const request = { adUnits: [ @@ -274,7 +275,9 @@ describe('auction/sendAuction', () => { }); it('returns empty array on network error', async () => { - globalThis.fetch = vi.fn().mockRejectedValue(new Error('network error')) as any; + globalThis.fetch = vi + .fn() + .mockRejectedValue(new Error('network error')) as unknown as typeof fetch; const bids = await sendAuction('/auction', { adUnits: [] }); expect(bids).toEqual([]); @@ -286,7 +289,7 @@ describe('auction/sendAuction', () => { status: 200, headers: { get: () => 'text/html' }, json: async () => ({}), - }) as any; + }) as unknown as typeof fetch; const bids = await sendAuction('/auction', { adUnits: [] }); expect(bids).toEqual([]); @@ -298,7 +301,7 @@ describe('auction/sendAuction', () => { status: 500, headers: { get: () => 'application/json' }, json: async () => ({}), - }) as any; + }) as unknown as typeof fetch; const bids = await sendAuction('/auction', { adUnits: [] }); expect(bids).toEqual([]); diff --git a/crates/trusted-server-js/lib/test/core/config.test.ts b/crates/trusted-server-js/lib/test/core/config.test.ts index 2b15d1929..f2d849320 100644 --- a/crates/trusted-server-js/lib/test/core/config.test.ts +++ b/crates/trusted-server-js/lib/test/core/config.test.ts @@ -16,7 +16,7 @@ describe('config', () => { setConfig({ debug: true }); expect(log.getLevel()).toBe('debug'); - setConfig({ logLevel: 'info' as any }); + setConfig({ logLevel: 'info' } as Parameters[0]); expect(log.getLevel()).toBe('info'); }); }); diff --git a/crates/trusted-server-js/lib/test/core/index.test.ts b/crates/trusted-server-js/lib/test/core/index.test.ts index dc77439b1..7b887474a 100644 --- a/crates/trusted-server-js/lib/test/core/index.test.ts +++ b/crates/trusted-server-js/lib/test/core/index.test.ts @@ -1,18 +1,24 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; -declare global { - interface Window { - tsjs?: any; - } +interface TsjsTestWindow { + tsjs?: { + que?: Array<() => void>; + version?: string; + setConfig?: unknown; + getConfig?: unknown; + log?: unknown; + } & Record; } +const testWindow = window as unknown as TsjsTestWindow; + const ORIGINAL_FETCH = global.fetch; describe('core/index', () => { beforeEach(async () => { await vi.resetModules(); document.body.innerHTML = ''; - delete (window as any).tsjs; + delete testWindow.tsjs; }); afterEach(() => { @@ -41,7 +47,7 @@ describe('core/index', () => { }); it('preserves edge-injected adSlots and bids set before the bundle loads', async () => { - (window as any).tsjs = { + testWindow.tsjs = { adSlots: [{ id: 'pre-injected' }], bids: { 'pre-injected': { hb_pb: '1.00' } }, }; @@ -56,7 +62,7 @@ describe('core/index', () => { const callback = vi.fn(function () { expect(this).toBe(window.tsjs); }); - (window as any).tsjs = { que: [callback] }; + testWindow.tsjs = { que: [callback] }; await import('../../src/core/index'); diff --git a/crates/trusted-server-js/lib/test/core/registry.test.ts b/crates/trusted-server-js/lib/test/core/registry.test.ts index 726f67797..7190a085b 100644 --- a/crates/trusted-server-js/lib/test/core/registry.test.ts +++ b/crates/trusted-server-js/lib/test/core/registry.test.ts @@ -17,7 +17,7 @@ describe('registry', () => { ], }, }, - } as any; + } as unknown as Parameters[0]; addAdUnits(unit); const all = getAllUnits(); diff --git a/crates/trusted-server-js/lib/test/core/request.test.ts b/crates/trusted-server-js/lib/test/core/request.test.ts index 2c56361dc..efc18948f 100644 --- a/crates/trusted-server-js/lib/test/core/request.test.ts +++ b/crates/trusted-server-js/lib/test/core/request.test.ts @@ -1,5 +1,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +/** Test view of the global scope with a mockable `fetch`. */ +const testGlobal = globalThis as unknown as { fetch: ReturnType }; + +type AddAdUnitsArg = Parameters[0]; + async function flushRequestAds(): Promise { await new Promise((resolve) => setTimeout(resolve, 0)); } @@ -21,7 +26,7 @@ describe('request.requestAds', () => { it('sends fetch and renders creatives via iframe from response', async () => { // mock fetch - returns creative HTML inline in adm field const creativeHtml = '
Test Creative
'; - (globalThis as any).fetch = vi.fn().mockResolvedValue({ + testGlobal.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, headers: { get: () => 'application/json' }, @@ -41,12 +46,15 @@ describe('request.requestAds', () => { const infoSpy = vi.spyOn(log, 'info').mockImplementation(() => undefined); document.body.innerHTML = '
'; - addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); + addAdUnits({ + code: 'slot1', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + } as unknown as AddAdUnitsArg); requestAds(); await flushRequestAds(); - expect((globalThis as any).fetch).toHaveBeenCalled(); + expect(testGlobal.fetch).toHaveBeenCalled(); // Verify iframe was created with creative HTML in srcdoc const iframe = document.querySelector('#slot1 iframe') as HTMLIFrameElement | null; @@ -67,7 +75,7 @@ describe('request.requestAds', () => { }); it('does not render on non-JSON response', async () => { - (globalThis as any).fetch = vi.fn().mockResolvedValue({ + testGlobal.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, headers: { get: () => 'text/plain' }, @@ -78,35 +86,41 @@ describe('request.requestAds', () => { const { requestAds } = await import('../../src/core/request'); document.body.innerHTML = '
'; - addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); + addAdUnits({ + code: 'slot1', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + } as unknown as AddAdUnitsArg); requestAds(); await flushRequestAds(); - expect((globalThis as any).fetch).toHaveBeenCalled(); + expect(testGlobal.fetch).toHaveBeenCalled(); expect(document.querySelector('iframe')).toBeNull(); }); it('ignores fetch rejection gracefully', async () => { - (globalThis as any).fetch = vi.fn().mockRejectedValue(new Error('network-error')); + testGlobal.fetch = vi.fn().mockRejectedValue(new Error('network-error')); const { addAdUnits } = await import('../../src/core/registry'); const { requestAds } = await import('../../src/core/request'); document.body.innerHTML = '
'; - addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); + addAdUnits({ + code: 'slot1', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + } as unknown as AddAdUnitsArg); requestAds(); await flushRequestAds(); - expect((globalThis as any).fetch).toHaveBeenCalled(); + expect(testGlobal.fetch).toHaveBeenCalled(); expect(document.querySelector('iframe')).toBeNull(); }); it('inserts an iframe with creative HTML from unified auction', async () => { // mock fetch for unified auction endpoint - returns inline HTML const creativeHtml = 'Ad'; - (globalThis as any).fetch = vi.fn().mockResolvedValue({ + testGlobal.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, headers: { get: () => 'application/json' }, @@ -129,7 +143,10 @@ describe('request.requestAds', () => { document.body.appendChild(div); // Add an ad unit and request - addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); + addAdUnits({ + code: 'slot1', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + } as unknown as AddAdUnitsArg); requestAds(); await flushRequestAds(); @@ -144,7 +161,7 @@ describe('request.requestAds', () => { it('renders creatives with safe URI markup', async () => { const creativeHtml = 'Contactad'; - (globalThis as any).fetch = vi.fn().mockResolvedValue({ + testGlobal.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, headers: { get: () => 'application/json' }, @@ -162,7 +179,10 @@ describe('request.requestAds', () => { const { requestAds } = await import('../../src/core/request'); document.body.innerHTML = '
'; - addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); + addAdUnits({ + code: 'slot1', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + } as unknown as AddAdUnitsArg); requestAds(); await flushRequestAds(); @@ -174,7 +194,7 @@ describe('request.requestAds', () => { }); it('rejects malformed non-string creative HTML without blanking the slot', async () => { - (globalThis as any).fetch = vi.fn().mockResolvedValue({ + testGlobal.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, headers: { get: () => 'application/json' }, @@ -194,7 +214,10 @@ describe('request.requestAds', () => { const warnSpy = vi.spyOn(log, 'warn').mockImplementation(() => undefined); document.body.innerHTML = '
existing
'; - addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); + addAdUnits({ + code: 'slot1', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + } as unknown as AddAdUnitsArg); requestAds(); await flushRequestAds(); @@ -221,7 +244,7 @@ describe('request.requestAds', () => { // Regression: multi-bid scenario where a rejected bid must not erase an earlier // successful render into the same slot. const goodCreative = '
Safe Ad
'; - (globalThis as any).fetch = vi.fn().mockResolvedValue({ + testGlobal.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, headers: { get: () => 'application/json' }, @@ -244,7 +267,10 @@ describe('request.requestAds', () => { const { requestAds } = await import('../../src/core/request'); document.body.innerHTML = '
'; - addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); + addAdUnits({ + code: 'slot1', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + } as unknown as AddAdUnitsArg); requestAds(); await flushRequestAds(); @@ -256,7 +282,7 @@ describe('request.requestAds', () => { }); it('rejects creatives that sanitize to empty markup', async () => { - (globalThis as any).fetch = vi.fn().mockResolvedValue({ + testGlobal.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, headers: { get: () => 'application/json' }, @@ -276,7 +302,10 @@ describe('request.requestAds', () => { const warnSpy = vi.spyOn(log, 'warn').mockImplementation(() => undefined); document.body.innerHTML = '
'; - addAdUnits({ code: 'slot1', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); + addAdUnits({ + code: 'slot1', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + } as unknown as AddAdUnitsArg); requestAds(); await flushRequestAds(); @@ -298,7 +327,7 @@ describe('request.requestAds', () => { it('skips iframe insertion when slot is missing', async () => { // mock fetch for unified auction endpoint - returns inline HTML - (globalThis as any).fetch = vi.fn().mockResolvedValue({ + testGlobal.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, headers: { get: () => 'application/json' }, @@ -314,7 +343,10 @@ describe('request.requestAds', () => { const { addAdUnits } = await import('../../src/core/registry'); const { requestAds } = await import('../../src/core/request'); - addAdUnits({ code: 'missing-slot', mediaTypes: { banner: { sizes: [[300, 250]] } } } as any); + addAdUnits({ + code: 'missing-slot', + mediaTypes: { banner: { sizes: [[300, 250]] } }, + } as unknown as AddAdUnitsArg); requestAds(); await flushRequestAds(); diff --git a/crates/trusted-server-js/lib/test/integrations/creative/click.test.ts b/crates/trusted-server-js/lib/test/integrations/creative/click.test.ts index 05dcd0e02..7cf31afa3 100644 --- a/crates/trusted-server-js/lib/test/integrations/creative/click.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/creative/click.test.ts @@ -17,7 +17,7 @@ describe('creative/click.ts', () => { it('repairs anchors via proxy rebuild fallback when fetch is unavailable', async () => { vi.useFakeTimers(); - global.fetch = undefined as any; + global.fetch = undefined as unknown as typeof fetch; const anchor = document.createElement('a'); anchor.setAttribute('data-tsclick', FIRST_PARTY_CLICK); diff --git a/crates/trusted-server-js/lib/test/integrations/creative/proxy_sign.test.ts b/crates/trusted-server-js/lib/test/integrations/creative/proxy_sign.test.ts index 41c86a873..7f31dbed9 100644 --- a/crates/trusted-server-js/lib/test/integrations/creative/proxy_sign.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/creative/proxy_sign.test.ts @@ -47,7 +47,7 @@ describe('creative/proxy_sign.ts', () => { }); it('returns null when fetch is unavailable', async () => { - global.fetch = undefined as any; + global.fetch = undefined as unknown as typeof fetch; const result = await signProxyUrl('https://cdn.example/asset.js'); expect(result).toBeNull(); }); diff --git a/crates/trusted-server-js/lib/test/integrations/datadome/script_guard.test.ts b/crates/trusted-server-js/lib/test/integrations/datadome/script_guard.test.ts index 795b442a6..70fe191fd 100644 --- a/crates/trusted-server-js/lib/test/integrations/datadome/script_guard.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/datadome/script_guard.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; + import { installDataDomeGuard, isGuardInstalled, diff --git a/crates/trusted-server-js/lib/test/integrations/didomi/index.test.ts b/crates/trusted-server-js/lib/test/integrations/didomi/index.test.ts index 487ff471f..bc347968c 100644 --- a/crates/trusted-server-js/lib/test/integrations/didomi/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/didomi/index.test.ts @@ -5,7 +5,7 @@ import { installDidomiSdkProxy } from '../../../src/integrations/didomi'; const ORIGINAL_WINDOW = global.window; type TestDidomiWindow = Window & { - didomiConfig?: any; + didomiConfig?: Record; __tsjs_didomi?: { proxyPath?: string }; }; @@ -20,11 +20,11 @@ describe('integrations/didomi', () => { beforeEach(() => { testWindow = createWindow('https://example.com/page'); - Object.assign(globalThis as any, { window: testWindow }); + Object.assign(globalThis as unknown as { window: unknown }, { window: testWindow }); }); afterEach(() => { - Object.assign(globalThis as any, { window: ORIGINAL_WINDOW }); + Object.assign(globalThis as unknown as { window: unknown }, { window: ORIGINAL_WINDOW }); }); it('initializes didomiConfig and forces sdkPath through trusted server proxy', () => { diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts index 406c6d1f5..9a3c774e1 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/index.test.ts @@ -1,5 +1,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +/** Window properties these tests read and write on the jsdom global. */ +interface GptTestWindow { + googletag?: unknown; + tsjs?: Record & { adInit?: () => void }; +} + +const gptTestWindow = window as unknown as GptTestWindow; + // We import installGptShim dynamically so each test can control whether the // GPT enable flag is present before module evaluation. @@ -219,14 +227,14 @@ describe('GPT – installSlimPrebidLoader', () => { describe('GPT – installTsAdInit', () => { beforeEach(() => { document.body.innerHTML = ''; - delete (window as any).tsjs; - delete (window as any).googletag; + delete gptTestWindow.tsjs; + delete gptTestWindow.googletag; }); afterEach(() => { document.body.innerHTML = ''; - delete (window as any).tsjs; - delete (window as any).googletag; + delete gptTestWindow.tsjs; + delete gptTestWindow.googletag; }); it('clears stale TS-managed targeting before applying a new route to a reused GPT slot', async () => { @@ -240,7 +248,13 @@ describe('GPT – installTsAdInit', () => { ['ts_initial', ['1']], ['pos', ['old-pos']], ]); - const gptSlot: any = { + interface GptTestSlot { + getSlotElementId: () => string; + getTargeting: (key: string) => string[]; + setTargeting: (key: string, value: string | string[]) => GptTestSlot; + clearTargeting: (key?: string) => GptTestSlot; + } + const gptSlot: GptTestSlot = { getSlotElementId: vi.fn(() => 'div-ad-homepage-header'), getTargeting: vi.fn((key: string) => slotTargeting.get(key) ?? []), setTargeting: vi.fn((key: string, value: string | string[]) => { @@ -269,14 +283,14 @@ describe('GPT – installTsAdInit', () => { }; document.body.innerHTML = '
'; - (window as any).googletag = { + gptTestWindow.googletag = { cmd, pubads: () => pubads, defineSlot: vi.fn(), destroySlots: vi.fn(), enableServices: vi.fn(), }; - (window as any).tsjs = { + gptTestWindow.tsjs = { prevSlotTargetingKeys: { 'div-ad-homepage-header': ['pos'], }, @@ -293,7 +307,7 @@ describe('GPT – installTsAdInit', () => { }; installTsAdInit(); - (window as any).tsjs.adInit(); + gptTestWindow.tsjs?.adInit?.(); expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_pb'); expect(gptSlot.clearTargeting).toHaveBeenCalledWith('hb_bidder'); diff --git a/crates/trusted-server-js/lib/test/integrations/lockr/script_guard.test.ts b/crates/trusted-server-js/lib/test/integrations/lockr/script_guard.test.ts index b9251b1e1..2f53c06b2 100644 --- a/crates/trusted-server-js/lib/test/integrations/lockr/script_guard.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/lockr/script_guard.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; + import { installLockrGuard, isGuardInstalled, diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 8827ddfea..665fddd42 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -10,6 +10,59 @@ const DEFAULT_BUNDLE_MANIFEST = { userIdModules: ['sharedIdSystem'], }; +/** Loose bid shape used by the requestBids shim tests. */ +interface TestBid { + bidder: string; + params?: Record; +} + +/** Loose ad unit shape used by the requestBids shim tests. */ +interface TestAdUnit { + code?: string; + bids?: TestBid[]; +} + +/** Window properties the prebid shim reads and writes in these tests. */ +interface PrebidTestWindow { + pbjs?: unknown; + tsjs?: unknown; + googletag?: unknown; + __tsjs_prebid?: Record; + __tsjs_prebid_bundle?: { adapters?: string[]; userIdModules?: string[] }; + __tsjs_prebid_diagnostics?: { + userIdModules?: { + includedModules: string[]; + configuredUserIdNames: string[]; + missingConfiguredUserIdNames: string[]; + }; + }; +} + +const testWindow = window as unknown as PrebidTestWindow; + +/** Argument type accepted by the shimmed `pbjs.requestBids`. */ +type RequestBidsArg = Parameters['requestBids']>[0]; + +/** The bid adapter spec object registered via `pbjs.registerBidAdapter`. */ +interface TestAdapterSpec { + code: string; + supportedMediaTypes: string[]; + isBidRequestValid: (bid: Record) => boolean; + buildRequests: ( + bidRequests: Array>, + bidderRequest?: Record + ) => { + method: string; + url: string; + data: Record; + options: Record; + }; + interpretResponse: ( + response: Record, + request?: Record + ) => Array>; +} + // Define mocks using vi.hoisted so they exist before the module under test is // imported. The shim reads Prebid.js from the `window.pbjs` global (owned by // the external bundle in production), so tests install the mock there instead @@ -38,15 +91,18 @@ const { registerBidAdapter: mockRegisterBidAdapter, getUserIdsAsEids: mockGetUserIdsAsEids, getConfig: mockGetConfig, - adUnits: [] as any[], + adUnits: [] as TestAdUnit[], + setTargetingForGPTAsync: undefined as ((adUnitCodes?: string[]) => void) | undefined, que: [] as Array<() => void>, cmd: [] as Array<() => void>, }; // Install the mock global BEFORE the shim module evaluates — the shim // captures `window.pbjs` at module scope. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const w = globalThis.window as any; + const w = globalThis.window as unknown as { + pbjs?: unknown; + __tsjs_prebid_bundle?: unknown; + }; w.pbjs = mockPbjs; w.__tsjs_prebid_bundle = { adapters: ['rubicon', 'openx', 'exampleBrowser', 'appnexus'], @@ -103,7 +159,7 @@ describe('prebid/collectBidders', () => { describe('prebid/getInjectedConfig', () => { afterEach(() => { - delete (window as any).__tsjs_prebid; + delete testWindow.__tsjs_prebid; }); it('returns undefined when window.__tsjs_prebid is not set', () => { @@ -111,7 +167,7 @@ describe('prebid/getInjectedConfig', () => { }); it('returns the injected config when present', () => { - (window as any).__tsjs_prebid = { accountId: 'server-42', timeout: 2000 }; + testWindow.__tsjs_prebid = { accountId: 'server-42', timeout: 2000 }; expect(getInjectedConfig()).toEqual({ accountId: 'server-42', timeout: 2000 }); }); }); @@ -217,8 +273,8 @@ describe('prebid/installPrebidNpm', () => { mockGetUserIdsAsEids.mockReturnValue([]); mockGetConfig.mockReset(); document.cookie = 'ts-eids=; Path=/; Max-Age=0'; - delete (window as any).__tsjs_prebid; - delete (window as any).__tsjs_prebid_diagnostics; + delete testWindow.__tsjs_prebid; + delete testWindow.__tsjs_prebid_diagnostics; }); afterEach(() => { @@ -268,7 +324,7 @@ describe('prebid/installPrebidNpm', () => { it('reports the User ID modules selected by the generated bundle', () => { installPrebidNpm(); - expect((window as any).__tsjs_prebid_diagnostics.userIdModules).toEqual({ + expect(testWindow.__tsjs_prebid_diagnostics.userIdModules).toEqual({ includedModules: ['sharedIdSystem'], configuredUserIdNames: [], missingConfiguredUserIdNames: [], @@ -285,7 +341,7 @@ describe('prebid/installPrebidNpm', () => { mockPbjs.requestBids({ adUnits: [] }); mockPbjs.requestBids({ adUnits: [] }); - expect((window as any).__tsjs_prebid_diagnostics.userIdModules).toEqual({ + expect(testWindow.__tsjs_prebid_diagnostics.userIdModules).toEqual({ includedModules: ['sharedIdSystem'], configuredUserIdNames: ['pairId', 'sharedId'], missingConfiguredUserIdNames: ['pairId'], @@ -301,9 +357,9 @@ describe('prebid/installPrebidNpm', () => { }); describe('adapter spec', () => { - function getAdapterSpec(): any { + function getAdapterSpec(): TestAdapterSpec { installPrebidNpm(); - return mockRegisterBidAdapter.mock.calls[0][2]; + return mockRegisterBidAdapter.mock.calls[0][2] as TestAdapterSpec; } it('isBidRequestValid always returns true', () => { @@ -581,18 +637,18 @@ describe('prebid/installPrebidNpm', () => { { bids: [{ bidder: 'appnexus', params: {} }] }, { bids: [{ bidder: 'rubicon', params: {} }] }, ]; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); // Each ad unit should have trustedServer added for (const unit of adUnits) { - const hasTsBidder = unit.bids.some((b: any) => b.bidder === 'trustedServer'); + const hasTsBidder = unit.bids.some((b: TestBid) => b.bidder === 'trustedServer'); expect(hasTsBidder).toBe(true); } - const trustedServerBid = adUnits[0].bids.find((b: any) => b.bidder === 'trustedServer'); + const trustedServerBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer'); expect(trustedServerBid.params.bidderParams).toEqual({ appnexus: {} }); - expect(adUnits[0].bids.map((b: any) => b.bidder)).toEqual(['trustedServer']); - expect(adUnits[1].bids.map((b: any) => b.bidder)).toEqual(['trustedServer']); + expect(adUnits[0].bids.map((b: TestBid) => b.bidder)).toEqual(['trustedServer']); + expect(adUnits[1].bids.map((b: TestBid) => b.bidder)).toEqual(['trustedServer']); // Should call through to original requestBids expect(mockRequestBids).toHaveBeenCalled(); @@ -602,9 +658,9 @@ describe('prebid/installPrebidNpm', () => { const pbjs = installPrebidNpm(); const adUnits = [{ bids: [{ bidder: 'trustedServer', params: {} }] }]; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); - const tsCount = adUnits[0].bids.filter((b: any) => b.bidder === 'trustedServer').length; + const tsCount = adUnits[0].bids.filter((b: TestBid) => b.bidder === 'trustedServer').length; expect(tsCount).toBe(1); }); @@ -619,15 +675,15 @@ describe('prebid/installPrebidNpm', () => { ], }, ]; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); - const trustedServerBid = adUnits[0].bids.find((b: any) => b.bidder === 'trustedServer'); + const trustedServerBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer'); expect(trustedServerBid).toBeDefined(); expect(trustedServerBid.params.bidderParams).toEqual({ appnexus: { placementId: 123 }, rubicon: { accountId: 'abc' }, }); - expect(adUnits[0].bids.map((b: any) => b.bidder)).toEqual(['trustedServer']); + expect(adUnits[0].bids.map((b: TestBid) => b.bidder)).toEqual(['trustedServer']); }); it('preserves captured bidder params when requestBids runs twice on the same ad unit', () => { @@ -643,16 +699,16 @@ describe('prebid/installPrebidNpm', () => { ], }, ]; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); // Second auction (refresh/re-auction) with the SAME ad unit object: the // server-side bidder entries were already pruned, so the shim must not // overwrite the captured params with an empty object. - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); const trustedServerBid = adUnits[0].bids.find( - (b: any) => b.bidder === 'trustedServer' - ) as any; + (b: TestBid) => b.bidder === 'trustedServer' + ) as TestBid; expect(trustedServerBid.params.bidderParams).toEqual({ appnexus: { placementId: 123 }, rubicon: { accountId: 'abc' }, @@ -662,8 +718,8 @@ describe('prebid/installPrebidNpm', () => { it('adds bids array to ad units that have none', () => { const pbjs = installPrebidNpm(); - const adUnits = [{ code: 'div-1' }] as any[]; - pbjs.requestBids({ adUnits } as any); + const adUnits = [{ code: 'div-1' }] as TestAdUnit[]; + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); expect(adUnits[0].bids).toHaveLength(1); expect(adUnits[0].bids[0].bidder).toBe('trustedServer'); @@ -684,12 +740,12 @@ describe('prebid/installPrebidNpm', () => { bids: [{ bidder: 'kargo', params: { placementId: '_def' } }], }, ]; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); - const tsBid0 = adUnits[0].bids.find((b: any) => b.bidder === 'trustedServer') as any; + const tsBid0 = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; expect(tsBid0.params.zone).toBe('header'); - const tsBid1 = adUnits[1].bids.find((b: any) => b.bidder === 'trustedServer') as any; + const tsBid1 = adUnits[1].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; expect(tsBid1.params.zone).toBe('fixed_bottom'); }); @@ -703,9 +759,9 @@ describe('prebid/installPrebidNpm', () => { bids: [{ bidder: 'appnexus', params: {} }], }, ]; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); - const tsBid = adUnits[0].bids.find((b: any) => b.bidder === 'trustedServer') as any; + const tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; expect(tsBid.params.zone).toBeUndefined(); }); @@ -713,9 +769,9 @@ describe('prebid/installPrebidNpm', () => { const pbjs = installPrebidNpm(); const adUnits = [{ bids: [{ bidder: 'rubicon', params: {} }] }]; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); - const tsBid = adUnits[0].bids.find((b: any) => b.bidder === 'trustedServer') as any; + const tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; expect(tsBid.params.zone).toBeUndefined(); }); @@ -733,16 +789,16 @@ describe('prebid/installPrebidNpm', () => { }, ]; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); - let tsBid = adUnits[0].bids.find((b: any) => b.bidder === 'trustedServer') as any; + let tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; expect(tsBid.params.zone).toBe('header'); expect(tsBid.params.custom).toBe('keep'); delete adUnits[0].mediaTypes.banner.name; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); - tsBid = adUnits[0].bids.find((b: any) => b.bidder === 'trustedServer') as any; + tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; expect(tsBid.params.zone).toBeUndefined(); expect(tsBid.params.custom).toBe('keep'); }); @@ -750,11 +806,11 @@ describe('prebid/installPrebidNpm', () => { it('falls back to pbjs.adUnits when requestObj has no adUnits', () => { const pbjs = installPrebidNpm(); - mockPbjs.adUnits = [{ bids: [{ bidder: 'openx', params: {} }] }] as any[]; - pbjs.requestBids({} as any); + mockPbjs.adUnits = [{ bids: [{ bidder: 'openx', params: {} }] }] as TestAdUnit[]; + pbjs.requestBids({} as RequestBidsArg); - const hasTsBidder = (mockPbjs.adUnits[0] as any).bids.some( - (b: any) => b.bidder === 'trustedServer' + const hasTsBidder = (mockPbjs.adUnits[0].bids ?? []).some( + (b: TestBid) => b.bidder === 'trustedServer' ); expect(hasTsBidder).toBe(true); }); @@ -774,7 +830,9 @@ describe('prebid/installPrebidNpm', () => { ]); const pbjs = installPrebidNpm(); - pbjs.requestBids({ adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }] } as any); + pbjs.requestBids({ + adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }], + } as unknown as RequestBidsArg); const cookieValue = document.cookie.match(/(?:^|; )ts-eids=([^;]+)/)?.[1]; expect(cookieValue).toBeDefined(); @@ -797,7 +855,9 @@ describe('prebid/installPrebidNpm', () => { mockGetUserIdsAsEids.mockReturnValue([]); const pbjs = installPrebidNpm(); - pbjs.requestBids({ adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }] } as any); + pbjs.requestBids({ + adUnits: [{ bids: [{ bidder: 'appnexus', params: {} }] }], + } as unknown as RequestBidsArg); expect(document.cookie).toBe(''); }); @@ -812,15 +872,15 @@ describe('prebid/installPrebidNpm with server-injected config', () => { mockGetUserIdsAsEids.mockReset(); mockGetUserIdsAsEids.mockReturnValue([]); document.cookie = 'ts-eids=; Path=/; Max-Age=0'; - delete (window as any).__tsjs_prebid; + delete testWindow.__tsjs_prebid; }); afterEach(() => { - delete (window as any).__tsjs_prebid; + delete testWindow.__tsjs_prebid; }); it('reads timeout and debug from window.__tsjs_prebid', () => { - (window as any).__tsjs_prebid = { timeout: 1500, debug: true }; + testWindow.__tsjs_prebid = { timeout: 1500, debug: true }; installPrebidNpm(); @@ -830,7 +890,7 @@ describe('prebid/installPrebidNpm with server-injected config', () => { }); it('explicit config overrides server-injected values', () => { - (window as any).__tsjs_prebid = { timeout: 1500, debug: true }; + testWindow.__tsjs_prebid = { timeout: 1500, debug: true }; installPrebidNpm({ timeout: 3000, debug: false }); @@ -853,13 +913,13 @@ describe('prebid/installRefreshHandler', () => { mockRequestBids.mockReset(); mockPbjs.requestBids = mockRequestBids; mockPbjs.adUnits = []; - (window as any).tsjs = undefined; - delete (window as any).googletag; + testWindow.tsjs = undefined; + delete testWindow.googletag; }); afterEach(() => { - (window as any).tsjs = undefined; - delete (window as any).googletag; + testWindow.tsjs = undefined; + delete testWindow.googletag; }); it('builds refresh ad units from injected slot metadata', () => { @@ -872,11 +932,11 @@ describe('prebid/installRefreshHandler', () => { refresh: originalRefresh, getSlots: vi.fn(() => [gptSlot]), }; - (window as any).googletag = { + testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - (window as any).tsjs = { + testWindow.tsjs = { adSlots: [ { id: 'homepage_header_ad', @@ -930,11 +990,11 @@ describe('prebid/installRefreshHandler', () => { refresh: originalRefresh, getSlots: vi.fn(() => [gptSlot]), }; - (window as any).googletag = { + testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - (window as any).tsjs = { + testWindow.tsjs = { adSlots: [ { id: 'prefix_ad', @@ -975,7 +1035,7 @@ describe('prebid/installRefreshHandler', () => { it('scopes the GPT targeting call to the refreshed slot code', () => { const setTargetingForGPTAsync = vi.fn(); - (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; + mockPbjs.setTargetingForGPTAsync = setTargetingForGPTAsync; // Run the bidsBackHandler synchronously so the targeting call fires. mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { opts?.bidsBackHandler?.(); @@ -991,11 +1051,11 @@ describe('prebid/installRefreshHandler', () => { refresh: originalRefresh, getSlots: vi.fn(() => [headerSlot]), }; - (window as any).googletag = { + testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - (window as any).tsjs = { + testWindow.tsjs = { adSlots: [ { id: 'header_ad', @@ -1021,11 +1081,11 @@ describe('prebid/installRefreshHandler', () => { expect(setTargetingForGPTAsync).toHaveBeenCalledWith(['div-ad-header']); expect(originalRefresh).toHaveBeenCalledWith([headerSlot], undefined); - delete (mockPbjs as any).setTargetingForGPTAsync; + mockPbjs.setTargetingForGPTAsync = undefined; }); it('includes configured client-side bidders in refresh ad units', () => { - (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon'] }; // Original publisher ad unit carries a client-side rubicon bid. mockPbjs.adUnits = [ { @@ -1045,11 +1105,11 @@ describe('prebid/installRefreshHandler', () => { refresh: originalRefresh, getSlots: vi.fn(() => [gptSlot]), }; - (window as any).googletag = { + testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - (window as any).tsjs = { + testWindow.tsjs = { adSlots: [ { id: 'homepage_header_ad', @@ -1078,7 +1138,7 @@ describe('prebid/installRefreshHandler', () => { }) ); - delete (window as any).__tsjs_prebid; + delete testWindow.__tsjs_prebid; mockPbjs.adUnits = []; }); @@ -1100,11 +1160,11 @@ describe('prebid/installRefreshHandler', () => { refresh: originalRefresh, getSlots: vi.fn(() => [gptSlot]), }; - (window as any).googletag = { + testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - (window as any).tsjs = { + testWindow.tsjs = { adSlots: [ { id: 'homepage_header_ad', @@ -1146,7 +1206,7 @@ describe('prebid/installRefreshHandler', () => { // publisher's Prebid ad unit is keyed by the inner div_id. The synthetic // refresh code stays the GPT element id (so GPT can match it), while params // and client-side bids are recovered from the injected div_id candidate. - (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon'] }; mockPbjs.adUnits = [ { code: 'div-ad-x', @@ -1165,11 +1225,11 @@ describe('prebid/installRefreshHandler', () => { refresh: originalRefresh, getSlots: vi.fn(() => [gptSlot]), }; - (window as any).googletag = { + testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - (window as any).tsjs = { + testWindow.tsjs = { adSlots: [ { id: 'x_ad', @@ -1205,7 +1265,7 @@ describe('prebid/installRefreshHandler', () => { }) ); - delete (window as any).__tsjs_prebid; + delete testWindow.__tsjs_prebid; mockPbjs.adUnits = []; }); @@ -1233,11 +1293,11 @@ describe('prebid/installRefreshHandler', () => { refresh: originalRefresh, getSlots: vi.fn(() => [gptSlot]), }; - (window as any).googletag = { + testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - (window as any).tsjs = { + testWindow.tsjs = { adSlots: [ { id: 'homepage_header_ad', @@ -1295,12 +1355,12 @@ describe('prebid/installRefreshHandler', () => { getSlots: vi.fn(() => [gptSlot]), }; const setTargetingForGPTAsync = vi.fn(); - (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; - (window as any).googletag = { + mockPbjs.setTargetingForGPTAsync = setTargetingForGPTAsync; + testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - (window as any).tsjs = { + testWindow.tsjs = { adSlots: [ { id: 'homepage_header_ad', @@ -1365,11 +1425,11 @@ describe('prebid/installRefreshHandler', () => { refresh: originalRefresh, getSlots: vi.fn(() => [gptSlot]), }; - (window as any).googletag = { + testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - (window as any).tsjs = { adInitRefreshInProgress: true }; + testWindow.tsjs = { adInitRefreshInProgress: true }; installRefreshHandler(750); pubads.refresh([gptSlot]); @@ -1390,11 +1450,11 @@ describe('prebid/installRefreshHandler', () => { refresh: originalRefresh, getSlots: vi.fn(() => [gptSlot]), }; - (window as any).googletag = { + testWindow.googletag = { cmd: { push: (fn: () => void) => fn() }, pubads: () => pubads, }; - (window as any).tsjs = { adInitRefreshInProgress: false }; + testWindow.tsjs = { adInitRefreshInProgress: false }; installRefreshHandler(750); pubads.refresh([gptSlot]); @@ -1412,16 +1472,16 @@ describe('prebid/client-side bidders', () => { mockGetUserIdsAsEids.mockReset(); mockGetUserIdsAsEids.mockReturnValue([]); // By default the manifest declares all adapters compiled in. - (window as any).__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; - delete (window as any).__tsjs_prebid; + testWindow.__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; + delete testWindow.__tsjs_prebid; }); afterEach(() => { - delete (window as any).__tsjs_prebid; + delete testWindow.__tsjs_prebid; }); it('excludes client-side bidders from trustedServer bidderParams', () => { - (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon'] }; const pbjs = installPrebidNpm(); @@ -1434,9 +1494,9 @@ describe('prebid/client-side bidders', () => { ], }, ]; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); - const tsBid = adUnits[0].bids.find((b: any) => b.bidder === 'trustedServer') as any; + const tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; expect(tsBid).toBeDefined(); // rubicon should NOT be in bidderParams — it runs client-side expect(tsBid.params.bidderParams).toEqual({ @@ -1446,7 +1506,7 @@ describe('prebid/client-side bidders', () => { }); it('preserves client-side bidder bids as standalone entries', () => { - (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon'] }; const pbjs = installPrebidNpm(); @@ -1458,17 +1518,17 @@ describe('prebid/client-side bidders', () => { ], }, ]; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); // rubicon bid should remain untouched as a standalone entry - const rubiconBid = adUnits[0].bids.find((b: any) => b.bidder === 'rubicon') as any; + const rubiconBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'rubicon') as TestBid; expect(rubiconBid).toBeDefined(); expect(rubiconBid.params).toEqual({ accountId: 'abc' }); - expect(adUnits[0].bids.find((b: any) => b.bidder === 'appnexus')).toBeUndefined(); + expect(adUnits[0].bids.find((b: TestBid) => b.bidder === 'appnexus')).toBeUndefined(); }); it('handles multiple client-side bidders', () => { - (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon', 'openx'] }; + testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon', 'openx'] }; const pbjs = installPrebidNpm(); @@ -1481,18 +1541,18 @@ describe('prebid/client-side bidders', () => { ], }, ]; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); - const tsBid = adUnits[0].bids.find((b: any) => b.bidder === 'trustedServer') as any; + const tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; // Only appnexus should be in bidderParams expect(tsBid.params.bidderParams).toEqual({ appnexus: { placementId: 123 }, }); // Both client-side bidders should remain - expect(adUnits[0].bids.find((b: any) => b.bidder === 'rubicon')).toBeDefined(); - expect(adUnits[0].bids.find((b: any) => b.bidder === 'openx')).toBeDefined(); - expect(adUnits[0].bids.find((b: any) => b.bidder === 'appnexus')).toBeUndefined(); + expect(adUnits[0].bids.find((b: TestBid) => b.bidder === 'rubicon')).toBeDefined(); + expect(adUnits[0].bids.find((b: TestBid) => b.bidder === 'openx')).toBeDefined(); + expect(adUnits[0].bids.find((b: TestBid) => b.bidder === 'appnexus')).toBeUndefined(); }); it('behaves normally when no client-side bidders are configured', () => { @@ -1507,9 +1567,9 @@ describe('prebid/client-side bidders', () => { ], }, ]; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); - const tsBid = adUnits[0].bids.find((b: any) => b.bidder === 'trustedServer') as any; + const tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; expect(tsBid.params.bidderParams).toEqual({ appnexus: { placementId: 123 }, rubicon: { accountId: 'abc' }, @@ -1517,7 +1577,7 @@ describe('prebid/client-side bidders', () => { }); it('behaves normally when client-side bidders list is empty', () => { - (window as any).__tsjs_prebid = { clientSideBidders: [] }; + testWindow.__tsjs_prebid = { clientSideBidders: [] }; const pbjs = installPrebidNpm(); @@ -1529,9 +1589,9 @@ describe('prebid/client-side bidders', () => { ], }, ]; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); - const tsBid = adUnits[0].bids.find((b: any) => b.bidder === 'trustedServer') as any; + const tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; expect(tsBid.params.bidderParams).toEqual({ appnexus: { placementId: 123 }, rubicon: { accountId: 'abc' }, @@ -1539,7 +1599,7 @@ describe('prebid/client-side bidders', () => { }); it('still injects trustedServer when all bidders are client-side', () => { - (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon', 'appnexus'] }; + testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon', 'appnexus'] }; const pbjs = installPrebidNpm(); @@ -1551,21 +1611,21 @@ describe('prebid/client-side bidders', () => { ], }, ]; - pbjs.requestBids({ adUnits } as any); + pbjs.requestBids({ adUnits } as unknown as RequestBidsArg); // trustedServer should still be present (even with empty bidderParams) - const tsBid = adUnits[0].bids.find((b: any) => b.bidder === 'trustedServer') as any; + const tsBid = adUnits[0].bids.find((b: TestBid) => b.bidder === 'trustedServer') as TestBid; expect(tsBid).toBeDefined(); expect(tsBid.params.bidderParams).toEqual({}); }); it('logs error when a client-side bidder has no adapter in the external bundle', () => { // rubicon is compiled into the external bundle, but openx is not - (window as any).__tsjs_prebid_bundle = { + testWindow.__tsjs_prebid_bundle = { ...DEFAULT_BUNDLE_MANIFEST, adapters: ['rubicon'], }; - (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon', 'openx'] }; + testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon', 'openx'] }; const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); @@ -1591,12 +1651,12 @@ describe('prebid/client-side bidders', () => { expect(hasRubiconError).toBe(false); errorSpy.mockRestore(); - (window as any).__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; + testWindow.__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; }); it('warns when the external bundle stamped no adapter manifest', () => { - delete (window as any).__tsjs_prebid_bundle; - (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + delete testWindow.__tsjs_prebid_bundle; + testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon'] }; const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); @@ -1608,11 +1668,11 @@ describe('prebid/client-side bidders', () => { expect(hasManifestWarn).toBe(true); warnSpy.mockRestore(); - (window as any).__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; + testWindow.__tsjs_prebid_bundle = DEFAULT_BUNDLE_MANIFEST; }); it('does not log errors when all client-side bidders have adapters', () => { - (window as any).__tsjs_prebid = { clientSideBidders: ['rubicon'] }; + testWindow.__tsjs_prebid = { clientSideBidders: ['rubicon'] }; const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); diff --git a/crates/trusted-server-js/lib/test/shared/beacon_guard.test.ts b/crates/trusted-server-js/lib/test/shared/beacon_guard.test.ts index 9ade23382..881a4515f 100644 --- a/crates/trusted-server-js/lib/test/shared/beacon_guard.test.ts +++ b/crates/trusted-server-js/lib/test/shared/beacon_guard.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; + import { createBeaconGuard, BeaconGuardConfig } from '../../src/shared/beacon_guard'; describe('Beacon Guard', () => { @@ -14,12 +15,10 @@ describe('Beacon Guard', () => { originalFetch = window.fetch; // Create spies that simulate real sendBeacon/fetch behaviour - sendBeaconSpy = vi.fn((_url: string | URL, _data?: BodyInit | null) => true); + sendBeaconSpy = vi.fn(() => true); navigator.sendBeacon = sendBeaconSpy; - fetchSpy = vi.fn((_input: RequestInfo | URL, _init?: RequestInit) => - Promise.resolve(new Response('', { status: 200 })) - ); + fetchSpy = vi.fn(() => Promise.resolve(new Response('', { status: 200 }))); window.fetch = fetchSpy; config = { From 5a4ef23ec67dea7ce6c9247af219257662a163d9 Mon Sep 17 00:00:00 2001 From: prk-Jr Date: Mon, 27 Jul 2026 13:52:21 +0530 Subject: [PATCH 125/198] Rename /__ts/page-bids to /_ts/page-bids MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SPA re-auction endpoint was the only internal route using a double-underscore prefix; every other internal path lives under /_ts/. Move it to /_ts/page-bids and switch the tsjs client fetch to match. A browser runs whichever tsjs bundle it was already served, so pages loaded before the rename — and cached bundles — keep requesting the old path. On a SPA that path delivers ads for in-session navigations, so /__ts/page-bids stays registered to the same handler as a transition alias until those bundles age out. Both paths are defined once in core as PAGE_BIDS_PATH and PAGE_BIDS_LEGACY_PATH so removal touches one const plus its four registrations. handle_page_bids logs an info line when the alias serves a gate-passed request. Without it nothing in the app distinguishes the two paths, so the "no remaining legacy traffic" precondition for removing the alias would only be answerable from edge access logs. Route coverage was missing for the page-bids GET registrations on Cloudflare and Spin, where GET and OPTIONS are registered separately and the preflight-denial parity test does not imply the GET side is wired. Mutation testing confirmed a broken registration previously passed every suite. The route-table assertions pin literal paths rather than the consts, since looking a route up by the same const it was registered with still passes when the const's value changes. --- crates/trusted-server-adapter-axum/src/app.rs | 18 +++- .../tests/routes.rs | 10 ++ .../src/app.rs | 56 ++++++----- .../tests/routes.rs | 10 ++ .../trusted-server-adapter-fastly/src/app.rs | 62 +++++++++++- crates/trusted-server-adapter-spin/src/app.rs | 26 +++-- .../tests/routes.rs | 50 ++++++++++ .../src/auction/endpoints.rs | 8 +- .../src/auction/orchestrator.rs | 4 +- .../src/auction/telemetry.rs | 2 +- .../src/integrations/gpt.rs | 2 +- crates/trusted-server-core/src/publisher.rs | 95 +++++++++++++++++-- .../tests/parity.rs | 38 ++++---- .../lib/src/integrations/gpt/index.ts | 4 +- .../test/integrations/gpt/spa_hook.test.ts | 6 +- 15 files changed, 311 insertions(+), 80 deletions(-) diff --git a/crates/trusted-server-adapter-axum/src/app.rs b/crates/trusted-server-adapter-axum/src/app.rs index 2f4329574..d4ec91047 100644 --- a/crates/trusted-server-adapter-axum/src/app.rs +++ b/crates/trusted-server-adapter-axum/src/app.rs @@ -19,8 +19,8 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, buffer_publisher_response_async, handle_page_bids, handle_publisher_request, - handle_tsjs_dynamic, page_bids_preflight_denied, + AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, buffer_publisher_response_async, + handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, @@ -140,7 +140,7 @@ where // --------------------------------------------------------------------------- /// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, -/// `/__ts/page-bids`, and the publisher fallback). +/// `/_ts/page-bids`, and the publisher fallback). /// /// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction /// Unknown, which fails the auction consent gate closed even for consented @@ -279,7 +279,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_routes() -> [NamedRoute; 12] { +fn named_routes() -> [NamedRoute; 13] { [ NamedRoute { path: "/.well-known/trusted-server.json", @@ -328,7 +328,15 @@ fn named_routes() -> [NamedRoute; 12] { // GET runs the SPA re-auction; OPTIONS is denied in-handler as a CORS // preflight guard for this side-effecting endpoint. NamedRoute { - path: "/__ts/page-bids", + path: PAGE_BIDS_PATH, + primary_methods: &[Method::GET, Method::OPTIONS], + handler: NamedRouteHandler::PageBids, + }, + // Deprecated double-underscore alias, kept so tsjs bundles served before + // the `/_ts/page-bids` rename keep getting ads on SPA navigations until + // they age out of browser caches. See `PAGE_BIDS_LEGACY_PATH`. + NamedRoute { + path: PAGE_BIDS_LEGACY_PATH, primary_methods: &[Method::GET, Method::OPTIONS], handler: NamedRouteHandler::PageBids, }, diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index c4bf7d990..4b15b4c6a 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -77,6 +77,16 @@ fn all_explicit_routes_are_registered() { ("POST", "/admin/keys/rotate"), ("POST", "/admin/keys/deactivate"), ("POST", "/auction"), + // SPA re-auction endpoint, plus its deprecated `/__ts/` alias. Both + // paths are spelled out as literals rather than referencing + // `PAGE_BIDS_PATH` / `PAGE_BIDS_LEGACY_PATH` so this test pins the + // actual URL the tsjs client fetches — asserting a const against itself + // would still pass if the const's value changed out from under the + // client. + ("GET", "/_ts/page-bids"), + ("OPTIONS", "/_ts/page-bids"), + ("GET", "/__ts/page-bids"), + ("OPTIONS", "/__ts/page-bids"), ("GET", "/first-party/proxy"), ("GET", "/first-party/click"), ("GET", "/first-party/sign"), diff --git a/crates/trusted-server-adapter-cloudflare/src/app.rs b/crates/trusted-server-adapter-cloudflare/src/app.rs index c931360f6..b3694fa5a 100644 --- a/crates/trusted-server-adapter-cloudflare/src/app.rs +++ b/crates/trusted-server-adapter-cloudflare/src/app.rs @@ -21,8 +21,9 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_page_bids, - handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, + AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherResponse, + buffer_publisher_response_async, handle_page_bids, handle_publisher_request, + handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, @@ -125,7 +126,7 @@ fn build_per_request_services(ctx: &RequestContext) -> RuntimeServices { } /// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, -/// `/__ts/page-bids`, and the publisher fallback). +/// `/_ts/page-bids`, and the publisher fallback). /// /// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction /// Unknown, which fails the auction consent gate closed even for consented @@ -480,28 +481,6 @@ fn build_router(state: &Arc) -> RouterService { .await }), ) - // SPA re-auction endpoint. The OPTIONS preflight for this - // side-effecting GET is denied so the GET handler's `X-TSJS-Page-Bids` - // gate stays trustworthy. - .route( - "/__ts/page-bids", - Method::OPTIONS, - make_handler(Arc::clone(&state), |_s, _services, _req| async move { - Ok(page_bids_preflight_denied()) - }), - ) - .get( - "/__ts/page-bids", - make_handler(Arc::clone(&state), |s, services, req| async move { - let ec_context = build_ec_context(&s.settings, &services, &req); - let auction = AuctionDispatch { - orchestrator: &s.orchestrator, - slots: s.settings.creative_opportunity_slots(), - registry: None, - }; - handle_page_bids(&s.settings, &services, None, auction, &ec_context, req).await - }), - ) .get( "/first-party/proxy", make_handler(Arc::clone(&state), |s, services, req| async move { @@ -533,6 +512,33 @@ fn build_router(state: &Arc) -> RouterService { }), ); + // SPA re-auction endpoint, registered on the canonical path and on the + // deprecated `PAGE_BIDS_LEGACY_PATH` double-underscore alias. The alias + // keeps tsjs bundles served before the `/_ts/page-bids` rename getting + // ads on SPA navigations until they age out of browser caches. + // + // The OPTIONS preflight is denied on both so the GET handler's + // `X-TSJS-Page-Bids` gate stays trustworthy — an alias that let the + // preflight fall through to a permissive origin would reopen exactly + // the cross-site hole the canonical path closes. + let page_bids = make_handler(Arc::clone(&state), |s, services, req| async move { + let ec_context = build_ec_context(&s.settings, &services, &req); + let auction = AuctionDispatch { + orchestrator: &s.orchestrator, + slots: s.settings.creative_opportunity_slots(), + registry: None, + }; + handle_page_bids(&s.settings, &services, None, auction, &ec_context, req).await + }); + let page_bids_preflight = + make_handler(Arc::clone(&state), |_s, _services, _req| async move { + Ok(page_bids_preflight_denied()) + }); + for path in [PAGE_BIDS_PATH, PAGE_BIDS_LEGACY_PATH] { + router = router.route(path, Method::GET, page_bids.clone()); + router = router.route(path, Method::OPTIONS, page_bids_preflight.clone()); + } + let legacy_admin_deny = make_handler(Arc::clone(&state), |_s, _services, _req| async move { Ok(legacy_admin_alias_denied()) diff --git a/crates/trusted-server-adapter-cloudflare/tests/routes.rs b/crates/trusted-server-adapter-cloudflare/tests/routes.rs index df2781945..d5eb98451 100644 --- a/crates/trusted-server-adapter-cloudflare/tests/routes.rs +++ b/crates/trusted-server-adapter-cloudflare/tests/routes.rs @@ -216,6 +216,16 @@ fn all_explicit_routes_are_registered() { ("POST", "/_ts/admin/keys/rotate"), ("POST", "/_ts/admin/keys/deactivate"), ("POST", "/auction"), + // SPA re-auction endpoint, plus its deprecated `/__ts/` alias. Both + // paths are spelled out as literals rather than referencing + // `PAGE_BIDS_PATH` / `PAGE_BIDS_LEGACY_PATH` so this test pins the + // actual URL the tsjs client fetches — asserting a const against itself + // would still pass if the const's value changed out from under the + // client. + ("GET", "/_ts/page-bids"), + ("OPTIONS", "/_ts/page-bids"), + ("GET", "/__ts/page-bids"), + ("OPTIONS", "/__ts/page-bids"), ("GET", "/first-party/proxy"), ("GET", "/first-party/click"), ("GET", "/first-party/sign"), diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 955ff235b..8b7de4000 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -116,8 +116,8 @@ use trusted_server_core::proxy::{ handle_first_party_proxy, handle_first_party_proxy_rebuild, handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, buffer_publisher_response_async, handle_page_bids, handle_publisher_request, - handle_tsjs_dynamic, page_bids_preflight_denied, + AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, buffer_publisher_response_async, + handle_page_bids, handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_deactivate_key, handle_rotate_key, handle_trusted_server_discovery, @@ -1083,7 +1083,17 @@ const NAMED_ROUTES: &[NamedRoute] = &[ // GET runs the SPA re-auction; OPTIONS is denied in-handler as a CORS // preflight guard for this side-effecting endpoint. NamedRoute { - path: "/__ts/page-bids", + path: PAGE_BIDS_PATH, + primary_methods: &[Method::GET, Method::OPTIONS], + handler: NamedRouteHandler::PageBids, + }, + // Deprecated double-underscore alias. tsjs bundles served before the + // `/_ts/page-bids` rename keep requesting this path from already-loaded + // pages and browser caches; dropping it would strand SPA navigations + // without ads until those bundles age out. See `PAGE_BIDS_LEGACY_PATH`; + // removal is tracked by IABTechLab/trusted-server#970. + NamedRoute { + path: PAGE_BIDS_LEGACY_PATH, primary_methods: &[Method::GET, Method::OPTIONS], handler: NamedRouteHandler::PageBids, }, @@ -1211,8 +1221,8 @@ mod tests { use std::sync::Arc; use super::{ - AppState, NAMED_ROUTES, NamedRouteHandler, TrustedServerApp, build_state_from_settings, - startup_error_router, + AppState, NAMED_ROUTES, NamedRouteHandler, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, + TrustedServerApp, build_state_from_settings, startup_error_router, }; use bytes::Bytes; use edgezero_core::body::Body; @@ -1624,6 +1634,48 @@ mod tests { } } + #[test] + fn page_bids_serves_canonical_path_and_deprecated_alias() { + // The SPA re-auction endpoint lives at the canonical single-underscore + // `/_ts/page-bids`, matching every other internal route. The deprecated + // `/__ts/page-bids` alias must stay registered to the same handler with + // the same methods until pre-rename tsjs bundles age out of browser + // caches — dropping it would leave those clients without ads on SPA + // navigations. + // + // The paths are literals, not `PAGE_BIDS_PATH` / `PAGE_BIDS_LEGACY_PATH`. + // Looking a route up by the same const it was registered with is + // tautological: it keeps passing if the const's value changes, which is + // exactly the break that would silently desync the server from the tsjs + // client's hardcoded fetch path. Pin the consts to their literals too so + // a rename has to be deliberate. + assert_eq!( + PAGE_BIDS_PATH, "/_ts/page-bids", + "canonical page-bids path must match the path tsjs fetches" + ); + assert_eq!( + PAGE_BIDS_LEGACY_PATH, "/__ts/page-bids", + "legacy alias must match the path pre-rename tsjs bundles fetch" + ); + + for path in ["/_ts/page-bids", "/__ts/page-bids"] { + let route = NAMED_ROUTES + .iter() + .find(|route| route.path == path) + .unwrap_or_else(|| panic!("{path} should be registered")); + + assert!( + matches!(route.handler, NamedRouteHandler::PageBids), + "{path} must map to the page-bids handler" + ); + assert_eq!( + route.primary_methods, + &[Method::GET, Method::OPTIONS], + "{path} must handle GET and OPTIONS directly, not fall through to the publisher" + ); + } + } + #[test] fn legacy_admin_aliases_denied_locally_not_proxied_to_publisher() { // Regression for the credential-leak finding: with a production-shaped diff --git a/crates/trusted-server-adapter-spin/src/app.rs b/crates/trusted-server-adapter-spin/src/app.rs index 2291fce74..9f9d3235f 100644 --- a/crates/trusted-server-adapter-spin/src/app.rs +++ b/crates/trusted-server-adapter-spin/src/app.rs @@ -20,8 +20,9 @@ use trusted_server_core::proxy::{ handle_first_party_proxy_sign, }; use trusted_server_core::publisher::{ - AuctionDispatch, PublisherResponse, buffer_publisher_response_async, handle_page_bids, - handle_publisher_request, handle_tsjs_dynamic, page_bids_preflight_denied, + AuctionDispatch, PAGE_BIDS_LEGACY_PATH, PAGE_BIDS_PATH, PublisherResponse, + buffer_publisher_response_async, handle_page_bids, handle_publisher_request, + handle_tsjs_dynamic, page_bids_preflight_denied, }; use trusted_server_core::request_signing::{ handle_trusted_server_discovery, handle_verify_signature, @@ -141,7 +142,7 @@ const LEGACY_ADMIN_DENY_METHODS: &[Method] = &[ Method::DELETE, ]; -fn named_fallback_paths() -> [(&'static str, &'static [Method]); 12] { +fn named_fallback_paths() -> [(&'static str, &'static [Method]); 13] { [ ("/.well-known/trusted-server.json", &[Method::GET]), ("/verify-signature", &[Method::POST]), @@ -150,7 +151,8 @@ fn named_fallback_paths() -> [(&'static str, &'static [Method]); 12] { ("/admin/keys/rotate", LEGACY_ADMIN_DENY_METHODS), ("/admin/keys/deactivate", LEGACY_ADMIN_DENY_METHODS), ("/auction", &[Method::POST]), - ("/__ts/page-bids", &[Method::GET, Method::OPTIONS]), + (PAGE_BIDS_PATH, &[Method::GET, Method::OPTIONS]), + (PAGE_BIDS_LEGACY_PATH, &[Method::GET, Method::OPTIONS]), ("/first-party/proxy", &[Method::GET]), ("/first-party/click", &[Method::GET]), ("/first-party/sign", &[Method::GET, Method::POST]), @@ -322,7 +324,7 @@ fn health_response() -> Response { } /// Builds the geo-aware [`EcContext`] for consent-gated endpoints (`/auction`, -/// `/__ts/page-bids`, and the publisher fallback). +/// `/_ts/page-bids`, and the publisher fallback). /// /// Mirrors the Fastly entry point: `EcContext::default()` leaves jurisdiction /// Unknown, which fails the auction consent gate closed even for consented @@ -541,7 +543,7 @@ fn build_router(state: &Arc) -> RouterService { } }; - // GET /__ts/page-bids — SPA re-auction endpoint. + // GET /_ts/page-bids — SPA re-auction endpoint. let s = Arc::clone(&state); let page_bids_handler = move |ctx: RequestContext| { let s = Arc::clone(&s); @@ -562,7 +564,7 @@ fn build_router(state: &Arc) -> RouterService { } }; - // OPTIONS /__ts/page-bids — deny the CORS preflight for this + // OPTIONS /_ts/page-bids — deny the CORS preflight for this // side-effecting GET so the `X-TSJS-Page-Bids` gate stays trustworthy. let page_bids_options_handler = |_ctx: RequestContext| async { Ok::(page_bids_preflight_denied()) @@ -731,9 +733,15 @@ fn build_router(state: &Arc) -> RouterService { .post("/_ts/admin/keys/rotate", admin_not_supported_handler) .post("/_ts/admin/keys/deactivate", admin_not_supported_handler) .post("/auction", auction_handler) - .get("/__ts/page-bids", page_bids_handler) + .get(PAGE_BIDS_PATH, page_bids_handler.clone()) + .route(PAGE_BIDS_PATH, Method::OPTIONS, page_bids_options_handler) + // Deprecated double-underscore alias, kept so tsjs bundles served + // before the `/_ts/page-bids` rename keep getting ads on SPA + // navigations until they age out of browser caches. See + // `PAGE_BIDS_LEGACY_PATH`. + .get(PAGE_BIDS_LEGACY_PATH, page_bids_handler) .route( - "/__ts/page-bids", + PAGE_BIDS_LEGACY_PATH, Method::OPTIONS, page_bids_options_handler, ) diff --git a/crates/trusted-server-adapter-spin/tests/routes.rs b/crates/trusted-server-adapter-spin/tests/routes.rs index 9b96dbd70..2e1f0f6e5 100644 --- a/crates/trusted-server-adapter-spin/tests/routes.rs +++ b/crates/trusted-server-adapter-spin/tests/routes.rs @@ -330,6 +330,56 @@ async fn auction_is_routed() { assert_ne!(resp.status().as_u16(), 404, "/auction must be routed"); } +/// `GET` on the SPA re-auction endpoint must reach the page-bids handler on +/// both the canonical path and its deprecated `/__ts/` alias. +/// +/// The alias is what pre-rename tsjs bundles still request, and on a SPA that +/// path is what delivers ads for in-session navigations — so a dropped or +/// misspelled registration silently costs revenue rather than erroring loudly. +/// Spin registers `GET` and `OPTIONS` separately, so the preflight-denial parity +/// test does not imply the `GET` side is wired. +/// +/// Paths are literals rather than `PAGE_BIDS_PATH` / `PAGE_BIDS_LEGACY_PATH`: +/// this pins the actual URL the client fetches, which asserting a const against +/// itself would not. +/// +/// These test settings configure no creative opportunities, so the handler's own +/// deterministic answer is a 404 `Creative opportunities not configured`. That +/// body is the anchor: an unregistered path would instead fall through to the +/// publisher fallback and attempt an outbound fetch to the (nonexistent) test +/// origin, which cannot produce this message. A bare `!= 404` check would be +/// wrong here — the handler legitimately returns 404 under this config. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn page_bids_get_is_routed_on_canonical_path_and_alias() { + let mut responses = Vec::new(); + + for path in ["/_ts/page-bids", "/__ts/page-bids"] { + let req = request_builder() + .method("GET") + .uri(path) + .header("sec-fetch-site", "same-origin") + .body(edgezero_core::body::Body::empty()) + .expect("should build request"); + let resp = route(test_router(), req).await; + let status = resp.status().as_u16(); + let body = String::from_utf8_lossy(&resp.into_body().into_bytes().unwrap_or_default()) + .into_owned(); + + assert!( + body.contains("Creative opportunities not configured"), + "GET {path} must reach the page-bids handler, \ + got status {status} body {body:?}" + ); + + responses.push((status, body)); + } + + assert_eq!( + responses[0], responses[1], + "the deprecated alias must answer identically to the canonical path" + ); +} + // --------------------------------------------------------------------------- // Publisher fallback method parity — non-GET/POST methods must reach the // publisher origin fallback (not a router-level 405), matching Fastly/Axum. diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 1b0ced7a7..1d959c05d 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -86,7 +86,7 @@ const MAX_AUCTION_BODY_SIZE: usize = 256 * 1024; /// callers** (e.g. slim-Prebid, native apps, server-to-server integrations). /// It is **not** the intended path for scroll or GPT refresh events. /// -/// **SPA navigation** is handled by `GET /__ts/page-bids`: the client-side SPA +/// **SPA navigation** is handled by `GET /_ts/page-bids`: the client-side SPA /// hook (`installSpaAuctionHook`) intercepts `pushState`/`replaceState`/`popstate` /// events and calls that endpoint to fetch fresh slots and bids for each new /// route, then invokes `window.tsjs.adInit()` with the updated data. @@ -171,7 +171,7 @@ pub async fn handle_auction( let consent_context = ec_context.consent().clone(); // Server-side auction consent gate. The publisher-navigation and - // `/__ts/page-bids` paths fail closed for GDPR/unknown jurisdictions that + // `/_ts/page-bids` paths fail closed for GDPR/unknown jurisdictions that // lack effective TCF Purpose 1. `/auction` is the programmatic entry point // for the same server-side auction, so it must gate identically: returning // a no-bid response here prevents outbound PBS/APS calls and the forwarding @@ -234,7 +234,7 @@ pub async fn handle_auction( // denied but a non-personalized auction may still run — could forward // persistent client EIDs from the body/cookie, since `gate_eids_by_consent` // only strips on TCF/GDPR signals. This matches the publisher and - // `/__ts/page-bids` paths, which also resolve client EIDs only when + // `/_ts/page-bids` paths, which also resolve client EIDs only when // `ec_id.is_some()`. let client_eids = if ec_id.is_some() { resolve_client_auction_eids( @@ -646,7 +646,7 @@ mod tests { // GDPR/unknown jurisdiction lacking effective TCF Purpose 1 must not run // a server-side auction. The /auction endpoint must short-circuit to a // no-bid response before dispatching to any provider — matching the - // publisher-navigation and /__ts/page-bids paths. + // publisher-navigation and /_ts/page-bids paths. let settings = create_test_settings(); let config = AuctionConfig { enabled: true, diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index bf9ecad7b..71afa7c50 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -367,7 +367,7 @@ impl AuctionOrchestrator { // restore nurl/burl/ad_id and PBS cache fields from the collected SSP // responses. The dispatched collect path already does this; the // synchronous mediation path used by POST /auction and - // /__ts/page-bids must match or mediated cache bids lose the metadata + // /_ts/page-bids must match or mediated cache bids lose the metadata // needed for creative rendering and win/billing beacons. let mediator_resp = mediator .parse_response_with_context( @@ -1779,7 +1779,7 @@ mod tests { // run_parallel_mediation must parse the mediator response via // parse_response_with_context so cache/nurl fields restored from SSP // responses survive the synchronous mediation path (POST /auction, - // /__ts/page-bids), matching the dispatched collect path. + // /_ts/page-bids), matching the dispatched collect path. let stub = Arc::new(StubHttpClient::new()); stub.push_response(200, b"{}".to_vec()); // bidder send_async stub.push_response(200, b"{}".to_vec()); // mediator send_async diff --git a/crates/trusted-server-core/src/auction/telemetry.rs b/crates/trusted-server-core/src/auction/telemetry.rs index d63445369..02752c6f9 100644 --- a/crates/trusted-server-core/src/auction/telemetry.rs +++ b/crates/trusted-server-core/src/auction/telemetry.rs @@ -25,7 +25,7 @@ const DYNAMIC_SEGMENT_REPLACEMENT: &str = ":id"; pub enum AuctionSource { /// Initial publisher navigation using server-side ad templates. InitialNavigation, - /// SPA navigation through `GET /__ts/page-bids`. + /// SPA navigation through `GET /_ts/page-bids`. SpaNavigation, /// Explicit `POST /auction` API. AuctionApi, diff --git a/crates/trusted-server-core/src/integrations/gpt.rs b/crates/trusted-server-core/src/integrations/gpt.rs index e21058a21..7783332d9 100644 --- a/crates/trusted-server-core/src/integrations/gpt.rs +++ b/crates/trusted-server-core/src/integrations/gpt.rs @@ -483,7 +483,7 @@ impl IntegrationHeadInjector for GptIntegration { /// for GPT refresh events, runs client-side auctions, and sets targeting for /// subsequent impressions. SPA navigation is handled separately by /// `installSpaAuctionHook()` in the GPT bundle, which re-runs the server-side - /// auction via `GET /__ts/page-bids` on pushState / replaceState / popstate + /// auction via `GET /_ts/page-bids` on pushState / replaceState / popstate /// route changes (see `auction/endpoints.rs`). /// The `POST /auction` endpoint is not involved in scroll or refresh flows. fn head_inserts(&self, _ctx: &IntegrationHtmlContext<'_>) -> Vec { diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 4d6e9d2ee..82767e323 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -2276,7 +2276,27 @@ fn is_supported_content_encoding(encoding: &str) -> bool { matches!(encoding, "" | "identity" | "gzip" | "deflate" | "br") } -/// Same-origin gate for `/__ts/page-bids`. +/// Canonical URL path of the SPA re-auction endpoint. +/// +/// Lives in the internal `/_ts/` namespace shared by every other Trusted +/// Server route. Adapters register this path; the tsjs SPA hook fetches it. +pub const PAGE_BIDS_PATH: &str = "/_ts/page-bids"; + +/// Deprecated double-underscore alias of [`PAGE_BIDS_PATH`]. +/// +/// The endpoint originally shipped as `/__ts/page-bids`, the only internal path +/// using a `__` prefix. Renaming it is atomic on the server, but a browser runs +/// whichever tsjs bundle it was already served: pages loaded before the rename — +/// and cached bundles — keep requesting this path, and on a SPA that path is what +/// delivers ads for in-session navigations. Adapters route it to the same handler +/// so those clients keep working. +/// +/// Removal is tracked by IABTechLab/trusted-server#970: drop this const and its +/// four adapter registrations once access logs show no remaining traffic on the +/// legacy path. +pub const PAGE_BIDS_LEGACY_PATH: &str = "/__ts/page-bids"; + +/// Same-origin gate for `/_ts/page-bids`. /// /// The endpoint is a side-effecting GET: it dispatches real PBS/APS auctions /// and forwards request-derived signals (IP, UA, geo, consent) to partners. @@ -2306,7 +2326,7 @@ fn page_bids_request_allowed(req: &Request) -> bool { } /// Builds the `403 Forbidden` returned when the side-effecting -/// `/__ts/page-bids` endpoint refuses a request — both the CORS preflight +/// `/_ts/page-bids` endpoint refuses a request — both the CORS preflight /// (`OPTIONS`) and the GET cross-site gate ([`page_bids_request_allowed`]) /// return this single denial shape. /// @@ -2315,7 +2335,8 @@ fn page_bids_request_allowed(req: &Request) -> bool { /// preflight; letting `OPTIONS` fall through to the publisher origin (which may /// return permissive CORS) would defeat that, allowing a cross-site page to /// trigger real PBS/APS auctions from a visitor's browser. Every adapter returns -/// this same response for `OPTIONS /__ts/page-bids`. +/// this same response for `OPTIONS /_ts/page-bids` and for its deprecated +/// `/__ts/page-bids` alias. pub fn page_bids_preflight_denied() -> Response { let mut response = Response::new(EdgeBody::from("Forbidden")); *response.status_mut() = StatusCode::FORBIDDEN; @@ -2340,7 +2361,7 @@ fn normalize_page_bids_path(raw: &str) -> String { } } -/// Handle `GET /__ts/page-bids?path=` — server-side auction for SPA navigation. +/// Handle `GET /_ts/page-bids?path=` — server-side auction for SPA navigation. /// /// Matches creative opportunity slots for the given path, runs a server-side /// auction (APS + PBS), and returns the slot definitions and winning bids as JSON. @@ -2380,6 +2401,22 @@ pub async fn handle_page_bids( return Ok(page_bids_preflight_denied()); } + // Deprecation signal for the transition alias. Logged only after the + // cross-site gate passes, so the count reflects genuine SPA clients still + // running a pre-rename tsjs bundle rather than anything a third-party page + // can inflate. This is the only in-app signal that + // `PAGE_BIDS_LEGACY_PATH` is still in use — the removal precondition in + // IABTechLab/trusted-server#970 is "no remaining traffic on the legacy + // path", which is otherwise only answerable from edge access logs. The line + // is self-limiting: it goes silent as old bundles age out, which is exactly + // the condition being waited on. + if req.uri().path() == PAGE_BIDS_LEGACY_PATH { + log::info!( + "page-bids: served deprecated alias {PAGE_BIDS_LEGACY_PATH} \ + (pre-rename tsjs bundle); see IABTechLab/trusted-server#970" + ); + } + let path_param = req .uri() .query() @@ -4947,11 +4984,15 @@ mod tests { } fn make_page_bids_request(path: &str) -> Request { + make_page_bids_request_on(PAGE_BIDS_PATH, path) + } + + /// Builds a page-bids request against an explicit endpoint path, so the + /// canonical route and its deprecated alias can be compared directly. + fn make_page_bids_request_on(endpoint: &str, path: &str) -> Request { let mut req = Request::builder() .method(Method::GET) - .uri(format!( - "https://test-publisher.com/_ts/page-bids?path={path}" - )) + .uri(format!("https://test-publisher.com{endpoint}?path={path}")) .body(EdgeBody::empty()) .expect("should build test request"); // Pass the same-origin gate the way a browser fetch from the @@ -5002,6 +5043,46 @@ mod tests { .expect("should return ok response") } + /// The deprecated `/__ts/page-bids` alias must be handled identically to + /// the canonical path — same status, same JSON body. + /// + /// The alias exists so pre-rename tsjs bundles keep getting ads on SPA + /// navigations. If the handler ever varied its output by request path + /// (slot matching reads the `path` *query parameter*, not the endpoint + /// path), those clients would silently get different results from the + /// ones on the canonical route. + #[tokio::test] + async fn deprecated_alias_response_matches_canonical_path() { + let settings = settings_with_co(); + let orchestrator = AuctionOrchestrator::new(settings.auction.clone()); + + let canonical = run_page_bids_response( + &settings, + &orchestrator, + &article_slot(), + make_page_bids_request_on(PAGE_BIDS_PATH, "/2024/01/my-article/"), + ) + .await; + let alias = run_page_bids_response( + &settings, + &orchestrator, + &article_slot(), + make_page_bids_request_on(PAGE_BIDS_LEGACY_PATH, "/2024/01/my-article/"), + ) + .await; + + assert_eq!( + canonical.status(), + alias.status(), + "alias must return the same status as the canonical path" + ); + assert_eq!( + canonical.into_body().into_bytes(), + alias.into_body().into_bytes(), + "alias must return the same body as the canonical path" + ); + } + #[tokio::test] async fn cross_site_fetch_metadata_is_rejected() { let settings = settings_with_co(); diff --git a/crates/trusted-server-integration-tests/tests/parity.rs b/crates/trusted-server-integration-tests/tests/parity.rs index e85b1d8d1..acf7f5f4b 100644 --- a/crates/trusted-server-integration-tests/tests/parity.rs +++ b/crates/trusted-server-integration-tests/tests/parity.rs @@ -696,28 +696,34 @@ async fn auction_not_challenged_by_auth_parity() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn page_bids_options_preflight_denied_parity() { - // OPTIONS /__ts/page-bids is a CORS preflight to a side-effecting endpoint. + // OPTIONS /_ts/page-bids is a CORS preflight to a side-effecting endpoint. // Every adapter must refuse it with 403 rather than proxy it to the origin: // a permissive origin preflight would let a cross-site page defeat the GET // handler's `X-TSJS-Page-Bids` gate and trigger real auctions in a visitor's // browser. The denial is unconditional (independent of creative-opportunity // configuration), so all adapters must agree on 403. - let (axum_status, _) = axum_options("/__ts/page-bids").await; - let (cf_status, _) = cf_options("/__ts/page-bids").await; - let (spin_status, _) = spin_options("/__ts/page-bids").await; + // + // The deprecated `/__ts/page-bids` alias routes to the same handler, so it + // must deny the preflight identically — an alias that fell through to the + // origin would reopen the hole the canonical path closes. + for path in ["/_ts/page-bids", "/__ts/page-bids"] { + let (axum_status, _) = axum_options(path).await; + let (cf_status, _) = cf_options(path).await; + let (spin_status, _) = spin_options(path).await; - assert_eq!( - axum_status, 403, - "Axum OPTIONS /__ts/page-bids must be denied with 403, got {axum_status}" - ); - assert_eq!( - cf_status, 403, - "Cloudflare OPTIONS /__ts/page-bids must be denied with 403, got {cf_status}" - ); - assert_eq!( - spin_status, 403, - "Spin OPTIONS /__ts/page-bids must be denied with 403, got {spin_status}" - ); + assert_eq!( + axum_status, 403, + "Axum OPTIONS {path} must be denied with 403, got {axum_status}" + ); + assert_eq!( + cf_status, 403, + "Cloudflare OPTIONS {path} must be denied with 403, got {cf_status}" + ); + assert_eq!( + spin_status, 403, + "Spin OPTIONS {path} must be denied with 403, got {spin_status}" + ); + } } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts index ca4689684..85e27b506 100644 --- a/crates/trusted-server-js/lib/src/integrations/gpt/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/gpt/index.ts @@ -696,7 +696,7 @@ function waitForSlotElements(slots: AuctionSlot[], signal: AbortSignal): Promise * * Patches `history.pushState` and `history.replaceState`, and listens to * `popstate`, so that after each client-side route change the trusted server - * fetches fresh slots + bids from `/__ts/page-bids?path=`, updates + * fetches fresh slots + bids from `/_ts/page-bids?path=`, updates * `window.tsjs.adSlots` / `window.tsjs.bids`, and calls `window.tsjs.adInit()`. * * Idempotent: guarded by `window.tsjs.spaHookInstalled` so multiple calls are safe. @@ -728,7 +728,7 @@ export function installSpaAuctionHook(): void { inflight = controller; try { - const res = await fetch(`/__ts/page-bids?path=${encodeURIComponent(path)}`, { + const res = await fetch(`/_ts/page-bids?path=${encodeURIComponent(path)}`, { credentials: 'include', // Non-simple header doubles as a CSRF token: the server rejects // requests that carry neither same-origin Fetch Metadata nor this diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts index 9a08defcb..7dc29989c 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/spa_hook.test.ts @@ -78,7 +78,7 @@ describe('installSpaAuctionHook', () => { await flushAsync(); expect(fetchStub).toHaveBeenCalledWith( - '/__ts/page-bids?path=%2Fnext-page', + '/_ts/page-bids?path=%2Fnext-page', expect.objectContaining({ credentials: 'include', headers: { 'X-TSJS-Page-Bids': '1' }, @@ -223,7 +223,7 @@ describe('installSpaAuctionHook', () => { history.replaceState({}, '', '/replaced'); await flushAsync(); expect(fetchStub).toHaveBeenCalledWith( - '/__ts/page-bids?path=%2Freplaced', + '/_ts/page-bids?path=%2Freplaced', expect.objectContaining({ credentials: 'include' }) ); }); @@ -241,7 +241,7 @@ describe('installSpaAuctionHook', () => { window.dispatchEvent(new PopStateEvent('popstate')); await flushAsync(); expect(fetchStub).toHaveBeenCalledWith( - '/__ts/page-bids?path=%2Fpopped', + '/_ts/page-bids?path=%2Fpopped', expect.objectContaining({ credentials: 'include' }) ); }); From 4ef2de6db5a8ac926fdde1b93ec0b127483adae4 Mon Sep 17 00:00:00 2001 From: Christian Date: Tue, 14 Jul 2026 14:07:32 -0500 Subject: [PATCH 126/198] Preserve Prebid ad units across GPT refreshes --- .../lib/src/integrations/prebid/index.ts | 239 +++++++- .../test/integrations/prebid/index.test.ts | 509 ++++++++++++++++++ 2 files changed, 731 insertions(+), 17 deletions(-) diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 342e4038d..29ac42399 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -238,6 +238,17 @@ type TrustedServerAdUnit = { mediaTypes?: { banner?: TrustedServerBanner }; bids?: TrustedServerBid[]; }; +type ClientSideBidSnapshot = { bidder: string; params: Record }; +type PublisherAdUnitSnapshot = { + bidderParams: Record>; + clientSideBids: ClientSideBidSnapshot[]; + zone?: string; +}; +type PublisherDeliveryContext = { remainingCodes: Set }; + +let publisherAdUnitSnapshots = new Map(); +let syntheticRefreshAdUnits = new WeakSet(); +const activePublisherDeliveryContexts: PublisherDeliveryContext[] = []; type TrustedServerBidRequest = { adUnitCode?: string; code?: string; @@ -373,6 +384,17 @@ function firstTargetingValue(values: string[] | undefined): string | undefined { * code in order and return the first matching ad unit, so container-backed slots * still recover the publisher's configured params and bidders. */ +function findRefreshSnapshot( + candidateCodes: Array +): PublisherAdUnitSnapshot | undefined { + for (const code of candidateCodes) { + if (!code) continue; + const snapshot = publisherAdUnitSnapshots.get(code); + if (snapshot) return snapshot; + } + return undefined; +} + function findRefreshAdUnit( candidateCodes: Array ): TrustedServerAdUnit | undefined { @@ -385,6 +407,89 @@ function findRefreshAdUnit( return undefined; } +function copyParamValue(value: unknown, seen = new WeakMap()): unknown { + if (Array.isArray(value)) { + const existing = seen.get(value); + if (existing) return existing; + const copy: unknown[] = []; + seen.set(value, copy); + value.forEach((entry) => copy.push(copyParamValue(entry, seen))); + return copy; + } + + if (value && typeof value === 'object') { + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) return value; + + const existing = seen.get(value); + if (existing) return existing; + const copy = Object.create(prototype) as Record; + seen.set(value, copy); + for (const [key, entry] of Object.entries(value)) { + Object.defineProperty(copy, key, { + value: copyParamValue(entry, seen), + enumerable: true, + configurable: true, + writable: true, + }); + } + return copy; + } + + return value; +} + +function copyParams(params: Record | undefined): Record { + return copyParamValue(params ?? {}) as Record; +} + +function foldedBidderParams( + bid: TrustedServerBid | undefined +): Record> { + const folded = (bid?.params?.[BIDDER_PARAMS_KEY] ?? {}) as Record< + string, + Record + >; + return Object.fromEntries( + Object.entries(folded).map(([bidder, params]) => [bidder, copyParams(params)]) + ); +} + +function capturePublisherAdUnitSnapshot( + unit: TrustedServerAdUnit, + clientSideBidders: Set +): PublisherAdUnitSnapshot | undefined { + if (typeof unit.code !== 'string' || unit.code.length === 0) return undefined; + + const rawBidderParams: Record> = {}; + const clientSideBids: ClientSideBidSnapshot[] = []; + let existingTsBid: TrustedServerBid | undefined; + + const bids = Array.isArray(unit.bids) ? unit.bids : []; + for (const bid of bids) { + if (!bid?.bidder) continue; + if (bid.bidder === ADAPTER_CODE) { + existingTsBid ??= bid; + continue; + } + if (clientSideBidders.has(bid.bidder)) { + clientSideBids.push({ bidder: bid.bidder, params: copyParams(bid.params) }); + continue; + } + rawBidderParams[bid.bidder] = copyParams(bid.params); + } + + const bidderParams = + Object.keys(rawBidderParams).length > 0 ? rawBidderParams : foldedBidderParams(existingTsBid); + const zone = unit.mediaTypes?.banner?.name; + + return { + bidderParams, + clientSideBids, + ...(zone ? { zone } : {}), + }; +} + /** * Collect the configured client-side bidder entries for a refreshing slot. * @@ -399,6 +504,14 @@ function findRefreshAdUnit( function clientSideBidsForRefresh( candidateCodes: Array ): Array<{ bidder: string; params: Record }> { + const snapshot = findRefreshSnapshot(candidateCodes); + if (snapshot) { + return snapshot.clientSideBids.map((bid) => ({ + bidder: bid.bidder, + params: copyParams(bid.params), + })); + } + const clientSideBidders = new Set(getInjectedConfig()?.clientSideBidders ?? []); if (clientSideBidders.size === 0) return []; @@ -408,7 +521,7 @@ function clientSideBidsForRefresh( const bids: Array<{ bidder: string; params: Record }> = []; for (const bid of match.bids) { if (bid?.bidder && clientSideBidders.has(bid.bidder)) { - bids.push({ bidder: bid.bidder, params: bid.params ?? {} }); + bids.push({ bidder: bid.bidder, params: copyParams(bid.params) }); } } return bids; @@ -430,6 +543,13 @@ function clientSideBidsForRefresh( function serverSideBidderParamsForRefresh( candidateCodes: Array ): Record> { + const snapshot = findRefreshSnapshot(candidateCodes); + if (snapshot) { + return Object.fromEntries( + Object.entries(snapshot.bidderParams).map(([bidder, params]) => [bidder, copyParams(params)]) + ); + } + const match = findRefreshAdUnit(candidateCodes); if (!match?.bids) return {}; @@ -466,6 +586,50 @@ function clearRefreshTargeting(slot: RefreshGptSlot): void { } } +function removePublisherDeliveryContext(context: PublisherDeliveryContext): void { + const index = activePublisherDeliveryContexts.lastIndexOf(context); + if (index >= 0) activePublisherDeliveryContexts.splice(index, 1); +} + +function consumeBarePublisherDeliveryContext(): boolean { + for (let index = activePublisherDeliveryContexts.length - 1; index >= 0; index -= 1) { + const context = activePublisherDeliveryContexts[index]; + if (context.remainingCodes.size === 0) continue; + context.remainingCodes.clear(); + return true; + } + return false; +} + +function consumeExplicitPublisherDeliveryContext(targetSlots: RefreshGptSlot[]): boolean { + if (targetSlots.length === 0) return false; + + for (let index = activePublisherDeliveryContexts.length - 1; index >= 0; index -= 1) { + const context = activePublisherDeliveryContexts[index]; + const coveredCodes: string[] = []; + let allCovered = true; + + for (const slot of targetSlots) { + const injectedSlot = findInjectedSlotForRefresh(slot); + const candidates = [refreshSlotElementId(slot), injectedSlot?.div_id]; + const coveredCode = candidates.find( + (code): code is string => !!code && context.remainingCodes.has(code) + ); + if (!coveredCode) { + allCovered = false; + break; + } + coveredCodes.push(coveredCode); + } + + if (!allCovered) continue; + coveredCodes.forEach((code) => context.remainingCodes.delete(code)); + return true; + } + + return false; +} + function collectAuctionEids(): AuctionEid[] | undefined { if (typeof pbjs.getUserIdsAsEids !== 'function') { return undefined; @@ -502,6 +666,10 @@ function collectAuctionEids(): AuctionEid[] | undefined { * 2. `config` argument — explicit overrides from the publisher's JS */ export function installPrebidNpm(config?: Partial): typeof pbjs { + publisherAdUnitSnapshots = new Map(); + syntheticRefreshAdUnits = new WeakSet(); + activePublisherDeliveryContexts.length = 0; + const injected = getInjectedConfig(); const merged: PrebidNpmConfig = { endpoint: config?.endpoint, @@ -574,9 +742,20 @@ export function installPrebidNpm(config?: Partial): typeof pbjs const opts = requestObj || {}; // eslint-disable-next-line @typescript-eslint/no-explicit-any const adUnits = ((opts as any).adUnits || pbjs.adUnits || []) as TrustedServerAdUnit[]; + const isSyntheticRefresh = + adUnits.length > 0 && adUnits.every((unit) => syntheticRefreshAdUnits.has(unit)); + const publisherAdUnitCodes = new Set(); // Ensure every ad unit has a trustedServer bid entry for (const unit of adUnits) { + if (!syntheticRefreshAdUnits.has(unit)) { + const snapshot = capturePublisherAdUnitSnapshot(unit, clientSideBidders); + if (snapshot && unit.code) { + publisherAdUnitSnapshots.set(unit.code, snapshot); + publisherAdUnitCodes.add(unit.code); + } + } + if (!Array.isArray(unit.bids)) { unit.bids = []; } @@ -660,8 +839,22 @@ export function installPrebidNpm(config?: Partial): typeof pbjs const originalBidsBack = opts.bidsBackHandler; opts.bidsBackHandler = function (...args: unknown[]) { syncPrebidEidsCookie(); - if (typeof originalBidsBack === 'function') { - originalBidsBack.apply(this, args); + if (typeof originalBidsBack !== 'function') return; + if (isSyntheticRefresh || publisherAdUnitCodes.size === 0) { + originalBidsBack.apply(this, args as Parameters); + return; + } + + const context: PublisherDeliveryContext = { + remainingCodes: new Set(publisherAdUnitCodes), + }; + // Delivery attribution is intentionally synchronous and ends as soon as + // the publisher's original callback returns. + activePublisherDeliveryContexts.push(context); + try { + originalBidsBack.apply(this, args as Parameters); + } finally { + removePublisherDeliveryContext(context); } }; @@ -745,6 +938,14 @@ export function installRefreshHandler(timeoutMs = 1500): void { const originalRefresh = pubads.refresh.bind(pubads); pubads.refresh = function (slots?: unknown[], opts?: unknown) { + // For bare refresh() calls (no slots arg), get all registered slots from GPT + // so we can auction the same concrete slot list and avoid stale targeting. + const targetSlots = ( + slots ?? + (pubads as { getSlots?: () => unknown[] }).getSlots?.() ?? + [] + ).filter((slot): slot is RefreshGptSlot => typeof slot === 'object' && slot !== null); + // One-shot bypass for adInit()'s internal refresh: that refresh delivers // freshly applied server-side targeting to GAM and must not be turned // into a client-side auction (which would clear the TS targeting). @@ -754,13 +955,14 @@ export function installRefreshHandler(timeoutMs = 1500): void { return originalRefresh(slots, opts); } - // For bare refresh() calls (no slots arg), get all registered slots from GPT - // so we can auction the same concrete slot list and avoid stale targeting. - const targetSlots = ( - slots ?? - (pubads as { getSlots?: () => unknown[] }).getSlots?.() ?? - [] - ).filter((slot): slot is RefreshGptSlot => typeof slot === 'object' && slot !== null); + const isExplicitSlotList = slots !== undefined; + const hasOnlyValidExplicitSlots = !isExplicitSlotList || targetSlots.length === slots.length; + const isPublisherDeliveryRefresh = isExplicitSlotList + ? hasOnlyValidExplicitSlots && consumeExplicitPublisherDeliveryContext(targetSlots) + : consumeBarePublisherDeliveryContext(); + if (isPublisherDeliveryRefresh) { + return originalRefresh(slots, opts); + } if (!targetSlots.length) { return originalRefresh(slots, opts); @@ -770,8 +972,16 @@ export function installRefreshHandler(timeoutMs = 1500): void { const adUnits = targetSlots.map((slot) => { const injectedSlot = findInjectedSlotForRefresh(slot); + const code = refreshSlotElementId(slot) ?? 'refresh-slot'; + // A TS-owned slot may be defined on `${div_id}-container`, so the GPT + // element id used as the synthetic refresh code can differ from the + // inner `div_id` the publisher keyed their ad unit by. Recover from both. + const candidateCodes = [code, injectedSlot?.div_id]; + const snapshot = findRefreshSnapshot(candidateCodes); const zone = - injectedSlot?.targeting?.[ZONE_KEY] ?? firstTargetingValue(slot.getTargeting?.(ZONE_KEY)); + injectedSlot?.targeting?.[ZONE_KEY] ?? + firstTargetingValue(slot.getTargeting?.(ZONE_KEY)) ?? + snapshot?.zone; const banner: TrustedServerBanner = { sizes: bannerSizesFromInjectedSlot(injectedSlot) ?? @@ -779,12 +989,6 @@ export function installRefreshHandler(timeoutMs = 1500): void { DEFAULT_REFRESH_SIZES, ...(zone ? { name: zone } : {}), }; - - const code = refreshSlotElementId(slot) ?? 'refresh-slot'; - // A TS-owned slot may be defined on `${div_id}-container`, so the GPT - // element id used as the synthetic refresh code can differ from the - // inner `div_id` the publisher keyed their ad unit by. Recover from both. - const candidateCodes = [code, injectedSlot?.div_id]; const tsParams: Record = zone ? { [ZONE_KEY]: zone } : {}; // Carry the publisher's inline server-side (PBS) bidder params captured // on the initial ad unit so refresh/scroll auctions don't drop them. @@ -807,6 +1011,7 @@ export function installRefreshHandler(timeoutMs = 1500): void { // unrelated GPT slots whose targeting this wrapper only cleared for // `targetSlots` — leaving their next request dependent on stale state. const refreshAdUnitCodes = adUnits.map((unit) => unit.code); + adUnits.forEach((unit) => syntheticRefreshAdUnits.add(unit)); pbjs.requestBids({ adUnits, bidsBackHandler: () => { diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 726f40b49..250bc0f0c 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -668,6 +668,15 @@ describe('prebid/installPrebidNpm', () => { expect(adUnits[0].bids[0].bidder).toBe('trustedServer'); }); + it('normalizes a truthy non-array bids value without throwing', () => { + const pbjs = installPrebidNpm(); + const adUnits = [{ code: 'example-malformed-slot', bids: { malformed: true } }] as any[]; + + expect(() => pbjs.requestBids({ adUnits } as any)).not.toThrow(); + + expect(adUnits[0].bids).toEqual([{ bidder: 'trustedServer', params: { bidderParams: {} } }]); + }); + it('includes zone from mediaTypes.banner.name in trustedServer params', () => { const pbjs = installPrebidNpm(); @@ -1403,6 +1412,506 @@ describe('prebid/installRefreshHandler', () => { }); }); +describe('prebid publisher snapshots and delivery refreshes', () => { + beforeEach(() => { + vi.clearAllMocks(); + mockRequestBids.mockReset(); + mockPbjs.requestBids = mockRequestBids; + mockPbjs.adUnits = []; + mockGetUserIdsAsEids.mockReset(); + mockGetUserIdsAsEids.mockReturnValue([]); + mockGetBidAdapter.mockReturnValue({}); + delete (mockPbjs as any).setTargetingForGPTAsync; + delete (window as any).__tsjs_prebid; + (window as any).tsjs = undefined; + delete (window as any).googletag; + }); + + afterEach(() => { + delete (window as any).__tsjs_prebid; + (window as any).tsjs = undefined; + delete (window as any).googletag; + }); + + function installGpt(slots: any[]) { + const originalRefresh = vi.fn(); + const pubads = { + refresh: originalRefresh, + getSlots: vi.fn(() => slots), + }; + (window as any).googletag = { + cmd: { push: (fn: () => void) => fn() }, + pubads: () => pubads, + }; + installRefreshHandler(640); + return { originalRefresh, pubads }; + } + + function refreshAdUnitFromLastRequest(): any { + const lastCall = mockRequestBids.mock.calls[mockRequestBids.mock.calls.length - 1]; + return lastCall?.[0]?.adUnits?.[0]; + } + + it('recovers inline params, ordered client bids, and zone when pbjs.adUnits is empty', () => { + (window as any).__tsjs_prebid = { clientSideBidders: ['exampleBrowser'] }; + const runtimeInstance = 'example-runtime-instance'; + const code = `example-slot-${runtimeInstance}`; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [{ getWidth: () => 320, getHeight: () => 100 }], + clearTargeting: vi.fn(), + }; + const { pubads } = installGpt([slot]); + const pbjs = installPrebidNpm(); + const firstParams = { placement: 'first' }; + const effectiveParams = { placement: 'effective' }; + + pbjs.requestBids({ + adUnits: [ + { + code, + mediaTypes: { banner: { name: 'example-zone', sizes: [[320, 100]] } }, + bids: [ + { bidder: 'exampleServer', params: firstParams }, + { bidder: 'exampleBrowser', params: { placement: 'browser-one' } }, + { bidder: 'exampleServer', params: effectiveParams }, + { bidder: 'exampleBrowser', params: { placement: 'browser-two' } }, + ], + }, + ], + } as any); + effectiveParams.placement = 'changed-after-auction'; + + pubads.refresh([slot]); + + expect(mockPbjs.adUnits).toEqual([]); + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(refreshAdUnitFromLastRequest()).toEqual({ + code, + mediaTypes: { banner: { name: 'example-zone', sizes: [[320, 100]] } }, + bids: [ + { + bidder: 'trustedServer', + params: { + bidderParams: { exampleServer: { placement: 'effective' } }, + zone: 'example-zone', + }, + }, + { bidder: 'exampleBrowser', params: { placement: 'browser-one' } }, + { bidder: 'exampleBrowser', params: { placement: 'browser-two' } }, + ], + }); + }); + + it('isolates nested bidder-param objects and arrays from later publisher mutation', () => { + (window as any).__tsjs_prebid = { clientSideBidders: ['exampleBrowser'] }; + const code = 'example-nested-params-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { pubads } = installGpt([slot]); + const pbjs = installPrebidNpm(); + const serverParams = { + placement: { + rules: [{ label: 'original-rule' }], + sizes: [300, 250], + }, + }; + const browserParams = { + groups: [{ values: ['original-value'] }], + }; + + pbjs.requestBids({ + adUnits: [ + { + code, + bids: [ + { bidder: 'exampleServer', params: serverParams }, + { bidder: 'exampleBrowser', params: browserParams }, + ], + }, + ], + } as any); + serverParams.placement.rules[0].label = 'changed-rule'; + serverParams.placement.sizes.push(999); + browserParams.groups[0].values[0] = 'changed-value'; + + pubads.refresh([slot]); + + const expectedBids = [ + { + bidder: 'trustedServer', + params: { + bidderParams: { + exampleServer: { + placement: { + rules: [{ label: 'original-rule' }], + sizes: [300, 250], + }, + }, + }, + }, + }, + { + bidder: 'exampleBrowser', + params: { groups: [{ values: ['original-value'] }] }, + }, + ]; + const firstRefreshBids = refreshAdUnitFromLastRequest().bids; + expect(firstRefreshBids).toEqual(expectedBids); + + firstRefreshBids[0].params.bidderParams.exampleServer.placement.rules[0].label = + 'changed-refresh-rule'; + firstRefreshBids[0].params.bidderParams.exampleServer.placement.sizes.push(777); + firstRefreshBids[1].params.groups[0].values[0] = 'changed-refresh-value'; + pubads.refresh([slot]); + + expect(refreshAdUnitFromLastRequest().bids).toEqual(expectedBids); + }); + + it('keeps snapshots across repeated synthetic refreshes and overwrites newer publisher config', () => { + const code = 'example-dynamic-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { pubads } = installGpt([slot]); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { + code, + mediaTypes: { banner: { name: 'example-zone-one', sizes: [[300, 250]] } }, + bids: [{ bidder: 'exampleServer', params: { placement: 'one' } }], + }, + ], + } as any); + pubads.refresh([slot]); + expect(refreshAdUnitFromLastRequest().bids[0].params).toEqual({ + bidderParams: { exampleServer: { placement: 'one' } }, + zone: 'example-zone-one', + }); + + pubads.refresh([slot]); + expect(refreshAdUnitFromLastRequest().bids[0].params).toEqual({ + bidderParams: { exampleServer: { placement: 'one' } }, + zone: 'example-zone-one', + }); + + pbjs.requestBids({ + adUnits: [ + { + code, + mediaTypes: { banner: { name: 'example-zone-two', sizes: [[300, 250]] } }, + bids: [{ bidder: 'exampleServer', params: { placement: 'two' } }], + }, + ], + } as any); + pubads.refresh([slot]); + + expect(refreshAdUnitFromLastRequest().bids[0].params).toEqual({ + bidderParams: { exampleServer: { placement: 'two' } }, + zone: 'example-zone-two', + }); + }); + + it('does not cross-contaminate dynamic-code snapshots and retains the global fallback', () => { + const slotOne = { + getSlotElementId: () => 'example-code-one', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const slotTwo = { + getSlotElementId: () => 'example-code-two', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const globalSlot = { + getSlotElementId: () => 'example-global-code', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { pubads } = installGpt([slotOne, slotTwo, globalSlot]); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { + code: 'example-code-one', + bids: [{ bidder: 'exampleServer', params: { placement: 'one' } }], + }, + { + code: 'example-code-two', + bids: [{ bidder: 'exampleServer', params: { placement: 'two' } }], + }, + ], + } as any); + mockPbjs.adUnits = [ + { + code: 'example-global-code', + bids: [{ bidder: 'exampleFallback', params: { placement: 'global' } }], + }, + ]; + + pubads.refresh([slotOne]); + expect(refreshAdUnitFromLastRequest().bids[0].params.bidderParams).toEqual({ + exampleServer: { placement: 'one' }, + }); + pubads.refresh([slotTwo]); + expect(refreshAdUnitFromLastRequest().bids[0].params.bidderParams).toEqual({ + exampleServer: { placement: 'two' }, + }); + pubads.refresh([globalSlot]); + expect(refreshAdUnitFromLastRequest().bids[0].params.bidderParams).toEqual({ + exampleFallback: { placement: 'global' }, + }); + }); + + it('bypasses explicit covered subset delivery refreshes without clearing targeting', () => { + const slotOne = { + getSlotElementId: () => 'example-covered-one', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const slotTwo = { + getSlotElementId: () => 'example-covered-two-container', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + (window as any).tsjs = { + adSlots: [{ div_id: 'example-covered-two', formats: [[300, 250]], targeting: {} }], + }; + const { originalRefresh, pubads } = installGpt([slotOne, slotTwo]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { code: 'example-covered-one', bids: [{ bidder: 'exampleServer', params: {} }] }, + { code: 'example-covered-two', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => { + pubads.refresh([slotOne]); + pubads.refresh([slotTwo]); + }, + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(slotOne.clearTargeting).not.toHaveBeenCalled(); + expect(slotTwo.clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenNthCalledWith(1, [slotOne], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [slotTwo], undefined); + }); + + it('bypasses a bare delivery refresh even when GPT includes a GAM-only extra slot', () => { + const coveredSlot = { + getSlotElementId: () => 'example-covered', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const gamOnlySlot = { + getSlotElementId: () => 'example-gam-only-interstitial', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([coveredSlot, gamOnlySlot]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code: 'example-covered', bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh(), + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(coveredSlot.clearTargeting).not.toHaveBeenCalled(); + expect(gamOnlySlot.clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith(undefined, undefined); + }); + + it('keeps explicit unrelated and mixed delivery lists on the synthetic path', () => { + const coveredSlot = { + getSlotElementId: () => 'example-covered', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const unrelatedSlot = { + getSlotElementId: () => 'example-unrelated', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([coveredSlot, unrelatedSlot]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code: 'example-covered', bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => { + pubads.refresh([unrelatedSlot]); + pubads.refresh([coveredSlot, unrelatedSlot]); + }, + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(3); + expect(mockRequestBids.mock.calls[1][0].adUnits.map((unit: any) => unit.code)).toEqual([ + 'example-unrelated', + ]); + expect(mockRequestBids.mock.calls[2][0].adUnits.map((unit: any) => unit.code)).toEqual([ + 'example-covered', + 'example-unrelated', + ]); + expect(coveredSlot.clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(unrelatedSlot.clearTargeting).toHaveBeenCalledWith('ts_initial'); + expect(unrelatedSlot.clearTargeting).toHaveBeenCalledWith('hb_cache_path'); + expect(originalRefresh).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenNthCalledWith(1, [unrelatedSlot], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [coveredSlot, unrelatedSlot], undefined); + }); + + it('treats a microtask refresh after publisher delivery as an independent auction', async () => { + const slot = { + getSlotElementId: () => 'example-deferred-refresh', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + let deferredRefresh: Promise | undefined; + + pbjs.requestBids({ + adUnits: [ + { code: 'example-deferred-refresh', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => { + deferredRefresh = Promise.resolve().then(() => pubads.refresh([slot])); + }, + } as any); + await deferredRefresh; + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(slot.clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); + + it('keeps nested publisher delivery contexts isolated during reentrant auctions', () => { + const outerSlot = { + getSlotElementId: () => 'example-outer-delivery', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const innerSlot = { + getSlotElementId: () => 'example-inner-delivery', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([outerSlot, innerSlot]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { code: 'example-outer-delivery', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => { + pbjs.requestBids({ + adUnits: [ + { code: 'example-inner-delivery', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => pubads.refresh([innerSlot]), + } as any); + pubads.refresh([outerSlot]); + }, + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(innerSlot.clearTargeting).not.toHaveBeenCalled(); + expect(outerSlot.clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenNthCalledWith(1, [innerSlot], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [outerSlot], undefined); + }); + + it('cleans delivery context after a publisher callback throws', () => { + const slot = { + getSlotElementId: () => 'example-throwing-callback', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + expect(() => + pbjs.requestBids({ + adUnits: [ + { + code: 'example-throwing-callback', + bids: [{ bidder: 'exampleServer', params: {} }], + }, + ], + bidsBackHandler: () => { + throw new Error('example callback failure'); + }, + } as any) + ).toThrow('example callback failure'); + + pubads.refresh([slot]); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(slot.clearTargeting).toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledTimes(1); + }); + + it('completes an internal synthetic refresh once without recursion', () => { + const slot = { + getSlotElementId: () => 'example-independent-refresh', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + installPrebidNpm(); + + pubads.refresh([slot]); + + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); +}); + describe('prebid/client-side bidders', () => { beforeEach(() => { vi.clearAllMocks(); From a94559839747f432f8aee3d1c636820351fc6722 Mon Sep 17 00:00:00 2001 From: Christian Date: Tue, 14 Jul 2026 16:21:19 -0500 Subject: [PATCH 127/198] Handle deferred Prebid delivery refreshes --- .../lib/src/integrations/prebid/index.ts | 101 ++++++++-- .../test/integrations/prebid/index.test.ts | 188 +++++++++++++++++- 2 files changed, 259 insertions(+), 30 deletions(-) diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 29ac42399..0432d8695 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -48,6 +48,7 @@ const TS_REFRESH_TARGETING_KEYS = [ 'hb_cache_host', 'hb_cache_path', ] as const; +const PUBLISHER_DELIVERY_CONTEXT_TIMEOUT_MS = 1000; /** Configuration options for the Prebid integration. */ export interface PrebidNpmConfig { @@ -244,7 +245,12 @@ type PublisherAdUnitSnapshot = { clientSideBids: ClientSideBidSnapshot[]; zone?: string; }; -type PublisherDeliveryContext = { remainingCodes: Set }; +type PublisherDeliveryContext = { + remainingCodes: Set; + retainForTargetedRefresh: boolean; + cleanupTimer?: ReturnType; +}; +type SetTargetingForGptAsync = (...args: unknown[]) => unknown; let publisherAdUnitSnapshots = new Map(); let syntheticRefreshAdUnits = new WeakSet(); @@ -587,15 +593,32 @@ function clearRefreshTargeting(slot: RefreshGptSlot): void { } function removePublisherDeliveryContext(context: PublisherDeliveryContext): void { + if (context.cleanupTimer !== undefined) { + clearTimeout(context.cleanupTimer); + context.cleanupTimer = undefined; + } const index = activePublisherDeliveryContexts.lastIndexOf(context); if (index >= 0) activePublisherDeliveryContexts.splice(index, 1); } +function targetingCoversPublisherDeliveryContext( + adUnitCodes: unknown, + context: PublisherDeliveryContext +): boolean { + if (adUnitCodes === undefined) return context.remainingCodes.size > 0; + const codes = typeof adUnitCodes === 'string' ? [adUnitCodes] : adUnitCodes; + return ( + Array.isArray(codes) && + codes.some((code) => typeof code === 'string' && context.remainingCodes.has(code)) + ); +} + function consumeBarePublisherDeliveryContext(): boolean { for (let index = activePublisherDeliveryContexts.length - 1; index >= 0; index -= 1) { const context = activePublisherDeliveryContexts[index]; if (context.remainingCodes.size === 0) continue; context.remainingCodes.clear(); + removePublisherDeliveryContext(context); return true; } return false; @@ -604,30 +627,35 @@ function consumeBarePublisherDeliveryContext(): boolean { function consumeExplicitPublisherDeliveryContext(targetSlots: RefreshGptSlot[]): boolean { if (targetSlots.length === 0) return false; - for (let index = activePublisherDeliveryContexts.length - 1; index >= 0; index -= 1) { - const context = activePublisherDeliveryContexts[index]; - const coveredCodes: string[] = []; - let allCovered = true; - - for (const slot of targetSlots) { - const injectedSlot = findInjectedSlotForRefresh(slot); - const candidates = [refreshSlotElementId(slot), injectedSlot?.div_id]; + // Publishers may include GAM-only slots in the same explicit refresh that + // delivers a completed Prebid auction. Attribute the call to delivery when + // any slot is covered, while consuming only the covered codes so an + // unrelated-only refresh still follows the synthetic auction path. + const matches = new Map>(); + for (const slot of targetSlots) { + const injectedSlot = findInjectedSlotForRefresh(slot); + const candidates = [refreshSlotElementId(slot), injectedSlot?.div_id]; + + for (let index = activePublisherDeliveryContexts.length - 1; index >= 0; index -= 1) { + const context = activePublisherDeliveryContexts[index]; const coveredCode = candidates.find( (code): code is string => !!code && context.remainingCodes.has(code) ); - if (!coveredCode) { - allCovered = false; - break; - } - coveredCodes.push(coveredCode); + if (!coveredCode) continue; + + const contextMatches = matches.get(context) ?? new Set(); + contextMatches.add(coveredCode); + matches.set(context, contextMatches); + break; } + } - if (!allCovered) continue; + if (matches.size === 0) return false; + for (const [context, coveredCodes] of matches) { coveredCodes.forEach((code) => context.remainingCodes.delete(code)); - return true; + if (context.remainingCodes.size === 0) removePublisherDeliveryContext(context); } - - return false; + return true; } function collectAuctionEids(): AuctionEid[] | undefined { @@ -668,7 +696,7 @@ function collectAuctionEids(): AuctionEid[] | undefined { export function installPrebidNpm(config?: Partial): typeof pbjs { publisherAdUnitSnapshots = new Map(); syntheticRefreshAdUnits = new WeakSet(); - activePublisherDeliveryContexts.length = 0; + [...activePublisherDeliveryContexts].forEach(removePublisherDeliveryContext); const injected = getInjectedConfig(); const merged: PrebidNpmConfig = { @@ -847,14 +875,43 @@ export function installPrebidNpm(config?: Partial): typeof pbjs const context: PublisherDeliveryContext = { remainingCodes: new Set(publisherAdUnitCodes), + retainForTargetedRefresh: false, + }; + const targetingPbjs = pbjs as unknown as { + setTargetingForGPTAsync?: SetTargetingForGptAsync; }; - // Delivery attribution is intentionally synchronous and ends as soon as - // the publisher's original callback returns. + const originalSetTargeting = targetingPbjs.setTargetingForGPTAsync; + let targetingWrapper: SetTargetingForGptAsync | undefined; + if (typeof originalSetTargeting === 'function') { + targetingWrapper = (...targetingArgs: unknown[]) => { + const result = originalSetTargeting.apply(targetingPbjs, targetingArgs); + if (targetingCoversPublisherDeliveryContext(targetingArgs[0], context)) { + context.retainForTargetedRefresh = true; + } + return result; + }; + targetingPbjs.setTargetingForGPTAsync = targetingWrapper; + } + activePublisherDeliveryContexts.push(context); try { originalBidsBack.apply(this, args as Parameters); } finally { - removePublisherDeliveryContext(context); + if (targetingWrapper && targetingPbjs.setTargetingForGPTAsync === targetingWrapper) { + targetingPbjs.setTargetingForGPTAsync = originalSetTargeting; + } + if (context.retainForTargetedRefresh && context.remainingCodes.size > 0) { + // Some publisher wrappers set targeting in bidsBackHandler, return, + // and schedule the matching GPT refresh shortly afterward. Retain + // this one-shot context only after that targeting signal, with a + // bounded expiry so a later independent refresh remains independent. + context.cleanupTimer = setTimeout( + () => removePublisherDeliveryContext(context), + PUBLISHER_DELIVERY_CONTEXT_TIMEOUT_MS + ); + } else { + removePublisherDeliveryContext(context); + } } }; diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 250bc0f0c..4c88a02c2 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -1745,7 +1745,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(originalRefresh).toHaveBeenCalledWith(undefined, undefined); }); - it('keeps explicit unrelated and mixed delivery lists on the synthetic path', () => { + it('keeps explicit unrelated lists synthetic and bypasses mixed delivery lists', () => { const coveredSlot = { getSlotElementId: () => 'example-covered', getTargeting: () => [], @@ -1772,15 +1772,11 @@ describe('prebid publisher snapshots and delivery refreshes', () => { }, } as any); - expect(mockRequestBids).toHaveBeenCalledTimes(3); + expect(mockRequestBids).toHaveBeenCalledTimes(2); expect(mockRequestBids.mock.calls[1][0].adUnits.map((unit: any) => unit.code)).toEqual([ 'example-unrelated', ]); - expect(mockRequestBids.mock.calls[2][0].adUnits.map((unit: any) => unit.code)).toEqual([ - 'example-covered', - 'example-unrelated', - ]); - expect(coveredSlot.clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(coveredSlot.clearTargeting).not.toHaveBeenCalled(); expect(unrelatedSlot.clearTargeting).toHaveBeenCalledWith('ts_initial'); expect(unrelatedSlot.clearTargeting).toHaveBeenCalledWith('hb_cache_path'); expect(originalRefresh).toHaveBeenCalledTimes(2); @@ -1788,7 +1784,183 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(originalRefresh).toHaveBeenNthCalledWith(2, [coveredSlot, unrelatedSlot], undefined); }); - it('treats a microtask refresh after publisher delivery as an independent auction', async () => { + it('bypasses an explicit delivery refresh with four covered slots and a GAM-only extra', () => { + const coveredSlots = Array.from({ length: 4 }, (_, index) => ({ + getSlotElementId: () => `example-covered-${index}`, + getTargeting: () => [], + clearTargeting: vi.fn(), + })); + const gamOnlySlot = { + getSlotElementId: () => 'example-gam-only-interstitial', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const refreshSlots = [...coveredSlots, gamOnlySlot]; + const { originalRefresh, pubads } = installGpt(refreshSlots); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: coveredSlots.map((_, index) => ({ + code: `example-covered-${index}`, + bids: [{ bidder: 'exampleServer', params: { placement: index } }], + })), + bidsBackHandler: () => pubads.refresh(refreshSlots), + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(1); + refreshSlots.forEach((slot) => expect(slot.clearTargeting).not.toHaveBeenCalled()); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith(refreshSlots, undefined); + }); + + it('bypasses a targeted delivery refresh shortly after the publisher callback returns', () => { + vi.useFakeTimers(); + try { + const coveredSlots = Array.from({ length: 4 }, (_, index) => ({ + getSlotElementId: () => `example-targeted-${index}`, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + })); + const gamOnlySlot = { + getSlotElementId: () => 'example-targeted-interstitial', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const refreshSlots = [...coveredSlots, gamOnlySlot]; + const { originalRefresh, pubads } = installGpt(refreshSlots); + const setTargetingForGPTAsync = vi.fn(); + (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; + let refreshAfterCallback: (() => void) | undefined; + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + const pendingRefresh = refreshAfterCallback; + refreshAfterCallback = undefined; + if (pendingRefresh) setTimeout(pendingRefresh, 750); + }); + const pbjs = installPrebidNpm(); + const coveredCodes = coveredSlots.map((slot) => slot.getSlotElementId()); + + pbjs.requestBids({ + adUnits: coveredCodes.map((code, index) => ({ + code, + bids: [{ bidder: 'exampleServer', params: { placement: index } }], + })), + bidsBackHandler: () => { + (pbjs as any).setTargetingForGPTAsync([gamOnlySlot.getSlotElementId(), ...coveredCodes]); + refreshAfterCallback = () => pubads.refresh(refreshSlots); + }, + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(setTargetingForGPTAsync).toHaveBeenCalledWith([ + gamOnlySlot.getSlotElementId(), + ...coveredCodes, + ]); + expect((mockPbjs as any).setTargetingForGPTAsync).toBe(setTargetingForGPTAsync); + + vi.advanceTimersByTime(750); + + refreshSlots.forEach((slot) => expect(slot.clearTargeting).not.toHaveBeenCalled()); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith(refreshSlots, undefined); + + vi.runOnlyPendingTimers(); + pubads.refresh([coveredSlots[0]]); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(coveredSlots[0].clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(originalRefresh).toHaveBeenCalledTimes(2); + } finally { + vi.runOnlyPendingTimers(); + vi.useRealTimers(); + delete (mockPbjs as any).setTargetingForGPTAsync; + } + }); + + it('expires a targeted delivery context before a later event-loop task', () => { + vi.useFakeTimers(); + try { + const slot = { + getSlotElementId: () => 'example-expiring-delivery', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + (mockPbjs as any).setTargetingForGPTAsync = vi.fn(); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { code: 'example-expiring-delivery', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => (pbjs as any).setTargetingForGPTAsync(['example-expiring-delivery']), + } as any); + vi.runOnlyPendingTimers(); + pubads.refresh([slot]); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(slot.clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + } finally { + vi.runOnlyPendingTimers(); + vi.useRealTimers(); + delete (mockPbjs as any).setTargetingForGPTAsync; + } + }); + + it('bypasses a mixed explicit delivery list spanning nested contexts', () => { + const outerSlot = { + getSlotElementId: () => 'example-outer-delivery', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const innerSlot = { + getSlotElementId: () => 'example-inner-delivery', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const gamOnlySlot = { + getSlotElementId: () => 'example-gam-only-interstitial', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const refreshSlots = [innerSlot, outerSlot, gamOnlySlot]; + const { originalRefresh, pubads } = installGpt(refreshSlots); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { + opts?.bidsBackHandler?.(); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { code: 'example-outer-delivery', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => { + pbjs.requestBids({ + adUnits: [ + { code: 'example-inner-delivery', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => pubads.refresh(refreshSlots), + } as any); + }, + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + refreshSlots.forEach((slot) => expect(slot.clearTargeting).not.toHaveBeenCalled()); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith(refreshSlots, undefined); + }); + + it('treats a microtask refresh without a targeting signal as an independent auction', async () => { const slot = { getSlotElementId: () => 'example-deferred-refresh', getTargeting: () => [], From 461cff9b8ee0208b50854b09a49812ee2d98fd77 Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 17 Jul 2026 11:54:55 -0500 Subject: [PATCH 128/198] Address Prebid refresh review feedback --- .../lib/src/integrations/prebid/index.ts | 359 ++++++------ .../test/integrations/prebid/index.test.ts | 543 ++++++++++++++---- 2 files changed, 604 insertions(+), 298 deletions(-) diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 0432d8695..007a7d40d 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -48,7 +48,8 @@ const TS_REFRESH_TARGETING_KEYS = [ 'hb_cache_host', 'hb_cache_path', ] as const; -const PUBLISHER_DELIVERY_CONTEXT_TIMEOUT_MS = 1000; +const MAX_PUBLISHER_AD_UNIT_SNAPSHOTS = 256; +const MAX_PENDING_PUBLISHER_BIDS = 2048; /** Configuration options for the Prebid integration. */ export interface PrebidNpmConfig { @@ -245,16 +246,14 @@ type PublisherAdUnitSnapshot = { clientSideBids: ClientSideBidSnapshot[]; zone?: string; }; -type PublisherDeliveryContext = { - remainingCodes: Set; - retainForTargetedRefresh: boolean; - cleanupTimer?: ReturnType; +type PendingPublisherBid = { + adUnitCode: string; }; -type SetTargetingForGptAsync = (...args: unknown[]) => unknown; +type RemoveAdUnit = (adUnitCode?: string | string[]) => unknown; let publisherAdUnitSnapshots = new Map(); +let pendingPublisherBids = new Map(); let syntheticRefreshAdUnits = new WeakSet(); -const activePublisherDeliveryContexts: PublisherDeliveryContext[] = []; type TrustedServerBidRequest = { adUnitCode?: string; code?: string; @@ -381,26 +380,41 @@ function firstTargetingValue(values: string[] | undefined): string | undefined { return values?.find((value) => value.length > 0); } -/** - * Find the publisher's original `pbjs.adUnits` entry for a refreshing slot. - * - * A TS-owned GPT slot may be defined on `${div_id}-container`, so the GPT - * element id used as the synthetic refresh ad unit code can differ from the - * inner `div_id` the publisher keyed their Prebid ad unit by. Try each candidate - * code in order and return the first matching ad unit, so container-backed slots - * still recover the publisher's configured params and bidders. - */ +/** Store a snapshot and evict the least-recently used entry when capacity is exceeded. */ +function storePublisherAdUnitSnapshot(code: string, snapshot: PublisherAdUnitSnapshot): void { + publisherAdUnitSnapshots.delete(code); + publisherAdUnitSnapshots.set(code, snapshot); + + if (publisherAdUnitSnapshots.size > MAX_PUBLISHER_AD_UNIT_SNAPSHOTS) { + const oldestCode = publisherAdUnitSnapshots.keys().next().value; + if (oldestCode !== undefined) publisherAdUnitSnapshots.delete(oldestCode); + } +} + +/** Find and touch a request-scoped publisher snapshot by candidate code. */ function findRefreshSnapshot( candidateCodes: Array ): PublisherAdUnitSnapshot | undefined { for (const code of candidateCodes) { if (!code) continue; const snapshot = publisherAdUnitSnapshots.get(code); - if (snapshot) return snapshot; + if (!snapshot) continue; + publisherAdUnitSnapshots.delete(code); + publisherAdUnitSnapshots.set(code, snapshot); + return snapshot; } return undefined; } +/** + * Find the publisher's live `pbjs.adUnits` entry for a refreshing slot. + * + * A TS-owned GPT slot may be defined on `${div_id}-container`, so the GPT + * element id used as the synthetic refresh ad unit code can differ from the + * inner `div_id` the publisher keyed their Prebid ad unit by. Try each candidate + * code in order and return the first matching ad unit, so container-backed slots + * still recover the publisher's configured params and bidders. + */ function findRefreshAdUnit( candidateCodes: Array ): TrustedServerAdUnit | undefined { @@ -413,6 +427,7 @@ function findRefreshAdUnit( return undefined; } +/** Deep-copy plain publisher params while preserving cycles and non-plain values. */ function copyParamValue(value: unknown, seen = new WeakMap()): unknown { if (Array.isArray(value)) { const existing = seen.get(value); @@ -449,6 +464,7 @@ function copyParams(params: Record | undefined): Record; } +/** Copy bidder params previously folded into a `trustedServer` bid. */ function foldedBidderParams( bid: TrustedServerBid | undefined ): Record> { @@ -461,6 +477,7 @@ function foldedBidderParams( ); } +/** Capture immutable request-scoped bidder and zone data before the shim mutates an ad unit. */ function capturePublisherAdUnitSnapshot( unit: TrustedServerAdUnit, clientSideBidders: Set @@ -503,34 +520,34 @@ function capturePublisherAdUnitSnapshot( * `requestBids` shim preserves a client-side bidder only when its bid entry is * already present on the ad unit, so without re-attaching them here publishers * that split demand between server-side and native Prebid adapters would lose - * all client-side demand on refresh/scroll impressions. Bids are sourced from - * the matching `pbjs.adUnits` entry (by candidate ad unit code) so the - * publisher's configured params are preserved. + * all client-side demand on refresh/scroll impressions. A live exact + * `pbjs.adUnits` match is authoritative; request-scoped snapshots are used only + * when no live unit exists. */ function clientSideBidsForRefresh( candidateCodes: Array ): Array<{ bidder: string; params: Record }> { - const snapshot = findRefreshSnapshot(candidateCodes); - if (snapshot) { - return snapshot.clientSideBids.map((bid) => ({ - bidder: bid.bidder, - params: copyParams(bid.params), - })); - } - const clientSideBidders = new Set(getInjectedConfig()?.clientSideBidders ?? []); - if (clientSideBidders.size === 0) return []; - const match = findRefreshAdUnit(candidateCodes); - if (!match?.bids) return []; + if (match) { + if (clientSideBidders.size === 0 || !Array.isArray(match.bids)) return []; - const bids: Array<{ bidder: string; params: Record }> = []; - for (const bid of match.bids) { - if (bid?.bidder && clientSideBidders.has(bid.bidder)) { - bids.push({ bidder: bid.bidder, params: copyParams(bid.params) }); + const bids: Array<{ bidder: string; params: Record }> = []; + for (const bid of match.bids) { + if (bid?.bidder && clientSideBidders.has(bid.bidder)) { + bids.push({ bidder: bid.bidder, params: copyParams(bid.params) }); + } } + return bids; } - return bids; + + const snapshot = findRefreshSnapshot(candidateCodes); + return ( + snapshot?.clientSideBids.map((bid) => ({ + bidder: bid.bidder, + params: copyParams(bid.params), + })) ?? [] + ); } /** @@ -539,49 +556,49 @@ function clientSideBidsForRefresh( * The synthetic refresh ad unit carries only the `trustedServer` bid, so the * `requestBids` shim has no original server-side bidder entries to collect into * `bidderParams` — without this, refresh/scroll `/auction` requests send `{}` - * and lose demand the publisher configured only on the initial ad unit. Source - * the params from the matching `pbjs.adUnits` entry by candidate code, covering - * both states the initial auction can leave that entry in: - * - raw server-side bidder entries (`{ bidder, params }`) not yet folded, and - * - params already folded into that unit's `trustedServer` bid `bidderParams` - * by a prior `requestBids` call. + * and lose demand the publisher configured only on the initial ad unit. A live + * exact `pbjs.adUnits` match is authoritative and covers both raw bidder entries + * and params already folded into a `trustedServer` bid. A request-scoped + * snapshot is used only when no live unit exists. */ function serverSideBidderParamsForRefresh( candidateCodes: Array ): Record> { - const snapshot = findRefreshSnapshot(candidateCodes); - if (snapshot) { - return Object.fromEntries( - Object.entries(snapshot.bidderParams).map(([bidder, params]) => [bidder, copyParams(params)]) - ); - } - const match = findRefreshAdUnit(candidateCodes); - if (!match?.bids) return {}; + if (match) { + if (!Array.isArray(match.bids)) return {}; - const clientSideBidders = new Set(getInjectedConfig()?.clientSideBidders ?? []); - const params: Record> = {}; + const clientSideBidders = new Set(getInjectedConfig()?.clientSideBidders ?? []); + const params: Record> = {}; - for (const bid of match.bids) { - if (!bid?.bidder) continue; - if (bid.bidder === ADAPTER_CODE) { - // Params captured and folded onto the trustedServer bid by an earlier - // requestBids call. - const folded = (bid.params?.[BIDDER_PARAMS_KEY] ?? {}) as Record< - string, - Record - >; - for (const [bidder, bidderParams] of Object.entries(folded)) { - params[bidder] = bidderParams; + for (const bid of match.bids) { + if (!bid?.bidder) continue; + if (bid.bidder === ADAPTER_CODE) { + Object.assign(params, foldedBidderParams(bid)); + continue; } - continue; + if (clientSideBidders.has(bid.bidder)) continue; + params[bid.bidder] = copyParams(bid.params); } - if (clientSideBidders.has(bid.bidder)) continue; - // Raw server-side bidder entry not yet folded by the shim. - params[bid.bidder] = bid.params ?? {}; + + return params; } - return params; + const snapshot = findRefreshSnapshot(candidateCodes); + return snapshot + ? Object.fromEntries( + Object.entries(snapshot.bidderParams).map(([bidder, params]) => [ + bidder, + copyParams(params), + ]) + ) + : {}; +} + +/** Return a live publisher zone, falling back to a request-scoped snapshot. */ +function publisherZoneForRefresh(candidateCodes: Array): string | undefined { + const match = findRefreshAdUnit(candidateCodes); + return match ? match.mediaTypes?.banner?.name : findRefreshSnapshot(candidateCodes)?.zone; } function clearRefreshTargeting(slot: RefreshGptSlot): void { @@ -592,70 +609,85 @@ function clearRefreshTargeting(slot: RefreshGptSlot): void { } } -function removePublisherDeliveryContext(context: PublisherDeliveryContext): void { - if (context.cleanupTimer !== undefined) { - clearTimeout(context.cleanupTimer); - context.cleanupTimer = undefined; +/** Store an auction-local bid ID for one-shot GPT delivery correlation. */ +function storePendingPublisherBid(adId: string, pendingBid: PendingPublisherBid): void { + pendingPublisherBids.delete(adId); + pendingPublisherBids.set(adId, pendingBid); + + if (pendingPublisherBids.size > MAX_PENDING_PUBLISHER_BIDS) { + const oldestAdId = pendingPublisherBids.keys().next().value; + if (oldestAdId !== undefined) pendingPublisherBids.delete(oldestAdId); } - const index = activePublisherDeliveryContexts.lastIndexOf(context); - if (index >= 0) activePublisherDeliveryContexts.splice(index, 1); } -function targetingCoversPublisherDeliveryContext( - adUnitCodes: unknown, - context: PublisherDeliveryContext -): boolean { - if (adUnitCodes === undefined) return context.remainingCodes.size > 0; - const codes = typeof adUnitCodes === 'string' ? [adUnitCodes] : adUnitCodes; - return ( - Array.isArray(codes) && - codes.some((code) => typeof code === 'string' && context.remainingCodes.has(code)) - ); +/** Remove every pending auction bid for an ad-unit code. */ +function removePendingPublisherBidsForCode(adUnitCode: string): void { + for (const [adId, pendingBid] of pendingPublisherBids) { + if (pendingBid.adUnitCode === adUnitCode) pendingPublisherBids.delete(adId); + } } -function consumeBarePublisherDeliveryContext(): boolean { - for (let index = activePublisherDeliveryContexts.length - 1; index >= 0; index -= 1) { - const context = activePublisherDeliveryContexts[index]; - if (context.remainingCodes.size === 0) continue; - context.remainingCodes.clear(); - removePublisherDeliveryContext(context); - return true; +/** Register bid IDs from the current `bidsBackHandler` callback only. */ +function registerPendingPublisherBids(bidResponses: unknown): void { + if (!bidResponses || typeof bidResponses !== 'object' || Array.isArray(bidResponses)) return; + + for (const [responseCode, responseGroup] of Object.entries(bidResponses)) { + if (!responseGroup || typeof responseGroup !== 'object') continue; + const bids = (responseGroup as { bids?: unknown }).bids; + if (!Array.isArray(bids)) continue; + + for (const bid of bids) { + if (!bid || typeof bid !== 'object') continue; + const response = bid as { adId?: unknown; adUnitCode?: unknown }; + const adId = typeof response.adId === 'string' ? response.adId : undefined; + const adUnitCode = + typeof response.adUnitCode === 'string' ? response.adUnitCode : responseCode; + if (!adId || !adUnitCode) continue; + + storePendingPublisherBid(adId, { adUnitCode }); + } } - return false; } -function consumeExplicitPublisherDeliveryContext(targetSlots: RefreshGptSlot[]): boolean { - if (targetSlots.length === 0) return false; +/** + * Partition slots by whether their current `hb_adid` belongs to a pending + * publisher auction, consuming every older pending bid for each matched code. + */ +function publisherDeliverySlots(targetSlots: RefreshGptSlot[]): Set { + const deliverySlots = new Set(); + const deliveredCodes = new Set(); - // Publishers may include GAM-only slots in the same explicit refresh that - // delivers a completed Prebid auction. Attribute the call to delivery when - // any slot is covered, while consuming only the covered codes so an - // unrelated-only refresh still follows the synthetic auction path. - const matches = new Map>(); for (const slot of targetSlots) { - const injectedSlot = findInjectedSlotForRefresh(slot); - const candidates = [refreshSlotElementId(slot), injectedSlot?.div_id]; + const adIds = slot.getTargeting?.('hb_adid'); + if (!Array.isArray(adIds)) continue; - for (let index = activePublisherDeliveryContexts.length - 1; index >= 0; index -= 1) { - const context = activePublisherDeliveryContexts[index]; - const coveredCode = candidates.find( - (code): code is string => !!code && context.remainingCodes.has(code) - ); - if (!coveredCode) continue; + const pendingBid = adIds + .filter((adId): adId is string => typeof adId === 'string' && adId.length > 0) + .map((adId) => pendingPublisherBids.get(adId)) + .find((bid): bid is PendingPublisherBid => bid !== undefined); + if (!pendingBid) continue; - const contextMatches = matches.get(context) ?? new Set(); - contextMatches.add(coveredCode); - matches.set(context, contextMatches); - break; - } + deliverySlots.add(slot); + deliveredCodes.add(pendingBid.adUnitCode); } - if (matches.size === 0) return false; - for (const [context, coveredCodes] of matches) { - coveredCodes.forEach((code) => context.remainingCodes.delete(code)); - if (context.remainingCodes.size === 0) removePublisherDeliveryContext(context); + deliveredCodes.forEach(removePendingPublisherBidsForCode); + return deliverySlots; +} + +/** Evict publisher state after Prebid removes one or more ad units. */ +function removePublisherState(adUnitCode?: string | string[]): void { + if (!adUnitCode) { + publisherAdUnitSnapshots.clear(); + pendingPublisherBids.clear(); + return; + } + + const adUnitCodes = Array.isArray(adUnitCode) ? adUnitCode : [adUnitCode]; + for (const code of adUnitCodes) { + publisherAdUnitSnapshots.delete(code); + removePendingPublisherBidsForCode(code); } - return true; } function collectAuctionEids(): AuctionEid[] | undefined { @@ -695,8 +727,18 @@ function collectAuctionEids(): AuctionEid[] | undefined { */ export function installPrebidNpm(config?: Partial): typeof pbjs { publisherAdUnitSnapshots = new Map(); + pendingPublisherBids = new Map(); syntheticRefreshAdUnits = new WeakSet(); - [...activePublisherDeliveryContexts].forEach(removePublisherDeliveryContext); + + const prebidWithRemoveAdUnit = pbjs as unknown as { removeAdUnit?: RemoveAdUnit }; + const originalRemoveAdUnit = prebidWithRemoveAdUnit.removeAdUnit; + if (typeof originalRemoveAdUnit === 'function') { + prebidWithRemoveAdUnit.removeAdUnit = function (adUnitCode?: string | string[]) { + const result = originalRemoveAdUnit.call(this, adUnitCode); + removePublisherState(adUnitCode); + return result; + }; + } const injected = getInjectedConfig(); const merged: PrebidNpmConfig = { @@ -772,15 +814,19 @@ export function installPrebidNpm(config?: Partial): typeof pbjs const adUnits = ((opts as any).adUnits || pbjs.adUnits || []) as TrustedServerAdUnit[]; const isSyntheticRefresh = adUnits.length > 0 && adUnits.every((unit) => syntheticRefreshAdUnits.has(unit)); - const publisherAdUnitCodes = new Set(); + const publisherAdUnitCodes = new Set( + adUnits + .filter((unit) => !syntheticRefreshAdUnits.has(unit)) + .map((unit) => unit.code) + .filter((code): code is string => typeof code === 'string' && code.length > 0) + ); // Ensure every ad unit has a trustedServer bid entry for (const unit of adUnits) { if (!syntheticRefreshAdUnits.has(unit)) { const snapshot = capturePublisherAdUnitSnapshot(unit, clientSideBidders); if (snapshot && unit.code) { - publisherAdUnitSnapshots.set(unit.code, snapshot); - publisherAdUnitCodes.add(unit.code); + storePublisherAdUnitSnapshot(unit.code, snapshot); } } @@ -868,51 +914,11 @@ export function installPrebidNpm(config?: Partial): typeof pbjs opts.bidsBackHandler = function (...args: unknown[]) { syncPrebidEidsCookie(); if (typeof originalBidsBack !== 'function') return; - if (isSyntheticRefresh || publisherAdUnitCodes.size === 0) { - originalBidsBack.apply(this, args as Parameters); - return; - } - - const context: PublisherDeliveryContext = { - remainingCodes: new Set(publisherAdUnitCodes), - retainForTargetedRefresh: false, - }; - const targetingPbjs = pbjs as unknown as { - setTargetingForGPTAsync?: SetTargetingForGptAsync; - }; - const originalSetTargeting = targetingPbjs.setTargetingForGPTAsync; - let targetingWrapper: SetTargetingForGptAsync | undefined; - if (typeof originalSetTargeting === 'function') { - targetingWrapper = (...targetingArgs: unknown[]) => { - const result = originalSetTargeting.apply(targetingPbjs, targetingArgs); - if (targetingCoversPublisherDeliveryContext(targetingArgs[0], context)) { - context.retainForTargetedRefresh = true; - } - return result; - }; - targetingPbjs.setTargetingForGPTAsync = targetingWrapper; - } - - activePublisherDeliveryContexts.push(context); - try { - originalBidsBack.apply(this, args as Parameters); - } finally { - if (targetingWrapper && targetingPbjs.setTargetingForGPTAsync === targetingWrapper) { - targetingPbjs.setTargetingForGPTAsync = originalSetTargeting; - } - if (context.retainForTargetedRefresh && context.remainingCodes.size > 0) { - // Some publisher wrappers set targeting in bidsBackHandler, return, - // and schedule the matching GPT refresh shortly afterward. Retain - // this one-shot context only after that targeting signal, with a - // bounded expiry so a later independent refresh remains independent. - context.cleanupTimer = setTimeout( - () => removePublisherDeliveryContext(context), - PUBLISHER_DELIVERY_CONTEXT_TIMEOUT_MS - ); - } else { - removePublisherDeliveryContext(context); - } + if (!isSyntheticRefresh) { + publisherAdUnitCodes.forEach(removePendingPublisherBidsForCode); + registerPendingPublisherBids(args[0]); } + originalBidsBack.apply(this, args as Parameters); }; return originalRequestBids(opts); @@ -1012,33 +1018,30 @@ export function installRefreshHandler(timeoutMs = 1500): void { return originalRefresh(slots, opts); } - const isExplicitSlotList = slots !== undefined; - const hasOnlyValidExplicitSlots = !isExplicitSlotList || targetSlots.length === slots.length; - const isPublisherDeliveryRefresh = isExplicitSlotList - ? hasOnlyValidExplicitSlots && consumeExplicitPublisherDeliveryContext(targetSlots) - : consumeBarePublisherDeliveryContext(); - if (isPublisherDeliveryRefresh) { + if (!targetSlots.length) { return originalRefresh(slots, opts); } - if (!targetSlots.length) { - return originalRefresh(slots, opts); + const deliverySlots = publisherDeliverySlots(targetSlots); + const independentSlots = targetSlots.filter((slot) => !deliverySlots.has(slot)); + if (deliverySlots.size > 0) { + originalRefresh([...deliverySlots], opts); } + if (independentSlots.length === 0) return; - targetSlots.forEach(clearRefreshTargeting); + independentSlots.forEach(clearRefreshTargeting); - const adUnits = targetSlots.map((slot) => { + const adUnits = independentSlots.map((slot) => { const injectedSlot = findInjectedSlotForRefresh(slot); const code = refreshSlotElementId(slot) ?? 'refresh-slot'; // A TS-owned slot may be defined on `${div_id}-container`, so the GPT // element id used as the synthetic refresh code can differ from the // inner `div_id` the publisher keyed their ad unit by. Recover from both. const candidateCodes = [code, injectedSlot?.div_id]; - const snapshot = findRefreshSnapshot(candidateCodes); const zone = injectedSlot?.targeting?.[ZONE_KEY] ?? firstTargetingValue(slot.getTargeting?.(ZONE_KEY)) ?? - snapshot?.zone; + publisherZoneForRefresh(candidateCodes); const banner: TrustedServerBanner = { sizes: bannerSizesFromInjectedSlot(injectedSlot) ?? @@ -1073,7 +1076,7 @@ export function installRefreshHandler(timeoutMs = 1500): void { adUnits, bidsBackHandler: () => { pbjs.setTargetingForGPTAsync?.(refreshAdUnitCodes); - originalRefresh(targetSlots, opts); + originalRefresh(independentSlots, opts); }, timeout: timeoutMs, }); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 4c88a02c2..92d2b07d9 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -8,6 +8,7 @@ const { mockRegisterBidAdapter, mockGetUserIdsAsEids, mockGetConfig, + mockRemoveAdUnit, mockPbjs, mockGetBidAdapter, mockAdapterManager, @@ -21,14 +22,34 @@ const { () => [] as Array<{ source: string; uids?: Array<{ id: string; atype?: number }> }> ); const mockGetConfig = vi.fn(); - const mockPbjs = { + let mockPbjs: { + setConfig: typeof mockSetConfig; + processQueue: typeof mockProcessQueue; + requestBids: typeof mockRequestBids; + registerBidAdapter: typeof mockRegisterBidAdapter; + getUserIdsAsEids: typeof mockGetUserIdsAsEids; + getConfig: typeof mockGetConfig; + removeAdUnit: ReturnType; + adUnits: any[]; + [key: string]: any; + }; + const mockRemoveAdUnit = vi.fn((adUnitCode?: string | string[]) => { + if (!adUnitCode) { + mockPbjs.adUnits = []; + return; + } + const codes = new Set(Array.isArray(adUnitCode) ? adUnitCode : [adUnitCode]); + mockPbjs.adUnits = mockPbjs.adUnits.filter((unit) => !codes.has(unit.code)); + }); + mockPbjs = { setConfig: mockSetConfig, processQueue: mockProcessQueue, requestBids: mockRequestBids, registerBidAdapter: mockRegisterBidAdapter, getUserIdsAsEids: mockGetUserIdsAsEids, getConfig: mockGetConfig, - adUnits: [] as any[], + removeAdUnit: mockRemoveAdUnit, + adUnits: [], }; const mockAdapterManager = { getBidAdapter: mockGetBidAdapter, @@ -40,6 +61,7 @@ const { mockRegisterBidAdapter, mockGetUserIdsAsEids, mockGetConfig, + mockRemoveAdUnit, mockPbjs, mockGetBidAdapter, mockAdapterManager, @@ -1413,10 +1435,18 @@ describe('prebid/installRefreshHandler', () => { }); describe('prebid publisher snapshots and delivery refreshes', () => { + let deliveryAdIds = new WeakMap(); + let installedGptSlots: any[] = []; + let auctionSequence = 0; + beforeEach(() => { vi.clearAllMocks(); + deliveryAdIds = new WeakMap(); + installedGptSlots = []; + auctionSequence = 0; mockRequestBids.mockReset(); mockPbjs.requestBids = mockRequestBids; + mockPbjs.removeAdUnit = mockRemoveAdUnit; mockPbjs.adUnits = []; mockGetUserIdsAsEids.mockReset(); mockGetUserIdsAsEids.mockReturnValue([]); @@ -1434,6 +1464,17 @@ describe('prebid publisher snapshots and delivery refreshes', () => { }); function installGpt(slots: any[]) { + installedGptSlots = slots; + for (const slot of slots) { + if (!slot || typeof slot !== 'object') continue; + const originalGetTargeting = slot.getTargeting?.bind(slot); + slot.getTargeting = (key: string) => { + const deliveryAdId = deliveryAdIds.get(slot); + if (key === 'hb_adid' && deliveryAdId) return [deliveryAdId]; + return originalGetTargeting?.(key) ?? []; + }; + } + const originalRefresh = vi.fn(); const pubads = { refresh: originalRefresh, @@ -1452,6 +1493,31 @@ describe('prebid publisher snapshots and delivery refreshes', () => { return lastCall?.[0]?.adUnits?.[0]; } + function completePublisherAuction( + opts?: { adUnits?: Array<{ code?: string }>; bidsBackHandler?: (...args: any[]) => void }, + options: { auctionId?: string; applyTargeting?: boolean } = {} + ): void { + const auctionId = options.auctionId ?? `example-auction-${auctionSequence++}`; + const bidResponses: Record = {}; + + for (const unit of opts?.adUnits ?? []) { + if (!unit.code) continue; + const adId = `${auctionId}-${unit.code}`; + bidResponses[unit.code] = { + bids: [{ adId, adUnitCode: unit.code, auctionId }], + }; + if (options.applyTargeting !== false) { + const slot = installedGptSlots.find((candidate) => { + const elementId = candidate?.getSlotElementId?.(); + return elementId === unit.code || elementId === `${unit.code}-container`; + }); + if (slot) deliveryAdIds.set(slot, adId); + } + } + + opts?.bidsBackHandler?.(bidResponses, false, auctionId); + } + it('recovers inline params, ordered client bids, and zone when pbjs.adUnits is empty', () => { (window as any).__tsjs_prebid = { clientSideBidders: ['exampleBrowser'] }; const runtimeInstance = 'example-runtime-instance'; @@ -1677,6 +1743,143 @@ describe('prebid publisher snapshots and delivery refreshes', () => { }); }); + it('prefers a rich live unit when a fresh same-code request overwrites the snapshot with empty bids', () => { + (window as any).__tsjs_prebid = { clientSideBidders: ['exampleBrowser'] }; + const code = 'example-live-rich-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { pubads } = installGpt([slot]); + const liveUnit = { + code, + bids: [ + { bidder: 'exampleServer', params: { placement: 'live-server' } }, + { bidder: 'exampleBrowser', params: { placement: 'live-browser' } }, + ], + }; + mockPbjs.adUnits = [liveUnit]; + const pbjs = installPrebidNpm(); + + pbjs.requestBids(); + pbjs.requestBids({ adUnits: [{ code, bids: [] }] } as any); + pubads.refresh([slot]); + + expect(refreshAdUnitFromLastRequest().bids).toEqual([ + { + bidder: 'trustedServer', + params: { bidderParams: { exampleServer: { placement: 'live-server' } } }, + }, + { bidder: 'exampleBrowser', params: { placement: 'live-browser' } }, + ]); + }); + + it('does not resurrect an older snapshot when the live unit is intentionally empty', () => { + const code = 'example-live-empty-slot'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { pubads } = installGpt([slot]); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: { placement: 'snapshot' } }] }], + } as any); + mockPbjs.adUnits = [{ code, bids: [] }]; + pubads.refresh([slot]); + + expect(refreshAdUnitFromLastRequest().bids).toEqual([ + { bidder: 'trustedServer', params: { bidderParams: {} } }, + ]); + }); + + it('evicts snapshots with the matching removeAdUnit lifecycle', () => { + const codes = ['example-remove-one', 'example-remove-two', 'example-remove-all']; + const slots = codes.map((code) => ({ + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + })); + const { pubads } = installGpt(slots); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: codes.map((code) => ({ + code, + bids: [{ bidder: 'exampleServer', params: { placement: code } }], + })), + } as any); + (pbjs as any).removeAdUnit(codes[0]); + (pbjs as any).removeAdUnit([codes[1]]); + + pubads.refresh([slots[0]]); + expect(refreshAdUnitFromLastRequest().bids[0].params).toEqual({ bidderParams: {} }); + pubads.refresh([slots[1]]); + expect(refreshAdUnitFromLastRequest().bids[0].params).toEqual({ bidderParams: {} }); + pubads.refresh([slots[2]]); + expect(refreshAdUnitFromLastRequest().bids[0].params.bidderParams).toEqual({ + exampleServer: { placement: codes[2] }, + }); + + (pbjs as any).removeAdUnit(); + pubads.refresh([slots[2]]); + expect(refreshAdUnitFromLastRequest().bids[0].params).toEqual({ bidderParams: {} }); + }); + + it('bounds snapshots with LRU eviction while retaining a recently refreshed entry', () => { + const capacity = 256; + const oldestCode = 'example-lru-0'; + const activeCode = `example-lru-${capacity - 1}`; + const oldestSlot = { + getSlotElementId: () => oldestCode, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const activeSlot = { + getSlotElementId: () => activeCode, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { pubads } = installGpt([oldestSlot, activeSlot]); + const pbjs = installPrebidNpm(); + + for (let index = 0; index < capacity; index += 1) { + pbjs.requestBids({ + adUnits: [ + { + code: `example-lru-${index}`, + bids: [{ bidder: 'exampleServer', params: { placement: index } }], + }, + ], + } as any); + } + + pubads.refresh([activeSlot]); + pbjs.requestBids({ + adUnits: [ + { + code: `example-lru-${capacity}`, + bids: [{ bidder: 'exampleServer', params: { placement: capacity } }], + }, + ], + } as any); + + pubads.refresh([oldestSlot]); + expect(refreshAdUnitFromLastRequest().bids[0].params).toEqual({ bidderParams: {} }); + pubads.refresh([activeSlot]); + expect(refreshAdUnitFromLastRequest().bids[0].params.bidderParams).toEqual({ + exampleServer: { placement: capacity - 1 }, + }); + }); + it('bypasses explicit covered subset delivery refreshes without clearing targeting', () => { const slotOne = { getSlotElementId: () => 'example-covered-one', @@ -1692,9 +1895,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { adSlots: [{ div_id: 'example-covered-two', formats: [[300, 250]], targeting: {} }], }; const { originalRefresh, pubads } = installGpt([slotOne, slotTwo]); - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); const pbjs = installPrebidNpm(); pbjs.requestBids({ @@ -1716,7 +1917,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(originalRefresh).toHaveBeenNthCalledWith(2, [slotTwo], undefined); }); - it('bypasses a bare delivery refresh even when GPT includes a GAM-only extra slot', () => { + it('partitions a bare delivery refresh from an unmatched GPT slot', () => { const coveredSlot = { getSlotElementId: () => 'example-covered', getTargeting: () => [], @@ -1728,9 +1929,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { clearTargeting: vi.fn(), }; const { originalRefresh, pubads } = installGpt([coveredSlot, gamOnlySlot]); - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); const pbjs = installPrebidNpm(); pbjs.requestBids({ @@ -1738,14 +1937,15 @@ describe('prebid publisher snapshots and delivery refreshes', () => { bidsBackHandler: () => pubads.refresh(), } as any); - expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(mockRequestBids).toHaveBeenCalledTimes(2); expect(coveredSlot.clearTargeting).not.toHaveBeenCalled(); - expect(gamOnlySlot.clearTargeting).not.toHaveBeenCalled(); - expect(originalRefresh).toHaveBeenCalledTimes(1); - expect(originalRefresh).toHaveBeenCalledWith(undefined, undefined); + expect(gamOnlySlot.clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(originalRefresh).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenNthCalledWith(1, [coveredSlot], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [gamOnlySlot], undefined); }); - it('keeps explicit unrelated lists synthetic and bypasses mixed delivery lists', () => { + it('keeps explicit unrelated lists synthetic and partitions mixed delivery lists', () => { const coveredSlot = { getSlotElementId: () => 'example-covered', getTargeting: () => [], @@ -1759,9 +1959,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { clearTargeting: vi.fn(), }; const { originalRefresh, pubads } = installGpt([coveredSlot, unrelatedSlot]); - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); const pbjs = installPrebidNpm(); pbjs.requestBids({ @@ -1772,19 +1970,23 @@ describe('prebid publisher snapshots and delivery refreshes', () => { }, } as any); - expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(mockRequestBids).toHaveBeenCalledTimes(3); expect(mockRequestBids.mock.calls[1][0].adUnits.map((unit: any) => unit.code)).toEqual([ 'example-unrelated', ]); + expect(mockRequestBids.mock.calls[2][0].adUnits.map((unit: any) => unit.code)).toEqual([ + 'example-unrelated', + ]); expect(coveredSlot.clearTargeting).not.toHaveBeenCalled(); expect(unrelatedSlot.clearTargeting).toHaveBeenCalledWith('ts_initial'); expect(unrelatedSlot.clearTargeting).toHaveBeenCalledWith('hb_cache_path'); - expect(originalRefresh).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenCalledTimes(3); expect(originalRefresh).toHaveBeenNthCalledWith(1, [unrelatedSlot], undefined); - expect(originalRefresh).toHaveBeenNthCalledWith(2, [coveredSlot, unrelatedSlot], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [coveredSlot], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(3, [unrelatedSlot], undefined); }); - it('bypasses an explicit delivery refresh with four covered slots and a GAM-only extra', () => { + it('partitions four delivered slots from an unmatched explicit slot', () => { const coveredSlots = Array.from({ length: 4 }, (_, index) => ({ getSlotElementId: () => `example-covered-${index}`, getTargeting: () => [], @@ -1797,9 +1999,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { }; const refreshSlots = [...coveredSlots, gamOnlySlot]; const { originalRefresh, pubads } = installGpt(refreshSlots); - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); const pbjs = installPrebidNpm(); pbjs.requestBids({ @@ -1810,111 +2010,125 @@ describe('prebid publisher snapshots and delivery refreshes', () => { bidsBackHandler: () => pubads.refresh(refreshSlots), } as any); - expect(mockRequestBids).toHaveBeenCalledTimes(1); - refreshSlots.forEach((slot) => expect(slot.clearTargeting).not.toHaveBeenCalled()); - expect(originalRefresh).toHaveBeenCalledTimes(1); - expect(originalRefresh).toHaveBeenCalledWith(refreshSlots, undefined); + expect(mockRequestBids).toHaveBeenCalledTimes(2); + coveredSlots.forEach((slot) => expect(slot.clearTargeting).not.toHaveBeenCalled()); + expect(gamOnlySlot.clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(originalRefresh).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenNthCalledWith(1, coveredSlots, undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [gamOnlySlot], undefined); }); - it('bypasses a targeted delivery refresh shortly after the publisher callback returns', () => { + it('correlates a targeted delivery refresh after more than one second without a timer race', () => { vi.useFakeTimers(); try { - const coveredSlots = Array.from({ length: 4 }, (_, index) => ({ - getSlotElementId: () => `example-targeted-${index}`, + const code = 'example-delayed-delivery'; + const auctionId = 'example-delayed-auction'; + const slot = { + getSlotElementId: () => code, getTargeting: () => [], getSizes: () => [[300, 250]], clearTargeting: vi.fn(), - })); - const gamOnlySlot = { - getSlotElementId: () => 'example-targeted-interstitial', - getTargeting: () => [], - clearTargeting: vi.fn(), }; - const refreshSlots = [...coveredSlots, gamOnlySlot]; - const { originalRefresh, pubads } = installGpt(refreshSlots); - const setTargetingForGPTAsync = vi.fn(); - (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; - let refreshAfterCallback: (() => void) | undefined; - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - const pendingRefresh = refreshAfterCallback; - refreshAfterCallback = undefined; - if (pendingRefresh) setTimeout(pendingRefresh, 750); - }); + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => + completePublisherAuction(opts, { auctionId, applyTargeting: false }) + ); const pbjs = installPrebidNpm(); - const coveredCodes = coveredSlots.map((slot) => slot.getSlotElementId()); pbjs.requestBids({ - adUnits: coveredCodes.map((code, index) => ({ - code, - bids: [{ bidder: 'exampleServer', params: { placement: index } }], - })), + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], bidsBackHandler: () => { - (pbjs as any).setTargetingForGPTAsync([gamOnlySlot.getSlotElementId(), ...coveredCodes]); - refreshAfterCallback = () => pubads.refresh(refreshSlots); + setTimeout(() => { + deliveryAdIds.set(slot, `${auctionId}-${code}`); + pubads.refresh([slot]); + }, 1500); }, } as any); - expect(mockRequestBids).toHaveBeenCalledTimes(1); - expect(setTargetingForGPTAsync).toHaveBeenCalledWith([ - gamOnlySlot.getSlotElementId(), - ...coveredCodes, - ]); - expect((mockPbjs as any).setTargetingForGPTAsync).toBe(setTargetingForGPTAsync); - - vi.advanceTimersByTime(750); + vi.advanceTimersByTime(1500); - refreshSlots.forEach((slot) => expect(slot.clearTargeting).not.toHaveBeenCalled()); + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(slot.clearTargeting).not.toHaveBeenCalled(); expect(originalRefresh).toHaveBeenCalledTimes(1); - expect(originalRefresh).toHaveBeenCalledWith(refreshSlots, undefined); - - vi.runOnlyPendingTimers(); - pubads.refresh([coveredSlots[0]]); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + pubads.refresh([slot]); expect(mockRequestBids).toHaveBeenCalledTimes(2); - expect(coveredSlots[0].clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(slot.clearTargeting).toHaveBeenCalledWith('hb_adid'); expect(originalRefresh).toHaveBeenCalledTimes(2); } finally { vi.runOnlyPendingTimers(); vi.useRealTimers(); - delete (mockPbjs as any).setTargetingForGPTAsync; } }); - it('expires a targeted delivery context before a later event-loop task', () => { - vi.useFakeTimers(); - try { - const slot = { - getSlotElementId: () => 'example-expiring-delivery', - getTargeting: () => [], - getSizes: () => [[300, 250]], - clearTargeting: vi.fn(), - }; - const { originalRefresh, pubads } = installGpt([slot]); - (mockPbjs as any).setTargetingForGPTAsync = vi.fn(); - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); - const pbjs = installPrebidNpm(); + it('correlates null and no-argument targeting with a custom GPT slot match', () => { + const code = 'example-custom-matched-code'; + const slot = { + getSlotElementId: () => 'example-different-gpt-slot', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + let auctionId = 'example-null-auction'; + const setTargetingForGPTAsync = vi.fn(() => { + deliveryAdIds.set(slot, `${auctionId}-${code}`); + }); + (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; + mockRequestBids.mockImplementation((opts) => + completePublisherAuction(opts, { auctionId, applyTargeting: false }) + ); + const pbjs = installPrebidNpm(); - pbjs.requestBids({ - adUnits: [ - { code: 'example-expiring-delivery', bids: [{ bidder: 'exampleServer', params: {} }] }, - ], - bidsBackHandler: () => (pbjs as any).setTargetingForGPTAsync(['example-expiring-delivery']), - } as any); - vi.runOnlyPendingTimers(); - pubads.refresh([slot]); + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => { + (pbjs as any).setTargetingForGPTAsync(null, () => () => true); + pubads.refresh([slot]); + }, + } as any); - expect(mockRequestBids).toHaveBeenCalledTimes(2); - expect(slot.clearTargeting).toHaveBeenCalledWith('hb_pb'); - expect(originalRefresh).toHaveBeenCalledTimes(1); - expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); - } finally { - vi.runOnlyPendingTimers(); - vi.useRealTimers(); - delete (mockPbjs as any).setTargetingForGPTAsync; - } + auctionId = 'example-no-argument-auction'; + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => { + (pbjs as any).setTargetingForGPTAsync(); + pubads.refresh([slot]); + }, + } as any); + + expect(setTargetingForGPTAsync).toHaveBeenNthCalledWith(1, null, expect.any(Function)); + expect(setTargetingForGPTAsync).toHaveBeenNthCalledWith(2); + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(slot.clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenNthCalledWith(1, [slot], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [slot], undefined); + delete (mockPbjs as any).setTargetingForGPTAsync; + }); + + it('uses the synthetic path when callback bid responses are missing or malformed', () => { + const slot = { + getSlotElementId: () => 'example-no-bid-delivery', + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: (...args: any[]) => void }) => { + opts?.bidsBackHandler?.({ 'example-no-bid-delivery': { bids: [null, {}] } }, false, 'bad'); + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [ + { code: 'example-no-bid-delivery', bids: [{ bidder: 'exampleServer', params: {} }] }, + ], + bidsBackHandler: () => pubads.refresh([slot]), + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(slot.clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); }); it('bypasses a mixed explicit delivery list spanning nested contexts', () => { @@ -1935,9 +2149,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { }; const refreshSlots = [innerSlot, outerSlot, gamOnlySlot]; const { originalRefresh, pubads } = installGpt(refreshSlots); - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); const pbjs = installPrebidNpm(); pbjs.requestBids({ @@ -1954,10 +2166,13 @@ describe('prebid publisher snapshots and delivery refreshes', () => { }, } as any); - expect(mockRequestBids).toHaveBeenCalledTimes(2); - refreshSlots.forEach((slot) => expect(slot.clearTargeting).not.toHaveBeenCalled()); - expect(originalRefresh).toHaveBeenCalledTimes(1); - expect(originalRefresh).toHaveBeenCalledWith(refreshSlots, undefined); + expect(mockRequestBids).toHaveBeenCalledTimes(3); + expect(innerSlot.clearTargeting).not.toHaveBeenCalled(); + expect(outerSlot.clearTargeting).not.toHaveBeenCalled(); + expect(gamOnlySlot.clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(originalRefresh).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenNthCalledWith(1, [innerSlot, outerSlot], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [gamOnlySlot], undefined); }); it('treats a microtask refresh without a targeting signal as an independent auction', async () => { @@ -1968,9 +2183,9 @@ describe('prebid publisher snapshots and delivery refreshes', () => { clearTargeting: vi.fn(), }; const { originalRefresh, pubads } = installGpt([slot]); - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); + mockRequestBids.mockImplementation((opts) => + completePublisherAuction(opts, { applyTargeting: false }) + ); const pbjs = installPrebidNpm(); let deferredRefresh: Promise | undefined; @@ -1990,6 +2205,98 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); }); + it('correlates targeting and refresh deferred together to a microtask', async () => { + const code = 'example-targeted-microtask'; + const auctionId = 'example-targeted-microtask-auction'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => + completePublisherAuction(opts, { auctionId, applyTargeting: false }) + ); + const pbjs = installPrebidNpm(); + let deferredRefresh: Promise | undefined; + + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => { + deferredRefresh = Promise.resolve().then(() => { + deliveryAdIds.set(slot, `${auctionId}-${code}`); + pubads.refresh([slot]); + }); + }, + } as any); + await deferredRefresh; + + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(slot.clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); + + it('consumes all overlapping pending bids for the same ad-unit code', () => { + const code = 'example-overlapping-code'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => {}, + } as any); + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => {}, + } as any); + + pubads.refresh([slot]); + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(slot.clearTargeting).not.toHaveBeenCalled(); + + deliveryAdIds.set(slot, `example-auction-0-${code}`); + pubads.refresh([slot]); + + expect(mockRequestBids).toHaveBeenCalledTimes(3); + expect(slot.clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(originalRefresh).toHaveBeenNthCalledWith(1, [slot], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [slot], undefined); + }); + + it('filters invalid explicit entries without duplicating or leaking a valid delivery', () => { + const code = 'example-valid-delivery'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh([slot, undefined, null] as any), + } as any); + + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(slot.clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + + pubads.refresh([slot]); + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(slot.clearTargeting).toHaveBeenCalledWith('hb_adid'); + }); + it('keeps nested publisher delivery contexts isolated during reentrant auctions', () => { const outerSlot = { getSlotElementId: () => 'example-outer-delivery', @@ -2002,9 +2309,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { clearTargeting: vi.fn(), }; const { originalRefresh, pubads } = installGpt([outerSlot, innerSlot]); - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); const pbjs = installPrebidNpm(); pbjs.requestBids({ @@ -2037,9 +2342,9 @@ describe('prebid publisher snapshots and delivery refreshes', () => { clearTargeting: vi.fn(), }; const { originalRefresh, pubads } = installGpt([slot]); - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); + mockRequestBids.mockImplementation((opts) => + completePublisherAuction(opts, { applyTargeting: false }) + ); const pbjs = installPrebidNpm(); expect(() => @@ -2071,9 +2376,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { clearTargeting: vi.fn(), }; const { originalRefresh, pubads } = installGpt([slot]); - mockRequestBids.mockImplementation((opts?: { bidsBackHandler?: () => void }) => { - opts?.bidsBackHandler?.(); - }); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); installPrebidNpm(); pubads.refresh([slot]); From e45b6b5a16e7274ac778fc9e6a0fa0f7f3c83d75 Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 24 Jul 2026 12:40:05 -0500 Subject: [PATCH 129/198] Resolve Prebid refresh review feedback --- .../lib/src/integrations/prebid/index.ts | 213 +++++++++++--- .../lib/test/integrations/gpt/ad_init.test.ts | 1 + .../test/integrations/prebid/index.test.ts | 267 ++++++++++++++++-- 3 files changed, 417 insertions(+), 64 deletions(-) diff --git a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts index 007a7d40d..9e0632e26 100644 --- a/crates/trusted-server-js/lib/src/integrations/prebid/index.ts +++ b/crates/trusted-server-js/lib/src/integrations/prebid/index.ts @@ -50,6 +50,7 @@ const TS_REFRESH_TARGETING_KEYS = [ ] as const; const MAX_PUBLISHER_AD_UNIT_SNAPSHOTS = 256; const MAX_PENDING_PUBLISHER_BIDS = 2048; +const PENDING_PUBLISHER_DELIVERY_TTL_MS = 5000; /** Configuration options for the Prebid integration. */ export interface PrebidNpmConfig { @@ -248,11 +249,23 @@ type PublisherAdUnitSnapshot = { }; type PendingPublisherBid = { adUnitCode: string; + expiresAt: number; + registrationId: number; +}; +type PendingPublisherCode = { + expiresAt: number; + registrationId: number; }; type RemoveAdUnit = (adUnitCode?: string | string[]) => unknown; +type PrebidWithRemoveAdUnit = { + removeAdUnit?: RemoveAdUnit; + __tsRemoveAdUnitWrapped?: boolean; +}; let publisherAdUnitSnapshots = new Map(); let pendingPublisherBids = new Map(); +let pendingPublisherCodes = new Map(); +let pendingPublisherRegistrationId = 0; let syntheticRefreshAdUnits = new WeakSet(); type TrustedServerBidRequest = { adUnitCode?: string; @@ -609,7 +622,45 @@ function clearRefreshTargeting(slot: RefreshGptSlot): void { } } -/** Store an auction-local bid ID for one-shot GPT delivery correlation. */ +/** Remove pending delivery state for an ad unit, optionally from one registration only. */ +function removePendingPublisherBidsForCode(adUnitCode: string, registrationId?: number): void { + const pendingCode = pendingPublisherCodes.get(adUnitCode); + if (registrationId !== undefined && pendingCode?.registrationId !== registrationId) return; + + pendingPublisherCodes.delete(adUnitCode); + for (const [adId, pendingBid] of pendingPublisherBids) { + if ( + pendingBid.adUnitCode === adUnitCode && + (registrationId === undefined || pendingBid.registrationId === registrationId) + ) { + pendingPublisherBids.delete(adId); + } + } +} + +/** Discard delivery state that outlived the publisher auction which created it. */ +function prunePendingPublisherBids(now = Date.now()): void { + for (const [adUnitCode, pendingCode] of pendingPublisherCodes) { + if (pendingCode.expiresAt <= now) removePendingPublisherBidsForCode(adUnitCode); + } + + for (const [adId, pendingBid] of pendingPublisherBids) { + if (pendingBid.expiresAt <= now) pendingPublisherBids.delete(adId); + } +} + +/** Store a short-lived pending publisher ad-unit code for delivery correlation. */ +function storePendingPublisherCode(adUnitCode: string, pendingCode: PendingPublisherCode): void { + pendingPublisherCodes.delete(adUnitCode); + pendingPublisherCodes.set(adUnitCode, pendingCode); + + if (pendingPublisherCodes.size > MAX_PENDING_PUBLISHER_BIDS) { + const oldestCode = pendingPublisherCodes.keys().next().value; + if (oldestCode !== undefined) removePendingPublisherBidsForCode(oldestCode); + } +} + +/** Store an auction-local bid ID for precise one-shot GPT delivery correlation. */ function storePendingPublisherBid(adId: string, pendingBid: PendingPublisherBid): void { pendingPublisherBids.delete(adId); pendingPublisherBids.set(adId, pendingBid); @@ -620,16 +671,23 @@ function storePendingPublisherBid(adId: string, pendingBid: PendingPublisherBid) } } -/** Remove every pending auction bid for an ad-unit code. */ -function removePendingPublisherBidsForCode(adUnitCode: string): void { - for (const [adId, pendingBid] of pendingPublisherBids) { - if (pendingBid.adUnitCode === adUnitCode) pendingPublisherBids.delete(adId); +/** Register every requested publisher code and any bid IDs returned for that auction. */ +function registerPendingPublisherBids( + publisherAdUnitCodes: Set, + bidResponses: unknown +): number { + prunePendingPublisherBids(); + const registrationId = ++pendingPublisherRegistrationId; + const expiresAt = Date.now() + PENDING_PUBLISHER_DELIVERY_TTL_MS; + + for (const adUnitCode of publisherAdUnitCodes) { + removePendingPublisherBidsForCode(adUnitCode); + storePendingPublisherCode(adUnitCode, { expiresAt, registrationId }); } -} -/** Register bid IDs from the current `bidsBackHandler` callback only. */ -function registerPendingPublisherBids(bidResponses: unknown): void { - if (!bidResponses || typeof bidResponses !== 'object' || Array.isArray(bidResponses)) return; + if (!bidResponses || typeof bidResponses !== 'object' || Array.isArray(bidResponses)) { + return registrationId; + } for (const [responseCode, responseGroup] of Object.entries(bidResponses)) { if (!responseGroup || typeof responseGroup !== 'object') continue; @@ -642,36 +700,53 @@ function registerPendingPublisherBids(bidResponses: unknown): void { const adId = typeof response.adId === 'string' ? response.adId : undefined; const adUnitCode = typeof response.adUnitCode === 'string' ? response.adUnitCode : responseCode; - if (!adId || !adUnitCode) continue; + if (!adId || !adUnitCode || !publisherAdUnitCodes.has(adUnitCode)) continue; - storePendingPublisherBid(adId, { adUnitCode }); + storePendingPublisherBid(adId, { adUnitCode, expiresAt, registrationId }); } } + + return registrationId; } /** - * Partition slots by whether their current `hb_adid` belongs to a pending - * publisher auction, consuming every older pending bid for each matched code. + * Partition slots by whether they belong to a pending publisher auction. + * + * A current `hb_adid` is the precise signal. When publishers intentionally + * omit that targeting, a short-lived requested-code match preserves delivery + * for no-bid and custom-targeting auctions. A non-empty unmatched ID remains + * independent so stale targeting cannot suppress a fresh auction. Every match + * is consumed once. */ function publisherDeliverySlots(targetSlots: RefreshGptSlot[]): Set { + prunePendingPublisherBids(); const deliverySlots = new Set(); const deliveredCodes = new Set(); for (const slot of targetSlots) { const adIds = slot.getTargeting?.('hb_adid'); - if (!Array.isArray(adIds)) continue; - - const pendingBid = adIds - .filter((adId): adId is string => typeof adId === 'string' && adId.length > 0) - .map((adId) => pendingPublisherBids.get(adId)) - .find((bid): bid is PendingPublisherBid => bid !== undefined); - if (!pendingBid) continue; + const pendingBid = Array.isArray(adIds) + ? adIds + .filter((adId): adId is string => typeof adId === 'string' && adId.length > 0) + .map((adId) => pendingPublisherBids.get(adId)) + .find((bid): bid is PendingPublisherBid => bid !== undefined) + : undefined; + const hasAdId = + Array.isArray(adIds) && adIds.some((adId) => typeof adId === 'string' && adId.length > 0); + const injectedSlot = findInjectedSlotForRefresh(slot); + const pendingCode = hasAdId + ? undefined + : [refreshSlotElementId(slot), injectedSlot?.div_id] + .filter((code): code is string => typeof code === 'string' && code.length > 0) + .find((code) => pendingPublisherCodes.has(code)); + const adUnitCode = pendingBid?.adUnitCode ?? pendingCode; + if (!adUnitCode) continue; deliverySlots.add(slot); - deliveredCodes.add(pendingBid.adUnitCode); + deliveredCodes.add(adUnitCode); } - deliveredCodes.forEach(removePendingPublisherBidsForCode); + deliveredCodes.forEach((adUnitCode) => removePendingPublisherBidsForCode(adUnitCode)); return deliverySlots; } @@ -680,6 +755,7 @@ function removePublisherState(adUnitCode?: string | string[]): void { if (!adUnitCode) { publisherAdUnitSnapshots.clear(); pendingPublisherBids.clear(); + pendingPublisherCodes.clear(); return; } @@ -728,16 +804,21 @@ function collectAuctionEids(): AuctionEid[] | undefined { export function installPrebidNpm(config?: Partial): typeof pbjs { publisherAdUnitSnapshots = new Map(); pendingPublisherBids = new Map(); + pendingPublisherCodes = new Map(); + pendingPublisherRegistrationId = 0; syntheticRefreshAdUnits = new WeakSet(); - const prebidWithRemoveAdUnit = pbjs as unknown as { removeAdUnit?: RemoveAdUnit }; - const originalRemoveAdUnit = prebidWithRemoveAdUnit.removeAdUnit; - if (typeof originalRemoveAdUnit === 'function') { - prebidWithRemoveAdUnit.removeAdUnit = function (adUnitCode?: string | string[]) { - const result = originalRemoveAdUnit.call(this, adUnitCode); - removePublisherState(adUnitCode); - return result; - }; + const prebidWithRemoveAdUnit = pbjs as unknown as PrebidWithRemoveAdUnit; + if (!prebidWithRemoveAdUnit.__tsRemoveAdUnitWrapped) { + const originalRemoveAdUnit = prebidWithRemoveAdUnit.removeAdUnit; + if (typeof originalRemoveAdUnit === 'function') { + prebidWithRemoveAdUnit.removeAdUnit = function (adUnitCode?: string | string[]) { + const result = originalRemoveAdUnit.call(this, adUnitCode); + removePublisherState(adUnitCode); + return result; + }; + prebidWithRemoveAdUnit.__tsRemoveAdUnitWrapped = true; + } } const injected = getInjectedConfig(); @@ -809,7 +890,7 @@ export function installPrebidNpm(config?: Partial): typeof pbjs log.debug('[tsjs-prebid] requestBids called'); recordUserIdModuleDiagnostics(); - const opts = requestObj || {}; + const opts = { ...(requestObj ?? {}) }; // eslint-disable-next-line @typescript-eslint/no-explicit-any const adUnits = ((opts as any).adUnits || pbjs.adUnits || []) as TrustedServerAdUnit[]; const isSyntheticRefresh = @@ -913,12 +994,21 @@ export function installPrebidNpm(config?: Partial): typeof pbjs const originalBidsBack = opts.bidsBackHandler; opts.bidsBackHandler = function (...args: unknown[]) { syncPrebidEidsCookie(); + const registrationId = isSyntheticRefresh + ? undefined + : registerPendingPublisherBids(publisherAdUnitCodes, args[0]); if (typeof originalBidsBack !== 'function') return; - if (!isSyntheticRefresh) { - publisherAdUnitCodes.forEach(removePendingPublisherBidsForCode); - registerPendingPublisherBids(args[0]); + + try { + originalBidsBack.apply(this, args as Parameters); + } catch (error) { + if (registrationId !== undefined) { + publisherAdUnitCodes.forEach((code) => + removePendingPublisherBidsForCode(code, registrationId) + ); + } + throw error; } - originalBidsBack.apply(this, args as Parameters); }; return originalRequestBids(opts); @@ -1018,16 +1108,15 @@ export function installRefreshHandler(timeoutMs = 1500): void { return originalRefresh(slots, opts); } - if (!targetSlots.length) { + if (!targetSlots.length || (slots !== undefined && targetSlots.length !== slots.length)) { return originalRefresh(slots, opts); } const deliverySlots = publisherDeliverySlots(targetSlots); const independentSlots = targetSlots.filter((slot) => !deliverySlots.has(slot)); - if (deliverySlots.size > 0) { - originalRefresh([...deliverySlots], opts); + if (independentSlots.length === 0) { + return originalRefresh(slots, opts); } - if (independentSlots.length === 0) return; independentSlots.forEach(clearRefreshTargeting); @@ -1072,14 +1161,44 @@ export function installRefreshHandler(timeoutMs = 1500): void { // `targetSlots` — leaving their next request dependent on stale state. const refreshAdUnitCodes = adUnits.map((unit) => unit.code); adUnits.forEach((unit) => syntheticRefreshAdUnits.add(unit)); - pbjs.requestBids({ - adUnits, - bidsBackHandler: () => { - pbjs.setTargetingForGPTAsync?.(refreshAdUnitCodes); - originalRefresh(independentSlots, opts); - }, - timeout: timeoutMs, - }); + + // Preserve GPT Single Request Architecture: when a publisher refresh + // includes both already-targeted delivery slots and independent slots, + // delay the whole original list until the independent auction completes. + // A one-shot fallback prevents a failed Prebid callback from dropping any + // slots, and a late callback cannot issue a second GAM request. + let completed = false; + let fallbackTimer: ReturnType | undefined; + function completeRefresh(): void { + if (completed) return; + completed = true; + if (fallbackTimer !== undefined) clearTimeout(fallbackTimer); + originalRefresh(slots, opts); + } + + try { + pbjs.requestBids({ + adUnits, + bidsBackHandler: () => { + if (completed) return; + try { + pbjs.setTargetingForGPTAsync?.(refreshAdUnitCodes); + } catch (error) { + log.error('[tsjs-prebid] refresh targeting failed', error); + } finally { + completeRefresh(); + } + }, + timeout: timeoutMs, + }); + // Prebid schedules its own timeout during requestBids(). Schedule this + // fallback afterward so its normal timeout callback gets first chance + // to apply targeting before the one-shot GPT completion path runs. + if (!completed) fallbackTimer = setTimeout(completeRefresh, timeoutMs); + } catch (error) { + log.error('[tsjs-prebid] refresh auction failed', error); + completeRefresh(); + } }; log.info('[tsjs-prebid] GPT refresh handler installed'); diff --git a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts index ac55b60de..733c43642 100644 --- a/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/gpt/ad_init.test.ts @@ -133,6 +133,7 @@ describe('installTsAdInit', () => { expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_cache_host', 'cache.example.com'); expect(mockSlot.setTargeting).toHaveBeenCalledWith('hb_cache_path', '/pbc/v1/cache'); expect(mockSlot.setTargeting).toHaveBeenCalledWith('ts_initial', '1'); + expect(mockPubads.enableSingleRequest).toHaveBeenCalledOnce(); expect(mockPubads.refresh).toHaveBeenCalled(); fetchSpy.mockRestore(); diff --git a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts index 92d2b07d9..ab253f758 100644 --- a/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts +++ b/crates/trusted-server-js/lib/test/integrations/prebid/index.test.ts @@ -1447,6 +1447,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { mockRequestBids.mockReset(); mockPbjs.requestBids = mockRequestBids; mockPbjs.removeAdUnit = mockRemoveAdUnit; + delete (mockPbjs as any).__tsRemoveAdUnitWrapped; mockPbjs.adUnits = []; mockGetUserIdsAsEids.mockReset(); mockGetUserIdsAsEids.mockReturnValue([]); @@ -1917,6 +1918,64 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(originalRefresh).toHaveBeenNthCalledWith(2, [slotTwo], undefined); }); + it('registers delivery state for a publisher auction without a bidsBackHandler', () => { + const code = 'example-handlerless-delivery'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + } as any); + pubads.refresh([slot]); + + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(slot.clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); + + it('preserves one mixed refresh request and its original options', () => { + const deliverySlot = { + getSlotElementId: () => 'example-sra-delivery', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const independentSlot = { + getSlotElementId: () => 'example-sra-independent', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const refreshOptions = { changeCorrelator: true }; + const { originalRefresh, pubads } = installGpt([deliverySlot, independentSlot]); + let syntheticBidsBackHandler: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + if (mockRequestBids.mock.calls.length === 1) { + completePublisherAuction(opts); + } else { + syntheticBidsBackHandler = opts.bidsBackHandler; + } + }); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code: 'example-sra-delivery', bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh([deliverySlot, independentSlot], refreshOptions), + } as any); + + expect(originalRefresh).not.toHaveBeenCalled(); + expect(independentSlot.clearTargeting).toHaveBeenCalledWith('hb_adid'); + + syntheticBidsBackHandler?.(); + + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith([deliverySlot, independentSlot], refreshOptions); + }); + it('partitions a bare delivery refresh from an unmatched GPT slot', () => { const coveredSlot = { getSlotElementId: () => 'example-covered', @@ -1940,9 +1999,8 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(mockRequestBids).toHaveBeenCalledTimes(2); expect(coveredSlot.clearTargeting).not.toHaveBeenCalled(); expect(gamOnlySlot.clearTargeting).toHaveBeenCalledWith('hb_adid'); - expect(originalRefresh).toHaveBeenCalledTimes(2); - expect(originalRefresh).toHaveBeenNthCalledWith(1, [coveredSlot], undefined); - expect(originalRefresh).toHaveBeenNthCalledWith(2, [gamOnlySlot], undefined); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith(undefined, undefined); }); it('keeps explicit unrelated lists synthetic and partitions mixed delivery lists', () => { @@ -1980,10 +2038,9 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(coveredSlot.clearTargeting).not.toHaveBeenCalled(); expect(unrelatedSlot.clearTargeting).toHaveBeenCalledWith('ts_initial'); expect(unrelatedSlot.clearTargeting).toHaveBeenCalledWith('hb_cache_path'); - expect(originalRefresh).toHaveBeenCalledTimes(3); + expect(originalRefresh).toHaveBeenCalledTimes(2); expect(originalRefresh).toHaveBeenNthCalledWith(1, [unrelatedSlot], undefined); - expect(originalRefresh).toHaveBeenNthCalledWith(2, [coveredSlot], undefined); - expect(originalRefresh).toHaveBeenNthCalledWith(3, [unrelatedSlot], undefined); + expect(originalRefresh).toHaveBeenNthCalledWith(2, [coveredSlot, unrelatedSlot], undefined); }); it('partitions four delivered slots from an unmatched explicit slot', () => { @@ -2013,9 +2070,39 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(mockRequestBids).toHaveBeenCalledTimes(2); coveredSlots.forEach((slot) => expect(slot.clearTargeting).not.toHaveBeenCalled()); expect(gamOnlySlot.clearTargeting).toHaveBeenCalledWith('hb_adid'); - expect(originalRefresh).toHaveBeenCalledTimes(2); - expect(originalRefresh).toHaveBeenNthCalledWith(1, coveredSlots, undefined); - expect(originalRefresh).toHaveBeenNthCalledWith(2, [gamOnlySlot], undefined); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith(refreshSlots, undefined); + }); + + it('expires an unconsumed publisher delivery before a later refresh', () => { + vi.useFakeTimers(); + try { + const code = 'example-expired-delivery'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => + completePublisherAuction(opts, { applyTargeting: false }) + ); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + } as any); + vi.advanceTimersByTime(5001); + pubads.refresh([slot]); + + expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(slot.clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + } finally { + vi.runOnlyPendingTimers(); + vi.useRealTimers(); + } }); it('correlates a targeted delivery refresh after more than one second without a timer race', () => { @@ -2106,7 +2193,7 @@ describe('prebid publisher snapshots and delivery refreshes', () => { delete (mockPbjs as any).setTargetingForGPTAsync; }); - it('uses the synthetic path when callback bid responses are missing or malformed', () => { + it('correlates requested no-bid slots without manufacturing unrelated bid state', () => { const slot = { getSlotElementId: () => 'example-no-bid-delivery', getTargeting: () => [], @@ -2126,6 +2213,30 @@ describe('prebid publisher snapshots and delivery refreshes', () => { bidsBackHandler: () => pubads.refresh([slot]), } as any); + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(slot.clearTargeting).not.toHaveBeenCalled(); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); + + it('does not use code fallback when a slot has an unmatched hb_adid', () => { + const code = 'example-stale-targeting'; + const slot = { + getSlotElementId: () => code, + getTargeting: (key: string) => (key === 'hb_adid' ? ['example-stale-ad-id'] : []), + getSizes: () => [[300, 250]], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => + completePublisherAuction(opts, { applyTargeting: false }) + ); + const pbjs = installPrebidNpm(); + + pbjs.requestBids({ + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + bidsBackHandler: () => pubads.refresh([slot]), + } as any); + expect(mockRequestBids).toHaveBeenCalledTimes(2); expect(slot.clearTargeting).toHaveBeenCalledWith('hb_adid'); expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); @@ -2170,12 +2281,11 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(innerSlot.clearTargeting).not.toHaveBeenCalled(); expect(outerSlot.clearTargeting).not.toHaveBeenCalled(); expect(gamOnlySlot.clearTargeting).toHaveBeenCalledWith('hb_adid'); - expect(originalRefresh).toHaveBeenCalledTimes(2); - expect(originalRefresh).toHaveBeenNthCalledWith(1, [innerSlot, outerSlot], undefined); - expect(originalRefresh).toHaveBeenNthCalledWith(2, [gamOnlySlot], undefined); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith(refreshSlots, undefined); }); - it('treats a microtask refresh without a targeting signal as an independent auction', async () => { + it('correlates a microtask refresh by its requested code without targeting', async () => { const slot = { getSlotElementId: () => 'example-deferred-refresh', getTargeting: () => [], @@ -2199,8 +2309,8 @@ describe('prebid publisher snapshots and delivery refreshes', () => { } as any); await deferredRefresh; - expect(mockRequestBids).toHaveBeenCalledTimes(2); - expect(slot.clearTargeting).toHaveBeenCalledWith('hb_pb'); + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(slot.clearTargeting).not.toHaveBeenCalled(); expect(originalRefresh).toHaveBeenCalledTimes(1); expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); }); @@ -2290,11 +2400,134 @@ describe('prebid publisher snapshots and delivery refreshes', () => { expect(mockRequestBids).toHaveBeenCalledTimes(1); expect(slot.clearTargeting).not.toHaveBeenCalled(); - expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + expect(originalRefresh).toHaveBeenCalledWith([slot, undefined, null], undefined); + + pubads.refresh([slot]); + expect(mockRequestBids).toHaveBeenCalledTimes(1); + expect(slot.clearTargeting).not.toHaveBeenCalled(); + }); + + it('does not mutate reused publisher request options', () => { + const code = 'example-reused-request'; + const slot = { + getSlotElementId: () => code, + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + const pbjs = installPrebidNpm(); + const request = { + adUnits: [{ code, bids: [{ bidder: 'exampleServer', params: {} }] }], + }; + pbjs.requestBids(request as any); + pbjs.requestBids(request as any); pubads.refresh([slot]); + + expect(request).not.toHaveProperty('bidsBackHandler'); expect(mockRequestBids).toHaveBeenCalledTimes(2); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); + + it('falls back to one GPT refresh when a synthetic auction throws', () => { + const slot = { + getSlotElementId: () => 'example-throwing-refresh', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation(() => { + throw new Error('example synthetic failure'); + }); + installPrebidNpm(); + + pubads.refresh([slot]); + expect(slot.clearTargeting).toHaveBeenCalledWith('hb_adid'); + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); + + it('falls back once when a synthetic auction never calls back', () => { + vi.useFakeTimers(); + try { + const slot = { + getSlotElementId: () => 'example-missing-refresh-callback', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + mockRequestBids.mockImplementation(() => undefined); + installPrebidNpm(); + + pubads.refresh([slot]); + expect(originalRefresh).not.toHaveBeenCalled(); + vi.advanceTimersByTime(640); + + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + } finally { + vi.runOnlyPendingTimers(); + vi.useRealTimers(); + } + }); + + it('ignores a synthetic callback that arrives after the fallback refresh', () => { + vi.useFakeTimers(); + try { + const slot = { + getSlotElementId: () => 'example-late-refresh-callback', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + const setTargetingForGPTAsync = vi.fn(); + (mockPbjs as any).setTargetingForGPTAsync = setTargetingForGPTAsync; + let syntheticBidsBackHandler: (() => void) | undefined; + mockRequestBids.mockImplementation((opts) => { + syntheticBidsBackHandler = opts.bidsBackHandler; + }); + installPrebidNpm(); + + pubads.refresh([slot]); + vi.advanceTimersByTime(640); + syntheticBidsBackHandler?.(); + + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(setTargetingForGPTAsync).not.toHaveBeenCalled(); + } finally { + vi.runOnlyPendingTimers(); + vi.useRealTimers(); + } + }); + + it('completes a synthetic refresh when targeting throws', () => { + const slot = { + getSlotElementId: () => 'example-throwing-targeting', + getTargeting: () => [], + clearTargeting: vi.fn(), + }; + const { originalRefresh, pubads } = installGpt([slot]); + (mockPbjs as any).setTargetingForGPTAsync = vi.fn(() => { + throw new Error('example targeting failure'); + }); + mockRequestBids.mockImplementation((opts) => completePublisherAuction(opts)); + installPrebidNpm(); + + pubads.refresh([slot]); + + expect(originalRefresh).toHaveBeenCalledTimes(1); + expect(originalRefresh).toHaveBeenCalledWith([slot], undefined); + }); + + it('does not stack the removeAdUnit lifecycle wrapper across installation', () => { + const pbjs = installPrebidNpm(); + installPrebidNpm(); + + (pbjs as any).removeAdUnit('example-reinstalled-slot'); + + expect(mockRemoveAdUnit).toHaveBeenCalledTimes(1); }); it('keeps nested publisher delivery contexts isolated during reentrant auctions', () => { From af46b98dfb432c80d0c9e2b4d73207fe307c03ca Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 15 Jul 2026 12:04:27 -0500 Subject: [PATCH 130/198] Make auction creative rewriting optional Allow operators to retain sanitizer-accepted external URLs in POST /auction adm while preserving mandatory server-side sanitization and the existing default behavior. --- CHANGELOG.md | 1 + .../src/auction/endpoints.rs | 7 +- .../src/auction/formats.rs | 141 +++++++++++++++++- .../src/auction/orchestrator.rs | 1 + .../src/auction_config_types.rs | 22 +++ .../trusted-server-core/src/config_payload.rs | 24 +++ crates/trusted-server-core/src/proxy.rs | 45 ++++++ crates/trusted-server-core/src/settings.rs | 35 +++++ docs/guide/auction-orchestration.md | 69 ++++++--- docs/guide/configuration.md | 26 +++- docs/guide/creative-processing.md | 54 +++++-- trusted-server.example.toml | 4 + 12 files changed, 382 insertions(+), 47 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bc49c80b..fddc8009d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added the default-true `[auction].rewrite_creatives` option. Setting it to `false` preserves mandatory `/auction` creative sanitization while skipping first-party resource/click URL rewriting and creative TSJS injection. - Added Osano consent mirror integration docs and public enablement guidance. - Implemented basic authentication for configurable endpoint paths (#73) - Added integrations guide with example `testlight` integration diff --git a/crates/trusted-server-core/src/auction/endpoints.rs b/crates/trusted-server-core/src/auction/endpoints.rs index 1b0ced7a7..e5796323f 100644 --- a/crates/trusted-server-core/src/auction/endpoints.rs +++ b/crates/trusted-server-core/src/auction/endpoints.rs @@ -76,9 +76,10 @@ const MAX_AUCTION_BODY_SIZE: usize = 256 * 1024; /// ## Response /// /// Returns an `OpenRTB 2.x` response. Creative HTML is inlined in each bid's -/// `adm` field after sanitisation and first-party URL rewriting. Response -/// headers include `X-TS-EC` (the caller's Edge Cookie ID) and -/// `X-TS-EC-Fresh` (a freshly generated ID for cookie renewal). +/// `adm` field after mandatory server-side sanitization. First-party resource +/// and click URL rewriting plus creative TSJS injection are enabled by default; +/// setting [`auction.rewrite_creatives`][`crate::auction_config_types::AuctionConfig::rewrite_creatives`] +/// to `false` skips only that rewrite pass. /// /// ## Scroll, refresh, and SPA navigation /// diff --git a/crates/trusted-server-core/src/auction/formats.rs b/crates/trusted-server-core/src/auction/formats.rs index 441828a18..71f9a290c 100644 --- a/crates/trusted-server-core/src/auction/formats.rs +++ b/crates/trusted-server-core/src/auction/formats.rs @@ -217,7 +217,8 @@ pub fn convert_tsjs_to_auction_request( /// Convert `OrchestrationResult` to `OpenRTB` response format. /// -/// Returns rewritten creative HTML directly in the `adm` field for inline delivery. +/// Always sanitizes creative HTML in the `adm` field and optionally rewrites it +/// according to the auction configuration. /// /// # Errors /// @@ -250,21 +251,34 @@ pub fn convert_to_openrtb_response( let width = to_openrtb_i32(bid.width, "width", &bid_context); let height = to_openrtb_i32(bid.height, "height", &bid_context); - // Process creative HTML if present - — sanitize dangerous markup first, then rewrite URLs. + // Process creative HTML if present — always sanitize dangerous markup first. let creative_html = if let Some(ref raw_creative) = bid.creative { let sanitized = creative::sanitize_creative_html(raw_creative); - let rewritten = creative::rewrite_creative_html(settings, &sanitized); + let sanitized_len = sanitized.len(); + let rewrite_creatives = settings.auction.rewrite_creatives; + let processed = if rewrite_creatives { + creative::rewrite_creative_html(settings, &sanitized) + } else { + sanitized + }; + let rewrite_mode = if rewrite_creatives { + "enabled" + } else { + "disabled" + }; log::debug!( - "Processed creative for auction {} slot {} ({} → {} → {} bytes)", + "Processed creative for auction {} slot {} bidder {} (rewrite {}, raw {} bytes, sanitized {} bytes, output {} bytes)", auction_request.id, slot_id, + bid.bidder, + rewrite_mode, raw_creative.len(), - sanitized.len(), - rewritten.len() + sanitized_len, + processed.len() ); - rewritten + processed } else { // No creative provided (e.g., from mediation layer that returns iframe URLs) log::warn!( @@ -445,6 +459,15 @@ mod tests { } } + fn make_complete_creative_bid() -> Bid { + let mut bid = make_bid("div-gpt-top", "appnexus", Some(2.75)); + bid.creative = Some( + r#""# + .to_string(), + ); + bid + } + fn make_result(bid: Bid) -> OrchestrationResult { OrchestrationResult { provider_responses: vec![AuctionResponse { @@ -466,6 +489,13 @@ mod tests { .expect("should parse JSON response") } + fn response_adm(response: Response) -> String { + response_json(response)["seatbid"][0]["bid"][0]["adm"] + .as_str() + .expect("should serialize adm as a string") + .to_string() + } + fn make_banner_body(config: Option) -> AdRequest { AdRequest { ad_units: vec![AdUnit { @@ -932,6 +962,103 @@ mod tests { ); } + #[test] + fn convert_to_openrtb_response_rewrites_sanitized_creative_by_default() { + let settings = make_settings(); + let auction_request = make_auction_request(); + let result = make_result(make_complete_creative_bid()); + + let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) + .expect("should convert creative with rewriting enabled"); + let adm = response_adm(response); + + assert!( + adm.matches("/first-party/proxy?tsurl=").count() >= 2, + "should rewrite image and inline CSS URLs through the proxy: {adm}" + ); + assert!( + adm.contains("/first-party/click?tsurl="), + "should rewrite click URLs: {adm}" + ); + assert!( + adm.contains("data-tsclick"), + "should add the click guard attribute: {adm}" + ); + assert!( + adm.contains("tsjs-unified.min.js"), + "should inject the unified creative runtime: {adm}" + ); + assert!( + !adm.contains(r#"src="https://cdn.example.com/ad.png""#), + "should not retain the image URL as a direct attribute: {adm}" + ); + assert!( + !adm.contains(r#"href="https://advertiser.example.com/landing""#), + "should not retain the click URL as a direct attribute: {adm}" + ); + assert!( + !adm.contains("url(https://styles.example.com/bg.png)"), + "should not retain the CSS URL as a direct value: {adm}" + ); + assert!( + !adm.contains("auction-script-marker"), + "should remove malicious script content before rewriting: {adm}" + ); + assert!( + !adm.contains("auction-handler-marker") && !adm.contains("onerror"), + "should remove event handlers before rewriting: {adm}" + ); + } + + #[test] + fn convert_to_openrtb_response_can_skip_rewriting_but_not_sanitization() { + let mut settings = make_settings(); + settings.auction.rewrite_creatives = false; + let auction_request = make_auction_request(); + let result = make_result(make_complete_creative_bid()); + + let response = convert_to_openrtb_response(&result, &settings, &auction_request, false) + .expect("should convert creative with rewriting disabled"); + let adm = response_adm(response); + + assert!( + adm.contains(r#"src="https://cdn.example.com/ad.png""#), + "should retain the sanitizer-accepted image URL: {adm}" + ); + assert!( + adm.contains(r#"href="https://advertiser.example.com/landing""#), + "should retain the sanitizer-accepted click URL: {adm}" + ); + assert!( + adm.contains("url(https://styles.example.com/bg.png)"), + "should retain the sanitizer-accepted CSS URL: {adm}" + ); + assert!( + !adm.contains("/first-party/proxy"), + "should not rewrite resource URLs: {adm}" + ); + assert!( + !adm.contains("/first-party/click"), + "should not rewrite click URLs: {adm}" + ); + assert!( + !adm.contains("data-tsclick"), + "should not add the click guard attribute: {adm}" + ); + assert!( + !adm.contains("tsjs-unified.min.js"), + "should not inject the unified creative runtime: {adm}" + ); + assert!( + !adm.contains("auction-script-marker"), + "should still remove malicious script content: {adm}" + ); + assert!( + !adm.contains("auction-handler-marker") && !adm.contains("onerror"), + "should still remove event handlers: {adm}" + ); + } + #[test] fn convert_to_openrtb_response_serializes_missing_creative_as_empty_adm() { let settings = make_settings(); diff --git a/crates/trusted-server-core/src/auction/orchestrator.rs b/crates/trusted-server-core/src/auction/orchestrator.rs index bf9ecad7b..69455e894 100644 --- a/crates/trusted-server-core/src/auction/orchestrator.rs +++ b/crates/trusted-server-core/src/auction/orchestrator.rs @@ -2102,6 +2102,7 @@ mod tests { futures::executor::block_on(async { let config = AuctionConfig { enabled: true, + rewrite_creatives: true, providers: vec![], mediator: None, timeout_ms: 2000, diff --git a/crates/trusted-server-core/src/auction_config_types.rs b/crates/trusted-server-core/src/auction_config_types.rs index 3bd747f64..f1d1a5cf0 100644 --- a/crates/trusted-server-core/src/auction_config_types.rs +++ b/crates/trusted-server-core/src/auction_config_types.rs @@ -11,6 +11,10 @@ pub struct AuctionConfig { #[serde(default)] pub enabled: bool, + /// Rewrite sanitized winning-bid creative HTML to first-party endpoints. + #[serde(default = "default_rewrite_creatives")] + pub rewrite_creatives: bool, + /// Provider names that participate in bidding /// Simply list the provider names (e.g., ["prebid", "aps"]) #[serde(default, deserialize_with = "crate::settings::vec_from_seq_or_map")] @@ -41,6 +45,7 @@ impl Default for AuctionConfig { fn default() -> Self { Self { enabled: false, + rewrite_creatives: default_rewrite_creatives(), providers: Vec::new(), mediator: None, timeout_ms: default_timeout(), @@ -54,6 +59,10 @@ fn default_timeout() -> u32 { 2000 } +fn default_rewrite_creatives() -> bool { + true +} + fn default_creative_store() -> String { "creative_store".to_owned() } @@ -79,3 +88,16 @@ impl AuctionConfig { self.mediator.is_some() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rewrite_creatives_defaults_to_true() { + assert!( + AuctionConfig::default().rewrite_creatives, + "should enable creative rewriting by default" + ); + } +} diff --git a/crates/trusted-server-core/src/config_payload.rs b/crates/trusted-server-core/src/config_payload.rs index dd0b35337..58c185381 100644 --- a/crates/trusted-server-core/src/config_payload.rs +++ b/crates/trusted-server-core/src/config_payload.rs @@ -78,6 +78,30 @@ mod tests { ); } + #[test] + fn legacy_blob_without_rewrite_creatives_preserves_rewriting() { + let mut data = + serde_json::to_value(test_settings()).expect("should serialize settings to JSON"); + let auction = data + .get_mut("auction") + .and_then(serde_json::Value::as_object_mut) + .expect("should serialize auction settings as an object"); + assert!( + auction.remove("rewrite_creatives").is_some(), + "should remove the newly serialized setting" + ); + let envelope = BlobEnvelope::new(data, "2026-01-01T00:00:00Z".to_string()); + let envelope_json = serde_json::to_string(&envelope).expect("should serialize envelope"); + + let reconstructed = + settings_from_config_blob(&envelope_json).expect("should reconstruct legacy settings"); + + assert!( + reconstructed.auction.rewrite_creatives, + "should enable creative rewriting for legacy blobs" + ); + } + #[test] fn strings_that_look_like_json_scalars_round_trip_as_strings() { let mut original = test_settings(); diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 43cff69bc..d5268388c 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -2884,6 +2884,51 @@ mod tests { assert_eq!(ct, "text/css; charset=utf-8"); } + #[test] + fn auction_rewrite_setting_does_not_change_proxied_html_or_css_rewriting() { + let mut settings = create_test_settings(); + settings.auction.rewrite_creatives = false; + let req = build_http_request(Method::GET, "https://edge.example/first-party/proxy"); + + let html = r#""#; + let mut html_response = build_http_response(StatusCode::OK, EdgeBody::from(html)); + html_response.headers_mut().insert( + header::CONTENT_TYPE, + HeaderValue::from_static("text/html; charset=utf-8"), + ); + let html_output = finalize( + &settings, + &req, + "https://cdn.example/creative.html", + html_response, + ) + .expect("should finalize proxied HTML"); + let html_body = response_body_string(html_output); + + let css = "body{background:url(https://cdn.example/bg.png)}"; + let mut css_response = build_http_response(StatusCode::OK, EdgeBody::from(css)); + css_response + .headers_mut() + .insert(header::CONTENT_TYPE, HeaderValue::from_static("text/css")); + let css_output = finalize( + &settings, + &req, + "https://cdn.example/creative.css", + css_response, + ) + .expect("should finalize proxied CSS"); + let css_body = response_body_string(css_output); + + assert!( + html_body.contains("/first-party/proxy?tsurl="), + "should keep rewriting proxied HTML when auction rewriting is disabled: {html_body}" + ); + assert!( + css_body.contains("/first-party/proxy?tsurl="), + "should keep rewriting proxied CSS when auction rewriting is disabled: {css_body}" + ); + } + #[test] fn html_response_rewrite_preserves_non_standard_port() { // Verify that HTML rewriting preserves non-standard ports in sub-resource URLs. diff --git a/crates/trusted-server-core/src/settings.rs b/crates/trusted-server-core/src/settings.rs index 9d4d352e6..be0216008 100644 --- a/crates/trusted-server-core/src/settings.rs +++ b/crates/trusted-server-core/src/settings.rs @@ -4384,6 +4384,41 @@ origin_host_header_overide = "www.example.com""#, assert!(!rewrite.is_excluded("")); } + #[test] + fn test_auction_rewrite_creatives_defaults_to_true_when_omitted() { + let toml_str = crate_test_settings_str() + + r#" + [auction] + enabled = true + providers = [] + "#; + + let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); + + assert!( + settings.auction.rewrite_creatives, + "should preserve creative rewriting when the setting is omitted" + ); + } + + #[test] + fn test_auction_rewrite_creatives_accepts_explicit_false() { + let toml_str = crate_test_settings_str() + + r#" + [auction] + enabled = true + providers = [] + rewrite_creatives = false + "#; + + let settings = Settings::from_toml(&toml_str).expect("should parse valid TOML"); + + assert!( + !settings.auction.rewrite_creatives, + "should disable creative rewriting when explicitly configured" + ); + } + #[test] fn test_auction_allowed_context_keys_defaults_to_empty() { let settings = create_test_settings(); diff --git a/docs/guide/auction-orchestration.md b/docs/guide/auction-orchestration.md index d75958812..c6c82dac3 100644 --- a/docs/guide/auction-orchestration.md +++ b/docs/guide/auction-orchestration.md @@ -12,7 +12,7 @@ Key capabilities: - **Strategy-based winner selection** — Automatic strategy detection based on configuration - **Mediator support** — Optional external mediator for decoding encoded prices (e.g., APS) and applying unified floor pricing - **Provider abstraction** — Pluggable provider interface for adding new demand sources -- **Creative rewriting** — Winning creatives automatically rewritten with first-party proxy URLs +- **Creative rewriting** — Winning creatives are sanitized and rewritten with first-party proxy URLs by default ## System Flow (Prebid + APS) @@ -147,7 +147,7 @@ sequenceDiagram Note over Client,Mock: Response Assembly activate TS activate Client - Orch->>Orch: Transform to OpenRTB response
Generate iframe creatives
Rewrite creative URLs
Add orchestrator metadata + Orch->>Orch: Transform to OpenRTB response
Sanitize creative HTML
Optionally rewrite creative URLs
Add orchestrator metadata Orch-->>TS: OpenRTB BidResponse Note right of Orch: { "id": "auction-response",
"seatbid": [{ "seat": "amazon-aps",
"bid": [{ "price": 2.50,
"adm": "