From 9f9d7970b281ec5e4e43cc4b84c619f999aca9ab Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:11:08 -0400 Subject: [PATCH 1/2] test(oom): bump python OOM function to 512 MB to fix flaky metric assertion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `integration-suite: [oom]` python case failed ~1 run in 3 with `out_of_memory` count 0 (all other runtimes passing). Datadog logs show the python function emitted no telemetry at all on failing runs — no logs and no metric — while the same function emits both on passing runs. At 256 MB the python3.13 runtime plus the `datadog_lambda` handler shim leave too little headroom, so when the function OOMs the kernel OOM-killer intermittently takes the extension too before it can run its end-of-invocation flush, dropping both logs and the enhanced metric. Bumping the python function to 512 MB keeps the extension alive to detect and flush. Detection paths are unchanged: the function still hits its memory cap and emits `MemoryError`. Co-Authored-By: Claude Opus 4.8 (1M context) --- integration-tests/lib/stacks/oom.ts | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/integration-tests/lib/stacks/oom.ts b/integration-tests/lib/stacks/oom.ts index c5a57dc2d..c8598a856 100644 --- a/integration-tests/lib/stacks/oom.ts +++ b/integration-tests/lib/stacks/oom.ts @@ -53,19 +53,20 @@ export class Oom extends cdk.Stack { const dotnetLayer = getDefaultDotnetLayer(this); const rubyLayer = getDefaultRubyLayer(this); - // 256 MB (not the customer's 192 MB from #1237) so the bottlecap - // extension has memory headroom to survive when the function process - // OOMs. At 192 MB the kernel OOM-killer often picks the extension - // instead of the function runtime (Lambda surfaces this as the - // `Extension.Crash` error type), and a dead extension can't emit the - // OOM metric. With 256 MB the function runtime's RSS dominates and - // kernel reliably kills it; the extension survives to detect/flush. - // The detection paths under test are unchanged — the functions still - // hit `max_memory_used == memory_size` in PlatformReport and still - // emit runtime-specific OOM error log lines. + // 256 MB (not the customer's 192 MB from #1237) to give the extension + // headroom to survive the function's OOM. If memory is too tight the + // kernel OOM-killer takes the extension instead (Lambda reports + // `Extension.Crash`), and a dead extension emits neither logs nor the OOM + // metric. Detection paths are unchanged: functions still hit + // `max_memory_used == memory_size` and emit runtime-specific OOM log lines. const oomMemorySize = 256; const oomTimeout = cdk.Duration.seconds(30); + // python3.13 + the `datadog_lambda` handler shim need more slack: at + // 256 MB the extension was intermittently killed before its flush, + // dropping the metric (~1 run in 3). 512 MB keeps it alive. + const pythonOomMemorySize = 512; + // Node case A — V8 heap exhaustion (log-line path). const nodeV8FunctionName = `${id}-node-v8-heap-lambda`; const nodeV8Function = new lambda.Function(this, nodeV8FunctionName, { @@ -122,7 +123,7 @@ export class Oom extends cdk.Stack { code: lambda.Code.fromAsset('./lambda/oom-python'), functionName: pythonFunctionName, timeout: oomTimeout, - memorySize: oomMemorySize, + memorySize: pythonOomMemorySize, environment: { ...defaultDatadogEnvVariables, DD_SERVICE: pythonFunctionName, From 8da8d855e36ce41f7780fa0bf2d2d02e5fe37831 Mon Sep 17 00:00:00 2001 From: Yiming Luo <10097700+lym953@users.noreply.github.com> Date: Wed, 15 Jul 2026 16:19:38 -0400 Subject: [PATCH 2/2] docs(oom): note python's 512 MB in the class-level comment Co-Authored-By: Claude Opus 4.8 (1M context) --- integration-tests/lib/stacks/oom.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/integration-tests/lib/stacks/oom.ts b/integration-tests/lib/stacks/oom.ts index c8598a856..b2a9b6544 100644 --- a/integration-tests/lib/stacks/oom.ts +++ b/integration-tests/lib/stacks/oom.ts @@ -38,9 +38,10 @@ import { * - oom-go : log line `fatal error: runtime: out of memory` * + PlatformReport memory equality (dedup) * - * Each function is configured with low memory (256 MB) and a short timeout - * (30 s) so the OOM fires quickly during the integration-test run. See the - * `oomMemorySize` comment for why 256 MB rather than the customer's 192 MB. + * Each function is configured with low memory (256 MB, except python at + * 512 MB) and a short timeout (30 s) so the OOM fires quickly during the + * integration-test run. See the `oomMemorySize` and `pythonOomMemorySize` + * comments for the memory rationale. */ export class Oom extends cdk.Stack { constructor(scope: Construct, id: string, props: cdk.StackProps) {