chaos: vthread-context-cascade antagonist + glibc heap-corruption defaults

.gitlab/reliability/chaos_check.sh

@@ -1,178 +1,17 @@
 #!/usr/bin/env bash
-
-set +e # Disable exit on error
+#
+# CI entry point for the chaos harness. Thin wrapper around
+# utils/run-chaos-harness.sh — this file only supplies the CI-specific
+# caching paths (so repeated scheduled runs on the same runner reuse the
+# downloaded JDK/agent jar) and the fixed hs_err.log location the pipeline's
+# `artifacts:` block expects at the repo root. All the actual build/run logic
+# lives in the standalone script, which is also runnable by hand.
 
 HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 ROOT="$( cd "${HERE}/../.." >/dev/null 2>&1 && pwd )"
 
-RUNTIME=${1}
-CONFIG=${2:-profiler+tracer}
-ALLOCATOR=${3:-gmalloc}
-
-echo "Chaos run: runtime=${RUNTIME}s config=${CONFIG} allocator=${ALLOCATOR}"
-
-CHAOS_JDK="${CHAOS_JDK:-21.0.3-tem}"
-# CHAOS_JDK uses sdkman notation (<version>-<dist>); extract major for Adoptium API.
-JDK_MAJOR="${CHAOS_JDK%%.*}"
-JDK_ARCH=$(uname -m | sed 's/x86_64/x64/')
-JDK_INSTALL_DIR="/opt/jdk-${CHAOS_JDK}"
-
-if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then
-  TMP=$(mktemp -d)
-  DL_URL="https://api.adoptium.net/v3/binary/latest/${JDK_MAJOR}/ga/linux/${JDK_ARCH}/jdk/hotspot/normal/eclipse"
-  echo "Downloading JDK ${CHAOS_JDK} (major ${JDK_MAJOR}) from Adoptium..."
-  if ! curl -fsSL --max-time 300 "${DL_URL}" -o "${TMP}/jdk.tar.gz"; then
-    echo "FAIL:JDK ${CHAOS_JDK} download failed" >&2
-    rm -rf "${TMP}"
-    exit 1
-  fi
-  mkdir -p "${JDK_INSTALL_DIR}"
-  tar -xzf "${TMP}/jdk.tar.gz" -C "${JDK_INSTALL_DIR}" --strip-components=1
-  rm -rf "${TMP}"
-fi
-
-if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then
-  echo "FAIL:JDK ${CHAOS_JDK} not available after install" >&2
-  exit 1
-fi
-export JAVA_HOME="${JDK_INSTALL_DIR}"
-export PATH="${JAVA_HOME}/bin:${PATH}"
-ACTIVE_JDK=$(java -version 2>&1 | head -1)
-# Check major version only — patch may differ from the Adoptium latest GA.
-if ! echo "${ACTIVE_JDK}" | grep -qE "\"${JDK_MAJOR}\."; then
-  echo "FAIL:wrong JDK active (expected major ${JDK_MAJOR}, got: ${ACTIVE_JDK})" >&2
-  exit 1
-fi
-
-# Resolve ddprof.jar: prefer local build artifact, fall back to Maven snapshot.
-# Running mvn from /tmp avoids the empty pom.xml at the repo root.
-DDPROF_JAR_LOCAL=$(ls "${ROOT}/ddprof-lib/build/libs/ddprof-"*.jar 2>/dev/null | head -1)
-if [ -n "${DDPROF_JAR_LOCAL}" ] && [ -f "${DDPROF_JAR_LOCAL}" ]; then
-  DDPROF_JAR="${DDPROF_JAR_LOCAL}"
-  echo "Using local ddprof jar: ${DDPROF_JAR}"
-else
-  if [ -z "${CURRENT_VERSION:-}" ]; then
-    echo "FAIL:CURRENT_VERSION is empty and no local jar found (get-versions dotenv missing)" >&2
-    exit 1
-  fi
-  echo "Local ddprof jar not found — downloading ${CURRENT_VERSION} from Maven snapshots"
-  (cd /tmp && mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \
-      -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \
-      -Dartifact=com.datadoghq:ddprof:${CURRENT_VERSION})
-  DDPROF_JAR="/root/.m2/repository/com/datadoghq/ddprof/${CURRENT_VERSION}/ddprof-${CURRENT_VERSION}.jar"
-fi
-
-if [ ! -f "${DDPROF_JAR}" ]; then
-  echo "FAIL:ddprof jar unavailable" >&2
-  exit 1
-fi
-
-mkdir -p /var/lib/datadog
-wget -q --timeout=120 --tries=3 -O /var/lib/datadog/dd-java-agent.jar 'https://dtdg.co/latest-java-tracer'
-
-# chaos.jar is produced once per pipeline by the chaos:build job (stresstest
-# stage) and pulled here as an artifact. Fall back to an inline build if the
-# artifact is absent (e.g. local repro outside CI).
-CHAOS_JAR="${ROOT}/ddprof-stresstest/build/libs/chaos.jar"
-if [ ! -f "${CHAOS_JAR}" ]; then
-  echo "chaos.jar artifact not present — building inline"
-  ( cd "${ROOT}" && ./gradlew :ddprof-stresstest:chaosJar -q )
-fi
-
-if [ ! -f "${CHAOS_JAR}" ]; then
-  echo "FAIL:chaos.jar unavailable" >&2
-  exit 1
-fi
-
-# Patch dd-java-agent.jar with the locally built ddprof contents so the agent's
-# (relocated) profiler classes match the version under test.
-DD_AGENT_JAR=/var/lib/datadog/dd-java-agent.jar \
-DDPROF_JAR=${DDPROF_JAR} \
-OUTPUT_JAR=/var/lib/datadog/dd-java-agent-patched.jar \
-"${ROOT}/utils/patch-dd-java-agent.sh"
-
-if [ ! -f /var/lib/datadog/dd-java-agent-patched.jar ]; then
-  echo "FAIL:dd-java-agent patching failed" >&2
-  exit 1
-fi
-
-PATCHED_AGENT=/var/lib/datadog/dd-java-agent-patched.jar
-
-case $CONFIG in
-  profiler)
-    echo "Running with profiler only"
-    ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false"
-    # @Trace is a no-op without the tracer, so trace-context is excluded here.
-    ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm"
-    ;;
-  profiler+tracer)
-    echo "Running with profiler and tracer"
-    ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true"
-    ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm"
-    ;;
-  *)
-    echo "Unknown configuration: $CONFIG"
-    exit 1
-    ;;
-esac
-
-case $ALLOCATOR in
-  gmalloc)
-    echo "Running with gmalloc"
-    ;;
-  tcmalloc)
-    echo "Running with tcmalloc"
-    export LD_PRELOAD=$(find /usr/lib/ -name 'libtcmalloc_minimal.so.4')
-    # thread-churn/dump-storm antagonists cycle many short-lived threads;
-    # tcmalloc's defaults are slow to return their per-thread caches to the
-    # OS, which was inflating container RSS past the OOM limit on aarch64.
-    export TCMALLOC_RELEASE_RATE=10
-    export TCMALLOC_AGGRESSIVE_DECOMMIT=1
-    ;;
-  jemalloc)
-    echo "Running with jemalloc"
-    export LD_PRELOAD=$(find /usr/lib/ -name 'libjemalloc.so')
-    ;;
-  *)
-    echo "Unknown allocator: $ALLOCATOR"
-    echo "Valid values are: gmalloc, tcmalloc, jemalloc"
-    exit 1
-    ;;
-esac
-
-echo "LD_PRELOAD=$LD_PRELOAD"
-
-timeout "$((RUNTIME + 300))" \
-java -javaagent:${PATCHED_AGENT} \
-     ${ENABLEMENT} \
-     -Ddd.profiling.upload.period=10 \
-     -Ddd.profiling.start-force-first=true \
-     -Ddd.profiling.ddprof.liveheap.enabled=true \
-     -Ddd.profiling.ddprof.alloc.enabled=true \
-     -Ddd.profiling.ddprof.wall.enabled=true \
-     -Ddd.profiling.ddprof.nativemem.enabled=true \
-     -Ddd.env=java-profiler-stability \
-     -Ddd.service=java-profiler-chaos \
-     -Xmx2g -Xms2g \
-     -XX:MaxMetaspaceSize=384m \
-     -XX:ErrorFile=${HERE}/../../hs_err.log \
-     -XX:OnError="${HERE}/../../dd_crash_uploader.sh %p" \
-     -jar ${CHAOS_JAR} \
-     --duration ${RUNTIME}s \
-     --antagonists ${ANTAGONISTS}
-
-RC=$?
-echo "RC=$RC"
+export CHAOS_JDK_DIR="${CHAOS_JDK_DIR:-/opt/jdk-${CHAOS_JDK}}"
+export CHAOS_WORK_DIR="${CHAOS_WORK_DIR:-/var/lib/datadog}"
+export CHAOS_ERROR_FILE="${ROOT}/hs_err.log"
 
-if [ $RC -ne 0 ]; then
-  CRASH_MSG="Chaos harness crashed (RC=${RC})"
-  HS_ERR="${HERE}/../../hs_err.log"
-  if [ -f "${HS_ERR}" ]; then
-    SIG=$(grep -m1 '^siginfo:' "${HS_ERR}" 2>/dev/null | tr -d '\n' | cut -c1-120)
-    FRAME=$(grep -m1 'libjavaProfiler\|AsyncProfiler' "${HS_ERR}" 2>/dev/null | sed 's/^[[:space:]]*//' | tr -d '\n' | cut -c1-120)
-    [ -n "${SIG}" ]   && CRASH_MSG="${CRASH_MSG};${SIG}"
-    [ -n "${FRAME}" ] && CRASH_MSG="${CRASH_MSG};${FRAME}"
-  fi
-  echo "FAIL:${CRASH_MSG}" >&2
-  exit 1
-fi
+exec "${ROOT}/utils/run-chaos-harness.sh" "$@"

build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt

@@ -3,6 +3,7 @@ package com.datadoghq.profiler
 
 import com.datadoghq.native.NativeBuildExtension
 import com.datadoghq.native.model.BuildConfiguration
+import com.datadoghq.native.model.Platform
 import com.datadoghq.native.util.PlatformUtils
 import org.gradle.api.DefaultTask
 import org.gradle.api.GradleException
@@ -204,6 +205,18 @@ class ProfilerTestPlugin : Plugin<Project> {
 
         // Environment variables (explicit for consistency across both paths)
         val envVars = buildMap<String, String> {
+            // Turn silent glibc heap corruption (e.g. a use-after-free write into a
+            // freed chunk) into an immediate, attributable SIGABRT instead of a crash
+            // much later in an unrelated allocation. Only meaningful against glibc's
+            // own malloc: skip on musl (no MALLOC_CHECK_ support), and skip whenever a
+            // sanitizer/allocator already replaces malloc via LD_PRELOAD.
+            if (PlatformUtils.currentPlatform == Platform.LINUX &&
+                !PlatformUtils.isMusl() &&
+                !testEnv.containsKey("LD_PRELOAD")
+            ) {
+                put("MALLOC_CHECK_", "3")
+                put("MALLOC_PERTURB_", (1..255).random().toString())
+            }
             putAll(testEnv)
             put("DDPROF_TEST_DISABLE_RATE_LIMIT", "1")
             put("CI", (project.hasProperty("CI") || System.getenv("CI")?.toBoolean() ?: false).toString())

ddprof-lib/src/main/cpp/frame.h

@@ -79,11 +79,13 @@ class FrameType {
     return (FrameTypeId)((bci >> TYPE_SHIFT) & FRAME_TYPE_MASK);
   }
 
-  // Raw pointers only exist on HotSpot (see hotspotSupport.cpp fillFrameRaw).
-  // On other VMs an unencoded, VM-supplied bci can coincidentally have bit 30
-  // set; that must not be mistaken for our raw-pointer encoding.
+  // Raw pointers only exist on HotSpot (see hotspotSupport.cpp fillFrameRaw),
+  // and RAW_POINTER_MASK is only ever set by encode(..., rawPointer=true),
+  // which unconditionally also sets ENCODED_MASK. Requiring both HotSpot and
+  // ENCODED_MASK prevents a raw, VM-supplied ASGCT BCI from false-positiving
+  // just because it happens to have bit 30 set.
   static inline bool isRawPointer(int bci) {
-    return VM::isHotspot() && bci > 0 && (bci & RAW_POINTER_MASK) != 0;
+    return VM::isHotspot() && (bci > 0 && (bci & ENCODED_MASK) != 0 && (bci & RAW_POINTER_MASK) != 0);
   }
 };
 

ddprof-lib/src/test/cpp/frame_ut.cpp

@@ -7,14 +7,29 @@
 #include "../../main/cpp/frame.h"
 #include "../../main/cpp/gtest_crash_handler.h"
 
-// VMTestAccessor — friend of VM, lets tests toggle VM::isHotspot() since
-// isRawPointer() now gates on it (raw pointers only exist on HotSpot).
+// Test-only friend accessor for VM internals. It exists solely so these unit
+// tests can exercise the VM-specific raw-pointer path in FrameType and must
+// never be reused from production code.
 class VMTestAccessor {
 public:
     static bool getHotspot() { return VM::_hotspot; }
     static void setHotspot(bool v) { VM::_hotspot = v; }
 };
 
+class VMHotspotGuard {
+private:
+    bool _saved;
+
+public:
+    explicit VMHotspotGuard(bool hotspot) : _saved(VMTestAccessor::getHotspot()) {
+        VMTestAccessor::setHotspot(hotspot);
+    }
+
+    ~VMHotspotGuard() {
+        VMTestAccessor::setHotspot(_saved);
+    }
+};
+
 static constexpr char FRAME_TEST_NAME[] = "FrameTest";
 
 class GlobalSetup {
@@ -64,13 +79,6 @@ TEST(FrameTypeEncodeTest, RawPointerBitNotSetByDefault) {
     EXPECT_EQ(encoded & (1 << 30), 0) << "rawPointer flag (bit 30) must not be set by default";
 }
 
-TEST(FrameTypeEncodeTest, EncodedValuesArePositive) {
-    for (int t = FRAME_INTERPRETED; t <= FRAME_TYPE_MAX; ++t) {
-        int encoded = FrameType::encode(t, 0);
-        EXPECT_GT(encoded, 0) << "encode() must return a positive value for type " << t;
-    }
-}
-
 // ---- decode ----------------------------------------------------------------
 
 TEST(FrameTypeDecodeTest, DecodeZeroReturnsJitCompiled) {
@@ -118,15 +126,6 @@ TEST(FrameTypeDecodeTest, RoundTripAllTypesNonZeroBci) {
     }
 }
 
-TEST(FrameTypeDecodeTest, DecodedTypeIsInValidRange) {
-    for (int t = FRAME_INTERPRETED; t <= FRAME_TYPE_MAX; ++t) {
-        int encoded = FrameType::encode(t, 42);
-        FrameTypeId decoded = FrameType::decode(encoded);
-        EXPECT_GE(decoded, FRAME_INTERPRETED);
-        EXPECT_LE(decoded, FRAME_TYPE_MAX);
-    }
-}
-
 // ---- isRawPointer ----------------------------------------------------------
 
 TEST(FrameTypeIsRawPointerTest, FalseForZero) {
@@ -150,26 +149,22 @@ TEST(FrameTypeIsRawPointerTest, FalseForEncodedWithoutFlag) {
 TEST(FrameTypeIsRawPointerTest, TrueWhenBit30IsSetOnHotspot) {
     // Manually set the raw-pointer flag (bit 30) on an encoded value.
     // Raw pointers only exist on HotSpot, so isRawPointer() must be gated on it.
-    bool saved = VMTestAccessor::getHotspot();
-    VMTestAccessor::setHotspot(true);
+    VMHotspotGuard hotspot(true);
     int base = FrameType::encode(FRAME_JIT_COMPILED, 0);
     int withFlag = base | (1 << 30);
     EXPECT_TRUE(FrameType::isRawPointer(withFlag))
         << "isRawPointer() must be true when bit 30 is set, value is positive, and VM is HotSpot";
-    VMTestAccessor::setHotspot(saved);
 }
 
 TEST(FrameTypeIsRawPointerTest, FalseWhenBit30IsSetOnNonHotspot) {
     // Non-HotSpot VMs can coincidentally produce an unencoded bci with bit 30
     // set (e.g. OpenJ9's raw AsyncGetCallTrace output); that must not be
     // mistaken for HotSpot's raw-pointer encoding.
-    bool saved = VMTestAccessor::getHotspot();
-    VMTestAccessor::setHotspot(false);
+    VMHotspotGuard hotspot(false);
     int base = FrameType::encode(FRAME_JIT_COMPILED, 0);
     int withFlag = base | (1 << 30);
     EXPECT_FALSE(FrameType::isRawPointer(withFlag))
         << "isRawPointer() must be false when VM is not HotSpot, even with bit 30 set";
-    VMTestAccessor::setHotspot(saved);
 }
 
 TEST(FrameTypeIsRawPointerTest, FalseWithOnlyBit30SetOnNegative) {
@@ -179,3 +174,15 @@ TEST(FrameTypeIsRawPointerTest, FalseWithOnlyBit30SetOnNegative) {
     EXPECT_LT(negativeWithBit30, 0);
     EXPECT_FALSE(FrameType::isRawPointer(negativeWithBit30));
 }
+
+TEST(FrameTypeIsRawPointerTest, FalseForRawAsgctBciWithBit30SetButNoEncodedMarker) {
+    // Regression test: a raw, unencoded ASGCT BCI (bit 20 / ENCODED_MASK not set)
+    // that incidentally has bit 30 set must NOT be reported as a raw pointer.
+    // Such values occur on non-HotSpot VMs (e.g. J9), which never call encode()
+    // and therefore never set ENCODED_MASK; only encode(..., rawPointer=true)
+    // (HotSpot-only, per its own assert) is allowed to set RAW_POINTER_MASK.
+    int rawAsgctBciWithBit30 = 1 << 30;
+    EXPECT_FALSE(FrameType::isRawPointer(rawAsgctBciWithBit30))
+        << "isRawPointer() must require ENCODED_MASK (bit 20) before trusting bit 30, "
+        << "otherwise raw ASGCT BCIs that never went through encode() can false-positive";
+}

ddprof-stresstest/src/chaos/README.md

@@ -15,6 +15,7 @@ the runner script.
 | `classloader-churn` | class unload racing stack walk, `CodeCache`/`Symbols` invalidation     |
 | `alloc-storm`     | Java alloc engine + GOT-patched libc malloc/free                         |
 | `trace-context`   | `setContext`/`clearContext` racing signals, span ID propagation          |
+| `vthread-context-cascade` | tracer-style context propagate/activate/restore across fan-out cascades of short-lived virtual threads, racing carrier-pin churn to trigger the `ContextStorageMode.THREAD` stale-carrier `DirectByteBuffer` use-after-free |
 
 ## Deferred
 

ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java

@@ -43,15 +43,15 @@ public static void main(String[] args) throws Exception {
         log("starting duration=" + parsed.duration + " antagonists=" + parsed.antagonists);
 
         List<Antagonist> running = new ArrayList<>();
-        for (String name : parsed.antagonists) {
-            Antagonist a = create(name);
-            a.start();
-            running.add(a);
-            log("antagonist started: " + a.name());
-        }
-
-        long deadlineNanos = System.nanoTime() + parsed.duration.toNanos();
         try {
+            for (String name : parsed.antagonists) {
+                Antagonist a = create(name);
+                a.start();
+                running.add(a);
+                log("antagonist started: " + a.name());
+            }
+
+            long deadlineNanos = System.nanoTime() + parsed.duration.toNanos();
             while (System.nanoTime() < deadlineNanos) {
                 Thread.sleep(1_000L);
             }
@@ -72,6 +72,8 @@ private static Antagonist create(String name) {
                 return new AllocStormAntagonist();
             case "vthread-churn":
                 return new VirtualThreadChurnAntagonist();
+            case "vthread-context-cascade":
+                return new VirtualThreadContextCascadeAntagonist();
             case "classloader-churn":
                 return new ClassLoaderChurnAntagonist();
             case "trace-context":