From 8c648fa095bfa7a4a9ada1e6721ce8a2d3da5c51 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 4 Aug 2026 18:23:52 +0200 Subject: [PATCH 01/28] chore(run-integration-test/kustomize): Upgrade chart and otel-collector image version --- .../container-log-scrape/opentelemetrycollector.yml | 10 +++++----- .../kubernetes-events/opentelemetrycollector.yml | 8 ++++---- .../kubernetes-objects/opentelemetrycollector.yml | 6 +++--- .../bases/opentelemetry-operator/kustomization.yml | 7 +++++-- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/run-integration-test/kustomize/bases/opentelemetry-collectors/container-log-scrape/opentelemetrycollector.yml b/run-integration-test/kustomize/bases/opentelemetry-collectors/container-log-scrape/opentelemetrycollector.yml index b318966..2fa557e 100644 --- a/run-integration-test/kustomize/bases/opentelemetry-collectors/container-log-scrape/opentelemetrycollector.yml +++ b/run-integration-test/kustomize/bases/opentelemetry-collectors/container-log-scrape/opentelemetrycollector.yml @@ -36,7 +36,7 @@ spec: config: receivers: # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/filelogreceiver - filelog/varlogpods: + file_log/varlogpods: # A storage extension (eg: redis, or file) can be used for storing log offsets. Otherwise it is held in memory. # See: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/storage # storage: {} @@ -105,7 +105,7 @@ spec: processors: # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/k8sattributesprocessor - k8sattributes: + k8s_attributes: filter: # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/k8sattributesprocessor/README.md#as-an-agent node_from_env_var: KUBE_NODE_NAME # this should be same as the var set from the downward API further up @@ -124,7 +124,7 @@ spec: # This key exists so it can be easily extended in an overlay: attributes: [] - resourcedetection/env: + resource_detection/env: detectors: [env] timeout: 2s override: false @@ -146,8 +146,8 @@ spec: extensions: [] pipelines: logs: - receivers: [filelog/varlogpods] + receivers: [file_log/varlogpods] # processors: [memory_limiter, batch] - processors: [k8sattributes, resourcedetection/env, resource] + processors: [k8s_attributes, resource_detection/env, resource] # Enable configured exporters in the overlay exporters: [] diff --git a/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-events/opentelemetrycollector.yml b/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-events/opentelemetrycollector.yml index ab11ec7..667b3b1 100644 --- a/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-events/opentelemetrycollector.yml +++ b/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-events/opentelemetrycollector.yml @@ -29,7 +29,7 @@ spec: processors: # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/k8sattributesprocessor - k8sattributes: + k8s_attributes: # Use the k8s attributes set by the receiver # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/k8sattributesprocessor#as-a-gateway passthrough: true @@ -51,7 +51,7 @@ spec: key: service.name value: kubernetes-events - resourcedetection/env: + resource_detection/env: detectors: [env] timeout: 2s override: false @@ -75,12 +75,12 @@ spec: logs: receivers: [k8s_events] # processors: [memory_limiter, batch] - processors: [k8sattributes, resourcedetection/env, resource] + processors: [k8s_attributes, resource_detection/env, resource] # Enable configured exporters in the overlay exporters: [] metrics: receivers: [k8s_cluster] # processors: [memory_limiter, batch] - processors: [k8sattributes, resourcedetection/env, resource] + processors: [k8s_attributes, resource_detection/env, resource] # Enable configured exporters in the overlay exporters: [] diff --git a/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml b/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml index 5113b56..2401074 100644 --- a/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml +++ b/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml @@ -27,7 +27,7 @@ spec: processors: # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/k8sattributesprocessor - k8sattributes: + k8s_attributes: # Use the k8s attributes set by the receiver # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/k8sattributesprocessor#as-a-gateway passthrough: false @@ -49,7 +49,7 @@ spec: key: service.name value: kubernetes-objects - resourcedetection/env: + resource_detection/env: detectors: [env] timeout: 2s override: false @@ -73,6 +73,6 @@ spec: logs: receivers: [k8sobjects] # processors: [memory_limiter, batch] - processors: [k8sattributes, resourcedetection/env, resource] + processors: [k8s_attributes, resource_detection/env, resource] # Enable configured exporters in the overlay exporters: [] diff --git a/run-integration-test/kustomize/bases/opentelemetry-operator/kustomization.yml b/run-integration-test/kustomize/bases/opentelemetry-operator/kustomization.yml index 53775d8..204de95 100644 --- a/run-integration-test/kustomize/bases/opentelemetry-operator/kustomization.yml +++ b/run-integration-test/kustomize/bases/opentelemetry-operator/kustomization.yml @@ -9,7 +9,10 @@ helmCharts: - name: opentelemetry-operator repo: https://open-telemetry.github.io/opentelemetry-helm-charts # Find the latest release here: https://github.com/open-telemetry/opentelemetry-helm-charts/releases - version: 0.93.0 + # helm repo add opentelemetry-helm-charts https://open-telemetry.github.io/opentelemetry-helm-charts --force-update + # helm search repo opentelemetry-helm-charts/opentelemetry-operator + # renovate: registryUrl=https://open-telemetry.github.io/opentelemetry-helm-charts + version: 0.120.0 # 0.156.0 releaseName: opentelemetry-operator includeCRDs: true skipTests: true @@ -22,7 +25,7 @@ helmCharts: # See which plugins the opentelemetry-collector-k8s image contains here: # https://github.com/open-telemetry/opentelemetry-collector-releases/blob/main/distributions/otelcol-k8s/manifest.yaml repository: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-k8s - tag: 0.132.4 # Often the chart has old values for the collector image + tag: 0.157.0 # Often the chart has old values for the collector image admissionWebhooks: certManager: enabled: false From 0e4e2d3c615887163e5f8556224f8c1aa2852210 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 4 Aug 2026 18:46:40 +0200 Subject: [PATCH 02/28] chore(run-integration-test): Use bearer auth for OTLP endpoint --- run-integration-test/action.yaml | 20 +++++++++++ .../overlays/replicated/kustomization.yml | 33 +++++++++++++++---- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index f4471c2..1490b34 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -17,6 +17,8 @@ inputs: # Tokens replicated-api-token: description: Replicated API token + otlp-bearer-token: + description: Bearer token for the OTLP ingester (otlp.stackable.build) # Tool versions interu-version: @@ -196,6 +198,24 @@ runs: kubectl -n opentelemetry-operator get pods echo "::endgroup::" + - name: Install OTLP ingester auth Secret + shell: bash + env: + OTLP_BEARER_TOKEN: ${{ inputs.otlp-bearer-token }} + run: | + set -euo pipefail + # No 'set -x' here: it would risk exposing the token. Mask defensively in case the + # value did not arrive via ${{ secrets.* }} (which GitHub redacts automatically). + echo "::add-mask::${OTLP_BEARER_TOKEN}" + + echo "::group::kubectl apply secret" + # The collectors read this via spec.env -> secretKeyRef (otlp-auth / bearer-token). + kubectl create secret generic otlp-auth \ + --namespace opentelemetry-operator \ + --from-literal=bearer-token="${OTLP_BEARER_TOKEN}" \ + --dry-run=client -o yaml | kubectl apply -f - + echo "::endgroup::" + - name: Apply OpenTelemetry Collectors configurations shell: bash env: diff --git a/run-integration-test/kustomize/overlays/replicated/kustomization.yml b/run-integration-test/kustomize/overlays/replicated/kustomization.yml index 7119214..2fc3102 100644 --- a/run-integration-test/kustomize/overlays/replicated/kustomization.yml +++ b/run-integration-test/kustomize/overlays/replicated/kustomization.yml @@ -12,14 +12,35 @@ patches: kind: OpenTelemetryCollector patch: |- - op: add - path: /spec/config/exporters/otlp + path: /spec/config/exporters/otlp_http value: - endpoint: otel-logs-source.nick.stackable.build:443 - # tls: - # insecure: true + # otlp_http appends the signal-specific path (eg: /v1/logs) to this endpoint. + # Note: the exporter was renamed from `otlphttp` to `otlp_http`; the old name + # is a deprecated alias that still works but will be removed in a future release. + endpoint: https://otlp.stackable.build + auth: + authenticator: bearertokenauth + # The ingester at otlp.stackable.build requires Bearer auth. The token is read from + # the OTLP_BEARER_TOKEN env var, sourced from the otlp-auth Secret (spec/env below). + # In CI this Secret is created by the run-integration-test action from a GHA secret. + - op: add + path: /spec/config/extensions/bearertokenauth + value: + token: ${env:OTLP_BEARER_TOKEN} + - op: add + path: /spec/config/service/extensions/- + value: bearertokenauth + - op: add + path: /spec/env/- + value: + name: OTLP_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: otlp-auth + key: bearer-token - op: add path: /spec/config/service/pipelines/logs/exporters/- - value: otlp + value: otlp_http - op: add path: /spec/envFrom/- value: @@ -63,7 +84,7 @@ patches: # - op: add # - op: add # path: /spec/config/service/pipelines/metrics/exporters/- - # value: otlphttp + # value: otlp_http configMapGenerator: # These get used by the attributes processor in each collector From b77472214c76fbb9f45e8c7744cf3e012dfc6f1d Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 4 Aug 2026 18:47:11 +0200 Subject: [PATCH 03/28] chore(run-integration-test/kustomize): Add local overlay --- run-integration-test/kustomize/.gitignore | 3 + .../overlays/local/kustomization.yml | 102 ++++++++++++++++++ .../local/otlp-auth.secret.env.example | 4 + 3 files changed, 109 insertions(+) create mode 100644 run-integration-test/kustomize/overlays/local/kustomization.yml create mode 100644 run-integration-test/kustomize/overlays/local/otlp-auth.secret.env.example diff --git a/run-integration-test/kustomize/.gitignore b/run-integration-test/kustomize/.gitignore index ee3892e..ec0187c 100644 --- a/run-integration-test/kustomize/.gitignore +++ b/run-integration-test/kustomize/.gitignore @@ -1 +1,4 @@ charts/ + +# Local-only secrets (eg: overlays/local/otlp-auth.secret.env). Never commit tokens. +*.secret.env diff --git a/run-integration-test/kustomize/overlays/local/kustomization.yml b/run-integration-test/kustomize/overlays/local/kustomization.yml new file mode 100644 index 0000000..fd255b8 --- /dev/null +++ b/run-integration-test/kustomize/overlays/local/kustomization.yml @@ -0,0 +1,102 @@ +--- +# Local testing overlay: a self-contained sibling of ../replicated for running on a +# local cluster (kind/minikube/k3d) without the run-integration-test GitHub Action. +# +# Differences from ../replicated: +# - Resource attributes are set to static literal values instead of ${env:...} vars +# populated by the CI action, so no integration-test-info.env / configMapGenerator +# is needed (the collector errors on startup if an ${env:...} var is unset). +# - Adds a `debug` exporter to the logs pipeline so telemetry is visible on the +# collector pod's stdout locally (kubectl logs) in addition to the OTLP/HTTP export. +namePrefix: local- + +resources: +- ../../bases/opentelemetry-collectors + +patches: +# Update all collectors to set the exporter endpoints and any other common overrides. +- target: + group: opentelemetry.io + version: v1beta1 + kind: OpenTelemetryCollector + patch: |- + - op: add + path: /spec/config/exporters/otlp_http + value: + # otlp_http appends the signal-specific path (eg: /v1/logs) to this endpoint. + endpoint: https://otlp.stackable.build + auth: + authenticator: bearertokenauth + - op: add + path: /spec/config/exporters/debug + value: + verbosity: detailed + # The ingester at otlp.stackable.build requires Bearer auth. The token is read from + # the OTLP_BEARER_TOKEN env var, sourced from the otlp-auth Secret (spec/env below). + # Locally that Secret is built by the secretGenerator from a gitignored otlp-auth.env. + - op: add + path: /spec/config/extensions/bearertokenauth + value: + token: ${env:OTLP_BEARER_TOKEN} + - op: add + path: /spec/config/service/extensions/- + value: bearertokenauth + - op: add + path: /spec/env/- + value: + name: OTLP_BEARER_TOKEN + valueFrom: + secretKeyRef: + name: otlp-auth + key: bearer-token + - op: add + path: /spec/config/service/pipelines/logs/exporters/- + value: otlp_http + - op: add + path: /spec/config/service/pipelines/logs/exporters/- + value: debug + - op: add + path: /spec/config/processors/resource/attributes/- + value: + action: upsert + key: k8s.cluster.name + value: local + - op: add + path: /spec/config/processors/resource/attributes/- + value: + action: upsert + key: k8s.cluster.distribution + value: local + - op: add + path: /spec/config/processors/resource/attributes/- + value: + action: upsert + key: k8s.cluster.version + value: local + - op: add + path: /spec/config/processors/resource/attributes/- + value: + action: upsert + key: github.actions.triggered_by + value: local + +# Specifically override config for the kubernetes-events collector. +- target: + group: opentelemetry.io + version: v1beta1 + kind: OpenTelemetryCollector + name: kubernetes-events + patch: |- + # OpenSearch Metrics collector is not yet available + - op: remove + path: /spec/config/service/pipelines/metrics + +secretGenerator: +# Local-only: builds the otlp-auth Secret from a gitignored otlp-auth.secret.env so the +# token is never committed. Copy otlp-auth.secret.env.example to otlp-auth.secret.env and +# set the real token. The generated (hashed) Secret name is rewritten into each collector's +# spec.env via the Secret nameReference in ../../bases/opentelemetry-collectors/crds.yml. +- name: otlp-auth + namespace: opentelemetry-operator + envs: + - otlp-auth.secret.env diff --git a/run-integration-test/kustomize/overlays/local/otlp-auth.secret.env.example b/run-integration-test/kustomize/overlays/local/otlp-auth.secret.env.example new file mode 100644 index 0000000..7ffa562 --- /dev/null +++ b/run-integration-test/kustomize/overlays/local/otlp-auth.secret.env.example @@ -0,0 +1,4 @@ +# Copy this file to otlp-auth.secret.env and set the real Bearer token for +# otlp.stackable.build. Files matching *.secret.env are gitignored and must never +# be committed. +bearer-token=REPLACE_WITH_BEARER_TOKEN From e5f521489c1b6bdc047b8ad1bdd31d5c4a197bfb Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 4 Aug 2026 18:53:02 +0200 Subject: [PATCH 04/28] docs(run-integration-test): Add new input --- run-integration-test/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/run-integration-test/README.md b/run-integration-test/README.md index d4eb274..9408b80 100644 --- a/run-integration-test/README.md +++ b/run-integration-test/README.md @@ -94,6 +94,7 @@ profiles: | Input | Required | Description | | ---------------------- | -------- | ---------------------------------------------------------------------- | | `replicated-api-token` | Yes | Replicated API Token, available via `secrets.REPLICATED_API_TOKEN` | +| `otlp-bearer-token` | Yes | Bearer token for the OTLP ingester (otlp.stackable.build) | | `test-mode` | Yes | Either run a `profile` or a `custom` test | | `test-mode-input` | Yes | The name of the profile or runner, based on the `test-mode` | | `test-suite` | No | The name of the BeKu test-suite (only used if running a `custom` test) | From a503a300ebd01662ea6fc22fc49cd4b3f07b8a06 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 4 Aug 2026 19:12:40 +0200 Subject: [PATCH 05/28] fix(run-integration-test): Remove bonk interpolation ![silly-sausage](https://github.com/user-attachments/assets/98ec0045-066f-4359-87aa-c6139abb3be9) --- run-integration-test/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index 1490b34..a83e6c0 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -205,7 +205,7 @@ runs: run: | set -euo pipefail # No 'set -x' here: it would risk exposing the token. Mask defensively in case the - # value did not arrive via ${{ secrets.* }} (which GitHub redacts automatically). + # value did not arrive via secrets.* (which GitHub redacts automatically). echo "::add-mask::${OTLP_BEARER_TOKEN}" echo "::group::kubectl apply secret" From 6689ae108a96288d3f77251ccab98122644da526 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 4 Aug 2026 19:18:29 +0200 Subject: [PATCH 06/28] fix(run-integration-test): Bump default helm version --- run-integration-test/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index a83e6c0..8f9ef23 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -40,7 +40,7 @@ inputs: helm-version: description: Version of helm # See https://github.com/helm/helm/releases for latest version - default: v3.19.0 + default: v3.21.1 stackablectl-version: description: Version of stackablectl # See https://github.com/stackabletech/stackable-cockpit/releases for latest version From d99cb1e2516f84134726aee023c8a71f04ed3203 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 4 Aug 2026 19:21:25 +0200 Subject: [PATCH 07/28] fix(run-integration-test): Disable helm signature verification because of expired pubkey --- run-integration-test/action.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index 8f9ef23..d585115 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -107,6 +107,8 @@ runs: KUBECTL_VERSION: ${{ inputs.kubectl-version }} KUTTL_VERSION: ${{ inputs.kuttl-version }} HELM_VERSION: ${{ inputs.helm-version }} + # The helm signature is expired since a couple of years... + VERIFY_SIGNATURE: "false" RUNNER_DEBUG: ${{ runner.debug }} shell: bash run: | From e61f7aeb9730e90bf1eabe3cae9366ae5187544f Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 4 Aug 2026 20:05:23 +0200 Subject: [PATCH 08/28] fix(run-integration-test): Remove RUNNER_DEBUG definition It is automatically set by GitHub Actions --- run-integration-test/action.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index d585115..72c0311 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -64,7 +64,6 @@ runs: - name: Install interu env: INTERU_VERSION: ${{ inputs.interu-version }} - RUNNER_DEBUG: ${{ runner.debug }} shell: bash run: "$GITHUB_ACTION_PATH/../.scripts/actions/install_interu.sh" @@ -75,7 +74,6 @@ runs: TEST_SUITE: ${{ inputs.test-suite }} TEST_MODE: ${{ inputs.test-mode }} TEST: ${{ inputs.test }} - RUNNER_DEBUG: ${{ runner.debug }} GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_RUN_ID: ${{ github.run_id }}" shell: bash @@ -109,7 +107,6 @@ runs: HELM_VERSION: ${{ inputs.helm-version }} # The helm signature is expired since a couple of years... VERIFY_SIGNATURE: "false" - RUNNER_DEBUG: ${{ runner.debug }} shell: bash run: | "$GITHUB_ACTION_PATH/../.scripts/actions/install_kubectl.sh" From 232b5eb3bfa70b618c4b19477d7b9ebb25eab37b Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 4 Aug 2026 20:06:43 +0200 Subject: [PATCH 09/28] fix(run-integration-test): Mask data from replicated actions I won't link it here, so people don't go snooping from the upstream issue linked as a comment in this commit. --- run-integration-test/action.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index 72c0311..850ea32 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -164,6 +164,12 @@ runs: - key: test-parallelism value: "${{ steps.extract.outputs.BEKU_TEST_PARALLELISM }}" + # Until this is resolved: https://github.com/replicatedhq/replicated-actions/issues/119 + - name: Mask data + shell: bash + run: | + echo "::add-mask::${{ steps.prepare-replicated-cluster.outputs.cluster-kubeconfig }}" + - name: Set Replicated kubeconfig env: KUBECONFIG: ${{ steps.prepare-replicated-cluster.outputs.cluster-kubeconfig }} From cd26349374b4665f3542004f2f6f3e9fd7cdd2e2 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 4 Aug 2026 21:45:29 +0200 Subject: [PATCH 10/28] fix(run-integration-test): Set missing env vars Missed in #112 --- run-integration-test/action.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index 850ea32..f06f3ad 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -265,6 +265,9 @@ runs: REF_NAME: ${{ github.ref_name }} GH_TOKEN: ${{ github.token }} OPERATOR_NAME: ${{ steps.extract_operator_name.outputs.OPERATOR_NAME }} + BEKU_TEST_PARALLELISM: ${{ steps.extract.outputs.BEKU_TEST_PARALLELISM }} + BEKU_TEST_SUITE: ${{ steps.extract.outputs.BEKU_TEST_SUITE }} + BEKU_TEST: ${{ steps.extract.outputs.BEKU_TEST }} shell: bash run: | set -euo pipefail From a859189f8be6185bbc7f3b64b0c814f8c4fbcd83 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 4 Aug 2026 22:04:52 +0200 Subject: [PATCH 11/28] Revert "fix(run-integration-test): Mask data from replicated actions" This reverts commit 232b5eb3bfa70b618c4b19477d7b9ebb25eab37b. --- run-integration-test/action.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index f06f3ad..a7c9ea6 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -164,12 +164,6 @@ runs: - key: test-parallelism value: "${{ steps.extract.outputs.BEKU_TEST_PARALLELISM }}" - # Until this is resolved: https://github.com/replicatedhq/replicated-actions/issues/119 - - name: Mask data - shell: bash - run: | - echo "::add-mask::${{ steps.prepare-replicated-cluster.outputs.cluster-kubeconfig }}" - - name: Set Replicated kubeconfig env: KUBECONFIG: ${{ steps.prepare-replicated-cluster.outputs.cluster-kubeconfig }} From ed065f20880e5777d7395bc2e2dd6ff43afb3eda Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 4 Aug 2026 22:06:30 +0200 Subject: [PATCH 12/28] fix(run-integration-test): Bump replicated action version This resolves the kubeconfig data masking problem. --- run-integration-test/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index a7c9ea6..2340a64 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -146,7 +146,7 @@ runs: - name: Prepare Replicated Cluster id: prepare-replicated-cluster - uses: replicatedhq/replicated-actions/create-cluster@291bef61a059631e39e84f8470f86152171c4c20 # v1.26.0 + uses: replicatedhq/replicated-actions/create-cluster@0fbf5395cf3e8c744f447bbc92b7de9135eead8e # v1.27.1 with: # See: https://github.com/replicatedhq/replicated-actions/tree/main/create-cluster#inputs api-token: ${{ inputs.replicated-api-token }} @@ -285,7 +285,7 @@ runs: if: always() # If the creation of the cluster failed, we don't want to error and abort continue-on-error: true - uses: replicatedhq/replicated-actions/remove-cluster@291bef61a059631e39e84f8470f86152171c4c20 # v1.26.0 + uses: replicatedhq/replicated-actions/remove-cluster@0fbf5395cf3e8c744f447bbc92b7de9135eead8e # v1.27.1 with: # See: https://github.com/replicatedhq/replicated-actions/tree/main/remove-cluster#inputs api-token: ${{ inputs.replicated-api-token }} From 73c69ec4aa7b067c12636d46a8aaea10546fa8ec Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Wed, 5 Aug 2026 11:46:42 +0200 Subject: [PATCH 13/28] feat(run-integration-test/kustomize): Enable k8s events metrics --- .../kustomize/overlays/replicated/kustomization.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/run-integration-test/kustomize/overlays/replicated/kustomization.yml b/run-integration-test/kustomize/overlays/replicated/kustomization.yml index 2fc3102..9247c04 100644 --- a/run-integration-test/kustomize/overlays/replicated/kustomization.yml +++ b/run-integration-test/kustomize/overlays/replicated/kustomization.yml @@ -78,13 +78,9 @@ patches: kind: OpenTelemetryCollector name: kubernetes-events patch: |- - # OpenSearch Metrics collector is not yet available - - op: remove - path: /spec/config/service/pipelines/metrics - # - op: add - # - op: add - # path: /spec/config/service/pipelines/metrics/exporters/- - # value: otlp_http + - op: add + path: /spec/config/service/pipelines/metrics/exporters/- + value: otlp_http configMapGenerator: # These get used by the attributes processor in each collector From ce42718191bafbd66b6371a2f4574a0769c4b7f7 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 6 Aug 2026 13:42:15 +0200 Subject: [PATCH 14/28] fix(send-slack-notification): Add missing TEST_RESULT env var --- send-slack-notification/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/send-slack-notification/action.yaml b/send-slack-notification/action.yaml index 90323c0..3ad5228 100644 --- a/send-slack-notification/action.yaml +++ b/send-slack-notification/action.yaml @@ -98,6 +98,7 @@ runs: BUILD_RESULT: ${{ steps.valid_inputs.outputs.BUILD_RESULT }} FAILED_TESTS: ${{ steps.valid_inputs.outputs.FAILED_TESTS }} TEST_HEALTH: ${{ steps.valid_inputs.outputs.TEST_HEALTH }} + TEST_RESULT: ${{ steps.valid_inputs.outputs.TEST_RESULT }} NOTIFICATION_TYPE: ${{ steps.valid_inputs.outputs.NOTIFICATION_TYPE }} GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} GITHUB_SERVER_URL: ${{ github.server_url }} From 1d006036a057ed46cf2dbcda9214fc0124694a44 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 6 Aug 2026 13:43:05 +0200 Subject: [PATCH 15/28] fix(send-slack-notification): Use correct variable in template --- send-slack-notification/templates/integration-test/success.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/send-slack-notification/templates/integration-test/success.tpl b/send-slack-notification/templates/integration-test/success.tpl index 7f6d8bc..af1de92 100644 --- a/send-slack-notification/templates/integration-test/success.tpl +++ b/send-slack-notification/templates/integration-test/success.tpl @@ -8,7 +8,7 @@ blocks: - type: "section" text: type: "mrkdwn" - text: "${HEALTH_SLACK_EMOJI} (${HEALTH_RATE}) The integration test for *${{ github.repository }}* succeeded." + text: "${HEALTH_SLACK_EMOJI} (${HEALTH_RATE}) The integration test for *${MESSAGE_SUBJECT}* succeeded." - type: "actions" elements: - type: button From e9111d1018f221343eb8fd54edee4273fe727d36 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 6 Aug 2026 16:31:42 +0200 Subject: [PATCH 16/28] feat(run-integration-test): Add run_attempt to attributes --- run-integration-test/action.yaml | 2 ++ .../kustomize/overlays/replicated/kustomization.yml | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index 2340a64..83b640f 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -224,6 +224,7 @@ runs: INTERU_KUBERNETES_DISTRIBUTION: ${{ steps.extract.outputs.INTERU_KUBERNETES_DISTRIBUTION }} INTERU_KUBERNETES_VERSION: ${{ steps.extract.outputs.INTERU_KUBERNETES_VERSION }} GITHUB_TRIGGERED_BY: ${{ github.triggering_actor }} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} run: | set -euo pipefail @@ -233,6 +234,7 @@ runs: echo "KUBERNETES_DISTRIBUTION=${INTERU_KUBERNETES_DISTRIBUTION}" | tee -a "$INTEGRATION_TEST_INFO" echo "KUBERNETES_VERSION=${INTERU_KUBERNETES_VERSION}" | tee -a "$INTEGRATION_TEST_INFO" echo "TRIGGERED_BY=${GITHUB_TRIGGERED_BY}" | tee -a "$INTEGRATION_TEST_INFO" + echo "RUN_ATTEMPT=${GITHUB_RUN_ATTEMPT}" | tee -a "$INTEGRATION_TEST_INFO" echo "::endgroup::" echo "::group::kubectl apply" diff --git a/run-integration-test/kustomize/overlays/replicated/kustomization.yml b/run-integration-test/kustomize/overlays/replicated/kustomization.yml index 9247c04..9d76d12 100644 --- a/run-integration-test/kustomize/overlays/replicated/kustomization.yml +++ b/run-integration-test/kustomize/overlays/replicated/kustomization.yml @@ -70,6 +70,12 @@ patches: action: upsert key: github.actions.triggered_by value: ${env:TRIGGERED_BY} + - op: add + path: /spec/config/processors/resource/attributes/- + value: + action: upsert + key: github.actions.run_attempt + value: ${env:RUN_ATTEMPT} # Specifically override config for the kubernetes-events collector. - target: From f86832aafbdf2b47f09b78eaf014ae9d5a1f6073 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 6 Aug 2026 17:15:07 +0200 Subject: [PATCH 17/28] feat(run-integration-test): Make cluster name unique per run attempt --- run-integration-test/action.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index 83b640f..48900da 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -75,6 +75,7 @@ runs: TEST_MODE: ${{ inputs.test-mode }} TEST: ${{ inputs.test }} GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}" GITHUB_RUN_ID: ${{ github.run_id }}" shell: bash run: | @@ -82,7 +83,7 @@ runs: [ -n "${RUNNER_DEBUG+set}" ] && set -x # Generate the cluster name - echo "KUBERNETES_CLUSTER_NAME=integration-test-${GITHUB_REPOSITORY}-${GITHUB_RUN_ID}" | tee -a "$GITHUB_OUTPUT" + echo "KUBERNETES_CLUSTER_NAME=integration-test-${GITHUB_REPOSITORY}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | tee -a "$GITHUB_OUTPUT" # Run interu to expand parameters into GITHUB_OUTPUT for use as env vars in later steps. if [ "$TEST_MODE" == "profile" ]; then From 847610772b078cd4cf57f72f315780c4ee000197 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 6 Aug 2026 20:00:56 +0200 Subject: [PATCH 18/28] chore(send-slack-notification): Add set -euo pipefail to run blocks --- send-slack-notification/action.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/send-slack-notification/action.yaml b/send-slack-notification/action.yaml index 3ad5228..4b88b1f 100644 --- a/send-slack-notification/action.yaml +++ b/send-slack-notification/action.yaml @@ -48,6 +48,8 @@ runs: TEST_HEALTH: ${{ inputs.test-health }} shell: bash run: | + set -euo pipefail + if [ -z "${NOTIFICATION_TYPE:-}" ]; then echo "The type input must be provided" exit 1 @@ -87,6 +89,8 @@ runs: if: steps.retrieve-slack-thread-id.outcome == 'success' shell: bash run: | + set -euo pipefail + echo "SLACK_THREAD_ID=$(cat slack-thread-id)" | tee -a "$GITHUB_OUTPUT" - name: Provide message template variables @@ -111,6 +115,8 @@ runs: MESSAGE_SUBJECT: ${{ inputs.message-subject }} shell: bash run: | + set -euo pipefail + export WORKFLOW_RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}" if [ "$NOTIFICATION_TYPE" == "container-image-build" ]; then @@ -166,6 +172,8 @@ runs: SLACK_THREAD_ID_OUTPUT: ${{ steps.send-notification.outputs.ts }} shell: bash run: | + set -euo pipefail + echo "$SLACK_THREAD_ID_OUTPUT" > slack-thread-id - name: Store Slack Thread ID as Artifact From 71b83e52ac3dff76c5083a6ba3c261ac242df9fd Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 6 Aug 2026 20:01:48 +0200 Subject: [PATCH 19/28] fix(send-slack-notification): Escape backtick in message text --- send-slack-notification/action.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/send-slack-notification/action.yaml b/send-slack-notification/action.yaml index 4b88b1f..4e1d7ed 100644 --- a/send-slack-notification/action.yaml +++ b/send-slack-notification/action.yaml @@ -134,24 +134,24 @@ runs: export MESSAGE_COLOR=10c400 fi - export MESSAGE_TEXT="*$MESSAGE_SUBJECT* $MESSAGE_VERB (attempt $GITHUB_RUN_ATTEMPT)" + export MESSAGE_TEXT="*${MESSAGE_SUBJECT}* ${MESSAGE_VERB} (attempt ${GITHUB_RUN_ATTEMPT})" PAYLOAD=$(envsubst < "${GITHUB_ACTION_PATH}/templates/container-image-build/failure.tpl") echo -e "PAYLOAD< Date: Wed, 5 Aug 2026 22:50:25 +0200 Subject: [PATCH 20/28] feat(run-integration-test/kustomize): Enable enrichment via the k8s API --- .../opentelemetrycollector.yml | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-events/opentelemetrycollector.yml b/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-events/opentelemetrycollector.yml index 667b3b1..69e4bfc 100644 --- a/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-events/opentelemetrycollector.yml +++ b/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-events/opentelemetrycollector.yml @@ -30,9 +30,24 @@ spec: processors: # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/processor/k8sattributesprocessor k8s_attributes: - # Use the k8s attributes set by the receiver - # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/k8sattributesprocessor#as-a-gateway - passthrough: true + # Enrich telemetry with pod metadata (including the owning workload) via the k8s API. + # The k8s_cluster/k8s_events receivers describe other objects and are not "sent" by + # those pods, so there is no source connection IP to match on. Associate by the + # k8s.pod.uid the k8s_cluster receiver stamps as a resource attribute instead. + passthrough: false + pod_association: + - sources: + - from: resource_attribute + name: k8s.pod.uid + extract: + metadata: + # Owning workload of each pod (deployment is resolved via its replicaset). + - k8s.deployment.name + - k8s.replicaset.name + - k8s.daemonset.name + - k8s.statefulset.name + - k8s.job.name + - k8s.cronjob.name # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/attributesprocessor # These are attributes for the specific spans, log events, etc... From 61e13866e9af97905d9f4eace183598ed554cc3a Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 6 Aug 2026 20:19:33 +0200 Subject: [PATCH 21/28] feat(run-integration-test/kustomize): Let's watch everything --- .../clusterrole-kubernetes-objects.yml | 90 +++++++++- .../opentelemetrycollector.yml | 155 +++++++++++++++++- 2 files changed, 237 insertions(+), 8 deletions(-) diff --git a/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/clusterrole-kubernetes-objects.yml b/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/clusterrole-kubernetes-objects.yml index 76e4664..e603899 100644 --- a/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/clusterrole-kubernetes-objects.yml +++ b/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/clusterrole-kubernetes-objects.yml @@ -6,20 +6,102 @@ metadata: labels: app: kubernetes-objects rules: - # TODO: Add Stackable objects, and other common k8s objects + # Read-only across the object types watched by the k8sobjects receiver. + # NOTE: core "secrets" is deliberately omitted - the receiver emits full object data. - apiGroups: - "" resources: - - events - pods + - services + - endpoints + - configmaps + - persistentvolumeclaims + - persistentvolumes + - namespaces + - nodes + - serviceaccounts + - resourcequotas + - limitranges verbs: - get - list - watch - apiGroups: - - "events.k8s.io" + - "apps" resources: - - events + - deployments + - statefulsets + - daemonsets + - replicasets verbs: + - get + - list - watch + - apiGroups: + - "batch" + resources: + - jobs + - cronjobs + verbs: + - get - list + - watch + - apiGroups: + - "networking.k8s.io" + resources: + - ingresses + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: + - "autoscaling" + resources: + - horizontalpodautoscalers + verbs: + - get + - list + - watch + - apiGroups: + - "policy" + resources: + - poddisruptionbudgets + verbs: + - get + - list + - watch + - apiGroups: + - "storage.k8s.io" + resources: + - storageclasses + verbs: + - get + - list + - watch + # All Stackable CRDs, by API group. Wildcard resources so this survives plural typos and + # new CRDs across SDP releases. Read-only. Groups are far more stable than plural names. + - apiGroups: + - airflow.stackable.tech + - druid.stackable.tech + - hbase.stackable.tech + - hdfs.stackable.tech + - hive.stackable.tech + - kafka.stackable.tech + - nifi.stackable.tech + - spark.stackable.tech + - superset.stackable.tech + - trino.stackable.tech + - zookeeper.stackable.tech + - opa.stackable.tech + - opensearch.stackable.tech + - authentication.stackable.tech + - s3.stackable.tech + - listeners.stackable.tech + - secrets.stackable.tech + resources: + - "*" + verbs: + - get + - list + - watch diff --git a/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml b/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml index 2401074..23109a9 100644 --- a/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml +++ b/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml @@ -13,14 +13,161 @@ spec: receivers: # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/k8sobjectsreceiver/testdata/config.yaml k8sobjects: + # This action runs across operator repos, so most Stackable CRDs below will not be + # installed on any given run. error_mode: ignore makes a missing/unknown object type + # non-fatal - the receiver logs it and skips it at startup instead of failing. + # See handleError() in the receiver's Start(): missing types hit `continue`. + error_mode: ignore objects: - # TODO: Add Stackable objects, and other common k8s objects + # NOTE: events are intentionally NOT watched here - they are already collected by the + # k8s_events receiver in the kubernetes-events collector. Core Secrets are intentionally + # excluded: k8sobjects emits full object data with no field stripping (would exfiltrate + # secret material, including this stack's own otlp-auth token). + + # --- Common core (v1) resources --- - name: pods mode: watch - - name: events + - name: services + mode: watch + - name: endpoints + mode: watch + - name: configmaps + mode: watch + - name: persistentvolumeclaims + mode: watch + - name: persistentvolumes + mode: watch + - name: namespaces + mode: watch + - name: nodes + mode: watch + - name: serviceaccounts + mode: watch + - name: resourcequotas + mode: watch + - name: limitranges + mode: watch + + # --- apps --- + - name: deployments + group: apps + mode: watch + - name: statefulsets + group: apps + mode: watch + - name: daemonsets + group: apps + mode: watch + - name: replicasets + group: apps + mode: watch + + # --- batch --- + - name: jobs + group: batch + mode: watch + - name: cronjobs + group: batch + mode: watch + + # --- networking / scaling / policy / storage --- + - name: ingresses + group: networking.k8s.io + mode: watch + - name: networkpolicies + group: networking.k8s.io + mode: watch + - name: horizontalpodautoscalers + group: autoscaling + mode: watch + - name: poddisruptionbudgets + group: policy + mode: watch + - name: storageclasses + group: storage.k8s.io + mode: watch + + # --- Stackable CRDs (https://hub.stackable.tech/crds) --- + # Verify plurals against `kubectl get crds` - a wrong plural is silently skipped + # (error_mode: ignore), so it fails as missing coverage rather than a crash. + - name: airflowclusters + group: airflow.stackable.tech + mode: watch + - name: druidclusters + group: druid.stackable.tech + mode: watch + - name: hbaseclusters + group: hbase.stackable.tech + mode: watch + - name: hdfsclusters + group: hdfs.stackable.tech + mode: watch + - name: hiveclusters + group: hive.stackable.tech + mode: watch + - name: kafkaclusters + group: kafka.stackable.tech + mode: watch + - name: nificlusters + group: nifi.stackable.tech + mode: watch + - name: sparkapplications + group: spark.stackable.tech + mode: watch + - name: sparkhistoryservers + group: spark.stackable.tech + mode: watch + - name: sparkconnectservers + group: spark.stackable.tech + mode: watch + - name: supersetclusters + group: superset.stackable.tech + mode: watch + - name: druidconnections + group: superset.stackable.tech + mode: watch + - name: trinoclusters + group: trino.stackable.tech + mode: watch + - name: trinocatalogs + group: trino.stackable.tech + mode: watch + - name: zookeeperclusters + group: zookeeper.stackable.tech + mode: watch + - name: zookeeperznodes + group: zookeeper.stackable.tech + mode: watch + - name: opaclusters + group: opa.stackable.tech + mode: watch + - name: opensearchclusters + group: opensearch.stackable.tech + mode: watch + - name: authenticationclasses + group: authentication.stackable.tech + mode: watch + - name: s3connections + group: s3.stackable.tech + mode: watch + - name: s3buckets + group: s3.stackable.tech + mode: watch + - name: listeners + group: listeners.stackable.tech + mode: watch + - name: listenerclasses + group: listeners.stackable.tech + mode: watch + - name: podlisteners + group: listeners.stackable.tech + mode: watch + - name: secretclasses + group: secrets.stackable.tech + mode: watch + - name: truststores + group: secrets.stackable.tech mode: watch - group: events.k8s.io - exclude_watch_type: [DELETED] # This key exists so it can be easily extended in an overlay: extensions: {} From b252060d298c743ad7a138b43da9d13bbb457074 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 6 Aug 2026 20:30:53 +0200 Subject: [PATCH 22/28] chore(run-integration-test/kustomize): Add todos for making k8sobjects.error_mode dynamic at runtime The means we won't have to change action hashes just to check which things we might be missing. --- run-integration-test/action.yaml | 10 ++++++++++ .../kubernetes-objects/opentelemetrycollector.yml | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index 48900da..1e14b4f 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -19,6 +19,12 @@ inputs: description: Replicated API token otlp-bearer-token: description: Bearer token for the OTLP ingester (otlp.stackable.build) + # TODO: Add an input to control the k8sobjects receiver error_mode at runtime, eg: + # otel-k8sobjects-error-mode: + # description: k8sobjects receiver error_mode (propagate|ignore|silent) + # default: ignore + # Wire it into integration-test-info.env (see the collectors apply step) and reference it + # via ${env:OTEL_K8SOBJECTS_ERROR_MODE:-ignore} in the kubernetes-objects collector config. # Tool versions interu-version: @@ -236,6 +242,10 @@ runs: echo "KUBERNETES_VERSION=${INTERU_KUBERNETES_VERSION}" | tee -a "$INTEGRATION_TEST_INFO" echo "TRIGGERED_BY=${GITHUB_TRIGGERED_BY}" | tee -a "$INTEGRATION_TEST_INFO" echo "RUN_ATTEMPT=${GITHUB_RUN_ATTEMPT}" | tee -a "$INTEGRATION_TEST_INFO" + # TODO: To make the k8sobjects receiver error_mode runtime-controllable, expose the + # otel-k8sobjects-error-mode input in this step's `env:` block above and write it here: + # echo "OTEL_K8SOBJECTS_ERROR_MODE=${OTEL_K8SOBJECTS_ERROR_MODE}" | tee -a "$INTEGRATION_TEST_INFO" + # Then reference it via ${env:OTEL_K8SOBJECTS_ERROR_MODE:-ignore} in the collector config. echo "::endgroup::" echo "::group::kubectl apply" diff --git a/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml b/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml index 23109a9..046a733 100644 --- a/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml +++ b/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml @@ -17,6 +17,11 @@ spec: # installed on any given run. error_mode: ignore makes a missing/unknown object type # non-fatal - the receiver logs it and skips it at startup instead of failing. # See handleError() in the receiver's Start(): missing types hit `continue`. + # TODO: Make error_mode runtime-controllable from the workflow (no action bump) by + # sourcing it from an env var with a safe default: + # error_mode: "${env:OTEL_K8SOBJECTS_ERROR_MODE:-ignore}" + # Requires: an action input, writing it into integration-test-info.env, and this + # collector receiving that ConfigMap via envFrom (already true in the replicated overlay). error_mode: ignore objects: # NOTE: events are intentionally NOT watched here - they are already collected by the From 3df3404d4aa62887c1613e4812f41a6136f746d7 Mon Sep 17 00:00:00 2001 From: Techassi Date: Thu, 6 Aug 2026 22:36:10 +0200 Subject: [PATCH 23/28] chore(send-slack-notification): What the hell happened here? --- send-slack-notification/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/send-slack-notification/action.yaml b/send-slack-notification/action.yaml index 4e1d7ed..c3361f2 100644 --- a/send-slack-notification/action.yaml +++ b/send-slack-notification/action.yaml @@ -149,7 +149,7 @@ runs: if [ "$TEST_RESULT" == "failure" ]; then export MESSAGE_TEXT="The integration test for *\`${MESSAGE_SUBJECT}\`* failed." PAYLOAD=$(envsubst < "${GITHUB_ACTION_PATH}/templates/integration-test/failure.tpl") - echo -e "PAYLOAD< Date: Fri, 7 Aug 2026 09:29:09 +0200 Subject: [PATCH 24/28] chore(run-integration-test): Output number of successes --- run-integration-test/action.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index 1e14b4f..80890ac 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -338,4 +338,6 @@ runs: [ "$INTEGRATION_TEST_CONCLUSION" == "success" ] && SUCCESSES=$(echo "$SUCCESSES+1" | bc) export SUCCESSES="$SUCCESSES" + echo "Number of successes (last ${LAST_TRIES_TOTAL} run(s) + this run): ${SUCCESSES}" + echo "HEALTH=$(cat "${GITHUB_ACTION_PATH}/health" | envsubst | bc)" | tee -a "$GITHUB_OUTPUT" From 1ab7c03a2d0fc0a33fc5f66af509f6ad54052bb5 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 11 Aug 2026 12:27:59 +0200 Subject: [PATCH 25/28] chore(run-integration-test/kustomize): Add github.actions.run_attempt attribute --- run-integration-test/action.yaml | 6 ++++++ .../kustomize/overlays/replicated/kustomization.yml | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index 80890ac..0dd0695 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -232,9 +232,14 @@ runs: INTERU_KUBERNETES_VERSION: ${{ steps.extract.outputs.INTERU_KUBERNETES_VERSION }} GITHUB_TRIGGERED_BY: ${{ github.triggering_actor }} GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_RUN_ID: ${{ github.run_id }} run: | set -euo pipefail + WORKFLOW_RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}" + echo "::group::get info" INTEGRATION_TEST_INFO="${GITHUB_ACTION_PATH}/kustomize/overlays/replicated/integration-test-info.env" echo "KUBERNETES_CLUSTER_NAME=${KUBERNETES_CLUSTER_NAME}" | tee -a "$INTEGRATION_TEST_INFO" @@ -242,6 +247,7 @@ runs: echo "KUBERNETES_VERSION=${INTERU_KUBERNETES_VERSION}" | tee -a "$INTEGRATION_TEST_INFO" echo "TRIGGERED_BY=${GITHUB_TRIGGERED_BY}" | tee -a "$INTEGRATION_TEST_INFO" echo "RUN_ATTEMPT=${GITHUB_RUN_ATTEMPT}" | tee -a "$INTEGRATION_TEST_INFO" + echo "RUN_URL=${WORKFLOW_RUN_URL}" | tee -a "$INTEGRATION_TEST_INFO" # TODO: To make the k8sobjects receiver error_mode runtime-controllable, expose the # otel-k8sobjects-error-mode input in this step's `env:` block above and write it here: # echo "OTEL_K8SOBJECTS_ERROR_MODE=${OTEL_K8SOBJECTS_ERROR_MODE}" | tee -a "$INTEGRATION_TEST_INFO" diff --git a/run-integration-test/kustomize/overlays/replicated/kustomization.yml b/run-integration-test/kustomize/overlays/replicated/kustomization.yml index 9d76d12..361d937 100644 --- a/run-integration-test/kustomize/overlays/replicated/kustomization.yml +++ b/run-integration-test/kustomize/overlays/replicated/kustomization.yml @@ -76,6 +76,12 @@ patches: action: upsert key: github.actions.run_attempt value: ${env:RUN_ATTEMPT} + - op: add + path: /spec/config/processors/resource/attributes/- + value: + action: upsert + key: github.actions.run_attempt + value: ${env:RUN_URL} # Specifically override config for the kubernetes-events collector. - target: From 32565c8813463337fe8dfb996f032b8b554e6535 Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 11 Aug 2026 12:26:05 +0200 Subject: [PATCH 26/28] fix(run-integration-test): Only count runs with conclusion It seems like that runs which were queued but never picked up to run result in the .conclusion field to show an empty string. We now ignore those when calculation the total attempts. In a follow-up PR, we ideally want to re-fetch more previous attempts if we detect one ore more runs with an empty conclusion to still get the "x out of 5" health status. --- run-integration-test/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index 0dd0695..ed0b776 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -339,7 +339,7 @@ runs: # turned into an appropriate "weather" emoji and provided as an output to follow-up steps. LAST_RUNS=$(gh run list --limit "$LAST_TRIES_LIMIT" --workflow "$WORKFLOW_NAME" --json conclusion) SUCCESSES=$(echo $LAST_RUNS | jq '[.[] | select(.conclusion == "success")] | length') - export LAST_TRIES_TOTAL=$(echo $LAST_RUNS | jq 'length') + export LAST_TRIES_TOTAL=$(echo $LAST_RUNS | jq '[.[] | select(.conclusion != "")] | length') [ "$INTEGRATION_TEST_CONCLUSION" == "success" ] && SUCCESSES=$(echo "$SUCCESSES+1" | bc) export SUCCESSES="$SUCCESSES" From 57fb892aa16f7364ff41004262c373c3b70cee65 Mon Sep 17 00:00:00 2001 From: Techassi Date: Tue, 11 Aug 2026 12:40:19 +0200 Subject: [PATCH 27/28] feat(run-integration-test)!: Record cluster start and end time --- run-integration-test/README.md | 6 ++++-- run-integration-test/action.yaml | 35 ++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/run-integration-test/README.md b/run-integration-test/README.md index 9408b80..62090a3 100644 --- a/run-integration-test/README.md +++ b/run-integration-test/README.md @@ -110,8 +110,10 @@ profiles: | Output | Description | | ------ | ----------- | -| `start-time` | The time the test started running, in the `%Y-%m-%dT%H:%M:%S` (eg. `2025-08-20T09:57:53`) format. | -| `end-time` | The time the test completed, in the same format as above. | +| `test-start-time` | The time the test started running, in the `%Y-%m-%dT%H:%M:%S` (eg. `2025-08-20T09:57:53`) format. | +| `test-end-time` | The time the test completed, in the same format as above. | +| `cluster-start-time` | The time the cluster started running, in the `%Y-%m-%dT%H:%M:%S` (eg. `2025-08-20T09:57:53`) format. | +| `cluster-end-time` | The time the cluster shut down, in the same format as above. | | `health` | The health of the integration tests. Contains three comma-separated values: Slack emoji, GitHub emoji, and success rate. | | `failed-tests` | A (potentially empty) plain text list of failed tests. | diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index ed0b776..18891ed 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -52,12 +52,18 @@ inputs: # See https://github.com/stackabletech/stackable-cockpit/releases for latest version default: 1.1.0 outputs: - start-time: + test-start-time: description: The date and time this integration test was started. - value: ${{ steps.start-time.outputs.START_TIME }} - end-time: + value: ${{ steps.test-start-time.outputs.TEST_START_TIME }} + test-end-time: description: The date and time this integration test finished. - value: ${{ steps.end-time.outputs.END_TIME }} + value: ${{ steps.test-end-time.outputs.TEST_END_TIME }} + cluster-start-time: + description: The date and time this cluster was started. + value: ${{ steps.cluster-start-time.outputs.CLUSTER_START_TIME }} + cluster-end-time: + description: The date and time this cluster was shut down. + value: ${{ steps.cluster-end-time.outputs.CLUSTER_END_TIME }} health: description: The health of this integration test over the last few tries. value: ${{ steps.health.outputs.HEALTH }} @@ -181,6 +187,12 @@ runs: mkdir ~/.kube echo "$KUBECONFIG" > ~/.kube/config + - name: Record Cluster Start Time + id: cluster-start-time + shell: bash + run: | + echo "CLUSTER_START_TIME=$(date +'%Y-%m-%dT%H:%M:%S')" | tee -a "$GITHUB_OUTPUT" + - name: Extract Operator Name id: extract_operator_name env: @@ -267,10 +279,10 @@ runs: echo "::endgroup::" - name: Record Test Start Time - id: start-time + id: test-start-time shell: bash run: | - echo "START_TIME=$(date +'%Y-%m-%dT%H:%M:%S')" | tee -a "$GITHUB_OUTPUT" + echo "TEST_START_TIME=$(date +'%Y-%m-%dT%H:%M:%S')" | tee -a "$GITHUB_OUTPUT" - name: Run Integration Test id: integration-test @@ -294,11 +306,11 @@ runs: python ./scripts/run-tests --skip-release --log-level debug --parallel "$BEKU_TEST_PARALLELISM" ${ARGS:-} | tee -a test-output.log - name: Record Test End Time - id: end-time + id: test-end-time if: always() shell: bash run: | - echo "END_TIME=$(date +'%Y-%m-%dT%H:%M:%S')" | tee -a "$GITHUB_OUTPUT" + echo "TEST_END_TIME=$(date +'%Y-%m-%dT%H:%M:%S')" | tee -a "$GITHUB_OUTPUT" - name: Destroy Replicated Cluster if: always() @@ -310,6 +322,13 @@ runs: api-token: ${{ inputs.replicated-api-token }} cluster-id: ${{ steps.prepare-replicated-cluster.outputs.cluster-id }} + - name: Record Cluster End Time + id: cluster-end-time + if: always() + shell: bash + run: | + echo "CLUSTER_END_TIME=$(date +'%Y-%m-%dT%H:%M:%S')" | tee -a "$GITHUB_OUTPUT" + - name: Extract Failed Tests id: failed-tests # The success() function is automatically used if no other status function is used. From 1fe91f39c769f885571fdb92c42a9be12b72d862 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Tue, 11 Aug 2026 12:33:40 +0200 Subject: [PATCH 28/28] chore(run-integration-test/kustomize): Rename otel receiver k8sobjects -> k8s_objects --- .../kubernetes-objects/opentelemetrycollector.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml b/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml index 046a733..96f61ec 100644 --- a/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml +++ b/run-integration-test/kustomize/bases/opentelemetry-collectors/kubernetes-objects/opentelemetrycollector.yml @@ -12,7 +12,7 @@ spec: config: receivers: # https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/k8sobjectsreceiver/testdata/config.yaml - k8sobjects: + k8s_objects: # This action runs across operator repos, so most Stackable CRDs below will not be # installed on any given run. error_mode: ignore makes a missing/unknown object type # non-fatal - the receiver logs it and skips it at startup instead of failing. @@ -26,7 +26,7 @@ spec: objects: # NOTE: events are intentionally NOT watched here - they are already collected by the # k8s_events receiver in the kubernetes-events collector. Core Secrets are intentionally - # excluded: k8sobjects emits full object data with no field stripping (would exfiltrate + # excluded: k8s_objects emits full object data with no field stripping (would exfiltrate # secret material, including this stack's own otlp-auth token). # --- Common core (v1) resources --- @@ -223,7 +223,7 @@ spec: extensions: [] pipelines: logs: - receivers: [k8sobjects] + receivers: [k8s_objects] # processors: [memory_limiter, batch] processors: [k8s_attributes, resource_detection/env, resource] # Enable configured exporters in the overlay