diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 461d99c..f768349 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,17 +1,52 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file - version: 2 updates: - - package-ecosystem: "docker" # See documentation for possible values - directory: "/" # Location of package manifests - target-branch: "development" + # Base images referenced by literal FROM tags (alpine/git, distroless). + # Dependabot bumps both the tag and the pinned @sha256 digest via PRs that + # the build-and-validate matrix gates before merge. ARG-interpolated + # python:* images cannot be tracked here; their distro line is chosen via + # build args and reviewed manually when moving Debian/Alpine releases. + - package-ecosystem: "docker" + directory: "/" schedule: interval: "weekly" - - package-ecosystem: "github-actions" # See documentation for possible values - directory: "/" # Location of package manifests - target-branch: "development" + open-pull-requests-limit: 5 + groups: + docker-base-images: + patterns: + - "*" + commit-message: + prefix: "docker" + labels: + - "dependencies" + - "docker" + + # Pinned build-time Python toolchain (ci/requirements-build.txt). + - package-ecosystem: "pip" + directory: "/ci" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + groups: + python-build-tools: + patterns: + - "*" + commit-message: + prefix: "pip" + labels: + - "dependencies" + - "python" + + - package-ecosystem: "github-actions" + directory: "/" schedule: interval: "weekly" + open-pull-requests-limit: 5 + groups: + github-actions: + patterns: + - "*" + commit-message: + prefix: "ci" + labels: + - "dependencies" + - "github-actions" diff --git a/.github/workflows/automatic-release.yaml b/.github/workflows/automatic-release.yaml index f4772a1..6b6797d 100644 --- a/.github/workflows/automatic-release.yaml +++ b/.github/workflows/automatic-release.yaml @@ -13,12 +13,17 @@ on: # Allows you to run this workflow manually from the Actions tab # workflow_dispatch: +# Deny all GITHUB_TOKEN scopes by default; jobs opt into the minimum they need. +permissions: {} + # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: # This workflow contains a single job called "build" get-labels: if: ${{github.event.pull_request.merged == true && !startsWith(github.head_ref, 'release/')}} runs-on: ubuntu-latest + permissions: + contents: read outputs: labels: ${{ steps.match-label.outputs.match }} steps: diff --git a/.github/workflows/build-and-validate.yaml b/.github/workflows/build-and-validate.yaml index 6e7d85f..73219a5 100644 --- a/.github/workflows/build-and-validate.yaml +++ b/.github/workflows/build-and-validate.yaml @@ -4,23 +4,86 @@ on: pull_request: branches: [main, development] +# Cancel any in-progress run for the same PR when a new commit is pushed. +# head_ref is the PR source branch; ref is the fallback for non-PR triggers. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +# Deny all GITHUB_TOKEN scopes by default; jobs opt into the minimum they need. +permissions: {} + env: REGISTRY: ghcr.io - PYTHON_BASE_IMAGE_NAME: ${{ github.repository }}/python-base jobs: - build-and-validate: + pr-matrix: strategy: fail-fast: false matrix: include: - - platform: linux/amd64 + - name: gtsam420-py311-trixie-slim-amd64 + platform: linux/amd64 runner: ubuntu-latest - arch: amd64 - - platform: linux/arm64 + python_version: "3.11" + gtsam_version: "4.2.0" + python_build_target: python-build-glibc + python_runtime_target: python-runtime-glibc-slim + python_suffix: glibc-trixie + runtime_suffix: glibc-trixie-slim + runtime_target: runtime-trixie-slim + validation: /examples/PlanarSLAMExample.py + experimental: false + - name: gtsam43a1-py314-trixie-slim-amd64 + platform: linux/amd64 + runner: ubuntu-latest + python_version: "3.14" + gtsam_version: "4.3a1" + python_build_target: python-build-glibc + python_runtime_target: python-runtime-glibc-slim + python_suffix: glibc-trixie + runtime_suffix: glibc-trixie-slim + runtime_target: runtime-trixie-slim + validation: /examples/PlanarSLAMExample.py + experimental: false + - name: gtsam43a1-py314-trixie-slim-arm64 + platform: linux/arm64 runner: ubuntu-24.04-arm - arch: arm64 + python_version: "3.14" + gtsam_version: "4.3a1" + python_build_target: python-build-glibc + python_runtime_target: python-runtime-glibc-slim + python_suffix: glibc-trixie + runtime_suffix: glibc-trixie-slim + runtime_target: runtime-trixie-slim + validation: /examples/validate_gtsam.py + experimental: false + - name: gtsam43a1-py314-distroless-amd64 + platform: linux/amd64 + runner: ubuntu-latest + python_version: "3.14" + gtsam_version: "4.3a1" + python_build_target: python-build-glibc + python_runtime_target: python-runtime-glibc-slim + python_suffix: glibc-trixie + runtime_suffix: glibc-trixie-slim + runtime_target: runtime-distroless + validation: /examples/PlanarSLAMExample.py + experimental: true + - name: gtsam43a1-py314-alpine-amd64 + platform: linux/amd64 + runner: ubuntu-latest + python_version: "3.14" + gtsam_version: "4.3a1" + python_build_target: python-build-musl + python_runtime_target: python-runtime-musl-alpine + python_suffix: musl-alpine + runtime_suffix: musl-alpine + runtime_target: runtime-alpine + validation: /examples/PlanarSLAMExample.py + experimental: true runs-on: ${{ matrix.runner }} + continue-on-error: ${{ matrix.experimental }} permissions: contents: read packages: read @@ -31,28 +94,50 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v4 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Python version from Dockerfile - id: python + - name: Build local Python build base run: | - VERSION=$(grep -oP 'ARG PYTHON_VERSION=\K[0-9.]+' Dockerfile.python-base) - echo "version=${VERSION}" >> $GITHUB_OUTPUT + docker build \ + --platform ${{ matrix.platform }} \ + -f Dockerfile.python-base \ + --target ${{ matrix.python_build_target }} \ + --build-arg PYTHON_VERSION=${{ matrix.python_version }} \ + -t python-build-local:py${{ matrix.python_version }}-${{ matrix.python_suffix }} \ + . - - name: Build image (${{ matrix.platform }}) + - name: Build local Python runtime base run: | - LOWER_IMAGE_NAME=$(echo "${{ env.PYTHON_BASE_IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') docker build \ --platform ${{ matrix.platform }} \ - --build-arg PYTHON_BASE_IMAGE=${{ env.REGISTRY }}/${LOWER_IMAGE_NAME}:${{ steps.python.outputs.version }}-trixie-${{ matrix.arch }} \ - -t gtsam_docker:latest . + -f Dockerfile.python-base \ + --target ${{ matrix.python_runtime_target }} \ + --build-arg PYTHON_VERSION=${{ matrix.python_version }} \ + -t python-runtime-local:py${{ matrix.python_version }}-${{ matrix.runtime_suffix }} \ + . - - name: Validate image (PlanarSLAM example) + - name: Build GTSAM runtime image run: | - chmod +x ./scripts/validate_container.sh - ./scripts/validate_container.sh gtsam_docker:latest /examples/PlanarSLAMExample.py + docker build \ + --platform ${{ matrix.platform }} \ + --target ${{ matrix.runtime_target }} \ + --build-arg PYTHON_VERSION=${{ matrix.python_version }} \ + --build-arg PYTHON_ABI=${{ matrix.python_version }} \ + --build-arg GTSAM_VERSION=${{ matrix.gtsam_version }} \ + --build-arg PYTHON_BUILD_IMAGE=python-build-local:py${{ matrix.python_version }}-${{ matrix.python_suffix }} \ + --build-arg PYTHON_RUNTIME_IMAGE=python-runtime-local:py${{ matrix.python_version }}-${{ matrix.runtime_suffix }} \ + --build-arg PYTHON_RUNTIME_TRIXIE_IMAGE=python-runtime-local:py${{ matrix.python_version }}-${{ matrix.runtime_suffix }} \ + --build-arg PYTHON_RUNTIME_SLIM_IMAGE=python-runtime-local:py${{ matrix.python_version }}-${{ matrix.runtime_suffix }} \ + --build-arg PYTHON_RUNTIME_ALPINE_IMAGE=python-runtime-local:py${{ matrix.python_version }}-${{ matrix.runtime_suffix }} \ + -t gtsam_docker:${{ matrix.name }} \ + . + + - name: Validate minimal GTSAM import + run: ./scripts/validate_container.sh gtsam_docker:${{ matrix.name }} /examples/validate_gtsam.py + + - name: Validate NumPy ABI + run: ./scripts/validate_container.sh gtsam_docker:${{ matrix.name }} /examples/validate_numpy_abi.py + + - name: Validate scenario script + run: ./scripts/validate_container.sh gtsam_docker:${{ matrix.name }} ${{ matrix.validation }} + + - name: Report image size + run: ./scripts/size-report.sh gtsam_docker:${{ matrix.name }} ${{ matrix.name }} diff --git a/.github/workflows/build-full-matrix.yaml b/.github/workflows/build-full-matrix.yaml new file mode 100644 index 0000000..cf2620c --- /dev/null +++ b/.github/workflows/build-full-matrix.yaml @@ -0,0 +1,257 @@ +name: Build Full Image Matrix + +on: + workflow_dispatch: + inputs: + release_version: + description: "Optional release tag prefix, e.g. v4.0.0" + required: false + default: "" + workflow_call: + inputs: + release_version: + description: "Optional release tag prefix, e.g. v4.0.0" + required: false + type: string + default: "" + schedule: + - cron: "21 8 * * 1" + +# Deny all GITHUB_TOKEN scopes by default; jobs opt into the minimum they need. +permissions: {} + +env: + REGISTRY: ghcr.io + +jobs: + build: + strategy: + fail-fast: false + matrix: + python_version: ["3.11", "3.12", "3.13", "3.14"] + gtsam_version: ["4.2.0", "4.2.1", "4.3a1"] + platform: + - platform: linux/amd64 + arch: amd64 + runner: ubuntu-latest + - platform: linux/arm64 + arch: arm64 + runner: ubuntu-24.04-arm + runtime: + - base: trixie + target: runtime-trixie + python_build_target: python-build-glibc + python_runtime_target: python-runtime-glibc-trixie + build_suffix: glibc-trixie + runtime_suffix: glibc-trixie + artifact_suffix: glibc-trixie + experimental: false + - base: trixie-slim + target: runtime-trixie-slim + python_build_target: python-build-glibc + python_runtime_target: python-runtime-glibc-slim + build_suffix: glibc-trixie + runtime_suffix: glibc-trixie-slim + artifact_suffix: glibc-trixie + experimental: false + - base: distroless-debian13 + target: runtime-distroless + python_build_target: python-build-glibc + python_runtime_target: python-runtime-glibc-slim + build_suffix: glibc-trixie + runtime_suffix: glibc-trixie-slim + artifact_suffix: glibc-trixie + experimental: false + - base: alpine + target: runtime-alpine + python_build_target: python-build-musl + python_runtime_target: python-runtime-musl-alpine + build_suffix: musl-alpine + runtime_suffix: musl-alpine + artifact_suffix: musl-alpine + experimental: true + runs-on: ${{ matrix.platform.runner }} + continue-on-error: ${{ matrix.runtime.experimental || (matrix.gtsam_version != '4.3a1' && (matrix.python_version == '3.13' || matrix.python_version == '3.14')) }} + permissions: + contents: read + packages: write + # SBOM/provenance attestations require the docker-container buildx driver, + # which cannot read images from the local Docker daemon. The freshly built + # Python bases are therefore pushed to a throwaway in-job registry that the + # container builder pulls from via the host network. + services: + registry: + image: registry:2 + ports: + - 5000:5000 + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + with: + driver-opts: network=host + buildkitd-config-inline: | + [registry."localhost:5000"] + http = true + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Normalize image names + id: image + run: | + repo="${GITHUB_REPOSITORY,,}" + echo "repo=${REGISTRY}/${repo}" >> "$GITHUB_OUTPUT" + echo "python_build=${REGISTRY}/${repo}/python-build" >> "$GITHUB_OUTPUT" + echo "python_runtime=${REGISTRY}/${repo}/python-runtime" >> "$GITHUB_OUTPUT" + echo "gtsam_build=${REGISTRY}/${repo}/gtsam-build" >> "$GITHUB_OUTPUT" + + - name: Build local Python build base + run: | + docker buildx build \ + --platform ${{ matrix.platform.platform }} \ + -f Dockerfile.python-base \ + --target ${{ matrix.runtime.python_build_target }} \ + --build-arg PYTHON_VERSION=${{ matrix.python_version }} \ + --provenance=false \ + -t localhost:5000/python-build:py${{ matrix.python_version }}-${{ matrix.runtime.build_suffix }} \ + --push . + + - name: Build local Python runtime base + run: | + docker buildx build \ + --platform ${{ matrix.platform.platform }} \ + -f Dockerfile.python-base \ + --target ${{ matrix.runtime.python_runtime_target }} \ + --build-arg PYTHON_VERSION=${{ matrix.python_version }} \ + --provenance=false \ + -t localhost:5000/python-runtime:py${{ matrix.python_version }}-${{ matrix.runtime.runtime_suffix }} \ + --push . + + - name: Publish GTSAM build artifact image + if: ${{ matrix.runtime.base == 'trixie-slim' || matrix.runtime.base == 'alpine' }} + run: | + set -euo pipefail + artifact_tag="gtsam${{ matrix.gtsam_version }}-py${{ matrix.python_version }}-${{ matrix.runtime.artifact_suffix }}-${{ matrix.platform.arch }}" + docker buildx build \ + --platform ${{ matrix.platform.platform }} \ + --target gtsam-build \ + --sbom=true \ + --provenance=mode=max \ + --build-arg PYTHON_VERSION=${{ matrix.python_version }} \ + --build-arg PYTHON_ABI=${{ matrix.python_version }} \ + --build-arg GTSAM_VERSION=${{ matrix.gtsam_version }} \ + --build-arg PYTHON_BUILD_IMAGE=localhost:5000/python-build:py${{ matrix.python_version }}-${{ matrix.runtime.build_suffix }} \ + --push \ + -t "${{ steps.image.outputs.gtsam_build }}:${artifact_tag}" \ + . + + - name: Build and push runtime image + run: | + set -euo pipefail + base_tag="gtsam${{ matrix.gtsam_version }}-py${{ matrix.python_version }}-${{ matrix.runtime.base }}" + arch_tag="${base_tag}-${{ matrix.platform.arch }}" + tags=("-t" "${{ steps.image.outputs.repo }}:${arch_tag}") + release_version="${{ inputs.release_version }}" + if [ -n "$release_version" ]; then + tags+=("-t" "${{ steps.image.outputs.repo }}:${release_version}-${arch_tag}") + fi + if [ "${{ matrix.gtsam_version }}" = "4.3a1" ] && [ "${{ matrix.python_version }}" = "3.14" ] && [ "${{ matrix.runtime.base }}" = "trixie-slim" ]; then + tags+=("-t" "${{ steps.image.outputs.repo }}:latest-${{ matrix.platform.arch }}") + fi + docker buildx build \ + --platform ${{ matrix.platform.platform }} \ + --target ${{ matrix.runtime.target }} \ + --sbom=true \ + --provenance=mode=max \ + --build-arg PYTHON_VERSION=${{ matrix.python_version }} \ + --build-arg PYTHON_ABI=${{ matrix.python_version }} \ + --build-arg GTSAM_VERSION=${{ matrix.gtsam_version }} \ + --build-arg PYTHON_BUILD_IMAGE=localhost:5000/python-build:py${{ matrix.python_version }}-${{ matrix.runtime.build_suffix }} \ + --build-arg PYTHON_RUNTIME_IMAGE=localhost:5000/python-runtime:py${{ matrix.python_version }}-${{ matrix.runtime.runtime_suffix }} \ + --build-arg PYTHON_RUNTIME_TRIXIE_IMAGE=localhost:5000/python-runtime:py${{ matrix.python_version }}-${{ matrix.runtime.runtime_suffix }} \ + --build-arg PYTHON_RUNTIME_SLIM_IMAGE=localhost:5000/python-runtime:py${{ matrix.python_version }}-${{ matrix.runtime.runtime_suffix }} \ + --build-arg PYTHON_RUNTIME_ALPINE_IMAGE=localhost:5000/python-runtime:py${{ matrix.python_version }}-${{ matrix.runtime.runtime_suffix }} \ + --push \ + "${tags[@]}" \ + . + + - name: Pull runtime image for validation + run: | + docker pull "${{ steps.image.outputs.repo }}:gtsam${{ matrix.gtsam_version }}-py${{ matrix.python_version }}-${{ matrix.runtime.base }}-${{ matrix.platform.arch }}" + docker tag "${{ steps.image.outputs.repo }}:gtsam${{ matrix.gtsam_version }}-py${{ matrix.python_version }}-${{ matrix.runtime.base }}-${{ matrix.platform.arch }}" gtsam_docker:validate + + - name: Validate minimal GTSAM import + run: ./scripts/validate_container.sh gtsam_docker:validate /examples/validate_gtsam.py + + - name: Validate NumPy ABI + run: ./scripts/validate_container.sh gtsam_docker:validate /examples/validate_numpy_abi.py + + - name: Validate PlanarSLAM + run: ./scripts/validate_container.sh gtsam_docker:validate /examples/PlanarSLAMExample.py + + - name: Report image size + run: ./scripts/size-report.sh gtsam_docker:validate gtsam${{ matrix.gtsam_version }}-py${{ matrix.python_version }}-${{ matrix.runtime.base }} + + manifest: + needs: build + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Create manifests for published images + run: | + set -euo pipefail + repo="${GITHUB_REPOSITORY,,}" + image="${REGISTRY}/${repo}" + gtsam_build="${REGISTRY}/${repo}/gtsam-build" + release_version="${{ inputs.release_version }}" + + # imagetools create assembles the multi-arch index while carrying over + # each arch's SBOM/provenance attestation manifests (docker manifest + # create would drop them). + create_manifest() { + local image_name="$1" + local tag="$2" + local amd64_tag="${tag}-amd64" + local arm64_tag="${tag}-arm64" + if docker buildx imagetools inspect "$image_name:$amd64_tag" >/dev/null 2>&1 && docker buildx imagetools inspect "$image_name:$arm64_tag" >/dev/null 2>&1; then + docker buildx imagetools create -t "$image_name:$tag" "$image_name:$amd64_tag" "$image_name:$arm64_tag" + else + echo "Skipping manifest $image_name:$tag because one or more arch tags are missing." + fi + } + + for py in 3.11 3.12 3.13 3.14; do + for gtsam in 4.2.0 4.2.1 4.3a1; do + for artifact in glibc-trixie musl-alpine; do + create_manifest "$gtsam_build" "gtsam${gtsam}-py${py}-${artifact}" + done + for base in trixie trixie-slim distroless-debian13 alpine; do + tag="gtsam${gtsam}-py${py}-${base}" + create_manifest "$image" "$tag" + if [ -n "$release_version" ]; then + create_manifest "$image" "${release_version}-${tag}" + fi + done + done + done + + create_manifest "$image" latest diff --git a/.github/workflows/build-python-base.yaml b/.github/workflows/build-python-base.yaml index 4379693..a854ea8 100644 --- a/.github/workflows/build-python-base.yaml +++ b/.github/workflows/build-python-base.yaml @@ -1,29 +1,74 @@ -name: Build Python Base Image +name: Build Python Base Images on: workflow_dispatch: inputs: python_version: - description: "Python version to build" - required: true - default: "3.11.2" + description: "Optional single Python minor version to build, e.g. 3.14" + required: false + default: "" push: paths: - "Dockerfile.python-base" + - ".github/workflows/build-python-base.yaml" branches: - main - development +# Deny all GITHUB_TOKEN scopes by default; jobs opt into the minimum they need. +permissions: {} + env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }}/python-base jobs: + # The matrix cannot be filtered with a job-level `if` because the `matrix` + # context is not available there. Instead we compute the Python-version list + # up front (one entry when workflow_dispatch passes python_version, otherwise + # the full set) and feed it into the build/manifest matrices via fromJSON. + setup: + runs-on: ubuntu-latest + outputs: + py_versions: ${{ steps.set.outputs.py_versions }} + steps: + - id: set + run: | + if [ -n "${{ inputs.python_version }}" ]; then + printf 'py_versions=["%s"]\n' "${{ inputs.python_version }}" >> "$GITHUB_OUTPUT" + else + echo 'py_versions=["3.11","3.12","3.13","3.14"]' >> "$GITHUB_OUTPUT" + fi + build: + needs: setup strategy: + fail-fast: false matrix: - platform: ["linux/amd64", "linux/arm64"] - runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} + python_version: ${{ fromJSON(needs.setup.outputs.py_versions) }} + platform: + - platform: linux/amd64 + arch: amd64 + runner: ubuntu-latest + - platform: linux/arm64 + arch: arm64 + runner: ubuntu-24.04-arm + base: + - image: python-build + target: python-build-glibc + suffix: glibc-trixie + - image: python-runtime + target: python-runtime-glibc-trixie + suffix: glibc-trixie + - image: python-runtime + target: python-runtime-glibc-slim + suffix: glibc-trixie-slim + - image: python-build + target: python-build-musl + suffix: musl-alpine + - image: python-runtime + target: python-runtime-musl-alpine + suffix: musl-alpine + runs-on: ${{ matrix.platform.runner }} permissions: contents: read packages: write @@ -41,48 +86,48 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set Python version - id: python + - name: Normalize image name + id: image run: | - # Use input if provided, otherwise extract from Dockerfile.python-base - if [ -n "${{ inputs.python_version }}" ]; then - echo "version=${{ inputs.python_version }}" >> $GITHUB_OUTPUT - else - VERSION=$(grep -oP 'ARG PYTHON_VERSION=\K[0-9.]+' Dockerfile.python-base) - echo "version=${VERSION}" >> $GITHUB_OUTPUT - fi - - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v6 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=raw,value=${{ steps.python.outputs.version }}-trixie-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + repo="${GITHUB_REPOSITORY,,}" + echo "name=${REGISTRY}/${repo}/${{ matrix.base.image }}" >> "$GITHUB_OUTPUT" - - name: Build and push Python base image for ${{ matrix.platform }} + - name: Build and push ${{ matrix.base.image }} (${{ matrix.python_version }}, ${{ matrix.base.suffix }}, ${{ matrix.platform.arch }}) uses: docker/build-push-action@v7 with: context: . file: Dockerfile.python-base + target: ${{ matrix.base.target }} push: true - provenance: false - platforms: ${{ matrix.platform }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + provenance: mode=max + sbom: true + platforms: ${{ matrix.platform.platform }} + tags: ${{ steps.image.outputs.name }}:py${{ matrix.python_version }}-${{ matrix.base.suffix }}-${{ matrix.platform.arch }} build-args: | - PYTHON_VERSION=${{ steps.python.outputs.version }} + PYTHON_VERSION=${{ matrix.python_version }} manifest: - needs: build + needs: [setup, build] runs-on: ubuntu-latest permissions: contents: read packages: write + strategy: + fail-fast: false + matrix: + python_version: ${{ fromJSON(needs.setup.outputs.py_versions) }} + base: + - image: python-build + suffix: glibc-trixie + - image: python-runtime + suffix: glibc-trixie + - image: python-runtime + suffix: glibc-trixie-slim + - image: python-build + suffix: musl-alpine + - image: python-runtime + suffix: musl-alpine steps: - - name: Checkout - uses: actions/checkout@v6 - - name: Log into registry ${{ env.REGISTRY }} uses: docker/login-action@v4 with: @@ -90,28 +135,17 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set Python version - id: python - run: | - if [ -n "${{ inputs.python_version }}" ]; then - echo "version=${{ inputs.python_version }}" >> $GITHUB_OUTPUT - else - VERSION=$(grep -oP 'ARG PYTHON_VERSION=\K[0-9.]+' Dockerfile.python-base) - echo "version=${VERSION}" >> $GITHUB_OUTPUT - fi + # imagetools create (unlike docker manifest create) preserves the per-arch + # SBOM/provenance attestation manifests when assembling the multi-arch index. + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 - name: Create multi-arch manifest run: | - LOWER_IMAGE_NAME=$(echo "${IMAGE_NAME}" | tr '[:upper:]' '[:lower:]') - PYTHON_VERSION="${{ steps.python.outputs.version }}" - - docker manifest create $REGISTRY/${LOWER_IMAGE_NAME}:${PYTHON_VERSION}-trixie \ - $REGISTRY/${LOWER_IMAGE_NAME}:${PYTHON_VERSION}-trixie-amd64 \ - $REGISTRY/${LOWER_IMAGE_NAME}:${PYTHON_VERSION}-trixie-arm64 - docker manifest push $REGISTRY/${LOWER_IMAGE_NAME}:${PYTHON_VERSION}-trixie - - # Also tag as latest - docker manifest create $REGISTRY/${LOWER_IMAGE_NAME}:latest \ - $REGISTRY/${LOWER_IMAGE_NAME}:${PYTHON_VERSION}-trixie-amd64 \ - $REGISTRY/${LOWER_IMAGE_NAME}:${PYTHON_VERSION}-trixie-arm64 - docker manifest push $REGISTRY/${LOWER_IMAGE_NAME}:latest + set -euo pipefail + repo="${GITHUB_REPOSITORY,,}" + image="${REGISTRY}/${repo}/${{ matrix.base.image }}" + tag="py${{ matrix.python_version }}-${{ matrix.base.suffix }}" + docker buildx imagetools create -t "$image:$tag" \ + "$image:$tag-amd64" \ + "$image:$tag-arm64" diff --git a/.github/workflows/pr-labels.yaml b/.github/workflows/pr-labels.yaml index c71510f..ba3be9f 100644 --- a/.github/workflows/pr-labels.yaml +++ b/.github/workflows/pr-labels.yaml @@ -8,10 +8,14 @@ on: branches: [ main ] types: [ opened, labeled, unlabeled, synchronize ] +permissions: {} + jobs: contains-labels: if: ${{!startsWith(github.head_ref, 'release/')}} runs-on: ubuntu-latest + permissions: + pull-requests: read steps: - uses: jesusvasquez333/verify-pr-label-action@v1.4.0 with: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index f6c9c55..40fc4a1 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -4,140 +4,33 @@ on: workflow_dispatch: inputs: version: - description: "Version Number (semver: 1.2.3)" + description: "Version Number (semver tag, e.g. v4.0.0)" required: true workflow_call: inputs: version: - description: "Version Number (semver: 1.2.3)" + description: "Version Number (semver tag, e.g. v4.0.0)" required: true type: string +# Deny all GITHUB_TOKEN scopes by default; jobs opt into the minimum they need. +permissions: {} + env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - PYTHON_BASE_IMAGE_NAME: ${{ github.repository }}/python-base jobs: - build: - strategy: - matrix: - include: - - platform: linux/amd64 - arch: amd64 - - platform: linux/arm64 - arch: arm64 - # Use GitHub-hosted runner for amd64; arm64 uses partner runner (ensure ubuntu-24.04-arm is enabled for the repo/org) - runs-on: ${{ matrix.platform == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - permissions: - contents: read - packages: write - steps: - - name: Checkout - uses: actions/checkout@v6 - - # Set up Docker Buildx - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v4 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Get Python version and base image - id: python - run: | - VERSION=$(grep -oP 'ARG PYTHON_VERSION=\K[0-9.]+' Dockerfile.python-base) - echo "version=${VERSION}" >> $GITHUB_OUTPUT - LOWER_BASE_IMAGE=$(echo "${{ env.PYTHON_BASE_IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') - echo "base_image=${LOWER_BASE_IMAGE}" >> $GITHUB_OUTPUT - - # Use docker/metadata-action to generate tags with an architecture suffix - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@v6 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=raw,value=latest-${{ matrix.arch }} - type=raw,value=${{ inputs.version }}-${{ matrix.arch }} - - - name: Build and push Docker image for ${{ matrix.platform }} - uses: docker/build-push-action@v7 - with: - context: . - push: true - provenance: false - platforms: ${{ matrix.platform }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - build-args: | - PYTHON_BASE_IMAGE=${{ env.REGISTRY }}/${{ steps.python.outputs.base_image }}:${{ steps.python.outputs.version }}-trixie-${{ matrix.arch }} - - validate: - needs: build - strategy: - fail-fast: false - matrix: - arch: [ amd64, arm64 ] - runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} - permissions: - contents: read - packages: read - steps: - - name: Checkout - uses: actions/checkout@v6 - - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v4 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Pull and validate image (${{ matrix.arch }}) - run: | - LOWER_IMAGE_NAME=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') - docker pull ${{ env.REGISTRY }}/${LOWER_IMAGE_NAME}:${{ inputs.version }}-${{ matrix.arch }} - docker tag ${{ env.REGISTRY }}/${LOWER_IMAGE_NAME}:${{ inputs.version }}-${{ matrix.arch }} gtsam_docker:latest - chmod +x ./scripts/validate_container.sh - ./scripts/validate_container.sh gtsam_docker:latest /examples/PlanarSLAMExample.py - - manifest: - needs: validate - runs-on: ubuntu-latest + build-full-matrix: permissions: contents: read packages: write - steps: - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v4 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Create multi-arch manifest for latest - run: | - LOWER_IMAGE_NAME=$(echo "${IMAGE_NAME}" | tr '[:upper:]' '[:lower:]') - docker manifest create $REGISTRY/${LOWER_IMAGE_NAME}:latest \ - $REGISTRY/${LOWER_IMAGE_NAME}:latest-amd64 \ - $REGISTRY/${LOWER_IMAGE_NAME}:latest-arm64 - docker manifest push $REGISTRY/${LOWER_IMAGE_NAME}:latest - - - name: Create multi-arch manifest for version tag - run: | - LOWER_IMAGE_NAME=$(echo "${IMAGE_NAME}" | tr '[:upper:]' '[:lower:]') - docker manifest create $REGISTRY/${LOWER_IMAGE_NAME}:${{ inputs.version }} \ - $REGISTRY/${LOWER_IMAGE_NAME}:${{ inputs.version }}-amd64 \ - $REGISTRY/${LOWER_IMAGE_NAME}:${{ inputs.version }}-arm64 - docker manifest push $REGISTRY/${LOWER_IMAGE_NAME}:${{ inputs.version }} + uses: ./.github/workflows/build-full-matrix.yaml + with: + release_version: ${{ inputs.version }} + secrets: inherit release: - needs: manifest + needs: build-full-matrix runs-on: ubuntu-latest permissions: contents: write @@ -151,12 +44,14 @@ jobs: prerelease: false title: ${{ inputs.version }} - create-release: - permissions: - pull-requests: write - contents: write + changelog-pr: needs: release runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + outputs: + main_pr: ${{ steps.create_main_pr.outputs.pull_request_number }} steps: - uses: actions/checkout@v6 @@ -173,28 +68,27 @@ jobs: with: version: ${{ inputs.version }} - - name: Commit changelog and version in package + - name: Commit changelog id: make-commit run: | git add CHANGELOG.md git commit --message "Prepare release ${{ inputs.version }}" - echo "commit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - name: Push commit - run: | - git push origin release/${{ inputs.version }} + run: git push origin release/${{ inputs.version }} - name: Create pull request into main + id: create_main_pr uses: thomaseizinger/create-pull-request@1.4.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} head: release/${{ inputs.version }} base: main title: ${{ inputs.version }} into main - reviewers: ${{ github.event.issue.user.login }} body: | - This PR was created when the Create Release workflow was run. - I've updated the version name and code commit: ${{ steps.make-commit.outputs.commit }}. + This PR was created by the Release workflow. + Changelog commit: ${{ steps.make-commit.outputs.commit }}. - name: Create pull request into development uses: thomaseizinger/create-pull-request@1.4.0 @@ -203,7 +97,58 @@ jobs: head: release/${{ inputs.version }} base: development title: ${{ inputs.version }} into development - reviewers: ${{ github.event.issue.user.login }} body: | - This PR was created when the Create Release workflow was run. - I've updated the version name and code commit: ${{ steps.make-commit.outputs.commit }}. + This PR was created by the Release workflow. + Changelog commit: ${{ steps.make-commit.outputs.commit }}. + + # Scan the just-published release images and post the results as a comment on + # the release PR. This is deliberately NON-GATING: the job does not fail on + # findings, it only surfaces them for review next to the changelog. + vuln-report: + needs: [build-full-matrix, changelog-pr] + if: ${{ needs.changelog-pr.outputs.main_pr != '' }} + # Never let scanning/commenting affect the release run's status. + continue-on-error: true + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + packages: read + pull-requests: write + steps: + - uses: actions/checkout@v6 + + - name: Install grype + run: | + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh \ + | sh -s -- -b /usr/local/bin + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate vulnerability report + env: + VULN_REPORT_VERSION: ${{ inputs.version }} + run: | + set -euo pipefail + repo="${GITHUB_REPOSITORY,,}" + image="${REGISTRY}/${repo}" + ver="${{ inputs.version }}" + imgs=() + for py in 3.11 3.12 3.13 3.14; do + for gtsam in 4.2.0 4.2.1 4.3a1; do + for base in trixie trixie-slim distroless-debian13 alpine; do + imgs+=("${image}:${ver}-gtsam${gtsam}-py${py}-${base}") + done + done + done + ./scripts/vuln-report.sh vuln-report.md "${imgs[@]}" + + - name: Comment report on release PR + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh pr comment "${{ needs.changelog-pr.outputs.main_pr }}" --body-file vuln-report.md diff --git a/Dockerfile b/Dockerfile index affb493..6b7b259 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,103 +1,224 @@ -# Default build produces the "runtime" stage (slim). Use --target gtsam for a dev image with build tools and shell. -# Pre-built Python base image (build once with Dockerfile.python-base, push to registry) -# To build locally without registry: docker build -f Dockerfile.python-base -t python-optimized:3.11.2-trixie . -ARG PYTHON_BASE_IMAGE=python-optimized:3.11.2-trixie -FROM ${PYTHON_BASE_IMAGE} AS dependencies - -# Disable GUI prompts -ENV DEBIAN_FRONTEND=noninteractive - -# Install GTSAM build dependencies (Python already installed in base image) -RUN apt-get update && apt-get install -y --no-install-recommends \ - apt-utils \ - build-essential \ - libboost-all-dev \ - cmake \ - libtbb-dev \ - flex \ - bison \ - dejagnu \ - libmpc-dev \ - libmpfr-dev \ - libgmp-dev \ - make \ - && rm -rf /var/lib/apt/lists/* - -# Set working directory +# GTSAM build artifact and runtime images. +# +# Common targets: +# gtsam-build build artifact image with GTSAM installed into /usr/local +# runtime-trixie final runtime on python-runtime:*-glibc-trixie +# runtime-trixie-slim final runtime on python-runtime:*-glibc-trixie-slim +# runtime-distroless final runtime on gcr.io/distroless/cc-debian13 +# runtime-alpine final runtime on python-runtime:*-musl-alpine +# runtime alias for runtime-trixie-slim +ARG PYTHON_VERSION=3.14 +ARG PYTHON_ABI=${PYTHON_VERSION} +ARG GTSAM_VERSION=4.3a1 +ARG NUMPY_SPEC= +ARG PYTHON_BUILD_IMAGE=python-build:py3.14-glibc-trixie +ARG PYTHON_RUNTIME_IMAGE=python-runtime:py3.14-glibc-trixie-slim +ARG PYTHON_RUNTIME_TRIXIE_IMAGE=python-runtime:py3.14-glibc-trixie +ARG PYTHON_RUNTIME_SLIM_IMAGE=python-runtime:py3.14-glibc-trixie-slim +ARG PYTHON_RUNTIME_ALPINE_IMAGE=python-runtime:py3.14-musl-alpine + +FROM alpine/git:2.52.0@sha256:4a0e72d49596a1f5d3701aeedafdadc5c0da4062be4657c7bdc4017387f591cc AS gtsam-source +ARG GTSAM_VERSION WORKDIR /usr/src - -# Use git to clone gtsam and specific GTSAM version -FROM alpine/git:2.52.0 AS gtsam-clone - -ARG GTSAM_VERSION=4.2.0 -WORKDIR /usr/src/ - -# Shallow clone specific tag for smaller, faster fetch -RUN git clone --depth 1 --branch ${GTSAM_VERSION} https://github.com/borglab/gtsam.git - -# Create new stage called gtsam for GTSAM building -FROM dependencies AS gtsam - -ARG PYTHON_VERSION=3.11.2 - -# Needed to link with GTSAM (ENV works in non-interactive shells; .bashrc does not) -ENV LD_LIBRARY_PATH=/usr/local/lib - -# Move gtsam data -COPY --from=gtsam-clone /usr/src/gtsam /usr/src/gtsam - +RUN git clone --quiet --depth 1 --branch "${GTSAM_VERSION}" https://github.com/borglab/gtsam.git + +FROM ${PYTHON_BUILD_IMAGE} AS gtsam-build +ARG GTSAM_VERSION +ARG NUMPY_SPEC +# Opt this stage into BuildKit SBOM scanning. GTSAM is compiled from source and +# links Debian/Alpine Boost, TBB and Eigen packages that leave no metadata in +# the final runtime image; scanning the build stage records that provenance in +# the SBOM attestation alongside the runtime stage's own scan. +ARG BUILDKIT_SBOM_SCAN_STAGE=true +ENV LD_LIBRARY_PATH=/usr/local/lib \ + PIP_NO_CACHE_DIR=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONUSERBASE=/usr/local \ + PIP_CONSTRAINT=/etc/pip-constraints.txt +COPY ci/requirements-build.txt /etc/pip-constraints.txt +COPY --from=gtsam-source /usr/src/gtsam /usr/src/gtsam WORKDIR /usr/src/gtsam/build - -# Install python wrapper requirements, then pin numpy for GTSAM ABI compatibility -RUN python3 -m pip install --no-cache-dir -U -r /usr/src/gtsam/python/requirements.txt && \ - python3 -m pip install --no-cache-dir "numpy==1.26.4" - -# Run cmake -RUN cmake \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DGTSAM_WITH_EIGEN_MKL=OFF \ - -DGTSAM_BUILD_EXAMPLES_ALWAYS=OFF \ - -DGTSAM_BUILD_TIMING_ALWAYS=OFF \ - -DGTSAM_BUILD_TESTS=OFF \ - -DGTSAM_BUILD_PYTHON=ON \ - -DGTSAM_BUILD_CONVENIENCE_LIBRARIES=OFF \ - -DGTSAM_PYTHON_VERSION=${PYTHON_VERSION} \ - -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ - .. - -# Build, install, strip binaries, and clean in one layer to reduce image size -RUN make -j$(nproc) install && \ - make python-install && \ - #find /usr/local -type f \( -name "*.so" -o -name "*.so.*" \) -exec strip --strip-unneeded {} \; 2>/dev/null || true && \ - #find /usr/local/bin /usr/local/lib -executable -type f -exec strip --strip-unneeded {} \; 2>/dev/null || true && \ - make clean && \ - ldconfig - -# Final cleanup (dependencies stage already cleared apt lists) -RUN rm -rf /tmp/* /var/tmp/* - -# ----------------------------------------------------------------------------- -# Slim runtime stage: copy only installed artifacts, no build tools or source -# ----------------------------------------------------------------------------- -FROM debian:trixie-slim AS runtime - -ENV DEBIAN_FRONTEND=noninteractive -ENV PATH="/usr/local/bin:${PATH}" -ENV LD_LIBRARY_PATH=/usr/local/lib - -# Runtime libs only. Python binary (ldd python3.11) needs only libc/libm/libpython; GTSAM needs Boost + TBB (see scripts/audit-runtime-deps.sh). -# Add back libssl3t64 libbz2-1.0 libreadline8t64 libsqlite3-0 libffi8 zlib1g libncursesw6 if you import ssl/sqlite3/readline/etc. -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - libtbb12 \ - libtbbmalloc2 \ - libboost-serialization1.83.0 \ - libboost-filesystem1.83.0 \ - libboost-timer1.83.0 \ - && rm -rf /var/lib/apt/lists/* - -COPY --from=gtsam /usr/local /usr/local - -RUN ldconfig - -CMD ["python3"] \ No newline at end of file +# numpy major is an ABI choice tied to the GTSAM version, so it stays a range +# here (verified at runtime by validate_numpy_abi.py). Everything else +# (pyparsing, pybind11_stubgen, pip/setuptools/wheel) is version-pinned via +# the PIP_CONSTRAINT file above. +RUN set -eu; \ + resolved_numpy_spec="${NUMPY_SPEC}"; \ + resolved_pybind11_stubgen=""; \ + if [ -z "$resolved_numpy_spec" ]; then \ + case "$GTSAM_VERSION" in \ + 4.3*) resolved_numpy_spec="numpy>=2,<3"; resolved_pybind11_stubgen="pybind11_stubgen" ;; \ + *) resolved_numpy_spec="numpy<2" ;; \ + esac; \ + fi; \ + python3 -m pip install -q --no-cache-dir -r /usr/src/gtsam/python/requirements.txt; \ + python3 -m pip install -q --no-cache-dir pyparsing "$resolved_numpy_spec" $resolved_pybind11_stubgen; +RUN set -eu; \ + cmake_log=/tmp/gtsam-cmake-configure.log; \ + python_exe="$(command -v python3)"; \ + python_include="$(python3 -c 'import sysconfig; print(sysconfig.get_path("include"))')"; \ + python_library="$(python3 -c 'import pathlib, sysconfig; print(pathlib.Path(sysconfig.get_config_var("LIBDIR"), sysconfig.get_config_var("LDLIBRARY")))')"; \ + if ! cmake \ + -Wno-dev \ + -Wno-deprecated \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DPython3_EXECUTABLE="$python_exe" \ + -DPython3_INCLUDE_DIR="$python_include" \ + -DPython3_LIBRARY="$python_library" \ + -DPython_EXECUTABLE="$python_exe" \ + -DPYTHON_EXECUTABLE="$python_exe" \ + -DPYTHON_INCLUDE_DIR="$python_include" \ + -DPYTHON_LIBRARY="$python_library" \ + -DGTSAM_WITH_EIGEN_MKL=OFF \ + -DGTSAM_BUILD_EXAMPLES_ALWAYS=OFF \ + -DGTSAM_BUILD_TIMING_ALWAYS=OFF \ + -DGTSAM_BUILD_TESTS=OFF \ + -DGTSAM_BUILD_PYTHON=ON \ + -DGTSAM_BUILD_CONVENIENCE_LIBRARIES=OFF \ + -DGTSAM_USE_SYSTEM_EIGEN=ON \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + .. > "$cmake_log" 2>&1; then \ + cat "$cmake_log"; \ + exit 1; \ + fi; \ + rm -f "$cmake_log" +RUN set -eu; \ + cmake --build . --parallel "$(nproc)" -- -s; \ + cmake --build . --target install -- -s; \ + cmake --build . --target python-install -- -s; \ + find /usr/local -type f \( -name '*.so' -o -name '*.so.*' \) -exec strip --strip-unneeded {} + 2>/dev/null || true; \ + find /usr/local -type f -name '*.a' -delete; \ + python_site="$(python3 -c 'import site; print(site.getsitepackages()[0])')"; \ + for pkg in pip setuptools wheel packaging pytest _pytest pluggy iniconfig pygments py pybind11_stubgen; do \ + rm -rf "$python_site/$pkg" "$python_site/$pkg".* "$python_site/$pkg"-*.dist-info; \ + done; \ + find /usr/local -type d \( -name '__pycache__' -o -name 'test' -o -name 'tests' \) -prune -exec rm -rf {} +; \ + rm -rf /usr/src/gtsam /tmp/* /var/tmp/* +CMD ["python3"] + +# Collect the exact set of third-party shared libraries the built GTSAM +# extensions link against (Boost, TBB, libstdc++, libgcc, ...), resolved +# dynamically via ldd so the list cannot drift when a base image bumps a +# Boost/TBB SONAME. Core libc/loader libraries are skipped because every +# runtime base already provides them. Original paths are preserved so the +# loader finds them with no ldconfig (needed for distroless). +# +# We also record package provenance for the copied libs into /sbom-meta so they +# show up in the runtime image's own SBOM (the canonical source vulnerability +# scanners read). Vendored .so files carry no package metadata, so we map each +# back to its owning dpkg/apk package and stage that package-database entry: +# - Debian: /var/lib/dpkg/status.d/ stanzas (the distroless convention, +# also read by syft on regular images). +# - Alpine: a /lib/apk/db/installed fragment appended in the runtime stage. +# The C/C++ toolchain libs (libstdc++/libgcc/libgomp) are excluded from the +# Debian metadata because the Debian runtime base already ships and catalogs +# them; emitting duplicates would double-count them in the SBOM. +FROM gtsam-build AS runtime-libs +RUN set -eu; \ + mkdir -p /runtime-libs /sbom-meta; \ + : > /tmp/deps.list; \ + targets="$(find /usr/local/lib -maxdepth 1 -name 'lib*gtsam*.so*' 2>/dev/null; \ + find /usr/local/lib/python*/site-packages/gtsam* -name '*.so' 2>/dev/null)"; \ + for so in $targets; do ldd "$so" 2>/dev/null || true; done \ + | awk '/=> \// {print $3}' | sort -u \ + | while IFS= read -r dep; do \ + [ -n "$dep" ] || continue; \ + case "$dep" in \ + /usr/local/*) continue ;; \ + */ld-linux*|*/ld-musl*|*/libc.so*|*/libm.so*|*/libdl.so*|*/libpthread.so*|*/librt.so*|*/libresolv.so*|*/libutil.so*) continue ;; \ + esac; \ + [ -e "$dep" ] || continue; \ + real="$(readlink -f "$dep")"; \ + dest="/runtime-libs$(dirname "$real")"; \ + mkdir -p "$dest"; \ + cp "$real" "$dest/$(basename "$dep")"; \ + printf '%s\n' "$real" >> /tmp/deps.list; \ + done; \ + if command -v dpkg-query >/dev/null 2>&1; then \ + mkdir -p /sbom-meta/var/lib/dpkg/status.d; \ + while IFS= read -r f; do \ + case "$(basename "$f")" in libstdc++*|libgcc_s*|libgomp*) continue ;; esac; \ + dpkg-query -S "$f" 2>/dev/null | awk -F': ' '{print $1}' | sed 's/:.*//'; \ + done < /tmp/deps.list | sort -u | while IFS= read -r p; do \ + [ -n "$p" ] || continue; \ + dpkg-query -s "$p" > "/sbom-meta/var/lib/dpkg/status.d/$p" 2>/dev/null || true; \ + done; \ + elif command -v apk >/dev/null 2>&1; then \ + names="$(while IFS= read -r f; do apk info -W "$f" 2>/dev/null; done < /tmp/deps.list \ + | sed -n 's/.* is owned by //p' | sed -E 's/-[0-9][^-]*-r[0-9]+$//' | sort -u)"; \ + awk -v RS='' -v names="$names" \ + 'BEGIN{c=split(names,a,"\n"); for(i=1;i<=c;i++) want[a[i]]=1} \ + { nm=""; n=split($0,L,"\n"); for(i=1;i<=n;i++) if(L[i] ~ /^P:/) nm=substr(L[i],3); if(nm in want) printf "%s\n\n", $0 }' \ + /lib/apk/db/installed > /sbom-meta/apk-fragment; \ + fi; \ + rm -f /tmp/deps.list + +FROM ${PYTHON_RUNTIME_TRIXIE_IMAGE} AS runtime-trixie +ARG PYTHON_ABI +ENV LD_LIBRARY_PATH=/usr/local/lib \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 +COPY --from=runtime-libs /runtime-libs/ / +COPY --from=runtime-libs /sbom-meta/ / +COPY --from=gtsam-build /usr/local/lib/lib*gtsam* /usr/local/lib/ +COPY --from=gtsam-build /usr/local/lib/python${PYTHON_ABI}/site-packages /usr/local/lib/python${PYTHON_ABI}/site-packages +RUN ldconfig && \ + find /usr/local -type d -name '__pycache__' -prune -exec rm -rf {} + && \ + find /usr/local -type f -name '*.a' -delete +CMD ["python3"] + +FROM ${PYTHON_RUNTIME_SLIM_IMAGE} AS runtime-trixie-slim +ARG PYTHON_ABI +ENV LD_LIBRARY_PATH=/usr/local/lib \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 +COPY --from=runtime-libs /runtime-libs/ / +COPY --from=runtime-libs /sbom-meta/ / +COPY --from=gtsam-build /usr/local/lib/lib*gtsam* /usr/local/lib/ +COPY --from=gtsam-build /usr/local/lib/python${PYTHON_ABI}/site-packages /usr/local/lib/python${PYTHON_ABI}/site-packages +RUN ldconfig && \ + find /usr/local -type d -name '__pycache__' -prune -exec rm -rf {} + && \ + find /usr/local -type f -name '*.a' -delete +CMD ["python3"] + +FROM ${PYTHON_RUNTIME_ALPINE_IMAGE} AS runtime-alpine +ARG PYTHON_ABI +ENV LD_LIBRARY_PATH=/usr/local/lib \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 +COPY --from=runtime-libs /runtime-libs/ / +COPY --from=runtime-libs /sbom-meta/apk-fragment /tmp/apk-fragment +COPY --from=gtsam-build /usr/local/lib/lib*gtsam* /usr/local/lib/ +COPY --from=gtsam-build /usr/local/lib/python${PYTHON_ABI}/site-packages /usr/local/lib/python${PYTHON_ABI}/site-packages +RUN { printf '\n'; cat /tmp/apk-fragment; } >> /lib/apk/db/installed && rm -f /tmp/apk-fragment && \ + find /usr/local -type d -name '__pycache__' -prune -exec rm -rf {} + && \ + find /usr/local -type f -name '*.a' -delete +CMD ["python3"] + +# The distroless base has no package manager, so stage the CPython interpreter +# (installed under /usr/local in the python base image) plus CA certificates. +# Third-party shared libs come from the runtime-libs collector below. +FROM ${PYTHON_RUNTIME_SLIM_IMAGE} AS distroless-rootfs +RUN set -eu; \ + mkdir -p /rootfs/usr/local; \ + cp -a /usr/local/. /rootfs/usr/local/; \ + mkdir -p /rootfs/etc/ssl; \ + cp -a /etc/ssl/certs /rootfs/etc/ssl/certs; \ + find /rootfs/usr/local -type d -name '__pycache__' -prune -exec rm -rf {} +; \ + find /rootfs/usr/local -type f -name '*.a' -delete + +FROM gcr.io/distroless/cc-debian13@sha256:a017e74bd2a12d98342dbecd33d121d2b160415ed777573dc1808969e989d94d AS runtime-distroless +ARG PYTHON_ABI +ENV LD_LIBRARY_PATH=/usr/local/lib \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 +COPY --from=distroless-rootfs /rootfs / +COPY --from=runtime-libs /runtime-libs/ / +COPY --from=runtime-libs /sbom-meta/ / +COPY --from=gtsam-build /usr/local/lib/lib*gtsam* /usr/local/lib/ +COPY --from=gtsam-build /usr/local/lib/python${PYTHON_ABI}/site-packages /usr/local/lib/python${PYTHON_ABI}/site-packages +CMD ["/usr/local/bin/python3"] + +FROM runtime-trixie-slim AS runtime diff --git a/Dockerfile.python-base b/Dockerfile.python-base index 3ecdfb2..9c3a214 100644 --- a/Dockerfile.python-base +++ b/Dockerfile.python-base @@ -1,42 +1,103 @@ -# Pre-built optimized Python base image for GTSAM builds. -# Build and push once per Python version: -# docker build -f Dockerfile.python-base -t ghcr.io/yourorg/python-optimized:3.11.2-trixie . -# docker push ghcr.io/yourorg/python-optimized:3.11.2-trixie -FROM debian:trixie-20260421 -ARG PYTHON_VERSION=3.11.2 +# Layered Python base images for GTSAM CI. +# +# Published targets: +# python-build-glibc -> python-build:py-glibc-trixie +# python-runtime-glibc-trixie -> python-runtime:py-glibc-trixie +# python-runtime-glibc-slim -> python-runtime:py-glibc-trixie-slim +# python-build-musl -> python-build:py-musl-alpine +# python-runtime-musl-alpine -> python-runtime:py-musl-alpine +ARG PYTHON_VERSION=3.14 +ARG PYTHON_GLIBC_VARIANT=trixie +ARG PYTHON_ALPINE_VARIANT=alpine3.24 -ENV DEBIAN_FRONTEND=noninteractive - -# Install Python build dependencies -RUN apt-get update && apt-get install -y --no-install-recommends \ +FROM python:${PYTHON_VERSION}-${PYTHON_GLIBC_VARIANT} AS python-build-glibc +ENV DEBIAN_FRONTEND=noninteractive \ + PIP_NO_CACHE_DIR=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 +RUN apt-get update -qq && apt-get install -qq -y --no-install-recommends \ ca-certificates \ + bison \ build-essential \ - wget \ - libssl-dev \ - libbz2-dev \ - libreadline-dev \ - libsqlite3-dev \ - libffi-dev \ - zlib1g-dev \ - libncursesw5-dev \ - tk-dev \ - libgdbm-dev \ - liblzma-dev \ - && rm -rf /var/lib/apt/lists/* - + cmake \ + dejagnu \ + flex \ + git \ + libboost-filesystem-dev \ + libboost-program-options-dev \ + libboost-regex-dev \ + libboost-serialization-dev \ + libboost-system-dev \ + libboost-thread-dev \ + libboost-timer-dev \ + libeigen3-dev \ + libgmp-dev \ + libmpc-dev \ + libmpfr-dev \ + libtbb-dev \ + ninja-build \ + pkg-config \ + > /dev/null && rm -rf /var/lib/apt/lists/* WORKDIR /usr/src +COPY ci/requirements-build.txt /etc/pip-constraints.txt +ENV PIP_CONSTRAINT=/etc/pip-constraints.txt +RUN python3 -m pip install -q --no-cache-dir --upgrade pip setuptools wheel + +# Runtime bases intentionally ship only CA certs. GTSAM's third-party shared +# libraries (Boost, TBB, libstdc++, ...) are injected per-build by the +# runtime-libs collector in the main Dockerfile, resolved via ldd, so these +# bases never need version-pinned Boost/TBB packages that break on distro bumps. +FROM python:${PYTHON_VERSION}-${PYTHON_GLIBC_VARIANT} AS python-runtime-glibc-trixie +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + LD_LIBRARY_PATH=/usr/local/lib +RUN apt-get update -qq && apt-get install -qq -y --no-install-recommends \ + ca-certificates \ + > /dev/null && rm -rf /var/lib/apt/lists/* +CMD ["python3"] -# Build Python with PGO and LTO optimizations (takes ~15 min but only done once) -RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \ - tar xvf Python-${PYTHON_VERSION}.tgz && \ - cd Python-${PYTHON_VERSION} && \ - ./configure --enable-optimizations --with-lto --enable-shared && \ - make -j$(nproc) && \ - make install && \ - cd .. && \ - rm -rf Python-${PYTHON_VERSION} Python-${PYTHON_VERSION}.tgz && \ - ldconfig +FROM python:${PYTHON_VERSION}-slim-${PYTHON_GLIBC_VARIANT} AS python-runtime-glibc-slim +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + LD_LIBRARY_PATH=/usr/local/lib +RUN apt-get update -qq && apt-get install -qq -y --no-install-recommends \ + ca-certificates \ + > /dev/null && rm -rf /var/lib/apt/lists/* +CMD ["python3"] + +FROM python:${PYTHON_VERSION}-${PYTHON_ALPINE_VARIANT} AS python-build-musl +ENV PIP_NO_CACHE_DIR=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 +RUN apk add -q --no-cache \ + bison \ + boost-dev \ + build-base \ + ca-certificates \ + cmake \ + eigen-dev \ + flex \ + git \ + gmp-dev \ + mpfr-dev \ + mpc1-dev \ + ninja \ + pkgconf \ + onetbb-dev +WORKDIR /usr/src +COPY ci/requirements-build.txt /etc/pip-constraints.txt +ENV PIP_CONSTRAINT=/etc/pip-constraints.txt +RUN python3 -m pip install -q --no-cache-dir --upgrade pip setuptools wheel -ENV PATH="/usr/local/bin:${PATH}" +FROM python:${PYTHON_VERSION}-${PYTHON_ALPINE_VARIANT} AS python-runtime-musl-alpine +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + LD_LIBRARY_PATH=/usr/local/lib +RUN apk add -q --no-cache \ + ca-certificates +CMD ["python3"] -RUN python3 -m pip install --no-cache-dir --upgrade pip +FROM python-build-glibc AS python-build +FROM python-runtime-glibc-slim AS python-runtime diff --git a/README.md b/README.md index 945735c..9db082d 100644 --- a/README.md +++ b/README.md @@ -1 +1,91 @@ -# gtsam_docker \ No newline at end of file +# gtsam_docker + +Layered CI images for GTSAM Python bindings across multiple Python, GTSAM, base-image, and architecture combinations. + +Published image families are built for `linux/amd64` and `linux/arm64`: + +| Family | Tags | +|--------|------| +| `ghcr.io///python-build` | `py-glibc-trixie`, `py-musl-alpine` | +| `ghcr.io///python-runtime` | `py-glibc-trixie`, `py-glibc-trixie-slim`, `py-musl-alpine` | +| `ghcr.io///gtsam-build` | `gtsam-py-glibc-trixie`, `gtsam-py-musl-alpine` | +| `ghcr.io//` | `gtsam-py-trixie`, `gtsam-py-trixie-slim`, `gtsam-py-distroless-debian13`, `gtsam-py-alpine` | + +Default matrix axes: + +- Python: `3.11`, `3.12`, `3.13`, `3.14` +- GTSAM: `4.2.0`, `4.2.1`, `4.3a1` +- Runtime bases: Debian trixie, Debian trixie slim, distroless Debian 13, Alpine +- Architectures: `linux/amd64`, `linux/arm64` + +`latest` remains a compatibility alias for the default Debian slim runtime: `gtsam4.3a1-py3.14-trixie-slim`. + +## Local build + +Build a local Python base, then build a GTSAM runtime from it: + +```bash +docker build -f Dockerfile.python-base \ + --target python-build-glibc \ + --build-arg PYTHON_VERSION=3.14 \ + -t python-build:py3.14-glibc-trixie . + +docker build -f Dockerfile.python-base \ + --target python-runtime-glibc-slim \ + --build-arg PYTHON_VERSION=3.14 \ + -t python-runtime:py3.14-glibc-trixie-slim . + +docker build \ + --target runtime-trixie-slim \ + --build-arg PYTHON_VERSION=3.14 \ + --build-arg PYTHON_ABI=3.14 \ + --build-arg GTSAM_VERSION=4.3a1 \ + --build-arg PYTHON_BUILD_IMAGE=python-build:py3.14-glibc-trixie \ + --build-arg PYTHON_RUNTIME_SLIM_IMAGE=python-runtime:py3.14-glibc-trixie-slim \ + -t gtsam_docker:latest . +``` + +For Alpine, use `python-build-musl`, `python-runtime-musl-alpine`, and `--target runtime-alpine`. Alpine jobs are currently experimental in CI. + +## Validate a container + +```bash +./scripts/validate_container.sh gtsam_docker:latest /examples/validate_gtsam.py +./scripts/validate_container.sh gtsam_docker:latest /examples/validate_numpy_abi.py +./scripts/validate_container.sh gtsam_docker:latest /examples/PlanarSLAMExample.py +``` + +The validation script uses vector-form Docker commands and `/usr/local/bin/python3`, so it works with distroless images that do not include a shell. + +## CI policy + +Pull requests run a tiered smoke matrix: + +- Blocking: `4.2.0` + Python `3.11` on trixie slim amd64. +- Blocking: `4.3a1` + Python `3.14` on trixie slim amd64 and arm64. +- Non-blocking: `4.3a1` + Python `3.14` on distroless and Alpine amd64. + +The full scheduled/manual/release workflow publishes the complete matrix. GTSAM `4.2.x` with Python `3.13`/`3.14` and all Alpine runtimes are allowed-failure until those compatibility paths are stable. + +## Size and dependency audits + +Report local image size and compare against committed baselines: + +```bash +./scripts/size-report.sh gtsam_docker:latest gtsam4.3a1-py3.14-trixie-slim +``` + +Baselines live in `ci/size-baselines.txt` and should be filled after the first successful full publish. Later CI runs fail images that grow more than 10% over their baseline. + +Audit linked runtime dependencies for a build artifact image: + +```bash +docker build --target gtsam-build -t gtsam-build \ + --build-arg PYTHON_BUILD_IMAGE=python-build:py3.14-glibc-trixie \ + --build-arg PYTHON_VERSION=3.14 \ + --build-arg PYTHON_ABI=3.14 \ + --build-arg GTSAM_VERSION=4.3a1 \ + . + +./scripts/audit-runtime-deps.sh gtsam-build 3.14 +``` diff --git a/ci/requirements-build.txt b/ci/requirements-build.txt new file mode 100644 index 0000000..d65f6ed --- /dev/null +++ b/ci/requirements-build.txt @@ -0,0 +1,15 @@ +# Pinned build-time Python toolchain, applied via PIP_CONSTRAINT in the +# Dockerfile build stages. These constraints pin versions without forcing +# installation, so they layer on top of GTSAM's own python/requirements.txt +# and the per-version numpy policy (numpy<2 for 4.2, numpy>=2,<3 for 4.3). +# +# numpy is intentionally NOT pinned here: its major is an ABI contract chosen +# per GTSAM version at build time and verified at runtime by validate_numpy_abi.py. +# +# Dependabot (pip ecosystem, see .github/dependabot.yml) bumps these via PRs +# that the build-and-validate matrix gates before merge. +pip==26.1.2 +setuptools==82.0.1 +wheel==0.47.0 +pyparsing==3.3.2 +pybind11_stubgen==2.5.5 diff --git a/ci/size-baselines.txt b/ci/size-baselines.txt new file mode 100644 index 0000000..3fbae5a --- /dev/null +++ b/ci/size-baselines.txt @@ -0,0 +1,51 @@ +# Runtime image size baselines are added after the first full successful publish. +# Format: +# gtsam-py- +gtsam4.2.0-py3.11-alpine 195921711 +gtsam4.2.0-py3.11-distroless-debian13 204698487 +gtsam4.2.0-py3.11-trixie 1251587269 +gtsam4.2.0-py3.11-trixie-slim 272521675 +gtsam4.2.0-py3.12-alpine 189893216 +gtsam4.2.0-py3.12-distroless-debian13 198727104 +gtsam4.2.0-py3.12-trixie 1255320517 +gtsam4.2.0-py3.12-trixie-slim 266550292 +gtsam4.2.0-py3.13-alpine 154345071 +gtsam4.2.0-py3.13-distroless-debian13 164696600 +gtsam4.2.0-py3.13-trixie 1225151319 +gtsam4.2.0-py3.13-trixie-slim 232519895 +gtsam4.2.0-py3.14-alpine 156556014 +gtsam4.2.0-py3.14-distroless-debian13 166084024 +gtsam4.2.0-py3.14-trixie 1232235392 +gtsam4.2.0-py3.14-trixie-slim 233909839 +gtsam4.2.1-py3.11-alpine 195934020 +gtsam4.2.1-py3.11-distroless-debian13 204710892 +gtsam4.2.1-py3.11-trixie 1251599674 +gtsam4.2.1-py3.11-trixie-slim 272534080 +gtsam4.2.1-py3.12-alpine 189909621 +gtsam4.2.1-py3.12-distroless-debian13 198747701 +gtsam4.2.1-py3.12-trixie 1255341114 +gtsam4.2.1-py3.12-trixie-slim 266570889 +gtsam4.2.1-py3.13-alpine 154361476 +gtsam4.2.1-py3.13-distroless-debian13 164717197 +gtsam4.2.1-py3.13-trixie 1225171916 +gtsam4.2.1-py3.13-trixie-slim 232540492 +gtsam4.2.1-py3.14-alpine 156572419 +gtsam4.2.1-py3.14-distroless-debian13 166100525 +gtsam4.2.1-py3.14-trixie 1232251893 +gtsam4.2.1-py3.14-trixie-slim 233926340 +gtsam4.3a1-py3.11-alpine 226743087 +gtsam4.3a1-py3.11-distroless-debian13 229560897 +gtsam4.3a1-py3.11-trixie 1276449946 +gtsam4.3a1-py3.11-trixie-slim 297384352 +gtsam4.3a1-py3.12-alpine 220750253 +gtsam4.3a1-py3.12-distroless-debian13 223576043 +gtsam4.3a1-py3.12-trixie 1280169723 +gtsam4.3a1-py3.12-trixie-slim 291399498 +gtsam4.3a1-py3.13-alpine 217324557 +gtsam4.3a1-py3.13-distroless-debian13 222264888 +gtsam4.3a1-py3.13-trixie 1282719874 +gtsam4.3a1-py3.13-trixie-slim 290088450 +gtsam4.3a1-py3.14-alpine 219605004 +gtsam4.3a1-py3.14-distroless-debian13 223730240 +gtsam4.3a1-py3.14-trixie 1289881875 +gtsam4.3a1-py3.14-trixie-slim 291556322 diff --git a/examples/validate_numpy_abi.py b/examples/validate_numpy_abi.py new file mode 100644 index 0000000..e1b5a18 --- /dev/null +++ b/examples/validate_numpy_abi.py @@ -0,0 +1,25 @@ +""" +Validate that the installed GTSAM Python bindings and NumPy agree on the array ABI. + +This intentionally exercises a small NumPy-backed GTSAM call that has exposed +mismatched NumPy/GTSAM builds in the past. +""" +import sys + +import numpy as np +import gtsam + + +def main() -> int: + sigmas = np.array([0.3, 0.3, 0.1], dtype=np.float64) + model = gtsam.noiseModel.Diagonal.Sigmas(sigmas) + returned_sigmas = model.sigmas() + assert isinstance(returned_sigmas, np.ndarray) + assert returned_sigmas.shape == (3,) + np.testing.assert_allclose(returned_sigmas, sigmas) + print(f"NUMPY ABI OK: numpy={np.__version__}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/audit-runtime-deps.sh b/scripts/audit-runtime-deps.sh new file mode 100755 index 0000000..7fbb10f --- /dev/null +++ b/scripts/audit-runtime-deps.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Find runtime .so dependencies used by Python + GTSAM in a built image. +# +# Usage: +# ./scripts/audit-runtime-deps.sh [IMAGE] [PYTHON_ABI] +# +# Examples: +# ./scripts/audit-runtime-deps.sh gtsam-build 3.14 +# ./scripts/audit-runtime-deps.sh ghcr.io/org/repo/gtsam-build:gtsam4.3a1-py3.14-glibc-trixie 3.14 + +set -euo pipefail + +IMAGE="${1:-gtsam-build}" +PYTHON_ABI="${2:-}" + +container_script='set -eu +python_bin="$(command -v python3 || true)" +if [ -z "$python_bin" ]; then + python_bin=/usr/local/bin/python3 +fi +python_abi="'"$PYTHON_ABI"'" +if [ -z "$python_abi" ]; then + python_abi="$($python_bin -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")" +fi +files="$python_bin /usr/local/lib/libgtsam.so /usr/local/lib/libgtsam_unstable.so" +site_dir="/usr/local/lib/python${python_abi}/site-packages" +if [ -d "$site_dir/gtsam" ]; then + files="$files $(find "$site_dir/gtsam" -type f -name "*.so" 2>/dev/null)" +fi +for f in $files; do + [ -f "$f" ] && ldd "$f" 2>/dev/null || true +done +' + +echo "=== 1. System shared libs actually linked (paths) ===" +docker run --rm "$IMAGE" sh -c "$container_script" \ + | awk '/=>/ {print $3}' \ + | grep -E '^/lib|^/usr/lib' \ + | sort -u || true + +echo "" +echo "=== 2. Debian packages that provide those libs (if dpkg is available) ===" +docker run --rm "$IMAGE" sh -c "$container_script" \ + | awk '/=>/ {print $3}' \ + | grep -E '^/lib|^/usr/lib' \ + | sort -u \ + | while read -r p; do r="$(readlink -f "$p" 2>/dev/null || printf '%s' "$p")"; echo "$r"; done \ + | docker run --rm -i "$IMAGE" sh -c 'if command -v dpkg >/dev/null 2>&1; then xargs -r dpkg -S 2>/dev/null | cut -d: -f1 | sort -u; else cat >/dev/null; echo "dpkg unavailable in image"; fi' || true diff --git a/scripts/run_test_matrix.sh b/scripts/run_test_matrix.sh new file mode 100755 index 0000000..871c74e --- /dev/null +++ b/scripts/run_test_matrix.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +set -u + +JOBS="${JOBS:-2}" +DOCKER_BUILD_FLAGS="${DOCKER_BUILD_FLAGS:---progress=plain}" +export DOCKER_BUILD_FLAGS + +PYTHONS="3.11 3.12 3.13 3.14" +GTSAMS="4.2.0 4.2.1 4.3a1" +RUNTIMES="trixie trixie-slim distroless-debian13 alpine" + +build_base() { + py="$1" + docker build $DOCKER_BUILD_FLAGS -f Dockerfile.python-base --target python-build-glibc \ + --build-arg PYTHON_VERSION="$py" -t "python-build-local:py${py}-glibc-trixie" . + docker build $DOCKER_BUILD_FLAGS -f Dockerfile.python-base --target python-runtime-glibc-trixie \ + --build-arg PYTHON_VERSION="$py" -t "python-runtime-local:py${py}-glibc-trixie" . + docker build $DOCKER_BUILD_FLAGS -f Dockerfile.python-base --target python-runtime-glibc-slim \ + --build-arg PYTHON_VERSION="$py" -t "python-runtime-local:py${py}-glibc-trixie-slim" . + docker build $DOCKER_BUILD_FLAGS -f Dockerfile.python-base --target python-build-musl \ + --build-arg PYTHON_VERSION="$py" -t "python-build-local:py${py}-musl-alpine" . + docker build $DOCKER_BUILD_FLAGS -f Dockerfile.python-base --target python-runtime-musl-alpine \ + --build-arg PYTHON_VERSION="$py" -t "python-runtime-local:py${py}-musl-alpine" . +} + +build_artifact() { + py="$1" + gtsam="$2" + libc="$3" + + case "$libc" in + glibc) suffix=glibc-trixie ;; + musl) suffix=musl-alpine ;; + esac + + docker build $DOCKER_BUILD_FLAGS \ + --target gtsam-build \ + --build-arg PYTHON_VERSION="$py" \ + --build-arg PYTHON_ABI="$py" \ + --build-arg GTSAM_VERSION="$gtsam" \ + --build-arg PYTHON_BUILD_IMAGE="python-build-local:py${py}-${suffix}" \ + -t "gtsam-build-local:gtsam${gtsam}-py${py}-${suffix}" . +} + +build_validate_runtime() { + py="$1" + gtsam="$2" + runtime="$3" + + case "$runtime" in + trixie) + target=runtime-trixie + build_suffix=glibc-trixie + runtime_suffix=glibc-trixie + ;; + trixie-slim) + target=runtime-trixie-slim + build_suffix=glibc-trixie + runtime_suffix=glibc-trixie-slim + ;; + distroless-debian13) + target=runtime-distroless + build_suffix=glibc-trixie + runtime_suffix=glibc-trixie-slim + ;; + alpine) + target=runtime-alpine + build_suffix=musl-alpine + runtime_suffix=musl-alpine + ;; + esac + + tag="gtsam_docker:gtsam${gtsam}-py${py}-${runtime}" + + docker build $DOCKER_BUILD_FLAGS \ + --target "$target" \ + --build-arg PYTHON_VERSION="$py" \ + --build-arg PYTHON_ABI="$py" \ + --build-arg GTSAM_VERSION="$gtsam" \ + --build-arg PYTHON_BUILD_IMAGE="python-build-local:py${py}-${build_suffix}" \ + --build-arg PYTHON_RUNTIME_TRIXIE_IMAGE="python-runtime-local:py${py}-${runtime_suffix}" \ + --build-arg PYTHON_RUNTIME_SLIM_IMAGE="python-runtime-local:py${py}-${runtime_suffix}" \ + --build-arg PYTHON_RUNTIME_ALPINE_IMAGE="python-runtime-local:py${py}-${runtime_suffix}" \ + -t "$tag" . + + ./scripts/validate_container.sh "$tag" /examples/validate_gtsam.py + ./scripts/validate_container.sh "$tag" /examples/validate_numpy_abi.py + ./scripts/validate_container.sh "$tag" /examples/PlanarSLAMExample.py + ./scripts/size-report.sh "$tag" "gtsam${gtsam}-py${py}-${runtime}" +} + +export -f build_base build_artifact build_validate_runtime + +printf "%s\n" $PYTHONS \ + | xargs -n 1 -P "$JOBS" bash -c 'build_base "$0"' + +for py in $PYTHONS; do + for gtsam in $GTSAMS; do + printf "%s %s glibc\n%s %s musl\n" "$py" "$gtsam" "$py" "$gtsam" + done +done | xargs -n 3 -P "$JOBS" bash -c 'build_artifact "$0" "$1" "$2"' + +for py in $PYTHONS; do + for gtsam in $GTSAMS; do + for runtime in $RUNTIMES; do + printf "%s %s %s\n" "$py" "$gtsam" "$runtime" + done + done +done | xargs -n 3 -P "$JOBS" bash -c 'build_validate_runtime "$0" "$1" "$2"' \ No newline at end of file diff --git a/scripts/size-report.sh b/scripts/size-report.sh new file mode 100755 index 0000000..0282443 --- /dev/null +++ b/scripts/size-report.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Report image size and optionally compare against a committed baseline. +# +# Baseline format: one image/tag key and size in bytes per line: +# gtsam4.3a1-py3.14-trixie-slim 123456789 +# +# Usage: +# ./scripts/size-report.sh IMAGE [BASELINE_KEY] [BASELINE_FILE] [MAX_GROWTH_PERCENT] + +set -euo pipefail + +IMAGE="${1:?usage: scripts/size-report.sh IMAGE [BASELINE_KEY] [BASELINE_FILE] [MAX_GROWTH_PERCENT]}" +BASELINE_KEY="${2:-$IMAGE}" +BASELINE_FILE="${3:-ci/size-baselines.txt}" +MAX_GROWTH_PERCENT="${4:-10}" + +size="" +if docker image inspect "$IMAGE" >/dev/null 2>&1; then + size="$(docker image inspect --format '{{.Size}}' "$IMAGE")" +elif docker buildx imagetools inspect "$IMAGE" >/dev/null 2>&1; then + echo "Image is remote or multi-arch; docker buildx imagetools can inspect it, but local byte comparison requires a pulled single-platform image." + docker buildx imagetools inspect "$IMAGE" + exit 0 +else + echo "ERROR: image not found locally or remotely: $IMAGE" >&2 + exit 1 +fi + +echo "IMAGE_SIZE_BYTES $BASELINE_KEY $size" + +if [[ ! -f "$BASELINE_FILE" ]]; then + echo "No baseline file found at $BASELINE_FILE; skipping threshold check." + exit 0 +fi + +baseline="$(awk -v key="$BASELINE_KEY" '$1 == key {print $2}' "$BASELINE_FILE" | tail -n 1)" +if [[ -z "$baseline" ]]; then + echo "No baseline entry for $BASELINE_KEY; skipping threshold check." + exit 0 +fi + +limit=$(( baseline + (baseline * MAX_GROWTH_PERCENT / 100) )) +if (( size > limit )); then + echo "ERROR: $BASELINE_KEY grew from $baseline to $size bytes; allowed limit is $limit bytes (${MAX_GROWTH_PERCENT}%)." >&2 + exit 1 +fi + +echo "OK: $BASELINE_KEY is within ${MAX_GROWTH_PERCENT}% of baseline ($baseline bytes)." diff --git a/scripts/validate_container.sh b/scripts/validate_container.sh index 17fbee4..7814bb8 100755 --- a/scripts/validate_container.sh +++ b/scripts/validate_container.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Run a GTSAM example inside the runtime container to validate the image. +# Run a validation script inside a GTSAM runtime container. # # Usage: # ./scripts/validate_container.sh [IMAGE_TAG] [EXAMPLE] @@ -8,20 +8,17 @@ # ./scripts/validate_container.sh # ./scripts/validate_container.sh gtsam_docker:latest # ./scripts/validate_container.sh gtsam_docker:latest /examples/PlanarSLAMExample.py -# -# Default EXAMPLE is /examples/validate_gtsam.py (minimal graph/values check). -# Use /examples/PlanarSLAMExample.py for the full PlanarSLAM example from borglab/gtsam. -# -# Prereq: build the runtime image first, e.g. docker build -t gtsam_docker:latest . +# PYTHON_CMD=/usr/local/bin/python3 ./scripts/validate_container.sh image /examples/validate_numpy_abi.py -set -e +set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" EXAMPLES_DIR="$REPO_ROOT/examples" IMAGE="${1:-gtsam_docker:latest}" EXAMPLE="${2:-/examples/validate_gtsam.py}" -# If EXAMPLE has no leading slash, treat as name under /examples/ +PYTHON_CMD="${PYTHON_CMD:-/usr/local/bin/python3}" + if [[ -n "$EXAMPLE" && "$EXAMPLE" != /* ]]; then EXAMPLE="/examples/$EXAMPLE" fi @@ -32,24 +29,23 @@ if [[ ! -d "$EXAMPLES_DIR" ]]; then fi if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then - echo "Image $IMAGE not found. Build it first, e.g.:" >&2 - echo " docker build -t gtsam_docker:latest ." >&2 + echo "Image $IMAGE not found. Build it first." >&2 exit 1 fi -echo "Running GTSAM example in container (image: $IMAGE, script: $EXAMPLE)..." +echo "Running container validation (image: $IMAGE, script: $EXAMPLE, python: $PYTHON_CMD)..." echo "---" docker run --rm \ -v "$EXAMPLES_DIR:/examples:ro" \ "$IMAGE" \ - python3 "$EXAMPLE" + "$PYTHON_CMD" "$EXAMPLE" -EXIT_CODE=$? +status=$? echo "---" -if [[ $EXIT_CODE -eq 0 ]]; then - echo "OK: Example finished with exit code 0." +if [[ $status -eq 0 ]]; then + echo "OK: validation finished with exit code 0." else - echo "FAIL: Example exited with code $EXIT_CODE." >&2 - exit $EXIT_CODE + echo "FAIL: validation exited with code $status." >&2 + exit "$status" fi diff --git a/scripts/vuln-report.sh b/scripts/vuln-report.sh new file mode 100755 index 0000000..94080c5 --- /dev/null +++ b/scripts/vuln-report.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# +# Collect container vulnerabilities for a set of images with grype and render a +# Markdown report. This is intentionally NON-GATING: it always exits 0 (unless +# misused) so it can be wired into release CI as an informational artifact that +# is posted to the release PR rather than a pass/fail gate. +# +# Usage: +# scripts/vuln-report.sh OUTPUT.md IMAGE [IMAGE ...] +# +# Environment: +# VULN_REPORT_VERSION Optional release label shown in the report header. +# +# Requires: grype, python3. Images must be pullable (run `docker login` first +# for private registries). +set -uo pipefail + +OUT="${1:-}" +if [ -z "$OUT" ]; then + echo "usage: $0 OUTPUT.md IMAGE [IMAGE ...]" >&2 + exit 2 +fi +shift +if [ "$#" -eq 0 ]; then + echo "$0: no images supplied" >&2 + exit 2 +fi +if ! command -v grype >/dev/null 2>&1; then + echo "$0: grype is not installed" >&2 + exit 2 +fi + +workdir="$(mktemp -d)" +trap 'rm -rf "$workdir"' EXIT +manifest="$workdir/manifest.tsv" +: > "$manifest" + +i=0 +for ref in "$@"; do + i=$((i + 1)) + json="$workdir/scan-$i.json" + status="ok" + echo "Scanning $ref ..." >&2 + if ! grype "$ref" -o json -q > "$json" 2> "$workdir/err-$i.log"; then + status="error" + echo " grype failed for $ref (see log); recording as not-scanned" >&2 + fi + printf '%s\t%s\t%s\n' "$ref" "$json" "$status" >> "$manifest" +done + +GRYPE_VERSION="$(grype version 2>/dev/null | awk -F': *' '/^Version:/{print $2}')" +export GRYPE_VERSION +export REPORT_VERSION="${VULN_REPORT_VERSION:-}" +export REPORT_MANIFEST="$manifest" + +python3 - "$OUT" <<'PY' +import collections, datetime, json, os, sys + +out_path = sys.argv[1] +manifest = os.environ["REPORT_MANIFEST"] +grype_ver = os.environ.get("GRYPE_VERSION", "") or "unknown" +release = os.environ.get("REPORT_VERSION", "") + +SEV = ["Critical", "High", "Medium", "Low", "Negligible", "Unknown"] +ABBR = {"Critical": "C", "High": "H", "Medium": "M", "Low": "L", + "Negligible": "N", "Unknown": "?"} + +rows = [] # (ref, Counter or None, fixable_int) +fixable = [] # (tag, pkg, version, severity, cve, fixed_in) +totals = collections.Counter() +total_fixable = 0 +unique_cves = set() +scanned = 0 +errors = 0 + +with open(manifest) as fh: + entries = [ln.rstrip("\n").split("\t") for ln in fh if ln.strip()] + +# Sort for stable, readable output. +entries.sort(key=lambda e: e[0]) + +for ref, jpath, status in entries: + if status != "ok" or not os.path.exists(jpath) or os.path.getsize(jpath) == 0: + rows.append((ref, None, 0)) + errors += 1 + continue + try: + data = json.load(open(jpath)) + except Exception: + rows.append((ref, None, 0)) + errors += 1 + continue + scanned += 1 + counts = collections.Counter() + fx = 0 + tag = ref.rsplit(":", 1)[-1] + for m in data.get("matches", []): + v = m["vulnerability"] + a = m["artifact"] + sev = v.get("severity") or "Unknown" + if sev not in SEV: + sev = "Unknown" + counts[sev] += 1 + totals[sev] += 1 + unique_cves.add(v.get("id", "?")) + is_fixed = (v.get("fix", {}) or {}).get("state") == "fixed" + if is_fixed: + fx += 1 + total_fixable += 1 + if sev in ("Critical", "High"): + fixed_in = ",".join((v.get("fix", {}) or {}).get("versions", [])) or "-" + fixable.append((tag, a.get("name", "?"), a.get("version", "?"), + sev, v.get("id", "?"), fixed_in)) + rows.append((ref, counts, fx)) + +now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC") +total_matches = sum(totals.values()) + +lines = [] +hdr = "## Container vulnerability report" +if release: + hdr += f" — `{release}`" +lines.append(hdr) +lines.append("") +lines.append( + f"Generated by grype `{grype_ver}` on {now}. " + f"**Informational only — this report does not gate the release.**" +) +lines.append("") +lines.append( + f"**{scanned} image(s) scanned** " + + (f"({errors} not published / not scanned) " if errors else "") + + f"· **{total_matches} matches** · **{len(unique_cves)} unique CVEs** " + f"· **{total_fixable} fixable**" +) +lines.append("") +sev_summary = " · ".join(f"{s} {totals.get(s, 0)}" for s in SEV) +lines.append(f"Severity totals: {sev_summary}") +lines.append("") + +# Per-image table. +header = "| Image | " + " | ".join(ABBR[s] for s in SEV) + " | Fixable |" +sep = "|---|" + "".join("--:|" for _ in SEV) + "--:|" +lines.append(header) +lines.append(sep) +for ref, counts, fx in rows: + tag = ref.rsplit(":", 1)[-1] + if counts is None: + lines.append(f"| `{tag}` | " + " | ".join("–" for _ in SEV) + " | – | _(not scanned)_") + continue + cells = " | ".join(str(counts.get(s, 0)) for s in SEV) + lines.append(f"| `{tag}` | {cells} | {fx} |") +lines.append("") +lines.append( + "Legend: **C**ritical · **H**igh · **M**edium · **L**ow · " + "**N**egligible · **?** Unknown." +) +lines.append("") + +# Fixable Critical/High detail. +lines.append("
") +if fixable: + lines.append(f"Fixable Critical/High vulnerabilities ({len(fixable)})") + lines.append("") + lines.append("| Image tag | Package | Installed | Severity | Vulnerability | Fixed in |") + lines.append("|---|---|---|---|---|---|") + fixable.sort(key=lambda r: (SEV.index(r[3]), r[0], r[1])) + for tag, pkg, ver, sev, cve, fixed_in in fixable: + lines.append(f"| `{tag}` | {pkg} | {ver} | {sev} | {cve} | {fixed_in} |") +else: + lines.append("Fixable Critical/High vulnerabilities (0)") + lines.append("") + lines.append("No fixable Critical/High vulnerabilities found " + "(remaining findings are either lower severity or have no vendor fix available).") +lines.append("
") +lines.append("") + +with open(out_path, "w") as fh: + fh.write("\n".join(lines) + "\n") + +print(f"Wrote {out_path}: {scanned} scanned, {total_matches} matches, " + f"{total_fixable} fixable, {errors} not scanned.") +PY