diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml new file mode 100644 index 0000000..b7cb979 --- /dev/null +++ b/.github/workflows/security-scan.yml @@ -0,0 +1,59 @@ +name: Security Scan + +on: + pull_request: + branches: [main] + push: + branches: [main] + schedule: + # Weekly rescan so newly published CVEs surface without a code change. + - cron: '0 6 * * 1' + +permissions: + contents: read + security-events: write + +jobs: + fs-scan: + name: Repo filesystem scan (deps + secrets + config) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Trivy filesystem scan + uses: aquasecurity/trivy-action@0.28.0 + with: + scan-type: fs + scan-ref: . + severity: CRITICAL,HIGH + exit-code: '1' + ignore-unfixed: true + format: sarif + output: trivy-fs.sarif + + - name: Upload SARIF + if: always() + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: trivy-fs.sarif + + api-image-scan: + name: API image scan + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build API image + run: docker build -t openprocessor-api:scan -f Dockerfile . + + - name: Trivy image scan + uses: aquasecurity/trivy-action@0.28.0 + with: + image-ref: openprocessor-api:scan + severity: CRITICAL,HIGH + exit-code: '1' + ignore-unfixed: true + + # The Triton image is ~20 GB (NGC base) — too heavy for hosted runners' + # disk on every PR. It is scanned locally via `make scan-triton` before + # any Docker Hub publish instead. diff --git a/Dockerfile b/Dockerfile index 7cf66e6..a789ec5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,8 +18,11 @@ FROM python:3.13-slim-trixie AS builder WORKDIR /build -# Install build dependencies (isolated to this stage) -RUN apt-get update && apt-get install -y --no-install-recommends \ +# Install build dependencies (isolated to this stage). +# apt-get upgrade pulls Debian point-release security fixes the base tag +# may lag behind. +RUN apt-get update && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends \ build-essential \ gcc \ g++ \ @@ -48,8 +51,10 @@ LABEL org.opencontainers.image.title="OpenProcessor FastAPI Service" \ org.opencontainers.image.source="https://github.com/davidamacey/OpenProcessor" \ org.opencontainers.image.documentation="https://github.com/davidamacey/OpenProcessor/blob/main/README.md" -# Runtime-only system packages (no build tools) -RUN apt-get update && apt-get install -y --no-install-recommends \ +# Runtime-only system packages (no build tools); upgrade first for +# Debian point-release security fixes. +RUN apt-get update && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends \ curl \ jq \ libgl1 \ diff --git a/Dockerfile.triton b/Dockerfile.triton index 544cde1..29db736 100644 --- a/Dockerfile.triton +++ b/Dockerfile.triton @@ -1,11 +1,15 @@ # ============================================================================= # Triton Inference Server with PyTorch for Python Backend (BLS) # ============================================================================= -# Based on NVIDIA Triton 25.10 with PyTorch + TorchVision for Python backend -# models (ocr_pipeline). +# Based on NVIDIA Triton 26.06 (CUDA 13.3, TensorRT 11.0, Ubuntu 24.04) with +# PyTorch + TorchVision for Python backend models (ocr_pipeline). +# +# NOTE: TensorRT engines (.plan) are bound to the TensorRT major version. +# Moving between Triton releases that change TRT versions requires +# re-exporting every engine — see docs/MIGRATION_TRITON_26.md. # ============================================================================= -FROM nvcr.io/nvidia/tritonserver:25.10-py3 +FROM nvcr.io/nvidia/tritonserver:26.06-py3 LABEL org.opencontainers.image.title="OpenProcessor Inference Server" \ org.opencontainers.image.description="NVIDIA Triton Inference Server with TensorRT models for visual AI" \ @@ -15,18 +19,30 @@ LABEL org.opencontainers.image.title="OpenProcessor Inference Server" \ org.opencontainers.image.source="https://github.com/davidamacey/OpenProcessor" \ org.opencontainers.image.documentation="https://github.com/davidamacey/OpenProcessor/blob/main/README.md" -# Install PyTorch and TorchVision (CUDA 12.x compatible) +# Pull in the latest Ubuntu security patches on every build; the NGC base +# lags point releases between monthly tags. +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install PyTorch and TorchVision (CUDA 13.x wheels) # Required for Python backend models (ROI align, per-box embeddings, OCR pipeline) -# Note: torch/torchvision use the PyTorch CUDA 12.4 wheel index RUN pip install --no-cache-dir \ - torch==2.5.1 \ - torchvision==0.20.1 \ - --index-url https://download.pytorch.org/whl/cu124 \ - && pip install --no-cache-dir opencv-python-headless==4.10.0.84 \ + torch==2.12.1 \ + torchvision==0.27.1 \ + --index-url https://download.pytorch.org/whl/cu130 \ + && pip install --no-cache-dir opencv-python-headless==4.13.0.92 \ && python3 -c "import torch; print(f'PyTorch {torch.__version__} installed, CUDA: {torch.cuda.is_available()}')" \ && python3 -c "import torchvision; print(f'TorchVision {torchvision.__version__} installed')" HEALTHCHECK --interval=30s --timeout=10s --start-period=120s --retries=3 \ CMD curl -f http://localhost:8000/v2/health/ready || exit 1 +# Run as the unprivileged user shipped with the NGC image (uid 1000). +# Triton's ports (8000-8002) are unprivileged and /models is a bind mount, +# so root is unnecessary. If the base image drops this user, the build +# fails loudly here rather than silently reverting to root. +USER triton-server + CMD ["tritonserver", "--model-store=/models"] diff --git a/INSTALLATION.md b/INSTALLATION.md index d7814e8..41351be 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -397,7 +397,7 @@ The project includes production-optimized Dockerfiles: | File | Purpose | Base Image | |------|---------|------------| | `Dockerfile` | FastAPI service | `python:3.13-slim-trixie` | -| `Dockerfile.triton` | Triton server | `nvcr.io/nvidia/tritonserver:25.10-py3` | +| `Dockerfile.triton` | Triton server | `nvcr.io/nvidia/tritonserver:26.06-py3` | ### Building Images diff --git a/Makefile b/Makefile index 94094ea..a98b18e 100644 --- a/Makefile +++ b/Makefile @@ -833,6 +833,22 @@ check-all: ## Full system health check (API + Triton + OpenSearch) @echo "" @echo "===================================================================================" +# ================================================================================== +# Security Scanning +# ================================================================================== + +.PHONY: scan +scan: ## Scan both Docker images for vulnerabilities (trivy/grype/dockle; FAIL_ON_CRITICAL=true) + bash $(SCRIPTS_DIR)/security-scan.sh all + +.PHONY: scan-api +scan-api: ## Scan only the API image + bash $(SCRIPTS_DIR)/security-scan.sh api + +.PHONY: scan-triton +scan-triton: ## Scan only the Triton image + bash $(SCRIPTS_DIR)/security-scan.sh triton + # ================================================================================== # Cleanup # ================================================================================== diff --git a/docker-compose.yml b/docker-compose.yml index 06dd323..08f7eff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -124,7 +124,7 @@ services: - triton_net triton-sdk: - image: nvcr.io/nvidia/tritonserver:25.10-py3-sdk + image: nvcr.io/nvidia/tritonserver:26.06-py3-sdk container_name: triton-sdk profiles: - benchmark @@ -136,7 +136,7 @@ services: - triton_net node-exporter: - image: prom/node-exporter:latest + image: prom/node-exporter:v1.10.2 container_name: triton-node-exporter restart: always command: @@ -170,7 +170,7 @@ services: - triton_net prometheus: - image: prom/prometheus:latest + image: prom/prometheus:v3.12.0 container_name: triton-prometheus restart: always ports: @@ -191,7 +191,7 @@ services: - node-exporter grafana: - image: grafana/grafana:latest + image: grafana/grafana:13.1.0 container_name: triton-grafana restart: always ports: @@ -216,10 +216,12 @@ services: - loki loki: - image: grafana/loki:latest + # Runs as the image's builtin non-root user (uid 10001). Existing + # root-owned loki_data volumes need a one-time chown — see + # docs/MIGRATION_TRITON_26.md. + image: grafana/loki:3.6.12 container_name: triton-loki restart: always - user: "0" # Run as root to avoid permission issues ports: - 4606:3100 # Loki volumes: @@ -229,15 +231,19 @@ services: networks: - triton_net - promtail: - image: grafana/promtail:latest - container_name: triton-promtail +# Log shipper: Grafana Alloy (Promtail reached EOL 2026-03-02). + alloy: + image: grafana/alloy:v1.17.1 + container_name: triton-alloy restart: always volumes: - - ./monitoring/promtail-config.yml:/etc/promtail/config.yml - - /var/run/docker.sock:/var/run/docker.sock + - ./monitoring/alloy-config.alloy:/etc/alloy/config.alloy:ro + - /var/run/docker.sock:/var/run/docker.sock:ro - /var/lib/docker/containers:/var/lib/docker/containers:ro - command: -config.file=/etc/promtail/config.yml + command: + - run + - /etc/alloy/config.alloy + - --storage.path=/tmp/alloy networks: - triton_net depends_on: @@ -248,7 +254,7 @@ services: # OpenSearch 3.x with k-NN plugin for vector similarity search # =================================================================== opensearch: - image: opensearchproject/opensearch:3.3.1 + image: opensearchproject/opensearch:3.6.0 container_name: triton-opensearch restart: always environment: @@ -276,7 +282,7 @@ services: - triton_net opensearch-dashboards: - image: opensearchproject/opensearch-dashboards:3.3.0 + image: opensearchproject/opensearch-dashboards:3.6.0 container_name: triton-opensearch-dashboards restart: always environment: diff --git a/docs/MIGRATION_TRITON_26.md b/docs/MIGRATION_TRITON_26.md new file mode 100644 index 0000000..d576f22 --- /dev/null +++ b/docs/MIGRATION_TRITON_26.md @@ -0,0 +1,92 @@ +# Migration Guide: Triton 26.06 / TensorRT 11 Upgrade + +This release moves the stack from Triton 25.10 (TensorRT 10.13) to +**Triton 26.06 (CUDA 13.3, TensorRT 11.0)** and pins every container in +`docker-compose.yml`. Existing deployments need the one-time steps below. + +## 1. Re-export ALL TensorRT engines (required) + +TensorRT serialized engines (`.plan`) are **not portable across TensorRT +major versions**. Every engine built under 25.10 (TRT 10.x) fails to load +on 26.06 (TRT 11.x) with a serialization-version error. + +```bash +# Rebuild images first +docker compose build + +# Start only what exports need +docker compose up -d triton-server yolo-api + +# Re-export every model (runs inside the API container) +docker compose exec yolo-api python /app/export/export_models.py --formats onnx_end2end trt_end2end +docker compose exec yolo-api python /app/export/export_scrfd.py +docker compose exec yolo-api python /app/export/export_face_recognition.py +docker compose exec yolo-api python /app/export/export_mobileclip_image_encoder.py +docker compose exec yolo-api python /app/export/export_mobileclip_text_encoder.py +docker compose exec yolo-api python /app/export/export_paddleocr_det.py +docker compose exec yolo-api python /app/export/export_paddleocr_rec.py + +# Restart Triton to load the new engines +docker compose restart triton-server +``` + +Notes: + +- The client-side `tensorrt-cu13==11.0.0.114` pin **must** match the TRT + bundled in the Triton image. Do not bump it independently; upgrade it + together with the Triton base tag. +- GPU requirements: driver R570+ (CUDA 13.x) and compute capability >= 7.5 + (Turing or newer). Volta/Pascal are no longer supported by this Triton + release line. + +## 2. Loki data volume ownership (one-time) + +Loki previously ran as root (`user: "0"`); it now runs as the image's +builtin non-root user (uid 10001). Existing volumes are root-owned and +will fail with permission errors until chowned: + +```bash +docker compose stop loki +docker run --rm -v openprocessor_loki_data:/loki alpine chown -R 10001:10001 /loki +docker compose up -d loki +``` + +Fresh installs need nothing. + +## 3. Promtail → Grafana Alloy + +Promtail reached end-of-life on 2026-03-02 and has been replaced by +**Grafana Alloy** (`monitoring/alloy-config.alloy`). The shipped pipeline +is equivalent (Docker service discovery for the Triton + API containers). + +If you customized `monitoring/promtail-config.yml`, convert it: + +```bash +docker run --rm -v $(pwd)/monitoring:/cfg grafana/alloy:v1.17.1 \ + convert --source-format=promtail --output=/cfg/alloy-config.alloy /cfg/promtail-config.yml +``` + +## 4. Health endpoint semantics + +- `/live` — process liveness only (new; used by the container HEALTHCHECK). +- `/ready` — actively probes Triton (gRPC `is_server_live`) and OpenSearch; + returns **503 with per-service detail** while any dependency is down. +- `/health` — now an alias of `/ready`. If you previously treated `/health` + as an always-200 liveness signal, point that consumer at `/live`. + +## 5. Pinned image matrix + +| Service | Image | +|---|---| +| triton-server | `nvcr.io/nvidia/tritonserver:26.06-py3` (via `Dockerfile.triton`) | +| opensearch | `opensearchproject/opensearch:3.6.0` | +| opensearch-dashboards | `opensearchproject/opensearch-dashboards:3.6.0` | +| prometheus | `prom/prometheus:v3.12.0` | +| grafana | `grafana/grafana:13.1.0` | +| loki | `grafana/loki:3.6.12` (non-root) | +| alloy (replaces promtail) | `grafana/alloy:v1.17.1` | +| node-exporter | `prom/node-exporter:v1.10.2` | +| dcgm-exporter | `nvcr.io/nvidia/k8s/dcgm-exporter:3.3.5-3.4.0-ubuntu22.04` | + +OpenSearch 3.3 → 3.6 is a same-major upgrade; existing indices roll +forward in place. Take a snapshot first if the data matters. diff --git a/export/export_face_recognition.py b/export/export_face_recognition.py index 7554cf1..4e5286c 100755 --- a/export/export_face_recognition.py +++ b/export/export_face_recognition.py @@ -277,12 +277,13 @@ def convert_to_tensorrt( try: import tensorrt as trt + from trt_utils import create_explicit_network TRT_LOGGER = trt.Logger(trt.Logger.WARNING) # Create builder builder = trt.Builder(TRT_LOGGER) - network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)) + network = create_explicit_network(builder) parser = trt.OnnxParser(network, TRT_LOGGER) # Parse ONNX diff --git a/export/export_mobileclip_image_encoder.py b/export/export_mobileclip_image_encoder.py index be75287..7f921d0 100755 --- a/export/export_mobileclip_image_encoder.py +++ b/export/export_mobileclip_image_encoder.py @@ -60,7 +60,7 @@ EMBEDDING_DIM = 512 # MobileCLIP2-S2 uses 512-dim embeddings # ONNX export settings -# TensorRT 10.x (Triton 25.10) supports opset 9-20 +# TensorRT 11.x (Triton 26.06) supports opset 9-20 # For Transformer/LayerNorm models with dynamic batch: opset 17+ recommended # See: https://docs.nvidia.com/deeplearning/tensorrt/latest/getting-started/support-matrix.html # See: https://torchpipe.github.io/docs/faq/onnx @@ -290,12 +290,13 @@ def convert_to_tensorrt(onnx_path, plan_path, fp16=True, max_batch_size=128): try: import tensorrt as trt + from trt_utils import create_explicit_network TRT_LOGGER = trt.Logger(trt.Logger.WARNING) # Create builder builder = trt.Builder(TRT_LOGGER) - network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)) + network = create_explicit_network(builder) parser = trt.OnnxParser(network, TRT_LOGGER) # Parse ONNX diff --git a/export/export_mobileclip_text_encoder.py b/export/export_mobileclip_text_encoder.py index 0c2ec3d..67de255 100755 --- a/export/export_mobileclip_text_encoder.py +++ b/export/export_mobileclip_text_encoder.py @@ -51,7 +51,7 @@ EMBEDDING_DIM = 512 # MobileCLIP2-S2 uses 512-dim embeddings (same as image encoder) # ONNX export settings -# TensorRT 10.x (Triton 25.10) supports opset 9-20 +# TensorRT 11.x (Triton 26.06) supports opset 9-20 # For Transformer/LayerNorm models with dynamic batch: opset 17+ recommended # See: https://docs.nvidia.com/deeplearning/tensorrt/latest/getting-started/support-matrix.html ONNX_OPSET_VERSION = 17 @@ -266,11 +266,12 @@ def convert_to_tensorrt(onnx_path, plan_path, fp16=True, max_batch_size=64): try: import tensorrt as trt + from trt_utils import create_explicit_network TRT_LOGGER = trt.Logger(trt.Logger.WARNING) builder = trt.Builder(TRT_LOGGER) - network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)) + network = create_explicit_network(builder) parser = trt.OnnxParser(network, TRT_LOGGER) # Parse ONNX diff --git a/export/export_models.py b/export/export_models.py index a108013..6c63b72 100755 --- a/export/export_models.py +++ b/export/export_models.py @@ -54,6 +54,7 @@ import onnx # noqa: E402 import tensorrt as trt # noqa: E402 import torch # noqa: E402 +from trt_utils import create_explicit_network # noqa: E402 from ultralytics import YOLO # noqa: E402 from ultralytics.cfg import get_cfg # noqa: E402 from ultralytics.engine.exporter import Exporter # noqa: E402 @@ -185,17 +186,11 @@ def setup_trt_builder( builder = trt.Builder(trt_logger) config = builder.create_builder_config() - # Set workspace size + # Set workspace size (TRT >= 10 memory-pool API; requirements pin TRT 11) workspace_bytes = int(workspace_gb * (1 << 30)) - is_trt10 = int(trt.__version__.split('.')[0]) >= 10 - if is_trt10: - config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, workspace_bytes) - else: - config.max_workspace_size = workspace_bytes + config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, workspace_bytes) - # Create network with explicit batch flag - flag = 1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH) - network = builder.create_network(flag) + network = create_explicit_network(builder) return builder, config, network, trt_logger @@ -635,23 +630,13 @@ def build_and_save_engine( logger.info('Building TensorRT engine (this may take 5-10 minutes)...') - is_trt10 = int(trt.__version__.split('.')[0]) >= 10 - try: - if is_trt10: - serialized_engine = builder.build_serialized_network(network, config) - if serialized_engine is None: - logger.error('Failed to build engine - builder returned None') - return False - with open(output_path, 'wb') as f: - f.write(serialized_engine) - else: - engine = builder.build_engine(network, config) - if engine is None: - logger.error('Failed to build engine - builder returned None') - return False - with open(output_path, 'wb') as f: - f.write(engine.serialize()) + serialized_engine = builder.build_serialized_network(network, config) + if serialized_engine is None: + logger.error('Failed to build engine - builder returned None') + return False + with open(output_path, 'wb') as f: + f.write(serialized_engine) file_size_mb = output_path.stat().st_size / (1024 * 1024) logger.info(f'Engine saved: {output_path} ({file_size_mb:.2f} MB)') diff --git a/export/export_paddleocr_det.py b/export/export_paddleocr_det.py index fcf6f7c..5e484b4 100755 --- a/export/export_paddleocr_det.py +++ b/export/export_paddleocr_det.py @@ -122,12 +122,13 @@ def convert_to_tensorrt( try: import tensorrt as trt + from trt_utils import create_explicit_network TRT_LOGGER = trt.Logger(trt.Logger.WARNING) # Create builder builder = trt.Builder(TRT_LOGGER) - network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)) + network = create_explicit_network(builder) parser = trt.OnnxParser(network, TRT_LOGGER) # Parse ONNX diff --git a/export/export_scrfd.py b/export/export_scrfd.py index 5191c1a..a2bc09d 100755 --- a/export/export_scrfd.py +++ b/export/export_scrfd.py @@ -442,12 +442,13 @@ def convert_to_tensorrt( try: import tensorrt as trt + from trt_utils import create_explicit_network trt.init_libnvinfer_plugins(None, '') TRT_LOGGER = trt.Logger(trt.Logger.WARNING) builder = trt.Builder(TRT_LOGGER) - network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)) + network = create_explicit_network(builder) parser = trt.OnnxParser(network, TRT_LOGGER) logger.info(' Parsing ONNX...') diff --git a/export/trt_utils.py b/export/trt_utils.py new file mode 100644 index 0000000..78c9259 --- /dev/null +++ b/export/trt_utils.py @@ -0,0 +1,22 @@ +"""Shared TensorRT builder helpers for the export scripts. + +Centralizes the version-sensitive bits of engine building so the per-model +export scripts don't each carry their own compatibility branches. +""" + +from __future__ import annotations + +import tensorrt as trt + + +def create_explicit_network(builder: trt.Builder) -> trt.INetworkDefinition: + """Create an explicit-batch network across TensorRT versions. + + TRT >= 10 networks are always explicit-batch; the + ``NetworkDefinitionCreationFlag.EXPLICIT_BATCH`` flag was deprecated in + 10.x and removed in newer majors. Pass it only where it still exists. + """ + flag = getattr(trt.NetworkDefinitionCreationFlag, 'EXPLICIT_BATCH', None) + if flag is not None: + return builder.create_network(1 << int(flag)) + return builder.create_network(0) diff --git a/monitoring/README.md b/monitoring/README.md index f9cfa75..5d858b0 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -8,7 +8,7 @@ Complete production-grade monitoring for NVIDIA Triton Inference Server. - **Node Exporter**: System-level CPU, memory, and hardware metrics - **Grafana**: Visualization dashboards - **Loki**: Log aggregation -- **Promtail**: Log shipping agent +- **Alloy**: Log shipping agent (Grafana Alloy; replaced EOL Promtail) ## Quick Start @@ -17,7 +17,7 @@ Complete production-grade monitoring for NVIDIA Triton Inference Server. docker compose up -d # Check monitoring services -docker compose ps prometheus grafana loki promtail +docker compose ps prometheus grafana loki alloy # View logs docker compose logs -f grafana @@ -323,14 +323,11 @@ curl http://localhost:4602/metrics ### Loki not receiving logs ```bash -# Check Promtail logs -docker compose logs promtail +# Check Alloy logs +docker compose logs alloy # Verify Loki is ready curl http://localhost:4606/ready - -# Check Promtail targets -curl http://localhost:9080/targets ``` ### Missing metrics in Grafana @@ -408,7 +405,7 @@ monitoring/ ├── grafana-dashboards.yml # Dashboard auto-provisioning config ├── grafana-alerting.yml # Grafana unified alerting rules & notifications ├── loki-config.yml # Loki log aggregation config -├── promtail-config.yml # Promtail log shipping config +├── alloy-config.alloy # Grafana Alloy log shipping config ├── alerts/ │ └── triton-alerts.yml # Prometheus alert rules (legacy) └── dashboards/ diff --git a/monitoring/alloy-config.alloy b/monitoring/alloy-config.alloy new file mode 100644 index 0000000..27f8f75 --- /dev/null +++ b/monitoring/alloy-config.alloy @@ -0,0 +1,76 @@ +// Grafana Alloy configuration — ships container logs to Loki. +// +// Replaces the former Promtail setup (Promtail is EOL since 2026-03-02). +// Same topology: discover containers over the Docker socket, keep only +// the Triton and FastAPI services, label with container + stream, and +// push to Loki. Regexes use a trailing wildcard so renamed variants +// (e.g. compose-project prefixes/suffixes) still match. + +discovery.docker "containers" { + host = "unix:///var/run/docker.sock" + refresh_interval = "5s" +} + +// Triton server logs +discovery.relabel "triton" { + targets = discovery.docker.containers.targets + + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/triton-server.*" + action = "keep" + } + rule { + source_labels = ["__meta_docker_container_name"] + target_label = "container" + } + rule { + source_labels = ["__meta_docker_container_log_stream"] + target_label = "stream" + } + rule { + target_label = "job" + replacement = "triton" + } +} + +// FastAPI logs +discovery.relabel "fastapi" { + targets = discovery.docker.containers.targets + + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(yolo-api|pytorch-api).*" + action = "keep" + } + rule { + source_labels = ["__meta_docker_container_name"] + target_label = "container" + } + rule { + source_labels = ["__meta_docker_container_log_stream"] + target_label = "stream" + } + rule { + target_label = "job" + replacement = "fastapi" + } +} + +loki.source.docker "triton" { + host = "unix:///var/run/docker.sock" + targets = discovery.relabel.triton.output + forward_to = [loki.write.default.receiver] +} + +loki.source.docker "fastapi" { + host = "unix:///var/run/docker.sock" + targets = discovery.relabel.fastapi.output + forward_to = [loki.write.default.receiver] +} + +loki.write "default" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} diff --git a/monitoring/promtail-config.yml b/monitoring/promtail-config.yml deleted file mode 100644 index 48faa9e..0000000 --- a/monitoring/promtail-config.yml +++ /dev/null @@ -1,38 +0,0 @@ -server: - http_listen_port: 9080 - grpc_listen_port: 0 - -positions: - filename: /tmp/positions.yaml - -clients: - - url: http://loki:3100/loki/api/v1/push - -scrape_configs: - # Triton server logs - - job_name: triton - docker_sd_configs: - - host: unix:///var/run/docker.sock - refresh_interval: 5s - relabel_configs: - - source_labels: ['__meta_docker_container_name'] - regex: '/triton-server' - action: keep - - source_labels: ['__meta_docker_container_name'] - target_label: container - - source_labels: ['__meta_docker_container_log_stream'] - target_label: stream - - # FastAPI logs - - job_name: fastapi - docker_sd_configs: - - host: unix:///var/run/docker.sock - refresh_interval: 5s - relabel_configs: - - source_labels: ['__meta_docker_container_name'] - regex: '/(yolo-api|pytorch-api)' - action: keep - - source_labels: ['__meta_docker_container_name'] - target_label: container - - source_labels: ['__meta_docker_container_log_stream'] - target_label: stream diff --git a/pyproject.toml b/pyproject.toml index 3514fcf..9aeac7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ dependencies = [ "python-multipart", "orjson", "pydantic-settings", - "ultralytics", + "ultralytics<8.4", "tritonclient[all]", "onnx>=1.12.0,<=1.19.1", "onnxslim>=0.1.71", @@ -37,7 +37,7 @@ dependencies = [ "onnxscript", "onnxruntime-gpu", "onnx-graphsurgeon", - "tensorrt-cu12==10.13.3.9", + "tensorrt-cu13==11.0.0.114", "nvidia-dali-cuda120", "torch", "torchvision", diff --git a/requirements.txt b/requirements.txt index 0df7831..b7db66e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ onnxsim>=0.4.33 onnxscript onnxruntime-gpu onnx-graphsurgeon # Required for End2End TensorRT NMS graph manipulation -tensorrt-cu12==10.13.3.9 # MUST match Triton 25.10 TensorRT version (10.13.3.9) for engine compatibility +tensorrt-cu13==11.0.0.114 # MUST match Triton 26.06 TensorRT version (11.0.0.114) for engine compatibility — do NOT bump to 11.1.x independently # PyTorch torch diff --git a/scripts/lib/gpu.sh b/scripts/lib/gpu.sh index f776002..cd6bad0 100755 --- a/scripts/lib/gpu.sh +++ b/scripts/lib/gpu.sh @@ -171,8 +171,8 @@ validate_cuda() { local major_version major_version=$(echo "$driver_version" | cut -d'.' -f1) - # Triton 25.10 requires driver >= 535 - local min_driver=535 + # Triton 26.06 (CUDA 13.x) requires driver >= 570 + local min_driver=570 if [[ "$major_version" -lt "$min_driver" ]]; then log_error "NVIDIA driver $driver_version is too old."