diff --git a/.github/scripts/deploy-release.sh b/.github/scripts/deploy-release.sh new file mode 100755 index 000000000..d4205b2b9 --- /dev/null +++ b/.github/scripts/deploy-release.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +release_tag=${1:?release tag is required} +deploy_dir=${2:?deployment directory is required} +source_dir=${3:?source directory is required} +pip_index_url=${4:-https://mirrors.aliyun.com/pypi/simple/} +pip_trusted_host=${5:-} + +case "$deploy_dir" in + /*) ;; + *) deploy_dir="$PWD/$deploy_dir" ;; +esac +case "$source_dir" in + /*) ;; + *) source_dir="$PWD/$source_dir" ;; +esac + +if ! [[ "$release_tag" =~ ^v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.]+)?$ ]]; then + echo "Invalid release tag: $release_tag" >&2 + exit 1 +fi + +cd "$source_dir" +git rev-parse --is-inside-work-tree >/dev/null + +if [ -n "$(git status --porcelain --untracked-files=no)" ]; then + echo "Source contains tracked local changes; refusing to overwrite production." >&2 + git status --short --untracked-files=no >&2 + exit 1 +fi + +if docker version >/dev/null 2>&1; then + docker_cmd=(docker) +elif sudo -n docker version >/dev/null 2>&1; then + docker_cmd=(sudo -n docker) +else + echo "Docker is unavailable to the deployment user." >&2 + exit 1 +fi +compose=("${docker_cmd[@]}" compose) +app_services=(backend-api backend-worker frontend) + +cd "$deploy_dir" +test -f .env +"${compose[@]}" config --quiet + +backend_container=$("${compose[@]}" ps -q backend-api) +worker_container=$("${compose[@]}" ps -q backend-worker) +frontend_container=$("${compose[@]}" ps -q frontend) +test -n "$backend_container" +test -n "$worker_container" +test -n "$frontend_container" + +old_backend_image=$("${docker_cmd[@]}" inspect -f '{{.Image}}' "$backend_container") +old_worker_image=$("${docker_cmd[@]}" inspect -f '{{.Image}}' "$worker_container") +old_frontend_image=$("${docker_cmd[@]}" inspect -f '{{.Image}}' "$frontend_container") +if [ "$old_backend_image" != "$old_worker_image" ]; then + echo "Backend API and worker do not use the same image; refusing deployment." >&2 + exit 1 +fi + +cd "$source_dir" +previous_commit=$(git rev-parse HEAD) +activation_started=false + +rollback() { + status=$? + trap - EXIT + if [ "$status" -ne 0 ]; then + echo "Deployment failed; restoring source $previous_commit." >&2 + set +e + cd "$source_dir" + git checkout --detach "$previous_commit" + if [ "$activation_started" = true ]; then + "${docker_cmd[@]}" tag "$old_backend_image" clawith-backend:latest + "${docker_cmd[@]}" tag "$old_frontend_image" clawith-frontend:latest + cd "$deploy_dir" + "${compose[@]}" up -d --no-deps --force-recreate "${app_services[@]}" + fi + fi + exit "$status" +} +trap rollback EXIT + +git fetch origin "refs/heads/main:refs/remotes/origin/main" +git fetch origin "refs/tags/$release_tag:refs/tags/$release_tag" + +release_commit=$(git rev-parse "$release_tag^{commit}") +if ! git merge-base --is-ancestor "$release_commit" origin/main; then + echo "Release $release_tag is not contained in origin/main." >&2 + exit 1 +fi + +git checkout --detach "$release_tag" + +backend_release_image="clawith-backend:$release_tag" +frontend_release_image="clawith-frontend:$release_tag" + +"${docker_cmd[@]}" build \ + --build-arg "CLAWITH_PIP_INDEX_URL=$pip_index_url" \ + --build-arg "CLAWITH_PIP_TRUSTED_HOST=$pip_trusted_host" \ + -t "$backend_release_image" \ + backend +"${docker_cmd[@]}" build -t "$frontend_release_image" frontend + +activation_started=true +"${docker_cmd[@]}" tag "$backend_release_image" clawith-backend:latest +"${docker_cmd[@]}" tag "$frontend_release_image" clawith-frontend:latest + +cd "$deploy_dir" +"${compose[@]}" up -d --no-deps --force-recreate "${app_services[@]}" + +expected_backend_id=$("${docker_cmd[@]}" image inspect -f '{{.Id}}' "$backend_release_image") +expected_frontend_id=$("${docker_cmd[@]}" image inspect -f '{{.Id}}' "$frontend_release_image") +running_api_id=$("${docker_cmd[@]}" inspect -f '{{.Image}}' "$("${compose[@]}" ps -q backend-api)") +running_worker_id=$("${docker_cmd[@]}" inspect -f '{{.Image}}' "$("${compose[@]}" ps -q backend-worker)") +running_frontend_id=$("${docker_cmd[@]}" inspect -f '{{.Image}}' "$("${compose[@]}" ps -q frontend)") + +if [ "$running_api_id" != "$expected_backend_id" ] || + [ "$running_worker_id" != "$expected_backend_id" ] || + [ "$running_frontend_id" != "$expected_frontend_id" ]; then + echo "Running containers do not use the newly built release images." >&2 + exit 1 +fi + +published_address=$("${compose[@]}" port frontend 3000 | tail -n 1) +frontend_port=${published_address##*:} +if ! [[ "$frontend_port" =~ ^[0-9]+$ ]]; then + echo "Unable to resolve the published frontend port." >&2 + exit 1 +fi + +expected_version=${release_tag#v} +health_response= +for attempt in $(seq 1 24); do + if health_response=$(curl -fsS --max-time 5 "http://127.0.0.1:$frontend_port/api/health"); then + health_status=$(printf '%s' "$health_response" | sed -n 's/.*"status"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p') + running_version=$(printf '%s' "$health_response" | sed -n 's/.*"version"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p') + if [ "$health_status" = ok ] && [ "$running_version" = "$expected_version" ]; then + echo "Successfully deployed $release_tag ($release_commit)." + "${compose[@]}" ps + activation_started=false + exit 0 + fi + fi + sleep 5 +done + +echo "Production health check failed after 120 seconds." >&2 +echo "Last health response: ${health_response:-}" >&2 +"${compose[@]}" ps >&2 +"${compose[@]}" logs --tail 200 backend frontend >&2 +exit 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 57409061e..c06d09b89 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -443,6 +443,8 @@ jobs: name: Publish release runs-on: ubuntu-latest if: github.event_name == 'pull_request' && github.event.pull_request.merged == true && startsWith(github.event.pull_request.head.ref, 'release/v') + outputs: + tag: ${{ steps.release_info.outputs.tag }} steps: - name: Checkout source uses: actions/checkout@v7 @@ -497,3 +499,77 @@ jobs: body_path: release-notes.extracted.md prerelease: ${{ github.event.pull_request.draft }} generate_release_notes: false + + deploy_release: + name: Deploy release to production + needs: publish_release + if: ${{ needs.publish_release.result == 'success' }} + runs-on: ubuntu-latest + environment: production + env: + DEPLOY_HOST: ${{ vars.CLAWITH_DEPLOY_HOST }} + DEPLOY_USER: ${{ vars.CLAWITH_DEPLOY_USER }} + DEPLOY_PORT: ${{ vars.CLAWITH_DEPLOY_PORT || '10022' }} + DEPLOY_PATH: ${{ vars.CLAWITH_DEPLOY_PATH || 'clawith_new' }} + SOURCE_PATH: ${{ vars.CLAWITH_SOURCE_PATH || 'Clawith' }} + PIP_INDEX_URL: ${{ vars.CLAWITH_PIP_INDEX_URL || 'https://mirrors.aliyun.com/pypi/simple/' }} + PIP_TRUSTED_HOST: ${{ vars.CLAWITH_PIP_TRUSTED_HOST }} + RELEASE_TAG: ${{ needs.publish_release.outputs.tag }} + SSH_PRIVATE_KEY: ${{ secrets.CLAWITH_DEPLOY_SSH_KEY }} + SSH_KNOWN_HOSTS: ${{ secrets.CLAWITH_DEPLOY_KNOWN_HOSTS }} + steps: + - name: Checkout deployment tooling + uses: actions/checkout@v7 + + - name: Validate deployment configuration + shell: bash + run: | + set -euo pipefail + + required=( + DEPLOY_HOST DEPLOY_USER DEPLOY_PORT DEPLOY_PATH SOURCE_PATH RELEASE_TAG + SSH_PRIVATE_KEY SSH_KNOWN_HOSTS + ) + for name in "${required[@]}"; do + if [ -z "${!name}" ]; then + echo "::error::Missing required production deployment setting: $name" + exit 1 + fi + done + + if ! [[ "$DEPLOY_PORT" =~ ^[0-9]+$ ]] || [ "$DEPLOY_PORT" -lt 1 ] || [ "$DEPLOY_PORT" -gt 65535 ]; then + echo "::error::CLAWITH_DEPLOY_PORT must be a valid TCP port" + exit 1 + fi + + - name: Configure SSH + shell: bash + run: | + set -euo pipefail + + install -d -m 0700 "$HOME/.ssh" + printf '%s\n' "$SSH_KNOWN_HOSTS" > "$HOME/.ssh/known_hosts" + chmod 0600 "$HOME/.ssh/known_hosts" + + key_file="$RUNNER_TEMP/clawith-deploy-key" + printf '%s\n' "$SSH_PRIVATE_KEY" > "$key_file" + chmod 0600 "$key_file" + echo "SSH_KEY_FILE=$key_file" >> "$GITHUB_ENV" + + - name: Update, build, and restart production + shell: bash + run: | + set -euo pipefail + + printf -v remote_command 'bash -s -- %q %q %q %q %q' \ + "$RELEASE_TAG" "$DEPLOY_PATH" "$SOURCE_PATH" \ + "$PIP_INDEX_URL" "$PIP_TRUSTED_HOST" + ssh \ + -i "$SSH_KEY_FILE" \ + -p "$DEPLOY_PORT" \ + -o BatchMode=yes \ + -o IdentitiesOnly=yes \ + -o StrictHostKeyChecking=yes \ + "$DEPLOY_USER@$DEPLOY_HOST" \ + "$remote_command" \ + < .github/scripts/deploy-release.sh diff --git a/deploy/RELEASE_DEPLOYMENT.md b/deploy/RELEASE_DEPLOYMENT.md new file mode 100644 index 000000000..5903c4936 --- /dev/null +++ b/deploy/RELEASE_DEPLOYMENT.md @@ -0,0 +1,81 @@ +# Production release deployment + +Merging an automated `release/vX.Y.Z` pull request publishes the GitHub Release +and then deploys that exact tag to production. GitHub Actions does not build or +upload application artifacts. The production server fetches the tag in its +source checkout, builds versioned backend and frontend images, points the +deployment's `latest` image tags to them, and restarts the existing Compose +project. Success requires the API and worker containers to use the newly built +backend image, the frontend container to use the newly built frontend image, +and `/api/health` to report both `status: ok` and the released version. + +The deployment keeps the server's ignored `.env`, `ss-nodes.json`, +`backend/agent_data`, and Docker volumes in place. It refuses to overwrite +tracked local changes. If the build, restart, or health check fails after the +checkout, it checks out the previous commit and rebuilds the previous version. + +## GitHub production configuration + +Configure these under **Settings > Environments > production**. Environment +protection rules can require approval before the deployment job starts. + +### Variables + +| Name | Current production value | +| --- | --- | +| `CLAWITH_DEPLOY_HOST` | `82.156.53.84` | +| `CLAWITH_DEPLOY_USER` | `qinrui` | +| `CLAWITH_DEPLOY_PORT` | `10022` | +| `CLAWITH_DEPLOY_PATH` | `clawith_new` | +| `CLAWITH_SOURCE_PATH` | `Clawith` | +| `CLAWITH_PIP_INDEX_URL` | `https://mirrors.aliyun.com/pypi/simple/` (optional default) | +| `CLAWITH_PIP_TRUSTED_HOST` | Optional; only for a mirror that requires pip trusted-host handling | + +### Secrets + +| Name | Value | +| --- | --- | +| `CLAWITH_DEPLOY_SSH_KEY` | Private key for a dedicated deployment identity | +| `CLAWITH_DEPLOY_KNOWN_HOSTS` | Verified OpenSSH `known_hosts` entry for the production server | + +The SSH key must be authorized for the deployment user and should be dedicated +to GitHub Actions. Verify the server fingerprint through a trusted channel +before storing the `known_hosts` entry; do not blindly trust an `ssh-keyscan` +result. + +## Server prerequisites + +The source directory must be a Git checkout whose `origin` points to this +repository. The deployment directory keeps the production `.env`, Compose +file, Nginx configuration, and other environment-specific files. The server +also needs: + +- Docker with the Compose plugin +- `curl` +- read access to the repository +- permission for the deployment user to run Docker directly or with + passwordless `sudo` +- an existing `.env` in `clawith_new` + +Validate the server once before enabling automatic deployments: + +```bash +ssh clawith +cd Clawith +git status --short +git remote -v +cd ../clawith_new +test -f .env +docker compose config --quiet +docker compose ps +curl -fsS http://127.0.0.1:3008/api/health +``` + +The PyPI mirror defaults to Aliyun and can be overridden with the two optional +production Environment variables above. Docker's existing image-layer cache +continues to skip dependency installation when `backend/pyproject.toml` has not +changed. + +The server checkout is intentionally left at a detached release tag after a +successful deployment. Never move or reuse a published tag; publish a new +version to roll forward.