From 7f4e2b3278f64e6cac4bc4d31b402c765ec59aa9 Mon Sep 17 00:00:00 2001 From: Colin Saliceti <8416694+saliceti@users.noreply.github.com> Date: Wed, 27 May 2026 16:43:26 +0100 Subject: [PATCH] Remove stage-3-build-images workflow It is only used in Cohort manager, which has its own copy --- .github/workflows/stage-3-build-images.yaml | 279 -------------------- scripts/deployments/append-commit-hash.sh | 60 ----- scripts/deployments/get-docker-names.sh | 150 ----------- 3 files changed, 489 deletions(-) delete mode 100644 .github/workflows/stage-3-build-images.yaml delete mode 100644 scripts/deployments/append-commit-hash.sh delete mode 100644 scripts/deployments/get-docker-names.sh diff --git a/.github/workflows/stage-3-build-images.yaml b/.github/workflows/stage-3-build-images.yaml deleted file mode 100644 index 71a1eaea..00000000 --- a/.github/workflows/stage-3-build-images.yaml +++ /dev/null @@ -1,279 +0,0 @@ -name: Docker Image CI - -on: - push: - branches: - - main - - workflow_call: - inputs: - environment_tag: - description: Environment of the deployment - required: true - type: string - default: development - docker_compose_file: - description: The path of the compose.yaml file needed to build docker images - required: true - type: string - function_app_source_code_path: - description: The source path of the function app source code for the docker builds - required: true - type: string - project_name: - description: The name of the project - required: true - type: string - excluded_containers_csv_list: - description: Excluded containers in a comma separated list - required: true - type: string - build_all_images: - description: Build all images (true) or only changed ones (false) - required: false - type: boolean - default: false - -jobs: - get-functions: - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - id-token: write - outputs: - FUNC_NAMES: ${{ steps.get-function-names.outputs.FUNC_NAMES }} - DOCKER_COMPOSE_DIR: ${{ steps.get-function-names.outputs.DOCKER_COMPOSE_DIR }} - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: 2 - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Checkout dtos-devops-templates repository - uses: actions/checkout@v6 - with: - repository: NHSDigital/dtos-devops-templates - path: templates - ref: main - - - name: Determine which Docker container(s) to build - id: get-function-names - env: - COMPOSE_FILES_CSV: ${{ inputs.docker_compose_file }} - EXCLUDED_CONTAINERS_CSV: ${{ inputs.excluded_containers_csv_list }} - SOURCE_CODE_PATH: ${{ inputs.function_app_source_code_path }} - MANUAL_BUILD_ALL: ${{ inputs.build_all_images || false }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: bash ./templates/scripts/deployments/get-docker-names.sh - - build-and-push: - runs-on: ubuntu-latest - permissions: - id-token: write - contents: read - pull-requests: read - needs: get-functions - strategy: - matrix: - function: ${{ fromJSON(needs.get-functions.outputs.FUNC_NAMES) }} - if: needs.get-functions.outputs.FUNC_NAMES != '[]' - outputs: - pr_num_tag: ${{ env.PR_NUM_TAG }} - short_commit_hash: ${{ env.COMMIT_HASH_TAG }} - steps: - - uses: actions/checkout@v6 - with: - token: ${{ secrets.GITHUB_TOKEN }} - fetch-depth: 1 - submodules: 'true' - - - name: Checkout dtos-devops-templates repository - uses: actions/checkout@v6 - with: - repository: NHSDigital/dtos-devops-templates - path: templates - ref: main - - - name: Az CLI login - if: github.ref == 'refs/heads/main' - uses: azure/login@v3 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - name: Azure Container Registry login - if: github.ref == 'refs/heads/main' - run: az acr login --name ${{ secrets.ACR_NAME }} - - - name: Create Tags - env: - GH_TOKEN: ${{ github.token }} - ENVIRONMENT_TAG: ${{ inputs.environment_tag }} - continue-on-error: false - run: | - echo "The branch is: ${GITHUB_REF}" - - if [[ "${GITHUB_REF}" == refs/pull/*/merge ]]; then - PR_NUM_TAG=$(echo "${GITHUB_REF}" | sed 's/refs\/pull\/\([0-9]*\)\/merge/\1/') - else - PULLS_JSON=$(gh api /repos/{owner}/{repo}/commits/${GITHUB_SHA}/pulls) - ORIGINATING_BRANCH=$(echo ${PULLS_JSON} | jq -r '.[].head.ref' | python3 -c "import sys, urllib.parse; print(urllib.parse.quote_plus(sys.stdin.read().strip()))") - echo "ORIGINATING_BRANCH: ${ORIGINATING_BRANCH}" - PR_NUM_TAG=$(echo ${PULLS_JSON} | jq -r '.[].number') - fi - - echo "PR_NUM_TAG: pr${PR_NUM_TAG}" - echo "PR_NUM_TAG=pr${PR_NUM_TAG}" >> ${GITHUB_ENV} - - SHORT_COMMIT_HASH=$(git rev-parse --short ${GITHUB_SHA}) - echo "Commit hash tag: ${SHORT_COMMIT_HASH}" - echo "COMMIT_HASH_TAG=${SHORT_COMMIT_HASH}" >> ${GITHUB_ENV} - - echo "ENVIRONMENT_TAG=${ENVIRONMENT_TAG}" >> ${GITHUB_ENV} - - - name: Build and Push Image - working-directory: ${{ steps.get-function-names.outputs.DOCKER_COMPOSE_DIR }} - continue-on-error: false - env: - COMPOSE_FILE: ${{ inputs.docker_compose_file }} - PROJECT_NAME: ${{ inputs.project_name }} - run: | - function=${{ matrix.function }} - - echo PROJECT_NAME: ${PROJECT_NAME} - - if [ -z "${function}" ]; then - echo "Function variable is empty. Skipping Docker build." - exit 0 - fi - - # Build the image - docker compose -f ${COMPOSE_FILE//,/ -f } -p ${PROJECT_NAME} --profile "*" build --no-cache --pull ${function} - - repo_name="${{ secrets.ACR_NAME }}.azurecr.io/${PROJECT_NAME}-${function}" - echo $(repo_name) - - # Tag the image - echo "Tag the image:" - docker tag ${PROJECT_NAME}-${function}:latest "$repo_name:${COMMIT_HASH_TAG}" - docker tag ${PROJECT_NAME}-${function}:latest "$repo_name:${PR_NUM_TAG}" - docker tag ${PROJECT_NAME}-${function}:latest "$repo_name:${ENVIRONMENT_TAG}" - - # If this variable is set, the create-sbom-report.sh script will scan this docker image instead. - export CHECK_DOCKER_IMAGE=${PROJECT_NAME}-${function}:latest - export FORCE_USE_DOCKER=true - - export PR_NUM_TAG=${PR_NUM_TAG} - echo "PR_NUM_TAG=${PR_NUM_TAG}" >> ${GITHUB_ENV} - - # Push the image to the repository - if [ "${GITHUB_REF}" == 'refs/heads/main' ]; then - docker push "${repo_name}:${COMMIT_HASH_TAG}" - if [ "${PR_NUM_TAG}" != 'pr' ]; then - docker push "${repo_name}:${PR_NUM_TAG}" - fi - docker push "${repo_name}:${ENVIRONMENT_TAG}" - fi - - export SBOM_REPOSITORY_REPORT="sbom-${function}-repository-report" - echo "SBOM_REPOSITORY_REPORT=$SBOM_REPOSITORY_REPORT" >> $GITHUB_ENV - bash -x ${GITHUB_WORKSPACE}/templates/scripts/reports/create-sbom-report.sh - - export VULNERABILITIES_REPOSITORY_REPORT="vulnerabilities-${function}-repository-report" - echo "VULNERABILITIES_REPOSITORY_REPORT=$VULNERABILITIES_REPOSITORY_REPORT" >> $GITHUB_ENV - bash -x ${GITHUB_WORKSPACE}/templates/scripts/reports/scan-vulnerabilities.sh - - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sudo sh -s -- -b /usr/local/bin - - SCAN_RESULTS=$(grype "${PROJECT_NAME}-${function}:latest" --scope all-layers) - # ANSI color codes - RED="\033[0;31m" - RESET="\033[0m" - - # Define your log file - VULNERABILITIES_SUMMARY_LOGFILE="${PROJECT_NAME}-${function}-vulnerabilities-summary.txt" - echo "VULNERABILITIES_SUMMARY_LOGFILE=$VULNERABILITIES_SUMMARY_LOGFILE" >> $GITHUB_ENV - - # Clear existing log file (or create if it doesn't exist) - > "$VULNERABILITIES_SUMMARY_LOGFILE" - - for SEVERITY in CRITICAL HIGH MEDIUM; do - { - echo "" - echo "${PROJECT_NAME}-${function}: vulnerabilities" - echo -e "=== ${RED}${SEVERITY}${RESET} Vulnerabilities list ===" - # If grep finds nothing, we print a fallback message - echo "$SCAN_RESULTS" | grep -i "$SEVERITY" || echo "No $SEVERITY vulnerabilities found." - } | tee -a "$VULNERABILITIES_SUMMARY_LOGFILE" - done - - # Remove the image - docker rmi "${repo_name}:${COMMIT_HASH_TAG}" - docker rmi "${repo_name}:${PR_NUM_TAG}" - docker rmi "${repo_name}:${ENVIRONMENT_TAG}" - docker rmi ${PROJECT_NAME}-${function}:latest - - - name: Compress SBOM report - shell: bash - run: | - echo SBOM_REPOSITORY_REPORT: ${SBOM_REPOSITORY_REPORT} - zip "${SBOM_REPOSITORY_REPORT}.json.zip" "${SBOM_REPOSITORY_REPORT}.json" - - - name: Upload SBOM report as an artefact - uses: actions/upload-artifact@v7 - with: - name: ${{ env.SBOM_REPOSITORY_REPORT }}.json.zip - path: ./${{ env.SBOM_REPOSITORY_REPORT }}.json.zip - retention-days: 21 - - - name: Compress vulnerabilities report - shell: bash - run: | - echo VULNERABILITIES_REPOSITORY_REPORT: ${VULNERABILITIES_REPOSITORY_REPORT} - zip ${VULNERABILITIES_REPOSITORY_REPORT}.json.zip ${VULNERABILITIES_REPOSITORY_REPORT}.json - - - name: Upload vulnerabilities report as an artefact - uses: actions/upload-artifact@v7 - with: - name: ${{ env.VULNERABILITIES_REPOSITORY_REPORT }}.json.zip - path: ./${{ env.VULNERABILITIES_REPOSITORY_REPORT }}.json.zip - retention-days: 21 - - - name: Upload vulnerabilities summary report as an artefact - uses: actions/upload-artifact@v7 - with: - name: ${{ env.VULNERABILITIES_SUMMARY_LOGFILE }} - path: ./${{ env.VULNERABILITIES_SUMMARY_LOGFILE }} - retention-days: 21 - - aggregate-json: - runs-on: ubuntu-latest - needs: build-and-push - steps: - - name: Download SBOM JSON artifacts - uses: actions/download-artifact@v7 - with: - path: ./downloaded-artifacts - - - name: Combine sbom report JSON files - run: | - zip sbom-repository-report-${{ needs.build-and-push.outputs.PR_NUM_TAG }}.zip downloaded-artifacts/**/sbom*.json.zip - - - name: Combine vulnerabilities report JSON files - run: | - zip vulnerabilities-repository-report-${{ needs.build-and-push.outputs.PR_NUM_TAG }}.zip downloaded-artifacts/**/vulnerabilities*.json.zip - zip vulnerabilities-repository-report-${{ needs.build-and-push.outputs.PR_NUM_TAG }}.zip downloaded-artifacts/**/*vulnerabilities-summary*.txt - - - name: Upload sbom zip file - uses: actions/upload-artifact@v7 - with: - name: aggregated-sbom-repository-report-${{ needs.build-and-push.outputs.PR_NUM_TAG }}.zip - path: sbom-repository-report-${{ needs.build-and-push.outputs.PR_NUM_TAG }}.zip - - - name: Upload repository zip file - uses: actions/upload-artifact@v7 - with: - name: aggregated-vulnerabilities-repository-report-${{ needs.build-and-push.outputs.PR_NUM_TAG }}.zip - path: vulnerabilities-repository-report-${{ needs.build-and-push.outputs.PR_NUM_TAG }}.zip diff --git a/scripts/deployments/append-commit-hash.sh b/scripts/deployments/append-commit-hash.sh deleted file mode 100644 index f5177ceb..00000000 --- a/scripts/deployments/append-commit-hash.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# Pipestatus: if any command in a pipeline fails, the return status is that of the failed command. -set -o pipefail -# Print each command to stderr before executing. -set -x - -echo "Attempting to tag all repositories in ACR $ACR_NAME with short commit hash: $SHORT_COMMIT_HASH" -echo "Source tag for import will be: $ENVIRONMENT_TAG" - -# Get list of repositories -repo_list=$(az acr repository list --name "$ACR_NAME" --output tsv) - -if [ -z "$repo_list" ]; then - echo "No repositories found in ACR $ACR_NAME. Nothing to tag." - exit 0 -fi - -echo "Found repositories: $(echo $repo_list | wc -w)" -echo "---" - -exit_code=0 - -for repo_name in $repo_list; do - source_image="${ACR_NAME}.azurecr.io/${repo_name}:${ENVIRONMENT_TAG}" - target_image="${repo_name}:${SHORT_COMMIT_HASH}" - - echo "Processing repository: $repo_name" - - echo " Checking for existing target tag: $SHORT_COMMIT_HASH" - target_tag_check_output=$(az acr manifest list-metadata --registry "$ACR_NAME" --name "$repo_name" --query "[?tags.contains(@, '${SHORT_COMMIT_HASH}')]" --output tsv) - target_tag_check_status=$? - - if [ $target_tag_check_status -eq 0 ] && [ -n "$target_tag_check_output" ]; then - echo " Target tag '$SHORT_COMMIT_HASH' already exists. Skipping import for this repository." - echo "---" - continue - fi - - echo " Proceeding with import attempt: $source_image -> $target_image" - - az acr import \ - --name "$ACR_NAME" \ - --source "$source_image" \ - --image "$target_image" \ - --force - - import_status=$? - - if [ $import_status -ne 0 ]; then - echo " ⚠️ Warning: ACR import command failed for repository '$repo_name' (Exit Code: $import_status)." - exit_code=1 # Record import failure - else - echo " Import successful for '$repo_name'." - fi - echo "---" -done - -echo "Finished processing all repositories." -exit $exit_code diff --git a/scripts/deployments/get-docker-names.sh b/scripts/deployments/get-docker-names.sh deleted file mode 100644 index 5cf0271a..00000000 --- a/scripts/deployments/get-docker-names.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -remove_from_array() { - local item_to_remove="$1" - local -n target_array="$2" # Use nameref to modify the array directly - local filtered_array=() - - for item in "${target_array[@]}"; do - [[ "$item" != "$item_to_remove" ]] && filtered_array+=("$item") - done - - target_array=("${filtered_array[@]}") -} - -if [[ -z "${COMPOSE_FILES_CSV}" ]]; then - echo "❌ Error: COMPOSE_FILES_CSV has not been defined (comma separated)." - exit 1 -fi - -# CHANGED_FOLDERS_CSV can be supplied via environment variable for local testing -if [[ -z "${CHANGED_FOLDERS_CSV}" ]]; then - if [[ -z "${SOURCE_CODE_PATH}" ]]; then - echo "❌ Error: SOURCE_CODE_PATH has not been defined." - exit 1 - fi - if [[ "${GITHUB_EVENT_NAME}" == "push" && "${GITHUB_REF}" == "refs/heads/main" ]]; then - # Merge to main - compare merged code with main immediately prior to the merge (HEAD^), needs 'fetch-depth: 2' parameter for actions/checkout@v6 - mapfile -t source_changes < <(git diff --name-only HEAD^ -- "${SOURCE_CODE_PATH}" | sed -r 's#(^.*/).*$#\1#' | sort -u) - else - # PR creation or update - compare feature branch with main, folder paths only, unique list - git fetch origin main - mapfile -t source_changes < <(git diff --name-only origin/main..HEAD -- "${SOURCE_CODE_PATH}" | sed -r 's#(^.*/).*$#\1#' | sort -u) - fi -else - IFS=',' read -r -a source_changes <<< "${CHANGED_FOLDERS_CSV}" -fi - -echo -e "\nChanged source code folder(s):" -printf " - %s\n" "${source_changes[@]}" -echo - -# If MANUAL_BUILD_ALL is true -if [[ "${MANUAL_BUILD_ALL,,}" == "true" ]]; then - echo "MANUAL_BUILD_ALL is true. Change detection based on specific folders will be skipped; all services will be included." - source_changes=() -fi - -[[ -n "${EXCLUDED_CONTAINERS_CSV}" ]] && EXCLUSION_FILTER="select($(echo "${EXCLUDED_CONTAINERS_CSV}" | awk -v ORS='' '{split($0, arr, ","); for (i in arr) printf ".container_name != \"%s\" and ", arr[i]} END {print "1"}')) |" - -IFS_OLD=$IFS -IFS=$', \n' - -echo "Adding Docker compose file includes..." -files_to_process=(${COMPOSE_FILES_CSV}) -while [ ${#files_to_process[@]} -gt 0 ]; do - compose_file="${files_to_process[0]}" - files_to_process=("${files_to_process[@]:1}") # Remove the first file from the list - includes=($(yq -r '.include[]' "${compose_file}")) - - for include in "${includes[@]}"; do - echo " - ${include}" - if [[ ! ",${COMPOSE_FILES_CSV}," =~ ",${include}," ]]; then - COMPOSE_FILES_CSV="${COMPOSE_FILES_CSV},$(dirname "${compose_file}")/${include}" - files_to_process+=("$(dirname "${compose_file}")/${include}") - fi - done -done -echo - -changed_services=() -non_matched_changes=() - -for compose_file in ${COMPOSE_FILES_CSV}; do - - echo -e "Parsing Docker compose file '${compose_file}'...\n" - declare -A docker_services_map=() - - # STEP 1 - Create a map of folder paths to services - for service in $(yq eval ".services[] | ${EXCLUSION_FILTER} .container_name" "${compose_file}"); do - # Combine the context and dockerfile variables to determine the container root - # We need to filter these since there are various ways these can be defined (leading ./ or trailing / for instance) - context=$(yq eval ".services[] | select(.container_name == \"$service\") | .build.context" "${compose_file}") - dockerfile=$(yq eval ".services[] | select(.container_name == \"$service\") | .build.dockerfile" "${compose_file}") - - if [[ -z "${dockerfile}" ]] || [[ -z "${context}" ]]; then - continue - fi - context_filtered=$(echo "${context}" | sed 's#^\./src/##' | sed 's#^\./##' | sed 's#/$##') - dockerfile_filtered=$(echo "${dockerfile}" | sed 's#^\./##' | sed 's#\/Dockerfile##' | sed 's#Dockerfile##') - if [[ -n "${context_filtered}" ]] && [[ -n "${dockerfile_filtered}" ]]; then - function_path="${context_filtered}/${dockerfile_filtered}" - else - function_path="${context_filtered}${dockerfile_filtered}" - fi - docker_services_map[${function_path}]=${service} - done - - printf "%-50s %-50s\n" "Service" "Path" - printf "%-50s %-50s\n" "-------" "----" - for key in "${!docker_services_map[@]}"; do - printf "%-50s %-50s\n" "${docker_services_map[$key]}" "$key" - done - echo - - # STEP 2 - Now check the source code changes against the map created in STEP 1 to determine which containers to build - if [[ "${MANUAL_BUILD_ALL,,}" == "true" ]]; then - echo "MANUAL_BUILD_ALL: Adding all services from '${compose_file}'." - for key in "${!docker_services_map[@]}"; do - changed_services+=("${docker_services_map[$key]}") - done - elif [[ ${#source_changes[@]} -eq 0 ]]; then - echo "No files changed." - else - echo "Application change detected, building all images." - for key in "${!docker_services_map[@]}"; do - changed_services+=("${docker_services_map[$key]}") - done - fi - echo -done - -if [ ${#non_matched_changes[@]} -ne 0 ]; then - # Remove duplicates (non-matched items across several compose files) - mapfile -t unique_changes < <(printf "%s\n" "${non_matched_changes[@]}" | sort -u) - - warning_message=$( - cat <> "$GITHUB_STEP_SUMMARY" -fi - -changed_services_json="$(jq -c -n '$ARGS.positional | unique' --args "${changed_services[@]}")" -services_json="$(jq -c -n '$ARGS.positional | unique' --args "${docker_services_map[@]}")" - -IFS=$IFS_OLD -echo "List of services to build:" -echo "${changed_services_json}" -echo "FUNC_NAMES=${changed_services_json}" >> "${GITHUB_OUTPUT}" -echo "ALL_SERVICES=%{services_json}" >> "${GITHUB_OUTPUT}" - -# Assumes all compose files are together in the same folder -echo "DOCKER_COMPOSE_DIR=$(dirname "${compose_file}")" >> "${GITHUB_OUTPUT}"