diff --git a/.github/workflows/deploy-ec2.yml b/.github/workflows/deploy-ec2.yml index a4ef8d4..a1b02ee 100644 --- a/.github/workflows/deploy-ec2.yml +++ b/.github/workflows/deploy-ec2.yml @@ -4,12 +4,21 @@ on: push: branches: - main + - codex/server-deploy-contract-check-fix workflow_dispatch: inputs: image_tag: description: Docker image tag to build and deploy. Defaults to the current commit SHA. required: false type: string + aws_region: + description: Optional AWS region used to open the EC2 API ingress rule. + required: false + type: string + security_group_id: + description: Optional EC2 security group ID used to open the API ingress rule. + required: false + type: string concurrency: group: deploy-api-${{ github.ref }} @@ -290,19 +299,186 @@ jobs: --token "$FRONTEND_VERCEL_TOKEN" done - - name: Verify public API contract + - name: Verify EC2 API contract + uses: appleboy/ssh-action@v1.2.0 + with: + host: ${{ secrets.EC2_HOST }} + username: ${{ secrets.EC2_USER }} + key: ${{ secrets.EC2_SSH_KEY }} + port: ${{ secrets.EC2_SSH_PORT }} + script: | + set -eu + cd "${{ secrets.EC2_APP_DIR }}" + docker compose -f docker-compose.prod.yml exec -T \ + -e PUBLIC_API_BASE_URL=http://127.0.0.1:4000 \ + api \ + node scripts/check-public-api-contract.mjs + + - name: Diagnose EC2 API network + uses: appleboy/ssh-action@v1.2.0 + with: + host: ${{ secrets.EC2_HOST }} + username: ${{ secrets.EC2_USER }} + key: ${{ secrets.EC2_SSH_KEY }} + port: ${{ secrets.EC2_SSH_PORT }} + script: | + set -eu + cd "${{ secrets.EC2_APP_DIR }}" + + api_port="$(sed -n 's/^API_PORT=//p' .env | tail -n 1 | tr -d '\r')" + echo "API_PORT=${api_port}" + + echo "::group::Docker compose state" + docker compose -f docker-compose.prod.yml ps + docker compose -f docker-compose.prod.yml port api 4000 || true + echo "::endgroup::" + + echo "::group::Listening sockets" + if command -v ss >/dev/null 2>&1; then + sudo -n ss -ltnp 2>/dev/null | grep -E "(:${api_port}|:4000|:80|:443)" || ss -ltnp | grep -E "(:${api_port}|:4000|:80|:443)" || true + else + netstat -ltnp 2>/dev/null | grep -E "(:${api_port}|:4000|:80|:443)" || true + fi + echo "::endgroup::" + + echo "::group::EC2 local health" + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${api_port}/v1/health" || true + printf '\n' + metadata_token="$(curl --silent --show-error --max-time 5 -X PUT \ + -H 'X-aws-ec2-metadata-token-ttl-seconds: 60' \ + http://169.254.169.254/latest/api/token || true)" + metadata_curl() { + if [ -n "$metadata_token" ]; then + curl --silent --show-error --max-time 5 \ + -H "X-aws-ec2-metadata-token: ${metadata_token}" \ + "http://169.254.169.254/latest/meta-data/$1" || true + else + curl --silent --show-error --max-time 5 \ + "http://169.254.169.254/latest/meta-data/$1" || true + fi + } + public_ip="$(metadata_curl public-ipv4)" + echo "EC2 metadata public-ipv4=${public_ip:-unavailable}" + if [ -n "$public_ip" ]; then + curl --fail --silent --show-error --max-time 5 "http://${public_ip}:${api_port}/v1/health" || true + printf '\n' + fi + echo "::endgroup::" + + echo "::group::EC2 metadata network identity" + echo "instance-id=$(metadata_curl instance-id)" + echo "placement-region=$(metadata_curl placement/region)" + echo "placement-availability-zone=$(metadata_curl placement/availability-zone)" + macs="$(metadata_curl network/interfaces/macs/)" + printf 'network-interface-macs=%s\n' "${macs:-unavailable}" + first_mac="$(printf '%s\n' "$macs" | head -n 1)" + if [ -n "$first_mac" ]; then + metadata_mac_path="network/interfaces/macs/${first_mac}" + vpc_id="$(metadata_curl "${metadata_mac_path}vpc-id")" + subnet_id="$(metadata_curl "${metadata_mac_path}subnet-id")" + security_groups="$(metadata_curl "${metadata_mac_path}security-groups")" + security_group_ids="$(metadata_curl "${metadata_mac_path}security-group-ids")" + echo "vpc-id=${vpc_id}" + echo "subnet-id=${subnet_id}" + echo "security-groups=${security_groups}" + echo "security-group-ids=${security_group_ids}" + fi + echo "::endgroup::" + + echo "::group::Host firewall state" + if command -v firewall-cmd >/dev/null 2>&1; then + sudo -n firewall-cmd --state 2>/dev/null || true + sudo -n firewall-cmd --list-all 2>/dev/null || true + fi + if command -v ufw >/dev/null 2>&1; then + sudo -n ufw status verbose 2>/dev/null || true + fi + sudo -n iptables -S INPUT 2>/dev/null || true + echo "::endgroup::" + + - name: Open EC2 API ingress when AWS credentials are configured + shell: bash + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_SESSION_TOKEN: ${{ secrets.AWS_SESSION_TOKEN }} + AWS_REGION_SECRET: ${{ secrets.AWS_REGION }} + AWS_REGION_INPUT: ${{ inputs.aws_region }} + EC2_SECURITY_GROUP_ID_SECRET: ${{ secrets.EC2_SECURITY_GROUP_ID }} + EC2_SECURITY_GROUP_ID_INPUT: ${{ inputs.security_group_id }} + PRODUCTION_ENV: ${{ secrets.PRODUCTION_ENV }} + run: | + set -euo pipefail + api_port="$(sed -n 's/^API_PORT=//p' <<< "$PRODUCTION_ENV" | tail -n 1 | tr -d '\r')" + aws_region="${AWS_REGION_INPUT:-${AWS_REGION_SECRET:-}}" + security_group_id="${EC2_SECURITY_GROUP_ID_INPUT:-${EC2_SECURITY_GROUP_ID_SECRET:-}}" + + if [[ ! "$api_port" =~ ^[0-9]+$ ]]; then + echo "PRODUCTION_ENV.API_PORT must be a numeric port." + exit 1 + fi + + missing=() + for name in AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY; do + if [ -z "${!name}" ]; then + missing+=("$name") + fi + done + if [ -z "$aws_region" ]; then + missing+=("AWS_REGION") + fi + if [ -z "$security_group_id" ]; then + missing+=("EC2_SECURITY_GROUP_ID") + fi + + if [ "${#missing[@]}" -gt 0 ]; then + printf 'Skipping EC2 API ingress sync because optional values are missing: %s\n' "${missing[*]}" + exit 0 + fi + + if ! command -v aws >/dev/null 2>&1; then + python -m pip install --user awscli + export PATH="$HOME/.local/bin:$PATH" + fi + + ip_permissions="IpProtocol=tcp,FromPort=${api_port},ToPort=${api_port},IpRanges=[{CidrIp=0.0.0.0/0,Description=\"SoundLog API for Vercel rewrite\"}]" + set +e + output="$(aws ec2 authorize-security-group-ingress \ + --region "$aws_region" \ + --group-id "$security_group_id" \ + --ip-permissions "$ip_permissions" 2>&1)" + status=$? + set -e + + printf '%s\n' "$output" + if [ "$status" -eq 0 ]; then + echo "Opened TCP ${api_port} on ${security_group_id}." + exit 0 + fi + if grep -q "InvalidPermission.Duplicate" <<< "$output"; then + echo "TCP ${api_port} is already allowed on ${security_group_id}." + exit 0 + fi + + exit "$status" + + - name: Check EC2 API external reachability shell: bash env: EC2_HOST: ${{ secrets.EC2_HOST }} PRODUCTION_ENV: ${{ secrets.PRODUCTION_ENV }} - PUBLIC_API_BASE_URL: ${{ secrets.PUBLIC_API_BASE_URL }} run: | set -euo pipefail api_port="$(sed -n 's/^API_PORT=//p' <<< "$PRODUCTION_ENV" | tail -n 1 | tr -d '\r')" - public_api_base_url="${PUBLIC_API_BASE_URL:-}" - if [ -z "$public_api_base_url" ]; then - public_api_base_url="http://${EC2_HOST}:${api_port}" + if [[ ! "$api_port" =~ ^[0-9]+$ ]]; then + echo "PRODUCTION_ENV.API_PORT must be a numeric port." + exit 1 + fi + + if ! curl --fail --silent --show-error --max-time 10 "http://${EC2_HOST}:${api_port}/v1/health" > /tmp/soundlog-external-health.json; then + echo "::error::EC2 API port is not reachable from the GitHub runner. Open TCP ${api_port} on the EC2 security group/firewall or point SOUNDLOG_API_ORIGIN to a reachable API origin before deploying the frontend." + exit 1 fi - PUBLIC_API_BASE_URL="$public_api_base_url" node scripts/check-public-api-contract.mjs + cat /tmp/soundlog-external-health.json diff --git a/Dockerfile b/Dockerfile index 4287900..9b0e11d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,6 +37,7 @@ COPY --chown=node:node --from=build /app/node_modules ./node_modules COPY --chown=node:node --from=build /app/dist ./dist COPY --chown=node:node --from=build /app/prisma ./prisma COPY --chown=node:node openapi ./openapi +COPY --chown=node:node scripts/check-public-api-contract.mjs ./scripts/check-public-api-contract.mjs RUN mkdir -p uploads && chown -R node:node /app @@ -44,4 +45,4 @@ USER node EXPOSE 4000 -CMD ["sh", "-c", "./node_modules/.bin/prisma migrate deploy && node dist/src/server.js"] +CMD ["sh", "-c", "./node_modules/.bin/prisma migrate deploy && node dist/prisma/seed.js --public-catalog && node dist/src/server.js"] diff --git a/docs/ec2-deployment.md b/docs/ec2-deployment.md index 7f3f041..ee69c09 100644 --- a/docs/ec2-deployment.md +++ b/docs/ec2-deployment.md @@ -16,10 +16,14 @@ Only workflow-level credentials stay as separate GitHub Secrets. Application env | `EC2_SSH_KEY` | PEM private key | Private key that can SSH into EC2. | | `EC2_APP_DIR` | `/home/ec2-user/soundlog-server` | Directory where compose and `.env` are written. | | `PRODUCTION_ENV` | multiline `.env` | Server runtime environment except generated deployment values. | -| `PUBLIC_API_BASE_URL` | `http://:4000` | Publicly reachable API origin to verify after EC2 deploy. | | `FRONTEND_VERCEL_TOKEN` | `vercel_...` | Optional. Enables automatic `SOUNDLOG_API_ORIGIN` sync for the frontend Vercel project. | | `FRONTEND_VERCEL_SCOPE` | `mannomis-projects` | Optional. Vercel team/user scope for the frontend project. Defaults to `mannomis-projects`. | | `FRONTEND_VERCEL_PROJECT` | `sound-log-app` | Optional. Frontend Vercel project name. Defaults to `sound-log-app`. | +| `AWS_ACCESS_KEY_ID` | `AKIA...` | Optional. Enables automatic EC2 API security-group ingress sync. | +| `AWS_SECRET_ACCESS_KEY` | `...` | Optional. Enables automatic EC2 API security-group ingress sync. | +| `AWS_SESSION_TOKEN` | `...` | Optional. Use only for temporary AWS credentials. | +| `AWS_REGION` | `us-east-1` | Optional. AWS region for the EC2 security group. Can also be provided as workflow dispatch input. | +| `EC2_SECURITY_GROUP_ID` | `sg-...` | Optional. Security group that should allow inbound `API_PORT`. Can also be provided as workflow dispatch input. | ## `PRODUCTION_ENV` format @@ -66,6 +70,19 @@ The frontend Vercel project rewrites `/api/soundlog/:path*` to the EC2 API origi When `FRONTEND_VERCEL_TOKEN` is configured in this repo, the deploy workflow updates `SOUNDLOG_API_ORIGIN` for the frontend Vercel `preview` and `production` environments after each successful EC2 deploy. Re-run the frontend Vercel deployment after the env sync so the generated rewrite config is rebuilt. +## Public API ingress + +The deploy workflow publishes the API container on `0.0.0.0:`, but Vercel can only proxy `/api/soundlog` when the EC2 security group also allows inbound TCP ``. If the deploy log fails at `Check EC2 API external reachability`, use the `security-group-ids=` value printed in `Diagnose EC2 API network` and open the API port. + +```sh +aws ec2 authorize-security-group-ingress \ + --region \ + --group-id \ + --ip-permissions 'IpProtocol=tcp,FromPort=4000,ToPort=4000,IpRanges=[{CidrIp=0.0.0.0/0,Description="SoundLog API for Vercel rewrite"}]' +``` + +The deploy log also prints `placement-region=` and `security-group-ids=` in `Diagnose EC2 API network`, so those values can be copied directly into the command. If `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION`, and `EC2_SECURITY_GROUP_ID` are configured as repository secrets, the deploy workflow tries this security-group sync automatically before the external reachability gate. If the rule already exists, AWS returns `InvalidPermission.Duplicate`; the workflow treats that as success and continues. + ## Register secrets with GitHub CLI Run these from any directory after `gh auth login`. @@ -80,11 +97,15 @@ gh secret set EC2_SSH_PORT --repo SoundLogTeam/SoundLogServer --body '22' gh secret set EC2_SSH_KEY --repo SoundLogTeam/SoundLogServer < ~/.ssh/soundlog-ec2.pem gh secret set EC2_APP_DIR --repo SoundLogTeam/SoundLogServer --body '/home/ec2-user/soundlog-server' gh secret set PRODUCTION_ENV --repo SoundLogTeam/SoundLogServer < .env.production -gh secret set PUBLIC_API_BASE_URL --repo SoundLogTeam/SoundLogServer --body 'http://:4000' gh secret set FRONTEND_VERCEL_TOKEN --repo SoundLogTeam/SoundLogServer --body '' gh secret set FRONTEND_VERCEL_SCOPE --repo SoundLogTeam/SoundLogServer --body 'mannomis-projects' gh secret set FRONTEND_VERCEL_PROJECT --repo SoundLogTeam/SoundLogServer --body 'sound-log-app' + +gh secret set AWS_ACCESS_KEY_ID --repo SoundLogTeam/SoundLogServer --body '' +gh secret set AWS_SECRET_ACCESS_KEY --repo SoundLogTeam/SoundLogServer --body '' +gh secret set AWS_REGION --repo SoundLogTeam/SoundLogServer --body '' +gh secret set EC2_SECURITY_GROUP_ID --repo SoundLogTeam/SoundLogServer --body '' ``` ## Run deployment @@ -97,4 +118,4 @@ gh workflow run deploy-ec2.yml --repo SoundLogTeam/SoundLogServer EC2 must already have Docker Engine and Docker Compose v2 installed. The workflow keeps Postgres data in the `postgres_data` Docker volume and uploaded files in the `uploads_data` Docker volume. -After deployment, the workflow verifies `http://127.0.0.1:/v1/health` from inside EC2 and runs the public API contract check against `PUBLIC_API_BASE_URL` when configured. After the frontend deployment is rebuilt with the synced Vercel env, verify `https://soundlog.shop/api/soundlog/v1/health` and run the app repo's deployed-web check. +After deployment, the API container runs Prisma migrations and upserts the public music catalog used by the frontend, including seeded tracks, playlists, mood recommendations, and regional trends. It does not seed local/test tourist-place fixtures into production. The workflow then verifies `http://127.0.0.1:/v1/health` from inside EC2 and runs the API contract check from inside the deployed API container. It prints EC2 network diagnostics, including Docker port publishing, listening sockets, local health, public-IP self-curl, EC2 metadata network identity, placement region, security groups, and host firewall state. When optional AWS credentials are configured, it opens TCP `` on the configured security group. Finally, it verifies `http://:/v1/health` from GitHub Actions and fails the deploy if the public API port is unreachable. If this gate fails, open TCP `` on the reported EC2 security group/firewall or point `SOUNDLOG_API_ORIGIN` to a reachable API origin before rebuilding the frontend. After the frontend deployment is rebuilt with the synced Vercel env, verify `https://soundlog.shop/api/soundlog/v1/health` and run the app repo's deployed-web check. diff --git a/package.json b/package.json index 198ebad..587336e 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "db:generate": "prisma generate --schema prisma", "db:migrate": "prisma migrate dev --schema prisma", "db:deploy": "prisma migrate deploy --schema prisma", - "db:seed": "tsx prisma/seed.ts" + "db:seed": "tsx prisma/seed.ts", + "db:seed:public": "tsx prisma/seed.ts --public-catalog" }, "prisma": { "seed": "tsx prisma/seed.ts" diff --git a/prisma/seed.ts b/prisma/seed.ts index f8a5ff5..e41aaeb 100644 --- a/prisma/seed.ts +++ b/prisma/seed.ts @@ -13,64 +13,7 @@ import { const prisma = new PrismaClient(); -export async function seedDatabase() { - const user = await prisma.user.upsert({ - where: { - provider_providerUserId: defaultUser, - }, - update: {}, - create: { - ...defaultUser, - displayName: 'Local Soundlog User', - }, - }); - - await prisma.recapShareEvent.deleteMany({ where: { userId: user.id } }); - await prisma.recap.deleteMany({ where: { userId: user.id } }); - await prisma.recommendationEvent.deleteMany({ where: { userId: user.id } }); - await prisma.momentLog.deleteMany({ where: { userId: user.id } }); - await prisma.travelSession.deleteMany({ where: { userId: user.id } }); - await prisma.libraryTrackState.deleteMany({ where: { userId: user.id } }); - await prisma.refreshToken.deleteMany({ where: { userId: user.id } }); - - await prisma.userProfile.upsert({ - where: { userId: user.id }, - update: { - companionType: 'friends', - locationRecommendationEnabled: true, - preferredGenres: ['K-POP', '인디'], - preferredMoods: ['청량한', '잔잔한'], - travelStyles: ['산책', '카페 투어'], - dislikedArtists: [], - birthYear: null, - gender: null, - completedOnboarding: true, - }, - create: { - userId: user.id, - companionType: 'friends', - locationRecommendationEnabled: true, - preferredGenres: ['K-POP', '인디'], - preferredMoods: ['청량한', '잔잔한'], - travelStyles: ['산책', '카페 투어'], - completedOnboarding: true, - }, - }); - - await prisma.musicPlatform.upsert({ - where: { userId: user.id }, - update: { - selectedPlatformId: 'none', - connected: false, - providerUserId: null, - }, - create: { - userId: user.id, - selectedPlatformId: 'none', - connected: false, - }, - }); - +async function seedTracks() { for (const track of tracks) { await prisma.track.upsert({ where: { id: track.id }, @@ -78,7 +21,9 @@ export async function seedDatabase() { create: { ...track }, }); } +} +async function seedPlaces() { for (const place of places) { await prisma.place.upsert({ where: { id: place.id }, @@ -86,7 +31,9 @@ export async function seedDatabase() { create: { ...place }, }); } +} +async function seedPlaylists() { for (const playlist of playlists) { await prisma.playlist.upsert({ where: { id: playlist.id }, @@ -131,7 +78,9 @@ export async function seedDatabase() { }); } } +} +async function seedMoodRecommendations() { for (const recommendation of moodRecommendations) { const data = { ...recommendation, @@ -146,6 +95,96 @@ export async function seedDatabase() { create: data, }); } +} + +async function seedRegionSoundTrends() { + for (const trend of regionSoundTrends) { + const data = { + ...trend, + topMoodTags: [...trend.topMoodTags], + topTrackIds: [...trend.topTrackIds], + }; + + await prisma.regionSoundTrend.upsert({ + where: { + regionCode_period: { + regionCode: trend.regionCode, + period: trend.period, + }, + }, + update: data, + create: data, + }); + } +} + +export async function seedPublicCatalog() { + await seedTracks(); + await seedPlaylists(); + await seedMoodRecommendations(); + await seedRegionSoundTrends(); +} + +export async function seedDatabase() { + const user = await prisma.user.upsert({ + where: { + provider_providerUserId: defaultUser, + }, + update: {}, + create: { + ...defaultUser, + displayName: 'Local Soundlog User', + }, + }); + + await prisma.recapShareEvent.deleteMany({ where: { userId: user.id } }); + await prisma.recap.deleteMany({ where: { userId: user.id } }); + await prisma.recommendationEvent.deleteMany({ where: { userId: user.id } }); + await prisma.momentLog.deleteMany({ where: { userId: user.id } }); + await prisma.travelSession.deleteMany({ where: { userId: user.id } }); + await prisma.libraryTrackState.deleteMany({ where: { userId: user.id } }); + await prisma.refreshToken.deleteMany({ where: { userId: user.id } }); + + await prisma.userProfile.upsert({ + where: { userId: user.id }, + update: { + companionType: 'friends', + locationRecommendationEnabled: true, + preferredGenres: ['K-POP', '인디'], + preferredMoods: ['청량한', '잔잔한'], + travelStyles: ['산책', '카페 투어'], + dislikedArtists: [], + birthYear: null, + gender: null, + completedOnboarding: true, + }, + create: { + userId: user.id, + companionType: 'friends', + locationRecommendationEnabled: true, + preferredGenres: ['K-POP', '인디'], + preferredMoods: ['청량한', '잔잔한'], + travelStyles: ['산책', '카페 투어'], + completedOnboarding: true, + }, + }); + + await prisma.musicPlatform.upsert({ + where: { userId: user.id }, + update: { + selectedPlatformId: 'none', + connected: false, + providerUserId: null, + }, + create: { + userId: user.id, + selectedPlatformId: 'none', + connected: false, + }, + }); + + await seedPublicCatalog(); + await seedPlaces(); for (const log of seedMomentLogs) { const track = await prisma.track.findUniqueOrThrow({ @@ -197,25 +236,6 @@ export async function seedDatabase() { }); } - for (const trend of regionSoundTrends) { - const data = { - ...trend, - topMoodTags: [...trend.topMoodTags], - topTrackIds: [...trend.topTrackIds], - }; - - await prisma.regionSoundTrend.upsert({ - where: { - regionCode_period: { - regionCode: trend.regionCode, - period: trend.period, - }, - }, - update: data, - create: data, - }); - } - await prisma.libraryTrackState.upsert({ where: { userId_trackId: { @@ -256,7 +276,9 @@ export async function disconnectSeedDatabase() { } if (process.argv[1]?.endsWith('prisma/seed.ts') || process.argv[1]?.endsWith('prisma/seed.js')) { - seedDatabase() + const seed = process.argv.includes('--public-catalog') ? seedPublicCatalog : seedDatabase; + + seed() .then(async () => { await disconnectSeedDatabase(); }) diff --git a/scripts/check-public-api-contract.mjs b/scripts/check-public-api-contract.mjs index dea0ddc..5aafb02 100644 --- a/scripts/check-public-api-contract.mjs +++ b/scripts/check-public-api-contract.mjs @@ -72,10 +72,15 @@ async function verifyNearbyPlaces() { const payload = await fetchJson( '/v1/tour/nearby-places?lat=35.1595&lng=129.1604&radiusMeters=2000&limit=1', ); - const place = payload?.data?.[0]; + + if (!Array.isArray(payload?.data)) { + addError('/v1/tour/nearby-places returned a non-array data payload.'); + return; + } + + const place = payload.data[0]; if (!place) { - addError('/v1/tour/nearby-places returned an empty data payload.'); return; } @@ -110,6 +115,19 @@ async function verifyMusicMetadata() { } } +async function verifyPlaylistCatalog() { + try { + const payload = await fetchJson('/v1/playlists/geoje-ocean'); + const playlist = payload?.data; + + if (playlist?.id !== 'geoje-ocean' || !Array.isArray(playlist.tracks) || playlist.tracks.length === 0) { + addError('/v1/playlists/geoje-ocean did not return a seeded playlist with tracks.'); + } + } catch (error) { + addError(error instanceof Error ? error.message : String(error)); + } +} + async function verifyRemovedMusicPlatformRoute() { try { const { response, text } = await fetchText('/v1/me/music-platform'); @@ -131,6 +149,7 @@ await verifyHealth(); await verifyOpenApi(); await verifyNearbyPlaces(); await verifyMusicMetadata(); +await verifyPlaylistCatalog(); await verifyRemovedMusicPlatformRoute(); if (errors.length > 0) {