feat(scrolls): expose download progress across catalog

.github/workflows/pr.yml

@@ -11,7 +11,7 @@ jobs:
       SCROLL_REGISTRY_API_KEY: ${{ secrets.SCROLL_REGISTRY_API_KEY }}
       SCROLL_REGISTRY_API_SECRET: ${{ secrets.SCROLL_REGISTRY_API_SECRET }}
       SCROLL_REGISTRY_BUCKET: ${{ secrets.SCROLL_REGISTRY_BUCKET_STAGING }}
-      DRUID_CLI_VERSION: v0.1.257
+      DRUID_CLI_VERSION: v0.1.258
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3

.github/workflows/release.yml

@@ -17,7 +17,7 @@ jobs:
       SCROLL_REGISTRY_HOST: ${{ secrets.SCROLL_REGISTRY_HOST }}
       SCROLL_REGISTRY_USER: ${{ secrets.SCROLL_REGISTRY_USER }}
       SCROLL_REGISTRY_PASSWORD: ${{ secrets.SCROLL_REGISTRY_PASSWORD }}
-      DRUID_CLI_VERSION: v0.1.257
+      DRUID_CLI_VERSION: v0.1.258
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3

generate-scrolls.go

@@ -20,6 +20,8 @@ type TemplateVars struct {
 	Artifact           string
 	Version            string
 	VersionEscaped     string
+	ProgressImage      string
+	ProgressSteamImage string
 	ColdstarterImage   string
 	SteamImage         string
 	Artifacts          map[string]string
@@ -101,6 +103,14 @@ func main() {
 	if err != nil {
 		log.Fatalln(err)
 	}
+	progressImage := os.Getenv("DRUID_PROGRESS_RUNTIME_IMAGE")
+	if progressImage == "" {
+		progressImage = "artifacts.druid.gg/druid-team/druid:v0.1.258"
+	}
+	progressSteamImage := os.Getenv("DRUID_PROGRESS_STEAM_RUNTIME_IMAGE")
+	if progressSteamImage == "" {
+		progressSteamImage = "artifacts.druid.gg/druid-team/druid:v0.1.258-steamcmd"
+	}
 	coldstarterImage := os.Getenv("DRUID_COLDSTARTER_IMAGE")
 	if coldstarterImage == "" {
 		coldstarterImage = "artifacts.druid.gg/druid-team/druid:v0.1.256"
@@ -117,6 +127,8 @@ func main() {
 		templateVars.Artifact = artifact
 		templateVars.Version = version
 		templateVars.VersionEscaped = strings.Replace(version, ".", "-", -1)
+		templateVars.ProgressImage = progressImage
+		templateVars.ProgressSteamImage = progressSteamImage
 		templateVars.ColdstarterImage = coldstarterImage
 		templateVars.SteamImage = steamImage
 		templateVars.Artifacts = GetArtifactsAbove(version, artifacts, true)

generate-scrolls_test.go

@@ -1,8 +1,13 @@
 package main
 
 import (
+	"fmt"
 	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
 	"slices"
+	"strings"
 	"testing"
 
 	"gopkg.in/yaml.v3"
@@ -80,6 +85,152 @@ func TestGeneratedSharedPortsRemainConcrete(t *testing.T) {
 	}
 }
 
+func TestReleasedScrollsHaveTruthfulInstallProgress(t *testing.T) {
+	for _, path := range releasedScrollPaths(t) {
+		path := path
+		t.Run(path, func(t *testing.T) {
+			scroll := readGeneratedScroll(t, filepath.Join(path, "scroll.yaml"))
+			commands := allGeneratedProcedureCommands(scroll)
+			for _, command := range commands {
+				joined := strings.Join(command, " ")
+				if rawDownloadCommandPattern.MatchString(joined) &&
+					!generatedCommandHasPrefix(command, "druid", "progress") {
+					t.Fatalf("raw payload download has no structured progress: %q", joined)
+				}
+			}
+			requireCompatibleProgressImages(t, scroll)
+			rejectRawDownloadScripts(t, path)
+
+			switch {
+			case strings.Contains(path, "/lgsm/"):
+				requireGeneratedCommandPrefix(t, commands, "druid", "progress", "steamcmd", "--")
+				requireGeneratedProcedureImage(
+					t,
+					scroll,
+					[]string{"sh", "install-lgsm.sh"},
+					"artifacts.druid.gg/druid-team/druid:v0.1.258-steamcmd",
+				)
+			case strings.Contains(path, "/rust/"):
+				requireGeneratedCommandPrefix(t, commands, "druid", "progress", "steamcmd", "--")
+				if strings.Contains(path, "rust-oxide") {
+					requireGeneratedCommandPrefix(t, commands, "druid", "progress", "download")
+				}
+			case strings.Contains(path, "/minecraft/"):
+				requireGeneratedCommandPrefix(t, commands, "druid", "progress", "download")
+				if strings.Contains(path, "/forge/") {
+					findGeneratedProcedure(t, scroll.Commands["install"].Procedures, "install-forge-server")
+				}
+			case strings.Contains(path, "/hytale/hytale-druid-gg"):
+				requireGeneratedCommandPrefix(t, commands, "druid", "progress", "download")
+				findGeneratedProcedure(t, scroll.Commands["install-server"].Procedures, "install-hytale-server")
+			case strings.Contains(path, "/hytale/hytale-standalone"):
+				installScript, err := os.ReadFile(filepath.Join(path, "data", "install.sh"))
+				if err != nil {
+					t.Fatal(err)
+				}
+				if !strings.Contains(string(installScript), "druid progress download") {
+					t.Fatal("Hytale standalone HSM download has no structured progress")
+				}
+				requireGeneratedProcedureImage(
+					t,
+					scroll,
+					[]string{"sh", "install.sh"},
+					"artifacts.druid.gg/druid-team/druid:v0.1.258",
+				)
+				findGeneratedProcedure(t, scroll.Commands["login"].Procedures, "authenticate-hytale")
+				findGeneratedProcedure(t, scroll.Commands["update"].Procedures, "install-hytale-server")
+			default:
+				t.Fatalf("released Scroll has no explicit HIG-24 progress classification: %s", path)
+			}
+		})
+	}
+}
+
+func TestPushRetargetsEmbeddedProgressImages(t *testing.T) {
+	if _, err := exec.LookPath("bash"); err != nil {
+		t.Skip("bash is required")
+	}
+
+	tests := []struct {
+		name          string
+		environment   []string
+		expectedImage string
+	}{
+		{
+			name: "explicit preview images",
+			environment: []string{
+				"DRUID_SCROLL_RUNTIME_IMAGE=local.example/druid:test",
+				"DRUID_SCROLL_STEAMCMD_IMAGE=local.example/druid:test-steamcmd",
+			},
+			expectedImage: "local.example/druid:test",
+		},
+		{
+			name: "plain HTTP local seed",
+			environment: []string{
+				"SCROLL_REGISTRY_HOST=http://druid-gs:8088",
+				"DRUID_REGISTRY_PLAIN_HTTP=true",
+			},
+			expectedImage: "druid:local",
+		},
+	}
+
+	for index, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			fakeDir := filepath.Join(".", fmt.Sprintf(".push-test-%d-%d", os.Getpid(), index))
+			if err := os.Mkdir(fakeDir, 0o755); err != nil {
+				t.Fatal(err)
+			}
+			t.Cleanup(func() { _ = os.RemoveAll(fakeDir) })
+			fakeDruid := filepath.Join(fakeDir, "druid")
+			script := `#!/usr/bin/env bash
+set -euo pipefail
+if [[ "$1" != "push" || "$2" == "category" ]]; then
+  exit 0
+fi
+source_dir="$3"
+scroll="$source_dir/scroll.yaml"
+if grep -Fq 'artifacts.druid.gg/druid-team/druid:v0.1.258' "$scroll"; then
+  echo "pinned progress image leaked into packaged Scroll: $scroll" >&2
+  exit 41
+fi
+if ! grep -Fq "$EXPECTED_PROGRESS_IMAGE" "$scroll"; then
+  echo "retargeted progress image missing from packaged Scroll: $scroll" >&2
+  exit 42
+fi
+`
+			if err := os.WriteFile(fakeDruid, []byte(script), 0o755); err != nil {
+				t.Fatal(err)
+			}
+
+			command := exec.Command("bash", "scripts/push.sh")
+			command.Env = append(
+				os.Environ(),
+				"DRUID_BIN="+filepath.ToSlash(fakeDruid),
+				"SCROLL_PUSH_CATEGORIES=0",
+				"SCROLL_PUSH_ARTIFACTS=1",
+				"EXPECTED_PROGRESS_IMAGE="+test.expectedImage,
+			)
+			command.Env = append(command.Env, test.environment...)
+			output, err := command.CombinedOutput()
+			if err != nil {
+				t.Fatalf("push.sh failed: %v\n%s", err, output)
+			}
+		})
+	}
+}
+
+func TestWorkflowsUseProgressCapableCLIVersion(t *testing.T) {
+	for _, path := range []string{".github/workflows/pr.yml", ".github/workflows/release.yml"} {
+		content, err := os.ReadFile(path)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !strings.Contains(string(content), "DRUID_CLI_VERSION: v0.1.258") {
+			t.Fatalf("%s does not use the progress-capable CLI release", path)
+		}
+	}
+}
+
 type generatedScroll struct {
 	Ports    []generatedPort             `yaml:"ports"`
 	Commands map[string]generatedCommand `yaml:"commands"`
@@ -96,6 +247,7 @@ type generatedCommand struct {
 
 type generatedProcedure struct {
 	ID            string                  `yaml:"id"`
+	Image         string                  `yaml:"image"`
 	ExpectedPorts []generatedExpectedPort `yaml:"expectedPorts"`
 	Command       []string                `yaml:"command"`
 }
@@ -136,3 +288,115 @@ func hasGeneratedExpectedPort(ports []generatedExpectedPort, name string) bool {
 	}
 	return false
 }
+
+func releasedScrollPaths(t *testing.T) []string {
+	t.Helper()
+	content, err := os.ReadFile("scripts/push.sh")
+	if err != nil {
+		t.Fatal(err)
+	}
+	pattern := regexp.MustCompile(`(?m)^\s*run druid push \S+\s+(\./scrolls/\S+)`)
+	matches := pattern.FindAllStringSubmatch(string(content), -1)
+	seen := map[string]bool{}
+	paths := make([]string, 0, len(matches))
+	for _, match := range matches {
+		path := strings.TrimPrefix(match[1], "./")
+		if !seen[path] {
+			seen[path] = true
+			paths = append(paths, path)
+		}
+	}
+	if len(paths) == 0 {
+		t.Fatal("release catalog did not contain any Scroll paths")
+	}
+	return paths
+}
+
+func allGeneratedProcedureCommands(scroll generatedScroll) [][]string {
+	commands := make([][]string, 0)
+	for _, command := range scroll.Commands {
+		for _, procedure := range command.Procedures {
+			commands = append(commands, procedure.Command)
+		}
+	}
+	return commands
+}
+
+func requireGeneratedCommandPrefix(t *testing.T, commands [][]string, prefix ...string) {
+	t.Helper()
+	for _, command := range commands {
+		if generatedCommandHasPrefix(command, prefix...) {
+			return
+		}
+	}
+	t.Fatalf("no procedure command starts with %q", strings.Join(prefix, " "))
+}
+
+func generatedCommandHasPrefix(command []string, prefix ...string) bool {
+	return len(command) >= len(prefix) && slices.Equal(command[:len(prefix)], prefix)
+}
+
+func requireCompatibleProgressImages(t *testing.T, scroll generatedScroll) {
+	t.Helper()
+	for _, command := range scroll.Commands {
+		for _, procedure := range command.Procedures {
+			if !generatedCommandHasPrefix(procedure.Command, "druid", "progress") {
+				continue
+			}
+			expected := "artifacts.druid.gg/druid-team/druid:v0.1.258"
+			if len(procedure.Command) > 2 && procedure.Command[2] == "steamcmd" {
+				expected += "-steamcmd"
+			}
+			if procedure.Image != expected {
+				t.Fatalf("progress command %q uses incompatible image %q; want %q",
+					strings.Join(procedure.Command, " "), procedure.Image, expected)
+			}
+		}
+	}
+}
+
+func requireGeneratedProcedureImage(
+	t *testing.T,
+	scroll generatedScroll,
+	command []string,
+	expectedImage string,
+) {
+	t.Helper()
+	for _, scrollCommand := range scroll.Commands {
+		for _, procedure := range scrollCommand.Procedures {
+			if slices.Equal(procedure.Command, command) {
+				if procedure.Image != expectedImage {
+					t.Fatalf("command %q uses incompatible image %q; want %q",
+						strings.Join(command, " "), procedure.Image, expectedImage)
+				}
+				return
+			}
+		}
+	}
+	t.Fatalf("procedure command %q not found", strings.Join(command, " "))
+}
+
+var rawDownloadCommandPattern = regexp.MustCompile(`(^|[[:space:];|&])(wget|curl|steamcmd)([[:space:];|&]|$)`)
+
+func rejectRawDownloadScripts(t *testing.T, root string) {
+	t.Helper()
+	err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() || filepath.Ext(path) != ".sh" {
+			return nil
+		}
+		content, err := os.ReadFile(path)
+		if err != nil {
+			return err
+		}
+		if rawDownloadCommandPattern.Match(content) {
+			t.Fatalf("raw payload download in published script %s has no structured progress", path)
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}