feat(runtime): expose precise SnapshotProgress during setup

apps/druid/adapters/cli/callback.go

@@ -11,6 +11,23 @@ type runtimeCallbackHandler struct {
 	callbacks *appservices.WorkerCallbackManager
 }
 
+func (h runtimeCallbackHandler) ReportProgress(c *fiber.Ctx) error {
+	var report struct {
+		Token      string `json:"token"`
+		Percentage *int64 `json:"percentage"`
+	}
+	if err := c.BodyParser(&report); err != nil || report.Percentage == nil {
+		return fiber.NewError(fiber.StatusBadRequest, "invalid progress report")
+	}
+	if *report.Percentage < 0 || *report.Percentage > 100 {
+		return fiber.NewError(fiber.StatusBadRequest, "percentage must be between 0 and 100")
+	}
+	if err := h.callbacks.ReportProgress(c.Params("runtime_id"), report.Token, *report.Percentage); err != nil {
+		return fiber.NewError(fiber.StatusUnauthorized, err.Error())
+	}
+	return c.SendStatus(fiber.StatusNoContent)
+}
+
 func (h runtimeCallbackHandler) CompleteWorker(c *fiber.Ctx, runtimeID callbackapi.Runtime) error {
 	var result callbackapi.WorkerResult
 	if err := c.BodyParser(&result); err != nil {

apps/druid/adapters/cli/callback_test.go

@@ -0,0 +1,40 @@
+package cli
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+	appservices "github.com/highcard-dev/daemon/apps/druid/core/services"
+)
+
+func TestRuntimeCallbackHandlerReportsProgress(t *testing.T) {
+	callbacks := appservices.NewWorkerCallbackManager()
+	token, _, err := callbacks.Register("runtime-1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	handler := runtimeCallbackHandler{callbacks: callbacks}
+	app := fiber.New()
+	app.Post("/internal/v1/workers/:runtime_id/progress", handler.ReportProgress)
+
+	request := httptest.NewRequest(
+		http.MethodPost,
+		"/internal/v1/workers/runtime-1/progress",
+		strings.NewReader(fmt.Sprintf(`{"token":%q,"percentage":42}`, token)),
+	)
+	request.Header.Set(fiber.HeaderContentType, fiber.MIMEApplicationJSON)
+	response, err := app.Test(request)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if response.StatusCode != http.StatusNoContent {
+		t.Fatalf("status = %d; want %d", response.StatusCode, http.StatusNoContent)
+	}
+	if progress, ok := callbacks.Progress("runtime-1"); !ok || progress != 42 {
+		t.Fatalf("progress = %v, %v; want 42, true", progress, ok)
+	}
+}

apps/druid/adapters/cli/daemon.go

@@ -170,7 +170,7 @@ func runRuntimeDaemon() error {
 	websocketHandler.SetAllowUnauthenticatedPublic(runtimeAllowUnauthenticatedPublic)
 	handlers := runtimehandlers.RouteHandlers{
 		Server: runtimehandlers.NewRuntimeServer(
-			runtimehandlers.NewHealthHandler(),
+			runtimehandlers.NewHealthHandlerWithProgress(callbacks.Progress),
 			scrollHandler,
 		),
 		Websocket: websocketHandler,
@@ -206,7 +206,9 @@ func runRuntimeDaemon() error {
 	if callbackListener != nil {
 		callbackApp = fiber.New(fiber.Config{DisableStartupMessage: true, ErrorHandler: runtimehandlers.ErrorHandler})
 		callbackApp.Use(runtimehandlers.RequestLogger)
-		callbackapi.RegisterHandlers(callbackApp, runtimeCallbackHandler{callbacks: callbacks})
+		callbackHandler := runtimeCallbackHandler{callbacks: callbacks}
+		callbackapi.RegisterHandlers(callbackApp, callbackHandler)
+		callbackApp.Post("/internal/v1/workers/:runtime_id/progress", callbackHandler.ReportProgress)
 	}
 	return listenRuntimeHTTP(managementApp, publicApp, callbackApp, callbackListener, runtime.Store.StateDir())
 }

apps/druid/adapters/cli/worker_progress_test.go

@@ -0,0 +1,53 @@
+package cli
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/highcard-dev/daemon/internal/core/domain"
+	"github.com/highcard-dev/daemon/internal/core/ports"
+)
+
+func TestWorkerProgressReporterReadsSnapshotProgress(t *testing.T) {
+	reports := make(chan int64, 1)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) {
+		var report struct {
+			Token      string `json:"token"`
+			Percentage int64  `json:"percentage"`
+		}
+		if err := json.NewDecoder(request.Body).Decode(&report); err != nil {
+			t.Error(err)
+		}
+		if report.Token != "token" {
+			t.Errorf("token = %q; want token", report.Token)
+		}
+		reports <- report.Percentage
+		w.WriteHeader(http.StatusNoContent)
+	}))
+	defer server.Close()
+
+	progress := domain.NewSnapshotProgress()
+	progress.Percentage.Store(37)
+	stop := startWorkerProgressReporter(
+		ports.RuntimeWorkerAction{
+			RuntimeID:     "runtime-1",
+			CallbackURL:   server.URL + "/internal/v1/workers/runtime-1/complete",
+			CallbackToken: "token",
+		},
+		progress,
+		time.Hour,
+	)
+	defer stop()
+
+	select {
+	case percentage := <-reports:
+		if percentage != 37 {
+			t.Fatalf("percentage = %d; want 37", percentage)
+		}
+	case <-time.After(time.Second):
+		t.Fatal("progress was not reported")
+	}
+}

apps/druid/adapters/cli/worker_pull.go

@@ -1,13 +1,16 @@
 package cli
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/highcard-dev/daemon/internal/callbackapi"
@@ -65,18 +68,22 @@ func runWorkerPull(action ports.RuntimeWorkerAction) ports.RuntimeWorkerResult {
 	if root == "" {
 		root = "/scroll"
 	}
+	progress := domain.NewSnapshotProgress()
+	stopProgress := startWorkerProgressReporter(action, progress, time.Second)
+	defer stopProgress()
+
 	oci := registry.NewOciClient(loadWorkerRegistryStore())
 	digest, err := oci.ResolveDigest(action.Artifact)
 	if err == nil {
 		result.ArtifactDigest = digest
 	}
 	switch action.Mode {
 	case ports.RuntimeWorkerModeUpdate:
-		err = pullWorkerUpdate(root, action.Artifact, oci)
+		err = pullWorkerUpdate(root, action.Artifact, oci, progress)
 	case ports.RuntimeWorkerModeRestore:
-		err = pullWorkerRestore(root, action.Artifact, oci)
+		err = pullWorkerRestore(root, action.Artifact, oci, progress)
 	default:
-		err = pullWorkerCreate(root, action.Artifact, oci)
+		err = pullWorkerCreate(root, action.Artifact, oci, progress)
 	}
 	if err != nil {
 		result.Error = err.Error()
@@ -115,7 +122,7 @@ func loadWorkerRegistryStore() *registry.CredentialStore {
 	return registry.NewCredentialStore(config.Registries)
 }
 
-func pullWorkerCreate(root string, artifact string, oci ports.OciRegistryInterface) error {
+func pullWorkerCreate(root string, artifact string, oci ports.OciRegistryInterface, progress *domain.SnapshotProgress) error {
 	if err := os.MkdirAll(root, 0755); err != nil {
 		return err
 	}
@@ -137,16 +144,16 @@ func pullWorkerCreate(root string, artifact string, oci ports.OciRegistryInterfa
 		}
 		return copyPath(artifact, root)
 	}
-	return oci.PullSelective(root, artifact, true, nil)
+	return oci.PullSelective(root, artifact, true, progress)
 }
 
-func pullWorkerUpdate(root string, artifact string, oci ports.OciRegistryInterface) error {
+func pullWorkerUpdate(root string, artifact string, oci ports.OciRegistryInterface, progress *domain.SnapshotProgress) error {
 	tmp, err := os.MkdirTemp("", "druid-worker-update-*")
 	if err != nil {
 		return err
 	}
 	defer os.RemoveAll(tmp)
-	if err := coreservices.MaterializeScrollArtifact(artifact, tmp, oci, true); err != nil {
+	if err := coreservices.MaterializeScrollArtifactWithProgress(artifact, tmp, oci, true, progress); err != nil {
 		return err
 	}
 	scrollYAML, err := os.ReadFile(filepath.Join(tmp, "scroll.yaml"))
@@ -162,13 +169,13 @@ func pullWorkerUpdate(root string, artifact string, oci ports.OciRegistryInterfa
 	return mergePulledRoot(tmp, root, skipData)
 }
 
-func pullWorkerRestore(root string, artifact string, oci ports.OciRegistryInterface) error {
+func pullWorkerRestore(root string, artifact string, oci ports.OciRegistryInterface, progress *domain.SnapshotProgress) error {
 	tmp, err := os.MkdirTemp("", "druid-worker-restore-*")
 	if err != nil {
 		return err
 	}
 	defer os.RemoveAll(tmp)
-	if err := coreservices.MaterializeScrollArtifact(artifact, tmp, oci, true); err != nil {
+	if err := coreservices.MaterializeScrollArtifactWithProgress(artifact, tmp, oci, true, progress); err != nil {
 		return err
 	}
 	if err := os.MkdirAll(root, 0755); err != nil {
@@ -313,6 +320,62 @@ func copyPath(src string, dst string) error {
 	return err
 }
 
+func startWorkerProgressReporter(action ports.RuntimeWorkerAction, progress *domain.SnapshotProgress, interval time.Duration) func() {
+	if action.CallbackURL == "" || action.CallbackToken == "" || progress == nil {
+		return func() {}
+	}
+	suffix := "/internal/v1/workers/" + action.RuntimeID + "/complete"
+	baseURL := strings.TrimSuffix(action.CallbackURL, suffix)
+	if baseURL == action.CallbackURL {
+		return func() {}
+	}
+	progressURL := baseURL + "/internal/v1/workers/" + action.RuntimeID + "/progress"
+	client := &http.Client{Timeout: 3 * time.Second}
+	done := make(chan struct{})
+	var wait sync.WaitGroup
+	wait.Add(1)
+	go func() {
+		defer wait.Done()
+		ticker := time.NewTicker(interval)
+		defer ticker.Stop()
+		lastPercentage := int64(-1)
+		report := func() {
+			percentage := progress.Percentage.Load()
+			if percentage == lastPercentage {
+				return
+			}
+			body, _ := json.Marshal(struct {
+				Token      string `json:"token"`
+				Percentage int64  `json:"percentage"`
+			}{action.CallbackToken, percentage})
+			request, _ := http.NewRequest(http.MethodPost, progressURL, bytes.NewReader(body))
+			request.Header.Set("Content-Type", "application/json")
+			response, err := client.Do(request)
+			if err == nil {
+				response.Body.Close()
+				if response.StatusCode < http.StatusBadRequest {
+					lastPercentage = percentage
+				}
+			}
+		}
+		report()
+		for {
+			select {
+			case <-ticker.C:
+				report()
+			case <-done:
+				report()
+				return
+			}
+		}
+	}()
+	var once sync.Once
+	return func() {
+		once.Do(func() { close(done) })
+		wait.Wait()
+	}
+}
+
 func reportWorkerResult(action ports.RuntimeWorkerAction, result ports.RuntimeWorkerResult) error {
 	if action.CallbackURL == "" {
 		body, err := json.Marshal(result)

apps/druid/adapters/cli/worker_test.go

@@ -57,7 +57,7 @@ func TestWorkerRestoreStagesBeforeReplacingRoot(t *testing.T) {
 	mustWrite(t, filepath.Join(root, "data", "old-only.txt"), "old")
 
 	oci := fakeRestoreOCI{t: t}
-	if err := pullWorkerRestore(root, "registry.local/backup:1", oci); err != nil {
+	if err := pullWorkerRestore(root, "registry.local/backup:1", oci, nil); err != nil {
 		t.Fatal(err)
 	}
 

apps/druid/adapters/http/handlers/health_handler.go

@@ -5,12 +5,27 @@ import (
 	"github.com/highcard-dev/daemon/internal/api"
 )
 
-type HealthHandler struct{}
+type ProgressLookup func(runtimeID string) (float64, bool)
+
+type HealthHandler struct {
+	progress ProgressLookup
+}
 
 func NewHealthHandler() *HealthHandler {
 	return &HealthHandler{}
 }
 
+func NewHealthHandlerWithProgress(progress ProgressLookup) *HealthHandler {
+	return &HealthHandler{progress: progress}
+}
+
 func (h *HealthHandler) GetHealthAuth(c *fiber.Ctx) error {
-	return c.JSON(api.HealthResponse{Mode: "ok"})
+	health := api.HealthResponse{Mode: "ok"}
+	if h.progress != nil {
+		if progress, ok := h.progress(c.Params("id")); ok {
+			value := float32(progress)
+			health.Progress = &value
+		}
+	}
+	return c.JSON(health)
 }

apps/druid/adapters/http/handlers/health_handler_test.go

@@ -0,0 +1,33 @@
+package handlers
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/highcard-dev/daemon/internal/api"
+)
+
+func TestGetHealthAuthIncludesPullProgress(t *testing.T) {
+	handler := NewHealthHandlerWithProgress(func(runtimeID string) (float64, bool) {
+		return 37, runtimeID == "scroll-1"
+	})
+	app := fiber.New()
+	app.Get("/:id/api/v1/health", handler.GetHealthAuth)
+
+	response, err := app.Test(httptest.NewRequest(http.MethodGet, "/scroll-1/api/v1/health", nil))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer response.Body.Close()
+
+	var health api.HealthResponse
+	if err := json.NewDecoder(response.Body).Decode(&health); err != nil {
+		t.Fatal(err)
+	}
+	if health.Progress == nil || *health.Progress != 37 {
+		t.Fatalf("progress = %v; want 37", health.Progress)
+	}
+}