feat: track applied migrations as a list instead of a scalar schema_version

Makefile

@@ -1,6 +1,6 @@
 SHELL = /bin/sh
 
-VERSION=1.8.0
+VERSION=2.0.0
 BUILD=`git rev-parse HEAD`
 
 LDFLAGS=-ldflags "-w -s \

UPGRADING.md

@@ -0,0 +1,91 @@
+# Upgrading testtrack-cli
+
+## 1.x to 2.0.0
+
+2.0.0 changes the on-disk schema format (`testtrack/schema.{yml,json}`).
+
+### What changed
+
+The schema's applied-migration high-water mark used to be a single scalar
+field:
+
+```yaml
+serializer_version: 1
+schema_version: "20200115000000"
+```
+
+Every new migration rewrote that one line, so any two branches that added
+migrations conflicted on it. 2.0.0 replaces the scalar with a list of every
+applied migration version (`serializer_version` bumps to `2`):
+
+```yaml
+serializer_version: 2
+schema_versions:
+- "20200601000000"
+- "20200115000000"
+```
+
+The list is ordered by a hash of each version (not chronologically), so
+concurrently added migrations scatter through the file instead of clustering
+on one line. The ordering is otherwise meaningless — don't rely on it.
+
+### This is a breaking format change
+
+A 2.0.0 CLI **refuses to read** an old (`serializer_version: 1`) schema file.
+Any command that reads the schema errors and tells you to run `testtrack schema
+upgrade` — `create`, `migrate`, `sync`, `decide`, the `destroy` commands, and
+`schema load` (essentially everything except `assign`/`unassign`, which use a
+different read path). This is deliberate: the old format can't be losslessly
+read into the new struct, so the CLI makes you convert it explicitly rather than
+silently round-tripping a lossy upgrade.
+
+Conversely, a pre-2.0.0 CLI does **not** understand `schema_versions`. If it
+reads a v2 schema it ignores the field, and the next time it regenerates the
+schema it rewrites the file back to the v1 format. So the whole team — and every
+CI/build environment — has to be on 2.0.0 before you commit a v2 schema, or the
+format will flip-flop between commits.
+
+Because a 2.0.0 CLI rejects a v1 schema and a 1.x CLI reverts a v2 schema, the
+switch is a flag day: the moment the v2 schema lands in the repo, every consumer
+of it must already be on 2.0.0. Order the rollout so the CLI bump and the schema
+commit line up, not in separate steps.
+
+1. **Make 2.0.0 available, but don't switch anything to it yet.** Cut the
+   `v2.0.0` release, update the brew formula, and make the pinned binary
+   available to CI — without yet changing the CI pin or asking devs to upgrade.
+
+2. **Land the schema conversion and the CI bump together, in one change.** In
+   your app root:
+
+   ```sh
+   testtrack schema upgrade
+   ```
+
+   This converts `testtrack/schema.{yml,json}` to the v2 format **in place**: it
+   keeps the materialized state already in the file (splits, decisions,
+   retirements, etc.) and only rebuilds the `schema_versions` list from the
+   migration filenames in `testtrack/migrate`. In the **same** PR, bump the
+   CI/build pin to `v2.0.0`. They must be atomic: if CI runs 2.0.0 against a
+   still-v1 schema, `testtrack migrate` errors; if the v2 schema lands while CI
+   still runs 1.x, the old binary rewrites it back to v1. Expect a large
+   one-time diff — the new `schema_versions` block lists every migration — but
+   the splits body is left untouched.
+
+3. **Developers upgrade as that change lands** (`brew upgrade testtrack-cli`). A
+   developer must be on 2.0.0 before running any testtrack command against the
+   upgraded schema — an older binary silently reverts it to v1.
+
+### Notes
+
+- **Use `schema upgrade`, not `schema generate`, to convert.** `generate`
+  rebuilds the schema by replaying every migration from scratch, which fails on
+  many long-lived apps — e.g. when `testtrack/migrate` predates some splits, or
+  contains a decision/retirement for a split that was created out of band in the
+  TestTrack admin (so there's no create migration to replay). `upgrade` doesn't
+  replay; it preserves the already-correct materialized state and just restamps
+  the format, so it works regardless.
+- `schema upgrade` reads the old file directly (it's the one command that
+  bypasses the version guard), so it's always the recovery path if you hit the
+  "older schema format" error.
+- No server-side change is required. The TestTrack server tracks applied
+  migrations itself; `schema_version`/`schema_versions` is a CLI-only artifact.

cmds/schema_upgrade.go

@@ -0,0 +1,40 @@
+package cmds
+
+import (
+	"github.com/Betterment/testtrack-cli/schema"
+	"github.com/spf13/cobra"
+)
+
+var schemaUpgradeDoc = `
+Upgrades testtrack/schema.{json,yml} to the schema format this CLI writes,
+in place, without replaying migrations.
+
+It keeps the materialized state already in the file (splits, decisions,
+retirements, feature completions, identifier types) and only rebuilds the
+applied-migration version list from the files in testtrack/migrate. This is the
+upgrade path to use when a newer CLI refuses to read an older-format schema.
+
+Unlike 'schema generate', upgrade does not rebuild the schema from scratch, so
+it works on apps whose migrations can't be replayed cleanly - e.g. when
+testtrack/migrate predates some splits, or references splits that were created
+out of band in the TestTrack admin.
+`
+
+func init() {
+	schemaCmd.AddCommand(schemaUpgradeCmd)
+}
+
+var schemaUpgradeCmd = &cobra.Command{
+	Use:   "upgrade",
+	Short: "Upgrade schema.{json,yml} to the current format in place",
+	Long:  schemaUpgradeDoc,
+	Args:  cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, _ []string) error {
+		return schemaUpgrade()
+	},
+}
+
+func schemaUpgrade() error {
+	_, err := schema.Upgrade()
+	return err
+}

fakeserver/server_test.go

@@ -20,8 +20,9 @@ import (
 )
 
 var testSchema = `
-serializer_version: 1
-schema_version: "2020011774023"
+serializer_version: 2
+schema_versions:
+- "2020011774023"
 splits:
 - name: test.test_experiment
   weights:
@@ -34,8 +35,8 @@ splits:
 `
 
 var otherTestSchema = `{
-  "serializer_version": 1,
-  "schema_version": "2020011774023",
+  "serializer_version": 2,
+  "schema_versions": ["2020011774023"],
   "splits": [
     {
       "name": "test.json_experiment",

migrationmanagers/migrationmanagers.go

@@ -94,8 +94,8 @@ func (m *MigrationManager) ApplyToSchema(migrationRepo migrations.Repository, id
 	}
 
 	appliedVersion := m.migration.MigrationVersion()
-	if appliedVersion != nil && m.schema.SchemaVersion < *appliedVersion {
-		m.schema.SchemaVersion = *appliedVersion
+	if appliedVersion != nil {
+		m.schema.AddVersion(*appliedVersion)
 	}
 	return nil
 }
@@ -160,10 +160,7 @@ func (m *MigrationManager) SyncVersion() error {
 		return fmt.Errorf("got %d status code", resp.StatusCode)
 	}
 
-	appliedVersion := m.migration.MigrationVersion()
-	if m.schema.SchemaVersion < *appliedVersion {
-		m.schema.SchemaVersion = *appliedVersion
-	}
+	m.schema.AddVersion(*m.migration.MigrationVersion())
 
 	return nil
 }

schema/schema.go

@@ -1,6 +1,8 @@
 package schema
 
 import (
+	"bytes"
+	"crypto/sha1"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -42,6 +44,65 @@ func Read() (*serializers.Schema, error) {
 	if err != nil {
 		return nil, err
 	}
+	if schema.SerializerVersion > serializers.SerializerVersion {
+		return nil, fmt.Errorf(
+			"%s was written by a newer testtrack CLI (serializer_version %d, this CLI supports %d). Please upgrade your testtrack CLI",
+			schemaPath, schema.SerializerVersion, serializers.SerializerVersion,
+		)
+	}
+	if schema.SerializerVersion < serializers.SerializerVersion {
+		// An older file predates a format change and may be missing data that
+		// can't be reconstructed by re-reading it (e.g. the v1 scalar
+		// schema_version carried no per-migration list). Refuse to read it
+		// rather than silently round-trip a lossy upgrade; `schema upgrade`
+		// converts it in place, preserving the materialized state.
+		return nil, fmt.Errorf(
+			"%s uses an older schema format (serializer_version %d, this CLI writes %d). Run `testtrack schema upgrade` to upgrade it",
+			schemaPath, schema.SerializerVersion, serializers.SerializerVersion,
+		)
+	}
+	return &schema, nil
+}
+
+// Upgrade converts an existing schema file to the current serializer format in
+// place. Unlike Generate it does not replay migrations, so it preserves the
+// already-materialized state (splits, decisions, retirements, etc.) and works
+// even on schemas that can't be rebuilt from scratch — e.g. apps whose
+// testtrack/migrate predates some splits or references ones created out of
+// band. The only thing it rebuilds is the applied-version list, which it reads
+// from the migration filenames on disk.
+func Upgrade() (*serializers.Schema, error) {
+	schemaPath, exists := findSchemaPath()
+	if !exists {
+		return nil, errors.New("no testtrack schema file to upgrade. Run testtrack schema generate to create one")
+	}
+	schemaBytes, err := os.ReadFile(schemaPath)
+	if err != nil {
+		return nil, err
+	}
+	var schema serializers.Schema
+	err = yaml.Unmarshal(schemaBytes, &schema)
+	if err != nil {
+		return nil, err
+	}
+	if schema.SerializerVersion > serializers.SerializerVersion {
+		return nil, fmt.Errorf(
+			"%s was written by a newer testtrack CLI (serializer_version %d, this CLI writes %d). Please upgrade your testtrack CLI",
+			schemaPath, schema.SerializerVersion, serializers.SerializerVersion,
+		)
+	}
+
+	migrationRepo, err := migrationloaders.Load()
+	if err != nil {
+		return nil, err
+	}
+	schema.SchemaVersions = migrationRepo.SortedVersions()
+	schema.SerializerVersion = serializers.SerializerVersion
+
+	err = Write(&schema)
+	if err != nil {
+		return nil, err
+	}
 	return &schema, nil
 }
 
@@ -63,9 +124,10 @@ func Generate() (*serializers.Schema, error) {
 	return schema, nil
 }
 
-// Write a schema to disk after alpha-sorting its resources
+// Write a schema to disk after sorting its resources into a stable order
 func Write(schema *serializers.Schema) error {
 	SortAlphabetically(schema)
+	sortSchemaVersions(schema)
 
 	schemaPath, _ := findSchemaPath()
 
@@ -204,12 +266,28 @@ func applyAllMigrationsToSchema(schema *serializers.Schema) error {
 			return err
 		}
 	}
-	if len(versions) != 0 {
-		schema.SchemaVersion = versions[len(versions)-1]
-	}
+	schema.SchemaVersions = versions
 	return nil
 }
 
+// sortSchemaVersions orders the applied-version list by the SHA-1 of each
+// version. The order is otherwise meaningless; hashing scatters newly added
+// versions through the list instead of clustering them by timestamp, so two
+// branches that each append a migration rarely touch the same lines. Hashes
+// are precomputed so each version is hashed once rather than on every
+// comparison.
+func sortSchemaVersions(schema *serializers.Schema) {
+	hashes := make(map[string][sha1.Size]byte, len(schema.SchemaVersions))
+	for _, version := range schema.SchemaVersions {
+		hashes[version] = sha1.Sum([]byte(version))
+	}
+	sort.Slice(schema.SchemaVersions, func(i, j int) bool {
+		a := hashes[schema.SchemaVersions[i]]
+		b := hashes[schema.SchemaVersions[j]]
+		return bytes.Compare(a[:], b[:]) < 0
+	})
+}
+
 // SortAlphabetically sorts the schema's resource slices by their natural keys
 func SortAlphabetically(schema *serializers.Schema) {
 	sort.Slice(schema.RemoteKills, func(i, j int) bool {