Add rebaseMigrationHistory feature. This supports a full reset of mig…

README.md

@@ -76,3 +76,13 @@ dbmigration/R__create_views_repeatable.sql
 
 "Version migrations" start with `V` (or not) and have a version number (1.2 etc) followed by double underscore `__` and then a comment.
 
+## Rebasing migration history
+
+Set `ebean.migration.rebaseMigrationHistory=true` for a one-time rebase after compacting
+existing migrations. The runner keeps its internal bootstrap row, deletes all other migration
+history, and records every migration currently found under `migrationPath` with its normal
+checksum without executing any migration SQL.
+
+This is destructive and must only be used when the existing database schema is known to match
+the replacement migrations. `dbinit` resources are not included. Remove the setting after the
+rebase; otherwise future migrations will be recorded without being executed.

ebean-migration/pom.xml

@@ -80,7 +80,7 @@
     <dependency>
       <groupId>com.microsoft.sqlserver</groupId>
       <artifactId>mssql-jdbc</artifactId>
-      <version>9.4.1.jre8</version>
+      <version>10.2.0.jre8</version>
       <scope>test</scope>
     </dependency>
 
@@ -143,21 +143,21 @@ mvn install:install-file -Dfile=/some/path/to/ojdbc7.jar -DgroupId=oracle \
     <dependency>
       <groupId>io.avaje</groupId>
       <artifactId>avaje-simple-logger</artifactId>
-      <version>0.5</version>
+      <version>2.0</version>
       <scope>test</scope>
     </dependency>
 
     <dependency>
       <groupId>io.ebean</groupId>
       <artifactId>ebean-test-containers</artifactId>
-      <version>7.14</version>
+      <version>8.4</version>
       <scope>test</scope>
     </dependency>
 
     <dependency>
       <groupId>io.avaje</groupId>
       <artifactId>junit</artifactId>
-      <version>1.6</version>
+      <version>1.8</version>
       <scope>test</scope>
     </dependency>
 

ebean-migration/src/main/java/io/ebean/migration/MigrationConfig.java

@@ -21,6 +21,7 @@ public class MigrationConfig {
 
   private boolean skipMigrationRun;
   private boolean skipChecksum;
+  private boolean rebaseMigrationHistory;
   private ClassLoader classLoader;
 
   private String dbUsername;
@@ -183,6 +184,26 @@ public void setSkipMigrationRun(boolean skipMigrationRun) {
     this.skipMigrationRun = skipMigrationRun;
   }
 
+  /**
+   * Return true if the migration history will be replaced with the migrations currently defined.
+   * <p>
+   * This does not execute migration scripts. It is intended for a one-time migration-history
+   * rebase after the existing migrations have been compacted.
+   */
+  public boolean isRebaseMigrationHistory() {
+    return rebaseMigrationHistory;
+  }
+
+  /**
+   * Set true to replace the migration history with checksum entries for the migrations currently defined.
+   * <p>
+   * This does not execute migration scripts. Remove this setting after the one-time rebase so that
+   * subsequent migrations execute normally.
+   */
+  public void setRebaseMigrationHistory(boolean rebaseMigrationHistory) {
+    this.rebaseMigrationHistory = rebaseMigrationHistory;
+  }
+
   /**
    * Return true if checksum check should be skipped (during development).
    */
@@ -472,6 +493,7 @@ public void load(Properties props) {
     fastMode = property("fastMode", fastMode);
     skipMigrationRun = property("skipMigrationRun", skipMigrationRun);
     skipChecksum = property("skipChecksum", skipChecksum);
+    rebaseMigrationHistory = property("rebaseMigrationHistory", rebaseMigrationHistory);
     earlyChecksumMode = property("earlyChecksumMode", earlyChecksumMode);
     createSchemaIfNotExists = property("createSchemaIfNotExists", createSchemaIfNotExists);
     setCurrentSchema = property("setCurrentSchema", setCurrentSchema);

ebean-migration/src/main/java/io/ebean/migration/runner/MigrationEngine.java

@@ -54,7 +54,12 @@ public List<MigrationResource> run(MigrationContext context) {
 
     long startMs = System.currentTimeMillis();
     LocalMigrationResources resources = new LocalMigrationResources(migrationConfig);
-    if (!resources.readResources() && !resources.readInitResources()) {
+    boolean rebaseHistory = migrationConfig.isRebaseMigrationHistory();
+    boolean hasMigrations = resources.readResources();
+    if (rebaseHistory && !hasMigrations) {
+      throw new MigrationException("Cannot rebase migration history without defined migrations");
+    }
+    if (!hasMigrations && !resources.readInitResources()) {
       log.log(DEBUG, "no migrations to check");
       return emptyList();
     }
@@ -63,7 +68,7 @@ public List<MigrationResource> run(MigrationContext context) {
     long splitMs = System.currentTimeMillis() - startMs;
     final var platform = derivePlatform(migrationConfig, connection);
     final var firstCheck = new FirstCheck(migrationConfig, context, platform);
-    if (fastMode && firstCheck.fastModeCheck(resources.versions())) {
+    if (!rebaseHistory && fastMode && firstCheck.fastModeCheck(resources.versions())) {
       long checkMs = System.currentTimeMillis() - startMs;
       log.log(INFO, "DB migrations completed in {0}ms - totalMigrations:{1} readResources:{2}ms", checkMs, firstCheck.count(), splitMs);
       return emptyList();
@@ -73,7 +78,9 @@ public List<MigrationResource> run(MigrationContext context) {
 
     final MigrationTable table = initialiseMigrationTable(firstCheck, connection);
     try {
-      List<MigrationResource> result = runMigrations(table, resources.versions());
+      List<MigrationResource> result = rebaseHistory
+        ? table.rebaseMigrationHistory(resources.versions())
+        : runMigrations(table, resources.versions());
       connection.commit();
       if (!checkStateOnly) {
         long commitMs = System.currentTimeMillis();

ebean-migration/src/main/java/io/ebean/migration/runner/MigrationTable.java

@@ -534,6 +534,49 @@ List<MigrationResource> runAll(List<LocalMigrationResource> localVersions) throw
     return checkMigrations;
   }
 
+  /**
+   * Replace the existing migration history with the currently defined migrations without executing them.
+   */
+  List<MigrationResource> rebaseMigrationHistory(List<LocalMigrationResource> localVersions) throws SQLException {
+    if (checkStateOnly) {
+      checkMigrations.addAll(localVersions);
+      return checkMigrations;
+    }
+
+    clearMigrationHistory();
+    for (LocalMigrationResource localVersion : localVersions) {
+      insertIntoHistory(localVersion, rebaseChecksum(localVersion), 0);
+    }
+    return checkMigrations;
+  }
+
+  private int rebaseChecksum(LocalMigrationResource local) {
+    if (local instanceof LocalUriMigrationResource) {
+      // index file migration with precomputed checksum
+      return ((LocalUriMigrationResource)local).checksum();
+    }
+    if (local instanceof LocalDdlMigrationResource) {
+      final var content = local.content();
+      final var script = convertScript(content);
+      // normal migration checksums without executing the script.
+      return Checksum.calculate(earlyChecksumMode ? content : script);
+    }
+    return ((LocalJdbcMigrationResource)local).checksum();
+  }
+
+  private void clearMigrationHistory() throws SQLException {
+    final var deleteSql = "delete from " + sqlTable + " where mversion <> ?";
+    try (var statement = context.connection().prepareStatement(deleteSql)) {
+      statement.setString(1, INIT_VER_0);
+      statement.executeUpdate();
+    }
+    migrations.clear();
+    currentVersion = null;
+    lastMigration = null;
+    priorVersion = null;
+    dbInitVersion = null;
+  }
+
   private void checkMinVersion() {
     if (minVersion != null && currentVersion != null && currentVersion.compareTo(minVersion) < 0) {
       StringBuilder sb = new StringBuilder();

ebean-migration/src/test/java/io/ebean/migration/MigrationConfigTest.java

@@ -27,6 +27,7 @@ public void testDefaults() {
     assertTrue(config.isSetCurrentSchema());
     assertFalse(config.isSkipChecksum());
     assertFalse(config.isSkipMigrationRun());
+    assertFalse(config.isRebaseMigrationHistory());
     assertEquals(config.getMetaTable(), "db_migration");
     assertNull(config.getRunPlaceholders());
     assertEquals(config.getMigrationPath(), "dbmigration");
@@ -45,6 +46,7 @@ public void loadProperties_ebean_migration() {
     props.setProperty("ebean.migration.setCurrentSchema","false");
     props.setProperty("ebean.migration.skipMigrationRun","true");
     props.setProperty("ebean.migration.skipChecksum","true");
+    props.setProperty("ebean.migration.rebaseMigrationHistory","true");
     props.setProperty("ebean.migration.driver","driver");
     props.setProperty("ebean.migration.url","url");
     props.setProperty("ebean.migration.metaTable","metaTable");
@@ -66,6 +68,7 @@ public void loadProperties() {
     props.setProperty("dbmigration.setCurrentSchema","false");
     props.setProperty("dbmigration.skipMigrationRun","true");
     props.setProperty("dbmigration.skipChecksum","true");
+    props.setProperty("dbmigration.rebaseMigrationHistory","true");
     props.setProperty("dbmigration.driver","driver");
     props.setProperty("dbmigration.url","url");
     props.setProperty("dbmigration.metaTable","metaTable");
@@ -88,6 +91,7 @@ private void assertLoadedProperties(Properties props) {
     assertFalse(config.isSetCurrentSchema());
     assertTrue(config.isSkipChecksum());
     assertTrue(config.isSkipMigrationRun());
+    assertTrue(config.isRebaseMigrationHistory());
     assertEquals(config.getMetaTable(), "metaTable");
     assertEquals(config.getRunPlaceholders(), "placeholders");
     assertEquals(config.getMigrationPath(), "migrationPath");
@@ -100,6 +104,7 @@ public void loadProperties_withName() {
     Properties props = new Properties();
     props.setProperty("ebean.mydb.migration.username","username");
     props.setProperty("ebean.mydb.migration.migrationPath","migrationPath");
+    props.setProperty("ebean.mydb.migration.rebaseMigrationHistory","true");
     props.setProperty("ebean.migration.password","password");
     props.setProperty("ebean.migration.schema","fooSchema");
     props.setProperty("dbmigration.url","url");
@@ -113,6 +118,7 @@ public void loadProperties_withName() {
     assertEquals(config.getDbUsername(), "username");
     assertEquals(config.getDbPassword(), "password");
     assertEquals(config.getDbSchema(), "fooSchema");
+    assertTrue(config.isRebaseMigrationHistory());
 
     assertEquals(config.getDbUrl(), "url");
     assertEquals(config.getMetaTable(), "metaTable");

ebean-migration/src/test/java/io/ebean/migration/MigrationRunnerTest.java

@@ -315,6 +315,106 @@ public void run_with_skipMigration() throws SQLException {
     }
   }
 
+  @Test
+  void run_rebaseMigrationHistory() throws SQLException {
+
+    var dataSource = DataSourcePool.builder()
+      .name("rebaseMigrationHistory")
+      .url("jdbc:h2:mem:rebaseMigrationHistory")
+      .username("sa")
+      .password("")
+      .build();
+    try {
+      MigrationConfig config = createMigrationConfig();
+      config.setMigrationPath("dbmig");
+      MigrationRunner runner = new MigrationRunner(config);
+      runner.run(dataSource);
+
+      config.setMigrationPath("dbmig_rebase");
+      config.setRebaseMigrationHistory(true);
+      // checkState with the rebase
+      assertThat(runner.checkState(dataSource)).hasSize(1);
+      try (Connection connection = dataSource.getConnection()) {
+        assertThat(migrationVersions(connection)).containsExactly("0", "hello", "1.1", "1.2", "1.2.1", "m2_view");
+      }
+
+      // perform the rebase, expect only 0 and 1.0 migrations
+      runner.run(dataSource);
+      try (Connection connection = dataSource.getConnection()) {
+        assertThat(migrationVersions(connection)).containsExactly("0", "1.0");
+        assertThat(singleQueryResult(connection, "select count(*) from m3")).containsExactly("1");
+      }
+
+      config.setRebaseMigrationHistory(false);
+      assertThat(runner.checkState(dataSource)).isEmpty();
+    } finally {
+      dataSource.shutdown();
+    }
+  }
+
+  @Test
+  public void run_rebaseMigrationHistory_bypassesFastMode() throws SQLException {
+
+    var dataSource = DataSourcePool.builder()
+      .name("rebaseMigrationHistoryFastMode")
+      .url("jdbc:h2:mem:rebaseMigrationHistoryFastMode")
+      .username("sa")
+      .password("")
+      .build();
+    try {
+      MigrationConfig config = createMigrationConfig();
+      config.setMigrationPath("dbmig");
+      MigrationRunner runner = new MigrationRunner(config);
+      runner.run(dataSource);
+
+      try (Connection connection = dataSource.getConnection();
+           PreparedStatement statement = connection.prepareStatement("update db_migration set mcomment = 'stale' where mversion = '1.1'")) {
+        statement.executeUpdate();
+        connection.commit();
+      }
+
+      config.setRebaseMigrationHistory(true);
+      runner.run(dataSource);
+
+      try (Connection connection = dataSource.getConnection()) {
+        assertThat(singleQueryResult(connection, "select mcomment from db_migration where mversion = '1.1'"))
+          .containsExactly("initial");
+        assertThat(migrationVersions(connection)).containsExactly("0", "hello", "1.1", "1.2", "1.2.1", "m2_view");
+      }
+    } finally {
+      dataSource.shutdown();
+    }
+  }
+
+  @Test
+  public void run_rebaseMigrationHistory_withoutMigrationsFails() throws SQLException {
+
+    var dataSource = DataSourcePool.builder()
+      .name("rebaseMigrationHistoryNoMigrations")
+      .url("jdbc:h2:mem:rebaseMigrationHistoryNoMigrations")
+      .username("sa")
+      .password("")
+      .build();
+    try {
+      MigrationConfig config = createMigrationConfig();
+      config.setMigrationPath("dbmig");
+      MigrationRunner runner = new MigrationRunner(config);
+      runner.run(dataSource);
+
+      config.setMigrationPath("missing-rebase-migrations");
+      config.setRebaseMigrationHistory(true);
+      assertThatThrownBy(() -> runner.run(dataSource))
+        .isInstanceOf(MigrationException.class)
+        .hasMessage("Cannot rebase migration history without defined migrations");
+
+      try (Connection connection = dataSource.getConnection()) {
+        assertThat(migrationNames(connection)).contains("<init>", "hello", "initial", "add_m3", "test", "m2_view");
+      }
+    } finally {
+      dataSource.shutdown();
+    }
+  }
+
 
   /**
    * Run this integration test manually against CockroachDB.
@@ -337,6 +437,10 @@ private List<String> migrationNames(Connection connection) throws SQLException {
     return singleQueryResult(connection, "select mcomment from db_migration");
   }
 
+  private List<String> migrationVersions(Connection connection) throws SQLException {
+    return singleQueryResult(connection, "select mversion from db_migration order by id");
+  }
+
   private List<String> singleQueryResult(Connection connection, String sql) throws SQLException {
     List<String> names = new ArrayList<>();
     try (final PreparedStatement statement = connection.prepareStatement(sql)) {

ebean-migration/src/test/java/io/ebean/migration/runner/MigrationTableAsyncTest.java

@@ -134,7 +134,7 @@ public void testMariaDb() throws Exception {
   @Test
   public void testSqlServer() throws Exception {
     // init sqlserver docker container
-    SqlServerContainer container = SqlServerContainer.builder("2017-GA-ubuntu")
+    SqlServerContainer container = SqlServerContainer.builder("2019-latest")
       .port(9435)
       .containerName("mig_async_sqlserver")
       .dbName("test_ebean")
@@ -147,7 +147,7 @@ public void testSqlServer() throws Exception {
     config.setMigrationPath("dbmig_sqlserver");
     config.setDbUsername("test_ebean");
     config.setDbPassword("SqlS3rv#r");
-    config.setDbUrl("jdbc:sqlserver://localhost:9435;databaseName=test_ebean;sendTimeAsDateTime=false");
+    config.setDbUrl(container.jdbcUrl() + ";sendTimeAsDateTime=false");
     runTest(true);
     runTest(false);
   }

ebean-migration/src/test/resources/dbmig_rebase/1.0__initial.sql

@@ -0,0 +1 @@
+drop table m3;