diff --git a/README.md b/README.md index c8cb636..31195d8 100644 --- a/README.md +++ b/README.md @@ -76,3 +76,13 @@ dbmigration/R__create_views_repeatable.sql "Version migrations" start with `V` (or not) and have a version number (1.2 etc) followed by double underscore `__` and then a comment. +## Rebasing migration history + +Set `ebean.migration.rebaseMigrationHistory=true` for a one-time rebase after compacting +existing migrations. The runner keeps its internal bootstrap row, deletes all other migration +history, and records every migration currently found under `migrationPath` with its normal +checksum without executing any migration SQL. + +This is destructive and must only be used when the existing database schema is known to match +the replacement migrations. `dbinit` resources are not included. Remove the setting after the +rebase; otherwise future migrations will be recorded without being executed. diff --git a/ebean-migration/pom.xml b/ebean-migration/pom.xml index f819dc6..eccc586 100644 --- a/ebean-migration/pom.xml +++ b/ebean-migration/pom.xml @@ -80,7 +80,7 @@ com.microsoft.sqlserver mssql-jdbc - 9.4.1.jre8 + 10.2.0.jre8 test @@ -143,21 +143,21 @@ mvn install:install-file -Dfile=/some/path/to/ojdbc7.jar -DgroupId=oracle \ io.avaje avaje-simple-logger - 0.5 + 2.0 test io.ebean ebean-test-containers - 7.14 + 8.4 test io.avaje junit - 1.6 + 1.8 test diff --git a/ebean-migration/src/main/java/io/ebean/migration/MigrationConfig.java b/ebean-migration/src/main/java/io/ebean/migration/MigrationConfig.java index 5b1d6bd..9526009 100644 --- a/ebean-migration/src/main/java/io/ebean/migration/MigrationConfig.java +++ b/ebean-migration/src/main/java/io/ebean/migration/MigrationConfig.java @@ -21,6 +21,7 @@ public class MigrationConfig { private boolean skipMigrationRun; private boolean skipChecksum; + private boolean rebaseMigrationHistory; private ClassLoader classLoader; private String dbUsername; @@ -183,6 +184,26 @@ public void setSkipMigrationRun(boolean skipMigrationRun) { this.skipMigrationRun = skipMigrationRun; } + /** + * Return true if the migration history will be replaced with the migrations currently defined. + *

+ * This does not execute migration scripts. It is intended for a one-time migration-history + * rebase after the existing migrations have been compacted. + */ + public boolean isRebaseMigrationHistory() { + return rebaseMigrationHistory; + } + + /** + * Set true to replace the migration history with checksum entries for the migrations currently defined. + *

+ * This does not execute migration scripts. Remove this setting after the one-time rebase so that + * subsequent migrations execute normally. + */ + public void setRebaseMigrationHistory(boolean rebaseMigrationHistory) { + this.rebaseMigrationHistory = rebaseMigrationHistory; + } + /** * Return true if checksum check should be skipped (during development). */ @@ -472,6 +493,7 @@ public void load(Properties props) { fastMode = property("fastMode", fastMode); skipMigrationRun = property("skipMigrationRun", skipMigrationRun); skipChecksum = property("skipChecksum", skipChecksum); + rebaseMigrationHistory = property("rebaseMigrationHistory", rebaseMigrationHistory); earlyChecksumMode = property("earlyChecksumMode", earlyChecksumMode); createSchemaIfNotExists = property("createSchemaIfNotExists", createSchemaIfNotExists); setCurrentSchema = property("setCurrentSchema", setCurrentSchema); diff --git a/ebean-migration/src/main/java/io/ebean/migration/runner/MigrationEngine.java b/ebean-migration/src/main/java/io/ebean/migration/runner/MigrationEngine.java index 8de37fb..98263bf 100644 --- a/ebean-migration/src/main/java/io/ebean/migration/runner/MigrationEngine.java +++ b/ebean-migration/src/main/java/io/ebean/migration/runner/MigrationEngine.java @@ -54,7 +54,12 @@ public List run(MigrationContext context) { long startMs = System.currentTimeMillis(); LocalMigrationResources resources = new LocalMigrationResources(migrationConfig); - if (!resources.readResources() && !resources.readInitResources()) { + boolean rebaseHistory = migrationConfig.isRebaseMigrationHistory(); + boolean hasMigrations = resources.readResources(); + if (rebaseHistory && !hasMigrations) { + throw new MigrationException("Cannot rebase migration history without defined migrations"); + } + if (!hasMigrations && !resources.readInitResources()) { log.log(DEBUG, "no migrations to check"); return emptyList(); } @@ -63,7 +68,7 @@ public List run(MigrationContext context) { long splitMs = System.currentTimeMillis() - startMs; final var platform = derivePlatform(migrationConfig, connection); final var firstCheck = new FirstCheck(migrationConfig, context, platform); - if (fastMode && firstCheck.fastModeCheck(resources.versions())) { + if (!rebaseHistory && fastMode && firstCheck.fastModeCheck(resources.versions())) { long checkMs = System.currentTimeMillis() - startMs; log.log(INFO, "DB migrations completed in {0}ms - totalMigrations:{1} readResources:{2}ms", checkMs, firstCheck.count(), splitMs); return emptyList(); @@ -73,7 +78,9 @@ public List run(MigrationContext context) { final MigrationTable table = initialiseMigrationTable(firstCheck, connection); try { - List result = runMigrations(table, resources.versions()); + List result = rebaseHistory + ? table.rebaseMigrationHistory(resources.versions()) + : runMigrations(table, resources.versions()); connection.commit(); if (!checkStateOnly) { long commitMs = System.currentTimeMillis(); diff --git a/ebean-migration/src/main/java/io/ebean/migration/runner/MigrationTable.java b/ebean-migration/src/main/java/io/ebean/migration/runner/MigrationTable.java index 0ab5b13..1f317bc 100644 --- a/ebean-migration/src/main/java/io/ebean/migration/runner/MigrationTable.java +++ b/ebean-migration/src/main/java/io/ebean/migration/runner/MigrationTable.java @@ -534,6 +534,49 @@ List runAll(List localVersions) throw return checkMigrations; } + /** + * Replace the existing migration history with the currently defined migrations without executing them. + */ + List rebaseMigrationHistory(List localVersions) throws SQLException { + if (checkStateOnly) { + checkMigrations.addAll(localVersions); + return checkMigrations; + } + + clearMigrationHistory(); + for (LocalMigrationResource localVersion : localVersions) { + insertIntoHistory(localVersion, rebaseChecksum(localVersion), 0); + } + return checkMigrations; + } + + private int rebaseChecksum(LocalMigrationResource local) { + if (local instanceof LocalUriMigrationResource) { + // index file migration with precomputed checksum + return ((LocalUriMigrationResource)local).checksum(); + } + if (local instanceof LocalDdlMigrationResource) { + final var content = local.content(); + final var script = convertScript(content); + // normal migration checksums without executing the script. + return Checksum.calculate(earlyChecksumMode ? content : script); + } + return ((LocalJdbcMigrationResource)local).checksum(); + } + + private void clearMigrationHistory() throws SQLException { + final var deleteSql = "delete from " + sqlTable + " where mversion <> ?"; + try (var statement = context.connection().prepareStatement(deleteSql)) { + statement.setString(1, INIT_VER_0); + statement.executeUpdate(); + } + migrations.clear(); + currentVersion = null; + lastMigration = null; + priorVersion = null; + dbInitVersion = null; + } + private void checkMinVersion() { if (minVersion != null && currentVersion != null && currentVersion.compareTo(minVersion) < 0) { StringBuilder sb = new StringBuilder(); diff --git a/ebean-migration/src/test/java/io/ebean/migration/MigrationConfigTest.java b/ebean-migration/src/test/java/io/ebean/migration/MigrationConfigTest.java index 62c721b..d1663e9 100644 --- a/ebean-migration/src/test/java/io/ebean/migration/MigrationConfigTest.java +++ b/ebean-migration/src/test/java/io/ebean/migration/MigrationConfigTest.java @@ -27,6 +27,7 @@ public void testDefaults() { assertTrue(config.isSetCurrentSchema()); assertFalse(config.isSkipChecksum()); assertFalse(config.isSkipMigrationRun()); + assertFalse(config.isRebaseMigrationHistory()); assertEquals(config.getMetaTable(), "db_migration"); assertNull(config.getRunPlaceholders()); assertEquals(config.getMigrationPath(), "dbmigration"); @@ -45,6 +46,7 @@ public void loadProperties_ebean_migration() { props.setProperty("ebean.migration.setCurrentSchema","false"); props.setProperty("ebean.migration.skipMigrationRun","true"); props.setProperty("ebean.migration.skipChecksum","true"); + props.setProperty("ebean.migration.rebaseMigrationHistory","true"); props.setProperty("ebean.migration.driver","driver"); props.setProperty("ebean.migration.url","url"); props.setProperty("ebean.migration.metaTable","metaTable"); @@ -66,6 +68,7 @@ public void loadProperties() { props.setProperty("dbmigration.setCurrentSchema","false"); props.setProperty("dbmigration.skipMigrationRun","true"); props.setProperty("dbmigration.skipChecksum","true"); + props.setProperty("dbmigration.rebaseMigrationHistory","true"); props.setProperty("dbmigration.driver","driver"); props.setProperty("dbmigration.url","url"); props.setProperty("dbmigration.metaTable","metaTable"); @@ -88,6 +91,7 @@ private void assertLoadedProperties(Properties props) { assertFalse(config.isSetCurrentSchema()); assertTrue(config.isSkipChecksum()); assertTrue(config.isSkipMigrationRun()); + assertTrue(config.isRebaseMigrationHistory()); assertEquals(config.getMetaTable(), "metaTable"); assertEquals(config.getRunPlaceholders(), "placeholders"); assertEquals(config.getMigrationPath(), "migrationPath"); @@ -100,6 +104,7 @@ public void loadProperties_withName() { Properties props = new Properties(); props.setProperty("ebean.mydb.migration.username","username"); props.setProperty("ebean.mydb.migration.migrationPath","migrationPath"); + props.setProperty("ebean.mydb.migration.rebaseMigrationHistory","true"); props.setProperty("ebean.migration.password","password"); props.setProperty("ebean.migration.schema","fooSchema"); props.setProperty("dbmigration.url","url"); @@ -113,6 +118,7 @@ public void loadProperties_withName() { assertEquals(config.getDbUsername(), "username"); assertEquals(config.getDbPassword(), "password"); assertEquals(config.getDbSchema(), "fooSchema"); + assertTrue(config.isRebaseMigrationHistory()); assertEquals(config.getDbUrl(), "url"); assertEquals(config.getMetaTable(), "metaTable"); diff --git a/ebean-migration/src/test/java/io/ebean/migration/MigrationRunnerTest.java b/ebean-migration/src/test/java/io/ebean/migration/MigrationRunnerTest.java index 8738b2f..83e013d 100644 --- a/ebean-migration/src/test/java/io/ebean/migration/MigrationRunnerTest.java +++ b/ebean-migration/src/test/java/io/ebean/migration/MigrationRunnerTest.java @@ -315,6 +315,106 @@ public void run_with_skipMigration() throws SQLException { } } + @Test + void run_rebaseMigrationHistory() throws SQLException { + + var dataSource = DataSourcePool.builder() + .name("rebaseMigrationHistory") + .url("jdbc:h2:mem:rebaseMigrationHistory") + .username("sa") + .password("") + .build(); + try { + MigrationConfig config = createMigrationConfig(); + config.setMigrationPath("dbmig"); + MigrationRunner runner = new MigrationRunner(config); + runner.run(dataSource); + + config.setMigrationPath("dbmig_rebase"); + config.setRebaseMigrationHistory(true); + // checkState with the rebase + assertThat(runner.checkState(dataSource)).hasSize(1); + try (Connection connection = dataSource.getConnection()) { + assertThat(migrationVersions(connection)).containsExactly("0", "hello", "1.1", "1.2", "1.2.1", "m2_view"); + } + + // perform the rebase, expect only 0 and 1.0 migrations + runner.run(dataSource); + try (Connection connection = dataSource.getConnection()) { + assertThat(migrationVersions(connection)).containsExactly("0", "1.0"); + assertThat(singleQueryResult(connection, "select count(*) from m3")).containsExactly("1"); + } + + config.setRebaseMigrationHistory(false); + assertThat(runner.checkState(dataSource)).isEmpty(); + } finally { + dataSource.shutdown(); + } + } + + @Test + public void run_rebaseMigrationHistory_bypassesFastMode() throws SQLException { + + var dataSource = DataSourcePool.builder() + .name("rebaseMigrationHistoryFastMode") + .url("jdbc:h2:mem:rebaseMigrationHistoryFastMode") + .username("sa") + .password("") + .build(); + try { + MigrationConfig config = createMigrationConfig(); + config.setMigrationPath("dbmig"); + MigrationRunner runner = new MigrationRunner(config); + runner.run(dataSource); + + try (Connection connection = dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement("update db_migration set mcomment = 'stale' where mversion = '1.1'")) { + statement.executeUpdate(); + connection.commit(); + } + + config.setRebaseMigrationHistory(true); + runner.run(dataSource); + + try (Connection connection = dataSource.getConnection()) { + assertThat(singleQueryResult(connection, "select mcomment from db_migration where mversion = '1.1'")) + .containsExactly("initial"); + assertThat(migrationVersions(connection)).containsExactly("0", "hello", "1.1", "1.2", "1.2.1", "m2_view"); + } + } finally { + dataSource.shutdown(); + } + } + + @Test + public void run_rebaseMigrationHistory_withoutMigrationsFails() throws SQLException { + + var dataSource = DataSourcePool.builder() + .name("rebaseMigrationHistoryNoMigrations") + .url("jdbc:h2:mem:rebaseMigrationHistoryNoMigrations") + .username("sa") + .password("") + .build(); + try { + MigrationConfig config = createMigrationConfig(); + config.setMigrationPath("dbmig"); + MigrationRunner runner = new MigrationRunner(config); + runner.run(dataSource); + + config.setMigrationPath("missing-rebase-migrations"); + config.setRebaseMigrationHistory(true); + assertThatThrownBy(() -> runner.run(dataSource)) + .isInstanceOf(MigrationException.class) + .hasMessage("Cannot rebase migration history without defined migrations"); + + try (Connection connection = dataSource.getConnection()) { + assertThat(migrationNames(connection)).contains("", "hello", "initial", "add_m3", "test", "m2_view"); + } + } finally { + dataSource.shutdown(); + } + } + /** * Run this integration test manually against CockroachDB. @@ -337,6 +437,10 @@ private List migrationNames(Connection connection) throws SQLException { return singleQueryResult(connection, "select mcomment from db_migration"); } + private List migrationVersions(Connection connection) throws SQLException { + return singleQueryResult(connection, "select mversion from db_migration order by id"); + } + private List singleQueryResult(Connection connection, String sql) throws SQLException { List names = new ArrayList<>(); try (final PreparedStatement statement = connection.prepareStatement(sql)) { diff --git a/ebean-migration/src/test/java/io/ebean/migration/runner/MigrationTableAsyncTest.java b/ebean-migration/src/test/java/io/ebean/migration/runner/MigrationTableAsyncTest.java index a6795d8..a704f92 100644 --- a/ebean-migration/src/test/java/io/ebean/migration/runner/MigrationTableAsyncTest.java +++ b/ebean-migration/src/test/java/io/ebean/migration/runner/MigrationTableAsyncTest.java @@ -134,7 +134,7 @@ public void testMariaDb() throws Exception { @Test public void testSqlServer() throws Exception { // init sqlserver docker container - SqlServerContainer container = SqlServerContainer.builder("2017-GA-ubuntu") + SqlServerContainer container = SqlServerContainer.builder("2019-latest") .port(9435) .containerName("mig_async_sqlserver") .dbName("test_ebean") @@ -147,7 +147,7 @@ public void testSqlServer() throws Exception { config.setMigrationPath("dbmig_sqlserver"); config.setDbUsername("test_ebean"); config.setDbPassword("SqlS3rv#r"); - config.setDbUrl("jdbc:sqlserver://localhost:9435;databaseName=test_ebean;sendTimeAsDateTime=false"); + config.setDbUrl(container.jdbcUrl() + ";sendTimeAsDateTime=false"); runTest(true); runTest(false); } diff --git a/ebean-migration/src/test/resources/dbmig_rebase/1.0__initial.sql b/ebean-migration/src/test/resources/dbmig_rebase/1.0__initial.sql new file mode 100644 index 0000000..a125077 --- /dev/null +++ b/ebean-migration/src/test/resources/dbmig_rebase/1.0__initial.sql @@ -0,0 +1 @@ +drop table m3;