diff --git a/README.md b/README.md index 3edfdbd..714609a 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,8 @@ ![Logo](./docs/assets/logo.png) -Simple database migration tool for Scala + Postgres with [skunk](https://typelevel.org/skunk/) that can be deployed on JVM and Native. +Simple database migration tool for [Postgres](https://www.postgresql.org) with [skunk](https://typelevel.org/skunk/). +Usable via command-line or as library in your Scala project targeting JVM or Native (see [usage example](#usage-example)). Supports a subset of [Flyway](https://flywaydb.org) features and keeps a Flyway compatible history state to allow you to switch to Flyway if necessary. You might also be able to simply switch from Flyway to Dumbo without any changes in migration files or the history state, depending on used Flyway features. @@ -58,6 +59,17 @@ key=value executeInTransaction=false ``` + ⚠️⚠️ + **NOTE**: Dumbo will attempt to execute each migration as a [simple query with multiple statements](https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-FLOW-MULTI-STATEMENT) + in a transaction by default (unlike Flyway which may decide not to do so). + To disable it you need to set it explicitly using the configuration above. + + Use with care and try to avoid where possible. Partially applied migrations will require manual intervention. + Dumbo is not going to update the history state in case of partial failures. + If you re-run the migration process it will attempt to execute the script the same way it did before it failed. + To fix it you'd need to roll back applied updates manually and then update the migration script and/or split it into multiple files before re-running the migration process. + ⚠️⚠️ + ## Usage example For usage via command line see [command-line](#command-line) section. diff --git a/build.sbt b/build.sbt index 0057871..513c51c 100644 --- a/build.sbt +++ b/build.sbt @@ -235,9 +235,9 @@ lazy val skunkVersion = "1.0.0-M5" lazy val epollcatVersion = "0.1.6" -lazy val munitVersion = "1.0.0-M11" +lazy val munitVersion = "1.0.0-RC1" -lazy val munitCEVersion = "2.0.0-M4" +lazy val munitCEVersion = "2.0.0-RC1" lazy val core = crossProject(JVMPlatform, NativePlatform) .crossType(CrossType.Full) @@ -344,7 +344,7 @@ lazy val tests = crossProject(JVMPlatform, NativePlatform) }, ) -lazy val flywayVersion = "10.11.1" +lazy val flywayVersion = "10.12.0" lazy val postgresqlVersion = "42.7.3" lazy val testsFlyway = project .in(file("modules/tests-flyway")) diff --git a/docker-compose.yaml b/docker-compose.yaml index 65f939f..d70fc0b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,13 +1,39 @@ services: - pg_1: - image: postgres:15-alpine + pg_latest_1: + image: postgres:16-alpine ports: - "5432:5432" environment: - POSTGRES_PASSWORD: postgres - pg_2: - image: postgres:15-alpine + POSTGRES_USER: root + POSTGRES_HOST_AUTH_METHOD: trust + pg_latest_2: + image: postgres:16-alpine ports: - "5433:5432" environment: - POSTGRES_PASSWORD: postgres \ No newline at end of file + POSTGRES_USER: root + POSTGRES_HOST_AUTH_METHOD: trust + pg_11_1: + image: postgres:11-alpine + ports: + - "5434:5432" + environment: + POSTGRES_USER: root + POSTGRES_HOST_AUTH_METHOD: trust + pg_11_2: + image: postgres:11-alpine + ports: + - "5435:5432" + environment: + POSTGRES_USER: root + POSTGRES_HOST_AUTH_METHOD: trust + cockroachdb_1: + image: cockroachdb/cockroach:v23.2.4 + ports: + - "5436:26257" + command: start-single-node --insecure + cockroachdb_2: + image: cockroachdb/cockroach:v23.2.4 + ports: + - "5437:26257" + command: start-single-node --insecure diff --git a/modules/core/shared/src/main/scala/dumbo/Dumbo.scala b/modules/core/shared/src/main/scala/dumbo/Dumbo.scala index 2133157..cff692d 100644 --- a/modules/core/shared/src/main/scala/dumbo/Dumbo.scala +++ b/modules/core/shared/src/main/scala/dumbo/Dumbo.scala @@ -63,40 +63,46 @@ final class DumboWithResourcesPartiallyApplied[F[_]](reader: ResourceReader[F]) schemas = schemas, schemaHistoryTable = schemaHistoryTable, validateOnMigrate = validateOnMigrate, - progressMonitor = Async[F].background { - Stream - .evalSeq( - sessionResource - .use( - _.execute( - sql"""SELECT ps.pid, ps.query_start, ps.state_change, ps.state, ps.wait_event_type, ps.wait_event, ps.query + progressMonitor = Resource + .eval(sessionResource.use(s => Dumbo.hasTableLockSupport(s, s"${defaultSchema}.${schemaHistoryTable}"))) + .flatMap { + case false => Resource.eval(Console[F].println("Progress monitor is not supported for current database")) + case true => + Async[F].background { + Stream + .evalSeq( + sessionResource + .use( + _.execute( + sql"""SELECT ps.pid, ps.query_start, ps.state_change, ps.state, ps.wait_event_type, ps.wait_event, ps.query FROM pg_locks l JOIN pg_stat_all_tables t ON t.relid = l.relation JOIN pg_stat_activity ps ON ps.pid = l.pid WHERE t.schemaname = '#${defaultSchema}' and t.relname = '#${schemaHistoryTable}'""" - .query(int4 *: timestamptz *: timestamptz *: text *: text.opt *: text.opt *: text) - ).map(_.groupByNel { case pid *: _ => pid }.toList.map(_._2.head)) - ) - ) - .evalMap { case pid *: start *: changed *: state *: eventType *: event *: query *: _ => - for { - now <- Clock[F].realTimeInstant - startedAgo = now.getEpochSecond() - start.toEpochSecond() - changedAgo = now.getEpochSecond() - changed.toEpochSecond() - queryLogSize = 150 - queryLog = query.take(queryLogSize) + (if (query.size > queryLogSize) "..." else "") - _ <- - Console[F].println( - s"Awaiting query with pid: $pid started: ${startedAgo}s ago (state: $state / last changed: ${changedAgo}s ago, " + - s"eventType: ${eventType.getOrElse("")}, event: ${event.getOrElse("")}):\n${queryLog}" + .query(int4 *: timestamptz *: timestamptz *: text *: text.opt *: text.opt *: text) + ).map(_.groupByNel { case pid *: _ => pid }.toList.map(_._2.head)) + ) ) - } yield () - } - .repeat - .metered(logMigrationStateAfter) - .compile - .drain - }.void, + .evalMap { case pid *: start *: changed *: state *: eventType *: event *: query *: _ => + for { + now <- Clock[F].realTimeInstant + startedAgo = now.getEpochSecond() - start.toEpochSecond() + changedAgo = now.getEpochSecond() - changed.toEpochSecond() + queryLogSize = 150 + queryLog = query.take(queryLogSize) + (if (query.size > queryLogSize) "..." else "") + _ <- + Console[F].println( + s"Awaiting query with pid: $pid started: ${startedAgo}s ago (state: $state / last changed: ${changedAgo}s ago, " + + s"eventType: ${eventType.getOrElse("")}, event: ${event.getOrElse("")}):\n${queryLog}" + ) + } yield () + } + .repeat + .metered(logMigrationStateAfter) + .compile + .drain + }.void + }, ) } @@ -117,7 +123,7 @@ final class DumboWithResourcesPartiallyApplied[F[_]](reader: ResourceReader[F]) user = connection.user, database = connection.database, password = connection.password, - strategy = Typer.Strategy.SearchPath, + strategy = Typer.Strategy.BuiltinsOnly, ssl = connection.ssl, parameters = params, ) @@ -201,6 +207,7 @@ class Dumbo[F[_]: Sync: Console]( private def migrateToNext( session: Session[F], fs: ResourceReader[F], + tableLockSupport: Boolean, )(sourceFiles: List[ResourceFile]): F[Option[(HistoryEntry, List[ResourceFile])]] = sourceFiles match { case Nil => none.pure[F] @@ -210,7 +217,8 @@ class Dumbo[F[_]: Sync: Console]( _ <- progressMonitor } yield txn).use { _ => for { - _ <- session.execute(sql"LOCK TABLE #${historyTable} IN ACCESS EXCLUSIVE MODE".command) + _ <- if (tableLockSupport) lockTable(session, historyTable).void + else session.executeDiscard(sql"SELECT * FROM #${historyTable} FOR UPDATE".command) latestInstalled <- session.unique(dumboHistory.findLatestInstalled).map(_.flatMap(_.sourceFileVersion)) result <- sourceFiles.dropWhile(s => latestInstalled.exists(s.version <= _)) match { case head :: tail => @@ -232,7 +240,7 @@ class Dumbo[F[_]: Sync: Console]( // it's supposed to be prevented by IF NOT EXISTS clause when running concurrently // but it doesn't always seem to prevent it, maybe better to lock another table instead of catching those? - // https://www.postgresql.org/docs/15/errcodes-appendix.html + // https://www.postgresql.org/docs/current/errcodes-appendix.html private val duplicateErrorCodes = Set( "42710", // duplicate_object "23505", // unique_violation @@ -242,6 +250,8 @@ class Dumbo[F[_]: Sync: Console]( def runMigration: F[MigrationResult] = sessionResource.use(migrateBySession) private def migrateBySession(session: Session[F]): F[Dumbo.MigrationResult] = for { + dbVersion <- session.unique(sql"SELECT version()".query(text)) + _ <- Console[F].println(s"Starting migration on $dbVersion") schemaRes <- allSchemas.toList .flatTraverse(schema => @@ -254,6 +264,7 @@ class Dumbo[F[_]: Sync: Console]( _ <- session.execute(dumboHistory.createTableCommand).void.recover { case e: skunk.exception.PostgresErrorException if duplicateErrorCodes.contains(e.code) => () } + tableLockSupport <- hasTableLockSupport(session, historyTable) _ <- schemaRes match { case e @ (_ :: _) => session.execute(dumboHistory.insertSchemaEntry)(e.mkString("\"", "\",\"", "\"")).void case _ => ().pure[F] @@ -272,11 +283,12 @@ class Dumbo[F[_]: Sync: Console]( Console[F].println(s"Found ${sourceFiles.size} versioned migration files$inLocation") } _ <- if (validateOnMigrate) validationGuard(session, sourceFiles) else ().pure[F] - migrationResult <- Stream - .unfoldEval(sourceFiles)(migrateToNext(session, resReader)) - .compile - .toList - .map(Dumbo.MigrationResult(_)) + migrationResult <- + Stream + .unfoldEval(sourceFiles)(migrateToNext(session, resReader, tableLockSupport)) + .compile + .toList + .map(Dumbo.MigrationResult(_)) } yield migrationResult _ <- migrationResult.migrations.sorted(Ordering[HistoryEntry].reverse).headOption match { @@ -355,6 +367,17 @@ object Dumbo extends internal.DumboPlatform { def withFilesIn[F[_]: Files](dir: Path): DumboWithResourcesPartiallyApplied[F] = new DumboWithResourcesPartiallyApplied[F](ResourceReader.fileFs(dir)) + private[dumbo] def hasTableLockSupport[F[_]: Sync](session: Session[F], table: String) = + session.transaction.use(_ => + lockTable(session, table).attempt.map { + case Right(Completion.LockTable) => true + case _ => false + } + ) + + private def lockTable[F[_]](session: Session[F], table: String) = + session.execute(sql"LOCK TABLE #${table} IN ACCESS EXCLUSIVE MODE".command) + private[dumbo] def listMigrationFiles[F[_]: Sync]( fs: ResourceReader[F] ): F[ValidatedNec[DumboValidationException, List[ResourceFile]]] = @@ -380,7 +403,7 @@ object Dumbo extends internal.DumboPlatform { fs: ResourceReader[F] ): Stream[F, Either[String, ResourceFile]] = fs.list - .filter(f => f.value.endsWith(".sql") || f.value.endsWith(".sql.conf")) + .filter(f => f.value.endsWith(".sql")) .evalMap { path => val confPath = path.append(".conf") diff --git a/modules/core/shared/src/main/scala/dumbo/History.scala b/modules/core/shared/src/main/scala/dumbo/History.scala index ea9ffba..dae38b9 100644 --- a/modules/core/shared/src/main/scala/dumbo/History.scala +++ b/modules/core/shared/src/main/scala/dumbo/History.scala @@ -51,21 +51,21 @@ object HistoryEntry { .to[HistoryEntry] val fieldNames = - "installed_rank, version, description, type, script, checksum, installed_by, installed_on, execution_time, success" + "installed_rank::INT4, version, description, type, script, checksum::INT4, installed_by, installed_on, execution_time::INT4, success" } class History(tableName: String) { val createTableCommand: Command[Void] = sql"""CREATE TABLE IF NOT EXISTS #${tableName} ( - installed_rank INT NOT NULL PRIMARY KEY, + installed_rank INT4 NOT NULL PRIMARY KEY, version VARCHAR(50) NULL, description VARCHAR(200) NOT NULL, type VARCHAR(20) NOT NULL, script VARCHAR(1000) NOT NULL, - checksum INT NULL, + checksum INT4 NULL, installed_by VARCHAR(100) NOT NULL, installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - execution_time INT NOT NULL, + execution_time INT4 NOT NULL, success BOOL NOT NULL )""".command diff --git a/modules/example/src/main/scala/ExampleApp.scala b/modules/example/src/main/scala/ExampleApp.scala index e6c8ce5..bfc4bb7 100644 --- a/modules/example/src/main/scala/ExampleApp.scala +++ b/modules/example/src/main/scala/ExampleApp.scala @@ -13,9 +13,9 @@ object ExampleApp extends IOApp.Simple { connection = ConnectionConfig( host = "localhost", port = 5432, - user = "postgres", + user = "root", database = "postgres", - password = Some("postgres"), + password = None, ssl = skunk.SSL.None, // skunk.SSL config, default is skunk.SSL.None ), defaultSchema = "dumbo", diff --git a/modules/tests-flyway/src/test/scala/DumboSpec.scala b/modules/tests-flyway/src/test/scala/DumboSpec.scala index ce9691a..5bf5b0e 100644 --- a/modules/tests-flyway/src/test/scala/DumboSpec.scala +++ b/modules/tests-flyway/src/test/scala/DumboSpec.scala @@ -4,14 +4,16 @@ package dumbo +import scala.io.AnsiColor + import cats.data.NonEmptyList import cats.effect.IO import fs2.io.file.Path import org.flywaydb.core.Flyway import org.flywaydb.core.api.output.MigrateResult -class DumboSpec extends ffstest.FTest { - override val postgresPort: Int = 5433 +trait DumboSpec extends ffstest.FTest { + def db: Db def flywayMigrate(defaultSchema: String, sourcesPath: Path, schemas: List[String] = Nil): IO[MigrateResult] = IO( Flyway @@ -21,8 +23,8 @@ class DumboSpec extends ffstest.FTest { .locations(sourcesPath.toString) .dataSource( s"jdbc:postgresql://localhost:$postgresPort/postgres?ssl=false", - "postgres", - "postgres", + "root", + null, ) .load() .migrate() @@ -66,23 +68,38 @@ class DumboSpec extends ffstest.FTest { val schema = "schema_1" for { - flywayRes <- flywayMigrate(schema, Path("db/test_failing_sql")).attempt - _ = assert(flywayRes.isLeft) - _ = assert(flywayRes.left.exists(_.getMessage().contains("relation \"test\" already exists"))) - historyFlyway <- loadHistory(schema) - _ <- dropSchemas - dumboRes <- dumboMigrate(schema, dumboWithResources("db/test_failing_sql")).attempt - _ = assert(dumboRes.isLeft) - _ = assert(dumboRes.left.exists(_.getMessage().contains("Relation \"test\" already exists"))) - historyDumbo <- loadHistory(schema) - _ = assertEqualHistory(historyFlyway, historyDumbo) + flywayRes <- flywayMigrate(schema, Path("db/test_failing_sql")).attempt + _ = assert(flywayRes.isLeft) + // Flyway does not provide more specific error message with CockroachDB in this case + _ = if (Set[Db](Db.Postgres(16), Db.Postgres(11)).contains(db)) { + assert(flywayRes.left.exists(_.getMessage().contains("relation \"test\" already exists"))) + } + historyFlyway <- loadHistory(schema).map(h => + db match { + case Db.Postgres(_) => h + // Flyway is not able to run it within a transaction and rollback, so it adds a history entry with success false in CockroachDB + // going to ignore it in the test for now... + case Db.CockroachDb => h.filter(_.success == true) + } + ) + _ <- dropSchemas + dumboRes <- dumboMigrate(schema, dumboWithResources("db/test_failing_sql")).attempt + _ = assert(dumboRes.isLeft) + _ = assert( + dumboRes.left.exists( + _.getMessage().linesIterator.exists(_.matches(""".*Relation ".*test" already exists.*""")) + ) + ) + historyDumbo <- loadHistory(schema) + _ = assertEqualHistory(historyFlyway, historyDumbo) } yield () } dbTest("Dumbo is compatible with Flyway history state") { - val path: Path = Path("db/test_1") - val withResources = dumboWithResources("db/test_1") - val defaultSchema = "test_a" + val path: Path = Path("db/test_1") + val withResources = dumboWithResources("db/test_1") + val withResourcesB = dumboWithResources("db/test_1_extended") + val defaultSchema = "test_a" for { flywayRes <- flywayMigrate(defaultSchema, path) @@ -92,12 +109,15 @@ class DumboSpec extends ffstest.FTest { resDumbo <- dumboMigrate(defaultSchema, withResources) _ = assertEquals(resDumbo.migrationsExecuted, 0) histB <- loadHistory(defaultSchema) - _ = assertEquals(histA, histB) // history unchanged + _ = assertEquals(histA, histB) // history unchanged + _ <- assertIO(dumboMigrate(defaultSchema, withResourcesB).map(_.migrationsExecuted), 1) + _ <- assertIO(loadHistory(defaultSchema).map(_.length), histB.length + 1) // history extended } yield () } dbTest("Flyway is compatible with Dumbo history state") { - val path: Path = Path("db/test_1") + val path = Path("db/test_1") + val pathB = Path("db/test_1_extended") val withResources = dumboWithResources("db/test_1") val defaultSchema = "test_a" @@ -109,7 +129,9 @@ class DumboSpec extends ffstest.FTest { _ = assert(flywayRes.success) _ = assertEquals(flywayRes.migrationsExecuted, 0) histB <- loadHistory(defaultSchema) - _ = assertEquals(histA, histB) // history unchanged + _ = assertEquals(histA, histB) // history unchanged + _ <- assertIO(flywayMigrate(defaultSchema, pathB).map(_.migrationsExecuted), 1) + _ <- assertIO(loadHistory(defaultSchema).map(_.length), histB.length + 1) // history extended } yield () } @@ -156,14 +178,21 @@ class DumboSpec extends ffstest.FTest { val schemas = NonEmptyList.of("schema_1", "schema_2") for { - flywayRes <- flywayMigrate(schemas.head, path, schemas.tail).attempt - _ = assert(flywayRes.isLeft) - flywayHistory <- loadHistory(schemas.head) - _ <- dropSchemas - dumboRes <- dumboMigrate(schemas.head, withResources, schemas.tail).attempt - _ = assert(dumboRes.isLeft) - dumboHistory <- loadHistory(schemas.head) - _ = assertEqualHistory(flywayHistory, dumboHistory) + flywayRes <- flywayMigrate(schemas.head, path, schemas.tail).attempt + _ = assert(flywayRes.isLeft) + flywayHistory <- loadHistory(schemas.head).map(h => + db match { + case Db.Postgres(_) => h + // Flyway is not able to run it within a transaction and rollback, so it adds a history entry with success false in CockroachDB + // going to ignore it in the test for now... + case Db.CockroachDb => h.filter(_.success == true) + } + ) + _ <- dropSchemas + dumboRes <- dumboMigrate(schemas.head, withResources, schemas.tail).attempt + _ = assert(dumboRes.isLeft) + dumboHistory <- loadHistory(schemas.head) + _ = assertEqualHistory(flywayHistory, dumboHistory) } yield () } @@ -189,26 +218,41 @@ class DumboSpec extends ffstest.FTest { val withResources = dumboWithResources("db/test_non_transactional") val schema = "schema_1" - for { - flywayRes <- flywayMigrate(schema, path).attempt - _ = assert( - flywayRes.left.exists(_.getMessage().contains("New enum values must be committed before they can be used")) - ) - flywayHistory <- loadHistory(schema) - _ <- dropSchemas - dumboRes <- dumboMigrate(schema, withResources).attempt - _ = assert( - dumboRes.left.exists(_.getMessage().contains("New enum values must be committed before they can be used")) - ) - dumboHistory <- loadHistory(schema) - _ = assertEqualHistory(flywayHistory, dumboHistory) - } yield () + // TODO: find a way to force Flyway to run the migration in a transaction on CockroachDb + if (db == Db.Postgres(11) || db == Db.Postgres(16)) + for { + flywayRes <- flywayMigrate(schema, path).attempt + flywayHistory <- loadHistory(schema) + _ <- dropSchemas + dumboRes <- dumboMigrate(schema, withResources).attempt + dumboHistory <- loadHistory(schema) + _ = assert(flywayRes.isLeft) + _ = assert(dumboRes.isLeft) + _ = List( + flywayRes.swap.toOption.get, + dumboRes.swap.toOption.get, + ).map(_.getMessage().toLowerCase().linesIterator).foreach { lines => + db match { + case Db.Postgres(11) => + assert(lines.exists(_.matches(".*alter type .* cannot run inside a transaction block.*"))) + case Db.Postgres(_) => + assert(lines.exists(_.matches(""".*unsafe use of new value ".*" of enum type.*"""))) + case Db.CockroachDb => + assert(lines.exists(_.matches(""".*enum value ".*" is not yet public.*"""))) + } + } + _ = assertEqualHistory(flywayHistory, dumboHistory) + } yield () + else + IO.println( + s"${AnsiColor.YELLOW}[$db] Skipping test 'Same behaviour on non-transactional operations' as Flyway can't run the statements in a transaction${AnsiColor.RESET}" + ) } dbTest("Same behavior on copy") { val path: Path = Path("db/test_copy_to") val withResources = dumboWithResources("db/test_copy_to") - val schema = "public" + val schema = "schema_1" for { flywayRes <- flywayMigrate(schema, path).attempt @@ -219,3 +263,24 @@ class DumboSpec extends ffstest.FTest { } yield () } } + +sealed trait Db +object Db { + case class Postgres(version: Int) extends Db + case object CockroachDb extends Db +} + +class DumboFlywaySpecPostgresLatest extends DumboSpec { + override val db: Db = Db.Postgres(16) + override val postgresPort: Int = 5433 +} + +class DumboFlywaySpecPostgres11 extends DumboSpec { + override val db: Db = Db.Postgres(11) + override val postgresPort: Int = 5435 +} + +class DumboFlywaySpecCockroachDb extends DumboSpec { + override val db: Db = Db.CockroachDb + override val postgresPort: Int = 5437 +} diff --git a/modules/tests/shared/src/main/scala/ffstest/FFramework.scala b/modules/tests/shared/src/main/scala/ffstest/FFramework.scala index 691deb0..e129965 100644 --- a/modules/tests/shared/src/main/scala/ffstest/FFramework.scala +++ b/modules/tests/shared/src/main/scala/ffstest/FFramework.scala @@ -5,6 +5,7 @@ package ffstest import scala.concurrent.duration.* +import scala.util.Random import cats.data.ValidatedNec import cats.effect.{IO, Resource, std} @@ -21,12 +22,14 @@ trait FTest extends CatsEffectSuite with FTestPlatform { def dbTest(name: String)(f: => IO[Unit]): Unit = test(name)(dropSchemas >> f) + def someSchemaName = s"schema_${Random.alphanumeric.take(10).mkString}" + lazy val connectionConfig: ConnectionConfig = ConnectionConfig( host = "localhost", port = postgresPort, - user = "postgres", + user = "root", database = "postgres", - password = Some("postgres"), + password = None, ) def session: Resource[IO, Session[IO]] = Session @@ -81,10 +84,13 @@ trait FTest extends CatsEffectSuite with FTestPlatform { def dropSchemas: IO[Unit] = session.use { s => for { customSchemas <- - s.execute(sql""" - SELECT schema_name + s.execute( + sql""" + SELECT schema_name::text FROM information_schema.schemata - WHERE schema_name NOT LIKE 'pg_%' AND schema_name != 'information_schema'""".query(skunk.codec.text.name)) + WHERE schema_name NOT LIKE 'pg_%' AND schema_name NOT LIKE 'crdb_%' AND schema_name NOT IN ('information_schema', 'public')""" + .query(skunk.codec.text.text) + ) _ <- IO.println(s"Dropping schemas ${customSchemas.mkString(", ")}") c <- customSchemas.traverse(schema => s.execute(sql"DROP SCHEMA IF EXISTS #${schema} CASCADE".command)) _ <- IO.println(s"Schema drop result ${c.mkString(", ")}") diff --git a/modules/tests/shared/src/test/resources/db/test_1/V2__test_b.sql b/modules/tests/shared/src/test/resources/db/test_1/V2__test_b.sql index 23c3a58..7ec6eda 100644 --- a/modules/tests/shared/src/test/resources/db/test_1/V2__test_b.sql +++ b/modules/tests/shared/src/test/resources/db/test_1/V2__test_b.sql @@ -6,6 +6,7 @@ CREATE TABLE test_v2 ( val_3 JSON, val_4 JSONB, val_5 INT[], + val_6 test_enum_type NOT NULL DEFAULT 'T1_ONE'::test_enum_type, date DATE, PRIMARY KEY (key_a, key_b) ); @@ -15,9 +16,3 @@ CREATE FUNCTION add(integer, integer) RETURNS integer LANGUAGE SQL IMMUTABLE RETURNS NULL ON NULL INPUT; - -CREATE OR REPLACE FUNCTION increment(i integer) RETURNS integer AS $$ - BEGIN - RETURN i + 1; - END; -$$ LANGUAGE plpgsql; diff --git a/modules/tests/shared/src/test/resources/db/test_1_extended/V1__test.sql b/modules/tests/shared/src/test/resources/db/test_1_extended/V1__test.sql new file mode 100644 index 0000000..37b8fd8 --- /dev/null +++ b/modules/tests/shared/src/test/resources/db/test_1_extended/V1__test.sql @@ -0,0 +1,14 @@ +CREATE TYPE test_enum_type AS ENUM ('T1_ONE', 't2_two', 't3_Three', 'T4_FOUR', 'T5_FIVE', 'T6Six', 'MULTIPLE_WORD_ENUM'); + +-- some comment +CREATE TABLE test ( + -- ignore this... + id SERIAL PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + name text, -- ignore this too + name_2 varchar NOT NULL, -- and this as well + number int, + template test_enum_type +); + +--SELECT 1 diff --git a/modules/tests/shared/src/test/resources/db/test_1_extended/V2__test_b.sql b/modules/tests/shared/src/test/resources/db/test_1_extended/V2__test_b.sql new file mode 100644 index 0000000..7ec6eda --- /dev/null +++ b/modules/tests/shared/src/test/resources/db/test_1_extended/V2__test_b.sql @@ -0,0 +1,18 @@ +CREATE TABLE test_v2 ( + key_a VARCHAR NOT NULL, + key_b VARCHAR NOT NULL, + val_1 VARCHAR[] NOT NULL, + val_2 INT[], + val_3 JSON, + val_4 JSONB, + val_5 INT[], + val_6 test_enum_type NOT NULL DEFAULT 'T1_ONE'::test_enum_type, + date DATE, + PRIMARY KEY (key_a, key_b) +); + +CREATE FUNCTION add(integer, integer) RETURNS integer + AS 'select $1 + $2;' + LANGUAGE SQL + IMMUTABLE + RETURNS NULL ON NULL INPUT; diff --git a/modules/tests/shared/src/test/resources/db/test_1_extended/V3__test_c.sql b/modules/tests/shared/src/test/resources/db/test_1_extended/V3__test_c.sql new file mode 100644 index 0000000..2a7a537 --- /dev/null +++ b/modules/tests/shared/src/test/resources/db/test_1_extended/V3__test_c.sql @@ -0,0 +1,11 @@ +CREATE TABLE test_v3 ( + key_a VARCHAR NOT NULL, + key_b VARCHAR NOT NULL, + val_1 VARCHAR[] NOT NULL, + val_2 INT[], + val_3 JSON, + val_4 JSONB, + val_5 INT[], + date DATE, + PRIMARY KEY (key_a, key_b) +); diff --git a/modules/tests/shared/src/test/resources/db/test_1_extended/V4__extend.sql b/modules/tests/shared/src/test/resources/db/test_1_extended/V4__extend.sql new file mode 100644 index 0000000..68d5dfa --- /dev/null +++ b/modules/tests/shared/src/test/resources/db/test_1_extended/V4__extend.sql @@ -0,0 +1,3 @@ +CREATE TABLE test_v4 (en test_enum_type); +INSERT INTO test_v4 VALUES ('T1_ONE'); +SELECT * FROM test_v4; diff --git a/modules/tests/shared/src/test/resources/db/test_copy_from/V1__test.sql b/modules/tests/shared/src/test/resources/db/test_copy_from/V1__test.sql index cd2d548..502e785 100644 --- a/modules/tests/shared/src/test/resources/db/test_copy_from/V1__test.sql +++ b/modules/tests/shared/src/test/resources/db/test_copy_from/V1__test.sql @@ -1,2 +1 @@ CREATE TABLE test (id VARCHAR); -COPY test FROM STDIN; diff --git a/modules/tests/shared/src/test/resources/db/test_copy_from/V2__copy.sql b/modules/tests/shared/src/test/resources/db/test_copy_from/V2__copy.sql new file mode 100644 index 0000000..da9a9a6 --- /dev/null +++ b/modules/tests/shared/src/test/resources/db/test_copy_from/V2__copy.sql @@ -0,0 +1 @@ +COPY test FROM STDIN; diff --git a/modules/tests/shared/src/test/resources/db/test_copy_to/V1__test.sql b/modules/tests/shared/src/test/resources/db/test_copy_to/V1__test.sql index 1190efd..502e785 100644 --- a/modules/tests/shared/src/test/resources/db/test_copy_to/V1__test.sql +++ b/modules/tests/shared/src/test/resources/db/test_copy_to/V1__test.sql @@ -1,3 +1 @@ CREATE TABLE test (id VARCHAR); -COPY test TO STDOUT; -CREATE TABLE test_2 (id VARCHAR); diff --git a/modules/tests/shared/src/test/resources/db/test_copy_to/V2__copy.sql b/modules/tests/shared/src/test/resources/db/test_copy_to/V2__copy.sql new file mode 100644 index 0000000..555a358 --- /dev/null +++ b/modules/tests/shared/src/test/resources/db/test_copy_to/V2__copy.sql @@ -0,0 +1 @@ +COPY test TO STDOUT; diff --git a/modules/tests/shared/src/test/resources/db/test_long_running/V1__long_running.sql b/modules/tests/shared/src/test/resources/db/test_long_running/V1__long_running.sql index b843675..d716a44 100644 --- a/modules/tests/shared/src/test/resources/db/test_long_running/V1__long_running.sql +++ b/modules/tests/shared/src/test/resources/db/test_long_running/V1__long_running.sql @@ -1 +1 @@ -DO $$ BEGIN PERFORM pg_sleep(2); END $$; \ No newline at end of file +SELECT pg_sleep(2); \ No newline at end of file diff --git a/modules/tests/shared/src/test/resources/db/test_non_transactional/V1__init.sql b/modules/tests/shared/src/test/resources/db/test_non_transactional/V1__init.sql new file mode 100644 index 0000000..0f868c1 --- /dev/null +++ b/modules/tests/shared/src/test/resources/db/test_non_transactional/V1__init.sql @@ -0,0 +1 @@ +CREATE TYPE test_enum_type AS ENUM ('T1'); diff --git a/modules/tests/shared/src/test/resources/db/test_non_transactional/V1__test.sql b/modules/tests/shared/src/test/resources/db/test_non_transactional/V2__test.sql similarity index 73% rename from modules/tests/shared/src/test/resources/db/test_non_transactional/V1__test.sql rename to modules/tests/shared/src/test/resources/db/test_non_transactional/V2__test.sql index b1e6760..c4175a9 100644 --- a/modules/tests/shared/src/test/resources/db/test_non_transactional/V1__test.sql +++ b/modules/tests/shared/src/test/resources/db/test_non_transactional/V2__test.sql @@ -1,7 +1,3 @@ -CREATE TYPE test_enum_type AS ENUM ('T1'); - -CREATE TABLE test (template test_enum_type); - ALTER TYPE test_enum_type ADD VALUE 'T2'; - +CREATE TABLE test (template test_enum_type); INSERT INTO test (template) VALUES ('T2'); diff --git a/modules/tests/shared/src/test/resources/db/test_non_transactional/V2__test.sql.conf b/modules/tests/shared/src/test/resources/db/test_non_transactional/V2__test.sql.conf new file mode 100644 index 0000000..9f016d3 --- /dev/null +++ b/modules/tests/shared/src/test/resources/db/test_non_transactional/V2__test.sql.conf @@ -0,0 +1 @@ +executeInTransaction=true \ No newline at end of file diff --git a/modules/tests/shared/src/test/resources/db/test_non_transactional_enabled/V1__non_transactional.sql b/modules/tests/shared/src/test/resources/db/test_non_transactional_enabled/V1__non_transactional.sql index e695a1e..d1299b9 100644 --- a/modules/tests/shared/src/test/resources/db/test_non_transactional_enabled/V1__non_transactional.sql +++ b/modules/tests/shared/src/test/resources/db/test_non_transactional_enabled/V1__non_transactional.sql @@ -15,13 +15,6 @@ CREATE FUNCTION add(integer, integer) RETURNS integer IMMUTABLE RETURNS NULL ON NULL INPUT; -CREATE OR REPLACE FUNCTION increment(i integer) RETURNS integer AS $$ - -- a comment in function body 2 - BEGIN - RETURN i + 1; - END; -$$ LANGUAGE plpgsql; - /* multiline; comment * with; nesting: /*; nested block comment */ ;*/ diff --git a/modules/tests/shared/src/test/scala/DumboSpec.scala b/modules/tests/shared/src/test/scala/DumboSpec.scala index 4ceccc6..a4b740b 100644 --- a/modules/tests/shared/src/test/scala/DumboSpec.scala +++ b/modules/tests/shared/src/test/scala/DumboSpec.scala @@ -17,8 +17,8 @@ import cats.effect.std.Console import cats.implicits.* import fs2.io.file.Path -class DumboSpec extends ffstest.FTest { - override val postgresPort: Int = 5432 +trait DumboSpec extends ffstest.FTest { + def db: Db def assertEqualHistory(histA: List[HistoryEntry], histB: List[HistoryEntry]): Unit = { def toCompare(h: HistoryEntry) = @@ -28,11 +28,10 @@ class DumboSpec extends ffstest.FTest { } test("Run multiple migrations concurrently") { - val schema = "schema_1" + dropSchemas >> (1 to 5).toList.traverse_ { _ => + val schema = someSchemaName - (1 to 5).toList.traverse_ { _ => for { - _ <- dropSchemas res <- (1 to 20).toList.parTraverse(_ => dumboMigrate(schema, dumboWithResources("db/test_1"))) ranks = res.flatMap(_.migrations.map(_.installedRank)).sorted _ = assertEquals(ranks, List(1, 2, 3)) @@ -43,7 +42,7 @@ class DumboSpec extends ffstest.FTest { } dbTest("Validate checksum with validation enabled") { - val schema = "schema_1" + val schema = someSchemaName for { _ <- dumboMigrate(schema, dumboWithResources("db/test_0")) @@ -59,7 +58,7 @@ class DumboSpec extends ffstest.FTest { } dbTest("Validate description with validation enabled") { - val schema = "schema_1" + val schema = someSchemaName for { _ <- dumboMigrate(schema, dumboWithResources("db/test_0")) @@ -86,7 +85,7 @@ class DumboSpec extends ffstest.FTest { } dbTest("Validate for missing files with validation enabled") { - val schema = "schema_1" + val schema = someSchemaName for { _ <- dumboMigrate(schema, dumboWithResources("db/test_0")) @@ -104,7 +103,7 @@ class DumboSpec extends ffstest.FTest { } dbTest("Ignore missing files or missing checksum on validation disabled") { - val schema = "schema_1" + val schema = someSchemaName for { _ <- dumboMigrate(schema, dumboWithResources("db/test_0")) @@ -116,7 +115,7 @@ class DumboSpec extends ffstest.FTest { } dbTest("Fail with CopyNotSupportedException") { - val schema = "public" + val schema = someSchemaName for { dumboResA <- dumboMigrate(schema, dumboWithResources("db/test_copy_from")).attempt @@ -195,6 +194,25 @@ class DumboSpec extends ffstest.FTest { } yield () } + dbTest("Fail on non-transactional operations") { + val withResources = dumboWithResources("db/test_non_transactional") + val schema = someSchemaName + + for { + dumboRes <- dumboMigrate(schema, withResources).attempt + _ = assert(dumboRes.isLeft) + errLines = dumboRes.swap.toOption.get.getMessage().linesIterator + _ = db match { + case Db.Postgres(11) => + assert(errLines.exists(_.matches(".*ALTER TYPE .* cannot run inside a transaction block.*"))) + case Db.Postgres(_) => + assert(errLines.exists(_.matches(""".*Unsafe use of new value ".*" of enum type.*"""))) + case Db.CockroachDb => + assert(errLines.exists(_.matches(""".*Enum value ".*" is not yet public.*"""))) + } + } yield () + } + { class TestConsole extends Console[IO] { val logs: AtomicReference[Vector[String]] = new AtomicReference(Vector.empty[String]) @@ -212,7 +230,7 @@ class DumboSpec extends ffstest.FTest { dbTest("don't log on waiting for lock release if under provided duration") { val testConsole = new TestConsole() for { - _ <- dumboMigrate("public", withResources, logMigrationStateAfter = 5.second)(testConsole) + _ <- dumboMigrate("schema_1", withResources, logMigrationStateAfter = 5.second)(testConsole) _ = assert(testConsole.logs.get().count(logMatch) == 0) } yield () } @@ -221,9 +239,34 @@ class DumboSpec extends ffstest.FTest { val testConsole = new TestConsole() for { - _ <- dumboMigrate("public", withResources, logMigrationStateAfter = 800.millis)(testConsole) - _ = assert(testConsole.logs.get().count(logMatch) >= 2) + _ <- dumboMigrate("schema_1", withResources, logMigrationStateAfter = 800.millis)(testConsole) + _ = db match { + case Db.Postgres(_) => assert(testConsole.logs.get().count(logMatch) >= 2) + case Db.CockroachDb => + assert(testConsole.logs.get().count(_.startsWith("Progress monitor is not supported")) == 1) + } } yield () } } } + +sealed trait Db +object Db { + case class Postgres(version: Int) extends Db + case object CockroachDb extends Db +} + +class DumboSpecPostgresLatest extends DumboSpec { + override val db: Db = Db.Postgres(16) + override val postgresPort: Int = 5432 +} + +class DumboSpecPostgres11 extends DumboSpec { + override val db: Db = Db.Postgres(11) + override val postgresPort: Int = 5434 +} + +class DumboSpecCockroachDb extends DumboSpec { + override val db: Db = Db.CockroachDb + override val postgresPort: Int = 5436 +}