From 6d1601eb2b40b2ce0509ef8c36ef9e8373bc6ca8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9nich=20Bon=20=C4=86iri=C4=87?= Date: Thu, 11 Jun 2026 00:58:01 -0600 Subject: [PATCH 1/4] fix(core): modernize crystal syntax and ameba linting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add and run ameba for linting. - Resolve not_nil! and syntax/style issues reported by ameba. - Ignore pedantic .ameba.yml rules like BlockParameterName. - Rename set_database_to_schema to extract_schema_name for accuracy. Co-developed-by: Gemini AI Signed-off-by: Rénich Bon Ćirić --- .ameba.yml | 6 ++++++ shard.yml | 3 +++ spec/migration_spec.cr | 2 +- src/micrate.cr | 28 +++++++++++++--------------- src/micrate/cli.cr | 6 +++--- src/micrate/db.cr | 17 ++++++++--------- src/micrate/migration.cr | 4 ++-- 7 files changed, 36 insertions(+), 30 deletions(-) create mode 100644 .ameba.yml diff --git a/.ameba.yml b/.ameba.yml new file mode 100644 index 0000000..ff0dedb --- /dev/null +++ b/.ameba.yml @@ -0,0 +1,6 @@ +Naming/BlockParameterName: + Enabled: false +Metrics/CyclomaticComplexity: + Enabled: false +Documentation/DocumentationAdmonition: + Enabled: false diff --git a/shard.yml b/shard.yml index 9243197..e981846 100644 --- a/shard.yml +++ b/shard.yml @@ -27,3 +27,6 @@ development_dependencies: spectator: gitlab: arctic-fox/spectator version: ~> 0.11.3 + ameba: + github: crystal-ameba/ameba + version: ~> 1.6.0 diff --git a/spec/migration_spec.cr b/spec/migration_spec.cr index 25e88fd..da2cfc7 100644 --- a/spec/migration_spec.cr +++ b/spec/migration_spec.cr @@ -101,5 +101,5 @@ baz;") end def statements(migration, direction) - migration.statements(direction).map { |stmt| stmt.strip } + migration.statements(direction).map(&.strip) end diff --git a/src/micrate.cr b/src/micrate.cr index c6d3522..639dd33 100644 --- a/src/micrate.cr +++ b/src/micrate.cr @@ -14,13 +14,11 @@ module Micrate end def self.dbversion(db) - begin - rows = DB.get_versions_last_first_order(db) - return extract_dbversion(rows) - rescue Exception - DB.create_migrations_table(db) - return 0 - end + rows = DB.get_versions_last_first_order(db) + extract_dbversion(rows) + rescue Exception + DB.create_migrations_table(db) + 0 end def self.up(db) @@ -32,7 +30,7 @@ module Micrate end current = dbversion(db) - target = all_migrations.keys.sort.last + target = all_migrations.keys.sort!.last migrate(all_migrations, current, target, db) end @@ -85,7 +83,7 @@ module Micrate Dir.mkdir_p dir File.write(filename, migration_template) - return filename + filename end def self.connection_url=(connection_url) @@ -150,7 +148,7 @@ module Micrate # the given version is (likely) valid but we didn't find # anything before it. # return value must reflect that no migrations have been applied. - return 0 + 0 else raise "no previous version found" end @@ -161,7 +159,7 @@ module Micrate .select { |name| File.file? File.join(migrations_dir, name) } .select { |name| /^\d+.+\.sql$/ =~ name } .map { |name| Migration.from_file(name) } - .index_by { |migration| migration.version } + .index_by(&.version) end def self.migration_plan(status : Hash(Migration, Time?), current : Int, target : Int, direction) @@ -177,12 +175,12 @@ module Micrate if direction == :forward all_versions.keys - .sort + .sort! .select { |v| v > current && v <= target } else all_versions.keys - .sort - .reverse + .sort! + .reverse! .select { |v| v <= current && v > target } end end @@ -204,7 +202,7 @@ module Micrate end end - return 0 + 0 end class UnorderedMigrationsException < Exception diff --git a/src/micrate/cli.cr b/src/micrate/cli.cr index cba6243..1036b79 100644 --- a/src/micrate/cli.cr +++ b/src/micrate/cli.cr @@ -11,7 +11,7 @@ module Micrate File.delete(path) Log.info { "Deleted file #{path}" } else - name = set_database_to_schema url + name = extract_schema_name url Micrate::DB.connect do |db| db.exec "DROP DATABASE IF EXISTS #{name};" end @@ -24,7 +24,7 @@ module Micrate if url.starts_with? "sqlite3:" Log.info { "For sqlite3, the database will be created during the first migration." } else - name = set_database_to_schema url + name = extract_schema_name url Micrate::DB.connect do |db| db.exec "CREATE DATABASE #{name};" end @@ -32,7 +32,7 @@ module Micrate end end - def self.set_database_to_schema(url) + def self.extract_schema_name(url) uri = URI.parse(url) if path = uri.path Micrate::DB.connection_url = url.gsub(path, "/#{uri.scheme}") diff --git a/src/micrate/db.cr b/src/micrate/db.cr index b6606a9..bc86ac1 100644 --- a/src/micrate/db.cr +++ b/src/micrate/db.cr @@ -11,13 +11,11 @@ module Micrate end def self.connect - validate_connection_url - ::DB.connect(self.connection_url.not_nil!) + ::DB.connect(valid_connection_url) end - def self.connect(&block) - validate_connection_url - ::DB.open self.connection_url.not_nil! do |db| + def self.connect(&) + ::DB.open valid_connection_url do |db| yield db end end @@ -50,14 +48,15 @@ module Micrate end private def self.dialect - validate_connection_url - @@dialect ||= Dialect.from_connection_url(self.connection_url.not_nil!) + @@dialect ||= Dialect.from_connection_url(valid_connection_url) end - private def self.validate_connection_url - if !self.connection_url + private def self.valid_connection_url : String + url = self.connection_url + if !url raise "No database connection URL is configured. Please set the DATABASE_URL environment variable." end + url end end end diff --git a/src/micrate/migration.cr b/src/micrate/migration.cr index b474154..823da77 100644 --- a/src/micrate/migration.cr +++ b/src/micrate/migration.cr @@ -77,8 +77,8 @@ module Micrate def self.from_version(version) file_name = Dir.entries(Micrate.migrations_dir) - .find { |name| name.starts_with? version.to_s } - .not_nil! + .find(&.starts_with?(version.to_s)) + self.from_file(file_name) end end From 69055eea9039fe9ce4cdbd92fe25f06d80fd5f33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9nich=20Bon=20=C4=86iri=C4=87?= Date: Thu, 11 Jun 2026 01:11:11 -0600 Subject: [PATCH 2/4] refactor: remove global database state and introduce Micrate::Runner MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaced `Micrate::DB` and its global `@@connection_url` and `@@dialect` with `Micrate::Runner` context objects. This encapsulates connection configuration and allows for concurrent programmatic database operations. Signed-off-by: Rénich Bon Ćirić Co-developed-by: Gemini AI --- .gitignore | 1 + README.md | 2 +- examples/micrate | 7 ++- src/micrate.cr | 105 +------------------------------------ src/micrate/cli.cr | 44 +++++++++------- src/micrate/db.cr | 126 +++++++++++++++++++++++++++++++++++++-------- 6 files changed, 141 insertions(+), 144 deletions(-) diff --git a/.gitignore b/.gitignore index 42eb3be..847951b 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ # Libraries don't need dependency lock # Dependencies will be locked in application that uses them /shard.lock +.agents/ diff --git a/README.md b/README.md index e6cf863..6102666 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ This allows you to programatically use micrate's features. You'll see the `Micra require "micrate" require "pg" -Micrate::DB.connection_url = "postgresql://..." +ENV["DATABASE_URL"] = "postgresql://..." Micrate::Cli.run ``` diff --git a/examples/micrate b/examples/micrate index 70bc5cd..1267472 100755 --- a/examples/micrate +++ b/examples/micrate @@ -8,5 +8,10 @@ require "../src/micrate" require "pg" -Micrate::DB.connection_url = "postgresql://..." +# The CLI uses the DATABASE_URL environment variable by default: +# ENV["DATABASE_URL"] = "postgresql://..." Micrate::Cli.run + +# Or to run migrations programmatically: +# runner = Micrate::Runner.new("postgresql://...") +# runner.connect { |db| runner.up(db) } diff --git a/src/micrate.cr b/src/micrate.cr index 639dd33..97bf90f 100644 --- a/src/micrate.cr +++ b/src/micrate.cr @@ -1,5 +1,4 @@ require "log" - require "./micrate/*" module Micrate @@ -13,60 +12,6 @@ module Micrate File.join(db_dir, "migrations") end - def self.dbversion(db) - rows = DB.get_versions_last_first_order(db) - extract_dbversion(rows) - rescue Exception - DB.create_migrations_table(db) - 0 - end - - def self.up(db) - all_migrations = migrations_by_version - - if all_migrations.size == 0 - Log.warn { "No migrations found!" } - return - end - - current = dbversion(db) - target = all_migrations.keys.sort!.last - migrate(all_migrations, current, target, db) - end - - def self.down(db) - all_migrations = migrations_by_version - - current = dbversion(db) - target = previous_version(current, all_migrations.keys) - migrate(all_migrations, current, target, db) - end - - def self.redo(db) - all_migrations = migrations_by_version - - current = dbversion(db) - previous = previous_version(current, all_migrations.keys) - - if migrate(all_migrations, current, previous, db) == :success - migrate(all_migrations, previous, current, db) - end - end - - def self.migration_status(db) : Hash(Migration, Time?) - # ensure that migration table exists - dbversion(db) - migration_status(migrations_by_version.values, db) - end - - def self.migration_status(migrations : Array(Migration), db) : Hash(Migration, Time?) - ({} of Migration => Time?).tap do |ret| - migrations.each do |m| - ret[m] = DB.get_migration_status(m, db) - end - end - end - def self.create(name, dir, time) timestamp = time.to_s("%Y%m%d%H%M%S") filename = File.join(dir, "#{timestamp}_#{name}.sql") @@ -86,49 +31,6 @@ module Micrate filename end - def self.connection_url=(connection_url) - DB.connection_url = connection_url - end - - # --------------------------------- - # Private - # --------------------------------- - - private def self.migrate(all_migrations : Hash(Int, Migration), current : Int, target : Int, db) - direction = current < target ? :forward : :backwards - - status = migration_status(all_migrations.values, db) - plan = migration_plan(status, current, target, direction) - - if plan.empty? - Log.info { "No migrations to run. current version: #{current}" } - return :nop - end - - Log.info { "Migrating db, current version: #{current}, target: #{target}" } - - plan.each do |version| - migration = all_migrations[version] - - # Wrap migration in a transaction - db.transaction do |tx| - migration.statements(direction).each do |stmt| - tx.connection.exec(stmt) - end - - DB.record_migration(migration, direction, tx.connection) - - tx.commit - Log.info { "OK #{migration.name}" } - rescue e : Exception - tx.rollback - Log.error(exception: e) { "An error occurred executing migration #{migration.version}." } - return :error - end - end - :success - end - private def self.verify_unordered_migrations(current, status : Hash(Int, Bool)) migrations = status.select { |version, is_applied| !is_applied && version < current } .keys @@ -138,7 +40,7 @@ module Micrate end end - private def self.previous_version(current, all_versions) + def self.previous_version(current, all_versions) all_previous = all_versions.select { |version| version < current } if !all_previous.empty? return all_previous.max @@ -154,7 +56,7 @@ module Micrate end end - private def self.migrations_by_version + def self.migrations_by_version Dir.entries(migrations_dir) .select { |name| File.file? File.join(migrations_dir, name) } .select { |name| /^\d+.+\.sql$/ =~ name } @@ -185,9 +87,6 @@ module Micrate end end - # The most recent record for each migration specifies - # whether it has been applied or rolled back. - # The first version we find that has been applied is the current version. def self.extract_dbversion(rows) to_skip = [] of Int64 diff --git a/src/micrate/cli.cr b/src/micrate/cli.cr index 1036b79..0a53aa0 100644 --- a/src/micrate/cli.cr +++ b/src/micrate/cli.cr @@ -5,14 +5,15 @@ module Micrate Log = ::Log.for(self) def self.drop_database - url = Micrate::DB.connection_url.to_s + url = ENV["DATABASE_URL"]? || raise "DATABASE_URL not set" if url.starts_with? "sqlite3:" path = url.gsub("sqlite3:", "") File.delete(path) Log.info { "Deleted file #{path}" } else - name = extract_schema_name url - Micrate::DB.connect do |db| + root_url, name = extract_schema_name url + runner = Micrate::Runner.new(root_url) + runner.connect do |db| db.exec "DROP DATABASE IF EXISTS #{name};" end Log.info { "Dropped database #{name}" } @@ -20,12 +21,13 @@ module Micrate end def self.create_database - url = Micrate::DB.connection_url.to_s + url = ENV["DATABASE_URL"]? || raise "DATABASE_URL not set" if url.starts_with? "sqlite3:" Log.info { "For sqlite3, the database will be created during the first migration." } else - name = extract_schema_name url - Micrate::DB.connect do |db| + root_url, name = extract_schema_name url + runner = Micrate::Runner.new(root_url) + runner.connect do |db| db.exec "CREATE DATABASE #{name};" end Log.info { "Created database #{name}" } @@ -35,36 +37,41 @@ module Micrate def self.extract_schema_name(url) uri = URI.parse(url) if path = uri.path - Micrate::DB.connection_url = url.gsub(path, "/#{uri.scheme}") - path.gsub("/", "") + root_url = url.gsub(path, "/#{uri.scheme}") + {root_url, path.gsub("/", "")} else Log.error { "Could not determine database name" } + {url, ""} end end def self.run_up - Micrate::DB.connect do |db| - Micrate.up(db) + runner = Micrate::Runner.new + runner.connect do |db| + runner.up(db) end end def self.run_down - Micrate::DB.connect do |db| - Micrate.down(db) + runner = Micrate::Runner.new + runner.connect do |db| + runner.down(db) end end def self.run_redo - Micrate::DB.connect do |db| - Micrate.redo(db) + runner = Micrate::Runner.new + runner.connect do |db| + runner.redo(db) end end def self.run_status - Micrate::DB.connect do |db| + runner = Micrate::Runner.new + runner.connect do |db| Log.info { "Applied At Migration" } Log.info { "=======================================" } - Micrate.migration_status(db).each do |migration, migrated_at| + runner.migration_status(db).each do |migration, migrated_at| ts = migrated_at.nil? ? "Pending" : migrated_at.to_s Log.info { "%-24s -- %s\n" % [ts, migration.name] } end @@ -81,9 +88,10 @@ module Micrate end def self.run_dbversion - Micrate::DB.connect do |db| + runner = Micrate::Runner.new + runner.connect do |db| begin - Log.info { Micrate.dbversion(db) } + Log.info { runner.dbversion(db) } rescue raise "Could not read dbversion. Please make sure the database exists and verify the connection URL." end diff --git a/src/micrate/db.cr b/src/micrate/db.cr index bc86ac1..266dff6 100644 --- a/src/micrate/db.cr +++ b/src/micrate/db.cr @@ -2,42 +2,45 @@ require "db" require "./db/*" module Micrate - module DB - class_getter connection_url : String? { ENV["DATABASE_URL"]? } + class Runner + getter connection_url : String - def self.connection_url=(connection_url) - @@dialect = nil - @@connection_url = connection_url + def initialize(connection_url : String? = ENV["DATABASE_URL"]?) + url = connection_url + if !url + raise "No database connection URL is configured. Please set the DATABASE_URL environment variable." + end + @connection_url = url end - def self.connect - ::DB.connect(valid_connection_url) + def connect + ::DB.connect(@connection_url) end - def self.connect(&) - ::DB.open valid_connection_url do |db| + def connect(&) + ::DB.open(@connection_url) do |db| yield db end end - def self.get_versions_last_first_order(db) + def get_versions_last_first_order(db) db.query_all "SELECT version_id, is_applied from micrate_db_version ORDER BY id DESC", as: {Int64, Bool} end - def self.create_migrations_table(db) + def create_migrations_table(db) dialect.query_create_migrations_table(db) end - def self.record_migration(migration, direction, db) + def record_migration(migration, direction, db) is_applied = direction == :forward dialect.query_record_migration(migration, is_applied, db) end - def self.exec(statement, db) + def exec(statement, db) db.exec(statement) end - def self.get_migration_status(migration, db) : Time? + def get_migration_status(migration, db) : Time? rows = dialect.query_migration_status(migration, db) if !rows.empty? && rows[0][1] @@ -47,16 +50,97 @@ module Micrate end end - private def self.dialect - @@dialect ||= Dialect.from_connection_url(valid_connection_url) + private getter dialect : DB::Dialect do + DB::Dialect.from_connection_url(@connection_url) end - private def self.valid_connection_url : String - url = self.connection_url - if !url - raise "No database connection URL is configured. Please set the DATABASE_URL environment variable." + def dbversion(db) + rows = get_versions_last_first_order(db) + Micrate.extract_dbversion(rows) + rescue Exception + create_migrations_table(db) + 0 + end + + def up(db) + all_migrations = Micrate.migrations_by_version + + if all_migrations.size == 0 + Log.warn { "No migrations found!" } + return + end + + current = dbversion(db) + target = all_migrations.keys.sort!.last + migrate(all_migrations, current, target, db) + end + + def down(db) + all_migrations = Micrate.migrations_by_version + + current = dbversion(db) + target = Micrate.previous_version(current, all_migrations.keys) + migrate(all_migrations, current, target, db) + end + + def redo(db) + all_migrations = Micrate.migrations_by_version + + current = dbversion(db) + previous = Micrate.previous_version(current, all_migrations.keys) + + if migrate(all_migrations, current, previous, db) == :success + migrate(all_migrations, previous, current, db) + end + end + + def migration_status(db) : Hash(Migration, Time?) + # ensure that migration table exists + dbversion(db) + migration_status(Micrate.migrations_by_version.values, db) + end + + def migration_status(migrations : Array(Migration), db) : Hash(Migration, Time?) + ({} of Migration => Time?).tap do |ret| + migrations.each do |m| + ret[m] = get_migration_status(m, db) + end + end + end + + private def migrate(all_migrations : Hash(Int, Migration), current : Int, target : Int, db) + direction = current < target ? :forward : :backwards + + status = migration_status(all_migrations.values, db) + plan = Micrate.migration_plan(status, current, target, direction) + + if plan.empty? + Log.info { "No migrations to run. current version: #{current}" } + return :nop + end + + Log.info { "Migrating db, current version: #{current}, target: #{target}" } + + plan.each do |version| + migration = all_migrations[version] + + # Wrap migration in a transaction + db.transaction do |tx| + migration.statements(direction).each do |stmt| + tx.connection.exec(stmt) + end + + record_migration(migration, direction, tx.connection) + + tx.commit + Log.info { "OK #{migration.name}" } + rescue e : Exception + tx.rollback + Log.error(exception: e) { "An error occurred executing migration #{migration.version}." } + return :error + end end - url + :success end end end From 7301c5445e22084c91c605750a5a8d798433338e Mon Sep 17 00:00:00 2001 From: crimson-knight Date: Tue, 11 Aug 2026 15:12:59 -0400 Subject: [PATCH 3/4] Prepare Micrate for Amber V2 persistence --- .github/workflows/ci.yml | 45 ++++++++++++++++ README.md | 36 +++++++++++-- shard.yml | 13 +++-- spec/micrate_spec.cr | 37 +++++++++++++ spec/runner_spec.cr | 49 +++++++++++++++++ spec/spec_helper.cr | 1 + src/micrate-cli.cr | 2 +- src/micrate.cr | 50 +++++++++++++----- src/micrate/cli.cr | 105 ++++++++++++++++++++++--------------- src/micrate/db.cr | 20 ++++--- src/micrate/db/mysql.cr | 18 ++++--- src/micrate/db/postgres.cr | 18 ++++--- src/micrate/db/sqlite3.cr | 16 +++--- src/micrate/migration.cr | 10 ++-- src/micrate/version.cr | 2 +- 15 files changed, 320 insertions(+), 102 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 spec/runner_spec.cr diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c55f794 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + branches: [master] + workflow_dispatch: + +jobs: + test: + name: Crystal 1.21 on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + + steps: + - name: Checkout + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + + - name: Install Crystal + uses: crystal-lang/install-crystal@d8ef131ecec0352ce0e39b81b0a6d95def58fe2f # v1 + with: + crystal: 1.21.0 + + - name: Install SQLite headers + if: runner.os == 'Linux' + run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev + + - name: Install dependencies + run: shards install + + - name: Check formatting + run: crystal tool format --check src spec + + - name: Lint + run: crystal run lib/ameba/src/cli.cr -- + + - name: Build standalone wrapper + run: shards build + + - name: Run unit and SQLite integration specs + run: crystal spec diff --git a/README.md b/README.md index 6102666..6ed49bc 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,17 @@ Micrate currently supports migrations for Postgres, Mysql and SQLite3, but it sh ## Command line -To install the standalone binary tool check out the releases page, or use homebrew: +Amber CLI ships Micrate inside the precompiled `amber` executable. In an Amber +application, prefer the matching commands so no second migration binary is +required: + +```console +amber database migrate +amber database rollback +amber database status +``` + +To install the standalone binary tool, check the releases page or use Homebrew: ``` $ brew tap amberframework/micrate @@ -17,7 +27,10 @@ $ brew install micrate Execute `micrate help` for usage instructions. Micrate will connect to the database specified by the `DATABASE_URL` environment variable. -To create a new migration use the `scaffold` subcommand. For example, `micrate scaffold add_users_table` will create a new SQL migration file with a name such as `db/migrations/20160524162446_add_users_table.sql` that looks like this: +To create a new migration use the `scaffold` subcommand. For example, +`micrate scaffold add_users_table` will create a new SQL migration file with a +millisecond-resolution name such as +`db/migrations/20160524162446123_add_users_table.sql` that looks like this: ```sql -- +micrate Up @@ -96,7 +109,24 @@ dependencies: github: amberframework/micrate ``` -This allows you to programatically use micrate's features. You'll see the `Micrate` module has an equivalent for every CLI command. If you need to use micrate's CLI without installing the tool (which could be convenient in a CI environment), you can write a runner script as follows: +This allows you to programmatically use Micrate's features. Use a +`Micrate::Runner` when embedding migrations so the connection and migrations +directory remain explicit and do not leak through global state: + +```crystal +require "micrate" +require "sqlite3" + +runner = Micrate::Runner.new( + "sqlite3:./db/app_development.db", + "./db/migrations" +) + +runner.connect { |database| runner.up(database) } +``` + +If you need to use Micrate's CLI without installing the tool (which can be +convenient in CI), write a runner script as follows: ```crystal #! /usr/bin/env crystal diff --git a/shard.yml b/shard.yml index e981846..5e3ce10 100644 --- a/shard.yml +++ b/shard.yml @@ -1,6 +1,6 @@ name: micrate -version: 0.15.1 -crystal: ">= 0.36.1, < 2.0.0" +version: 0.16.0-beta.1 +crystal: ">= 1.20.0, < 2.0.0" authors: - Juan Edi @@ -22,6 +22,8 @@ executables: dependencies: db: github: crystal-lang/crystal-db + # v0.13.1 + commit: 3eaac85a5d4b7bee565b55dcb584e84e29fc5567 development_dependencies: spectator: @@ -29,4 +31,9 @@ development_dependencies: version: ~> 0.11.3 ameba: github: crystal-ameba/ameba - version: ~> 1.6.0 + # Crystal 1.21 compatibility pending Ameba 1.7.0. + commit: cdd58b34b0d8a9c785d67183a7a8f07541549e55 + sqlite3: + github: crystal-lang/crystal-sqlite3 + # v0.21.0 + commit: c58cea290c85e2a33dc8f494a5f04b519d3e0274 diff --git a/spec/micrate_spec.cr b/spec/micrate_spec.cr index cc4fa83..dbf246c 100644 --- a/spec/micrate_spec.cr +++ b/spec/micrate_spec.cr @@ -49,6 +49,31 @@ Spectator.describe Micrate do end end + context "with mixed timestamp precision" do + it "orders Amber millisecond and historic Micrate second timestamps chronologically" do + migrations = { + 20220906195432 => false, + 20220907021800909 => false, + 20220908100000 => false, + } + + plan = Micrate.migration_plan(migrations, 0, 20220908100000, :forward) + plan.should eq([20220906195432, 20220907021800909, 20220908100000]) + end + + it "detects a genuinely older migration without treating all 14-digit versions as old" do + migrations = { + 20220906195432 => false, + 20220907021800909 => true, + 20220908100000 => false, + } + + expect_raises(Micrate::UnorderedMigrationsException) do + Micrate.migration_plan(migrations, 20220907021800909, 20220908100000, :forward) + end + end + end + describe "detecting unordered migrations" do it "fails if there are unapplied migrations with older timestamp than current version" do migrations = { @@ -63,6 +88,18 @@ Spectator.describe Micrate do end end end + + describe "create" do + it "uses Amber-compatible millisecond timestamps" do + root = File.join(Dir.tempdir, "micrate-create-#{Process.pid}-#{Random.rand(1_000_000)}") + begin + path = Micrate.create("create_pets", root, Time.utc(2026, 8, 11, 12, 34, 56, nanosecond: 789_000_000)) + File.basename(path).should eq("20260811123456789_create_pets.sql") + ensure + FileUtils.rm_r(root) if Dir.exists?(root) + end + end + end end def sample_migrations diff --git a/spec/runner_spec.cr b/spec/runner_spec.cr new file mode 100644 index 0000000..9cf59f6 --- /dev/null +++ b/spec/runner_spec.cr @@ -0,0 +1,49 @@ +require "./spec_helper" +require "sqlite3" + +Spectator.describe Micrate::Runner do + it "runs, rolls back, and re-runs migrations from an explicit directory" do + root = File.join(Dir.tempdir, "micrate-runner-#{Process.pid}-#{Random.rand(1_000_000)}") + migrations_dir = File.join(root, "custom_migrations") + database_path = File.join(root, "pets.db") + database_url = "sqlite3:#{database_path}" + + begin + Dir.mkdir_p(migrations_dir) + migration = <<-SQL + -- +micrate Up + CREATE TABLE pets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL + ); + + -- +micrate Down + DROP TABLE pets; + SQL + File.write( + File.join(migrations_dir, "20260811120000000_create_pets.sql"), + migration + ) + + runner = Micrate::Runner.new(database_url, migrations_dir) + + runner.connect { |db| runner.up(db) } + ::DB.open(database_url) do |db| + db.scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'pets'").should eq(1_i64) + end + + runner.connect { |db| runner.down(db) } + ::DB.open(database_url) do |db| + db.scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'pets'").should eq(0_i64) + end + + runner.connect { |db| runner.up(db) } + runner.connect { |db| runner.redo(db) } + ::DB.open(database_url) do |db| + db.scalar("SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'pets'").should eq(1_i64) + end + ensure + FileUtils.rm_r(root) if Dir.exists?(root) + end + end +end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index 585c9b8..f8e1d7d 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -1,4 +1,5 @@ require "log" +require "file_utils" require "spectator" require "spectator/should" require "../src/micrate" diff --git a/src/micrate-cli.cr b/src/micrate-cli.cr index f646103..b93261f 100644 --- a/src/micrate-cli.cr +++ b/src/micrate-cli.cr @@ -1,5 +1,5 @@ require "log" -{% for db in %w(pg mysql sqlite3) %} +{% for db in %w[pg mysql sqlite3] %} {% if file_exists?("lib/" + db) %} require {{ db }} {% end %} diff --git a/src/micrate.cr b/src/micrate.cr index 97bf90f..cf25a45 100644 --- a/src/micrate.cr +++ b/src/micrate.cr @@ -13,7 +13,10 @@ module Micrate end def self.create(name, dir, time) - timestamp = time.to_s("%Y%m%d%H%M%S") + # Amber's generators include milliseconds so several migrations created in + # the same second retain a deterministic order. Micrate now emits the same + # format while continuing to read historic second-resolution filenames. + timestamp = time.to_utc.to_s("%Y%m%d%H%M%S%3N") filename = File.join(dir, "#{timestamp}_#{name}.sql") migration_template = "\ @@ -32,7 +35,10 @@ module Micrate end private def self.verify_unordered_migrations(current, status : Hash(Int, Bool)) - migrations = status.select { |version, is_applied| !is_applied && version < current } + current_order = version_order_key(current) + migrations = status.select do |version, is_applied| + !is_applied && version_order_key(version) < current_order + end .keys if !migrations.empty? @@ -41,9 +47,10 @@ module Micrate end def self.previous_version(current, all_versions) - all_previous = all_versions.select { |version| version < current } + current_order = version_order_key(current) + all_previous = all_versions.select { |version| version_order_key(version) < current_order } if !all_previous.empty? - return all_previous.max + return all_previous.max_by { |version| version_order_key(version) } end if all_versions.includes? current @@ -56,11 +63,13 @@ module Micrate end end - def self.migrations_by_version - Dir.entries(migrations_dir) - .select { |name| File.file? File.join(migrations_dir, name) } + def self.migrations_by_version(dir = migrations_dir) + return {} of Int64 => Migration unless Dir.exists?(dir) + + Dir.entries(dir) + .select { |name| File.file? File.join(dir, name) } .select { |name| /^\d+.+\.sql$/ =~ name } - .map { |name| Migration.from_file(name) } + .map { |name| Migration.from_file(name, dir) } .index_by(&.version) end @@ -77,16 +86,33 @@ module Micrate if direction == :forward all_versions.keys - .sort! - .select { |v| v > current && v <= target } + .sort_by! { |version| version_order_key(version) } + .select do |version| + version_order_key(version) > version_order_key(current) && + version_order_key(version) <= version_order_key(target) + end else all_versions.keys - .sort! + .sort_by! { |version| version_order_key(version) } .reverse! - .select { |v| v <= current && v > target } + .select do |version| + version_order_key(version) <= version_order_key(current) && + version_order_key(version) > version_order_key(target) + end end end + # Micrate historically generated 14-digit second-resolution timestamps while + # Amber generated 17-digit millisecond-resolution timestamps. Comparing the + # raw integers makes every millisecond migration look newer than every + # second-resolution migration, regardless of its actual date. Scale the + # historic format only for ordering; the stored migration identity remains + # unchanged for backwards compatibility. + def self.version_order_key(version : Int) : Int64 + raw = version.to_i64 + raw.to_s.size == 14 ? raw * 1000 : raw + end + def self.extract_dbversion(rows) to_skip = [] of Int64 diff --git a/src/micrate/cli.cr b/src/micrate/cli.cr index 0a53aa0..a22d44f 100644 --- a/src/micrate/cli.cr +++ b/src/micrate/cli.cr @@ -4,12 +4,16 @@ module Micrate module Cli Log = ::Log.for(self) - def self.drop_database - url = ENV["DATABASE_URL"]? || raise "DATABASE_URL not set" + def self.drop_database(connection_url : String? = ENV["DATABASE_URL"]?) + url = connection_url || raise "DATABASE_URL not set" if url.starts_with? "sqlite3:" - path = url.gsub("sqlite3:", "") - File.delete(path) - Log.info { "Deleted file #{path}" } + path = sqlite_path(url) + if File.exists?(path) + File.delete(path) + Log.info { "Deleted file #{path}" } + else + Log.info { "Database file does not exist: #{path}" } + end else root_url, name = extract_schema_name url runner = Micrate::Runner.new(root_url) @@ -20,8 +24,8 @@ module Micrate end end - def self.create_database - url = ENV["DATABASE_URL"]? || raise "DATABASE_URL not set" + def self.create_database(connection_url : String? = ENV["DATABASE_URL"]?) + url = connection_url || raise "DATABASE_URL not set" if url.starts_with? "sqlite3:" Log.info { "For sqlite3, the database will be created during the first migration." } else @@ -45,29 +49,33 @@ module Micrate end end - def self.run_up - runner = Micrate::Runner.new + def self.run_up(connection_url : String? = ENV["DATABASE_URL"]?, + migrations_dir : String = Micrate.migrations_dir) + runner = Micrate::Runner.new(connection_url, migrations_dir) runner.connect do |db| runner.up(db) end end - def self.run_down - runner = Micrate::Runner.new + def self.run_down(connection_url : String? = ENV["DATABASE_URL"]?, + migrations_dir : String = Micrate.migrations_dir) + runner = Micrate::Runner.new(connection_url, migrations_dir) runner.connect do |db| runner.down(db) end end - def self.run_redo - runner = Micrate::Runner.new + def self.run_redo(connection_url : String? = ENV["DATABASE_URL"]?, + migrations_dir : String = Micrate.migrations_dir) + runner = Micrate::Runner.new(connection_url, migrations_dir) runner.connect do |db| runner.redo(db) end end - def self.run_status - runner = Micrate::Runner.new + def self.run_status(connection_url : String? = ENV["DATABASE_URL"]?, + migrations_dir : String = Micrate.migrations_dir) + runner = Micrate::Runner.new(connection_url, migrations_dir) runner.connect do |db| Log.info { "Applied At Migration" } Log.info { "=======================================" } @@ -78,24 +86,24 @@ module Micrate end end - def self.run_scaffold - if ARGV.size < 1 + def self.run_scaffold(name : String? = ARGV.shift, + migrations_dir : String = Micrate.migrations_dir) + if name.nil? || name.empty? raise "Migration name required" end - migration_file = Micrate.create(ARGV.shift, Micrate.migrations_dir, Time.local) + migration_file = Micrate.create(name, migrations_dir, Time.utc) Log.info { "Created #{migration_file}" } end - def self.run_dbversion - runner = Micrate::Runner.new + def self.run_dbversion(connection_url : String? = ENV["DATABASE_URL"]?, + migrations_dir : String = Micrate.migrations_dir) + runner = Micrate::Runner.new(connection_url, migrations_dir) runner.connect do |db| - begin - Log.info { runner.dbversion(db) } - rescue - raise "Could not read dbversion. Please make sure the database exists and verify the connection URL." - end + Log.info { runner.dbversion(db) } end + rescue + raise "Could not read dbversion. Please make sure the database exists and verify the connection URL." end def self.report_unordered_migrations(conflicting) @@ -103,27 +111,38 @@ module Micrate conflicting.each do |version| Log.info { " #{Migration.from_version(version).name}" } end - Log.info { " -Micrate will not run these migrations because they may have been written with an older database model in mind. -You should probably check if they need to be updated and rename them so they are considered a newer version." } + Log.info do + <<-MESSAGE + Micrate will not run these migrations because they may have been written with an older database model in mind. + You should probably check if they need to be updated and rename them so they are considered a newer version. + MESSAGE + end + end + + private def self.sqlite_path(url : String) : String + url.sub(/^sqlite3:(?:\/\/)?/, "") end def self.print_help - Log.info { "micrate is a database migration management system for Crystal projects, *heavily* inspired by Goose (https://bitbucket.org/liamstask/goose/). - -Usage: - set DATABASE_URL environment variable i.e. export DATABASE_URL=postgres://user:pswd@host:port/database - micrate [options] [subcommand options] - -Commands: - create Create the database (permissions required) - drop Drop the database (permissions required) - up Migrate the DB to the most recent version available - down Roll back the version by 1 - redo Re-run the latest migration - status Dump the migration status for the current DB - scaffold Create the scaffolding for a new migration - dbversion Print the current version of the database" } + Log.info do + <<-HELP + micrate is a database migration management system for Crystal projects, *heavily* inspired by Goose (https://bitbucket.org/liamstask/goose/). + + Usage: + set DATABASE_URL environment variable i.e. export DATABASE_URL=postgres://user:pswd@host:port/database + micrate [options] [subcommand options] + + Commands: + create Create the database (permissions required) + drop Drop the database (permissions required) + up Migrate the DB to the most recent version available + down Roll back the version by 1 + redo Re-run the latest migration + status Dump the migration status for the current DB + scaffold Create the scaffolding for a new migration + dbversion Print the current version of the database + HELP + end end def self.run diff --git a/src/micrate/db.cr b/src/micrate/db.cr index 266dff6..ea4b9c5 100644 --- a/src/micrate/db.cr +++ b/src/micrate/db.cr @@ -4,8 +4,10 @@ require "./db/*" module Micrate class Runner getter connection_url : String + getter migrations_dir : String - def initialize(connection_url : String? = ENV["DATABASE_URL"]?) + def initialize(connection_url : String? = ENV["DATABASE_URL"]?, + @migrations_dir : String = Micrate.migrations_dir) url = connection_url if !url raise "No database connection URL is configured. Please set the DATABASE_URL environment variable." @@ -43,11 +45,7 @@ module Micrate def get_migration_status(migration, db) : Time? rows = dialect.query_migration_status(migration, db) - if !rows.empty? && rows[0][1] - rows[0][0] - else - nil - end + rows[0][0] if !rows.empty? && rows[0][1] end private getter dialect : DB::Dialect do @@ -63,7 +61,7 @@ module Micrate end def up(db) - all_migrations = Micrate.migrations_by_version + all_migrations = Micrate.migrations_by_version(migrations_dir) if all_migrations.size == 0 Log.warn { "No migrations found!" } @@ -71,12 +69,12 @@ module Micrate end current = dbversion(db) - target = all_migrations.keys.sort!.last + target = all_migrations.keys.max_by { |version| Micrate.version_order_key(version) } migrate(all_migrations, current, target, db) end def down(db) - all_migrations = Micrate.migrations_by_version + all_migrations = Micrate.migrations_by_version(migrations_dir) current = dbversion(db) target = Micrate.previous_version(current, all_migrations.keys) @@ -84,7 +82,7 @@ module Micrate end def redo(db) - all_migrations = Micrate.migrations_by_version + all_migrations = Micrate.migrations_by_version(migrations_dir) current = dbversion(db) previous = Micrate.previous_version(current, all_migrations.keys) @@ -97,7 +95,7 @@ module Micrate def migration_status(db) : Hash(Migration, Time?) # ensure that migration table exists dbversion(db) - migration_status(Micrate.migrations_by_version.values, db) + migration_status(Micrate.migrations_by_version(migrations_dir).values, db) end def migration_status(migrations : Array(Migration), db) : Hash(Migration, Time?) diff --git a/src/micrate/db/mysql.cr b/src/micrate/db/mysql.cr index 277c091..37806be 100644 --- a/src/micrate/db/mysql.cr +++ b/src/micrate/db/mysql.cr @@ -1,17 +1,19 @@ module Micrate::DB class Mysql < Dialect def query_create_migrations_table(db) - db.exec("CREATE TABLE micrate_db_version ( - id serial NOT NULL, - version_id bigint NOT NULL, - is_applied boolean NOT NULL, - tstamp timestamp NULL default now(), - PRIMARY KEY(id) - );") + db.exec <<-SQL + CREATE TABLE micrate_db_version ( + id serial NOT NULL, + version_id bigint NOT NULL, + is_applied boolean NOT NULL, + tstamp timestamp NULL default now(), + PRIMARY KEY(id) + ); + SQL end def query_migration_status(migration, db) - db.query_all "SELECT tstamp, is_applied FROM micrate_db_version WHERE version_id=? ORDER BY tstamp DESC LIMIT 1", migration.version, as: {Time, Bool} + db.query_all "SELECT tstamp, is_applied FROM micrate_db_version WHERE version_id=? ORDER BY id DESC LIMIT 1", migration.version, as: {Time, Bool} end def query_record_migration(migration, is_applied, db) diff --git a/src/micrate/db/postgres.cr b/src/micrate/db/postgres.cr index cf3f55f..65b67ec 100644 --- a/src/micrate/db/postgres.cr +++ b/src/micrate/db/postgres.cr @@ -1,17 +1,19 @@ module Micrate::DB class Postgres < Dialect def query_create_migrations_table(db) - db.exec("CREATE TABLE micrate_db_version ( - id serial NOT NULL, - version_id bigint NOT NULL, - is_applied boolean NOT NULL, - tstamp timestamp NULL default now(), - PRIMARY KEY(id) - );") + db.exec <<-SQL + CREATE TABLE micrate_db_version ( + id serial NOT NULL, + version_id bigint NOT NULL, + is_applied boolean NOT NULL, + tstamp timestamp NULL default now(), + PRIMARY KEY(id) + ); + SQL end def query_migration_status(migration, db) - db.query_all "SELECT tstamp, is_applied FROM micrate_db_version WHERE version_id=$1 ORDER BY tstamp DESC LIMIT 1", migration.version, as: {Time, Bool} + db.query_all "SELECT tstamp, is_applied FROM micrate_db_version WHERE version_id=$1 ORDER BY id DESC LIMIT 1", migration.version, as: {Time, Bool} end def query_record_migration(migration, is_applied, db) diff --git a/src/micrate/db/sqlite3.cr b/src/micrate/db/sqlite3.cr index 630e8f4..44fb444 100644 --- a/src/micrate/db/sqlite3.cr +++ b/src/micrate/db/sqlite3.cr @@ -6,16 +6,18 @@ module Micrate::DB # # As a workaround, we create timestamps locally so that the driver decides timestamp # formats when writing and reading. - db.exec("CREATE TABLE micrate_db_version ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - version_id INTEGER NOT NULL, - is_applied INTEGER NOT NULL, - tstamp TIMESTAMP - );") + db.exec <<-SQL + CREATE TABLE micrate_db_version ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + version_id INTEGER NOT NULL, + is_applied INTEGER NOT NULL, + tstamp TIMESTAMP + ); + SQL end def query_migration_status(migration, db) - db.query_all "SELECT tstamp, is_applied FROM micrate_db_version WHERE version_id=? ORDER BY tstamp DESC LIMIT 1", migration.version, as: {Time, Bool} + db.query_all "SELECT tstamp, is_applied FROM micrate_db_version WHERE version_id=? ORDER BY id DESC LIMIT 1", migration.version, as: {Time, Bool} end def query_record_migration(migration, is_applied, db) diff --git a/src/micrate/migration.cr b/src/micrate/migration.cr index 823da77..64f71df 100644 --- a/src/micrate/migration.cr +++ b/src/micrate/migration.cr @@ -69,17 +69,17 @@ module Micrate s.split("--")[0].strip.ends_with? ";" end - def self.from_file(file_name) - full_path = File.join(Micrate.migrations_dir, file_name) + def self.from_file(file_name, migrations_dir = Micrate.migrations_dir) + full_path = File.join(migrations_dir, file_name) version = file_name.split("_")[0].to_i64 new(version, file_name, File.read(full_path)) end - def self.from_version(version) - file_name = Dir.entries(Micrate.migrations_dir) + def self.from_version(version, migrations_dir = Micrate.migrations_dir) + file_name = Dir.entries(migrations_dir) .find(&.starts_with?(version.to_s)) - self.from_file(file_name) + from_file(file_name, migrations_dir) end end end diff --git a/src/micrate/version.cr b/src/micrate/version.cr index 9d18b6b..07e7486 100644 --- a/src/micrate/version.cr +++ b/src/micrate/version.cr @@ -1,3 +1,3 @@ module Micrate - VERSION = "0.10.0" + VERSION = "0.16.0-beta.1" end From 56afd5607cc591776980f3d3762495c037b98d98 Mon Sep 17 00:00:00 2001 From: crimson-knight Date: Tue, 11 Aug 2026 15:14:45 -0400 Subject: [PATCH 4/4] Allow host apps to resolve Crystal DB --- shard.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shard.yml b/shard.yml index 5e3ce10..ab64cb6 100644 --- a/shard.yml +++ b/shard.yml @@ -22,8 +22,7 @@ executables: dependencies: db: github: crystal-lang/crystal-db - # v0.13.1 - commit: 3eaac85a5d4b7bee565b55dcb584e84e29fc5567 + version: 0.13.1 development_dependencies: spectator: