dergrimm/mentorenwahl
dergrimm/mentorenwahl
2
1
Fork 0
Code Issues 6 Pull requests Projects 3 Releases Wiki Activity

Switch to micrate for db migrations

Browse source
This commit is contained in:
Dominic Grimm 2022-04-14 18:22:07 +02:00
parent 84f15abf55
commit 48b25adb07
No known key found for this signature in database
GPG key ID: 27C59510125F3C8A
37 changed files with 150 additions and 1231 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
docker/backend/Dockerfile
View file

@ -14,28 +14,42 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
FROM crystallang/crystal:1.4-alpine as deps
WORKDIR /app
COPY ./shard.yml ./shard.lock ./
FROM crystallang/crystal:1.4-alpine as micrate-deps
WORKDIR /src
COPY ./micrate/shard.yml ./micrate/shard.lock ./
RUN shards install --production
FROM tdewolff/minify as public-minify
FROM crystallang/crystal:1.4-alpine as micrate-builder
WORKDIR /src
# RUN apk add --no-cache sqlite-static
COPY --from=micrate-deps /src/shard.yml /src/shard.lock ./
COPY --from=micrate-deps /src/lib ./lib
COPY ./micrate/src ./src
RUN shards build --release --static --verbose -s -p -t
FROM tdewolff/minify as public
WORKDIR /src
COPY ./public ./src
RUN minify -r -o ./tmp_dist ./src
RUN mv ./tmp_dist/src ./dist
RUN rm -rf ./tmp_dist
FROM crystallang/crystal:1.4-alpine as deps
WORKDIR /src
COPY ./shard.yml ./shard.lock ./
RUN shards install --production
FROM crystallang/crystal:1.4-alpine as builder
ARG BUILD_ENV
WORKDIR /src/mentorenwahl
RUN apk add --no-cache pcre2-dev
COPY --from=deps /app/shard.yml /app/shard.lock ./
COPY --from=deps /app/lib ./lib
COPY --from=deps /src/shard.yml /src/shard.lock ./
COPY --from=deps /src/lib ./lib
COPY ./LICENSE .
COPY ./Makefile .
COPY ./src ./src
COPY --from=public-minify /src/dist ./public
COPY ./db ./db
COPY --from=public /src/dist ./public
RUN if [ "${BUILD_ENV}" = "development" ]; then \
make dev; \
else \
@ -43,7 +57,9 @@ RUN if [ "${BUILD_ENV}" = "development" ]; then \
fi
FROM scratch as runner
COPY --from=micrate-builder /src/bin/micrate /bin/micrate
COPY --from=builder /src/mentorenwahl/bin /bin
COPY --from=builder /src/mentorenwahl/db ./db
EXPOSE 80
ENTRYPOINT [ "backend" ]
CMD [ "run" ]

82
docker/backend/db/migrations/20220414171336_create_users.sql Normal file
View file

@ -0,0 +1,82 @@
/*
Mentorenwahl: A fullstack application for assigning mentors to students based on their whishes.
Copyright (C) 2022 Dominic Grimm
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
-- +micrate Up
-- SQL in section ' Up ' is executed when this migration is applied
CREATE TYPE user_roles AS ENUM ('teacher', 'student');
CREATE TABLE users(
id BIGSERIAL PRIMARY KEY,
username TEXT UNIQUE NOT NULL,
role user_roles NOT NULL,
skif BOOLEAN NOT NULL,
admin BOOLEAN NOT NULL
);
CREATE TABLE teachers(
id BIGSERIAL PRIMARY KEY,
user_id BIGINT NOT NULL UNIQUE REFERENCES users(id),
max_students INT NOT NULL
);
CREATE TABLE students(
id BIGSERIAL PRIMARY KEY,
user_id BIGINT NOT NULL UNIQUE REFERENCES users(id)
);
ALTER TABLE
users
ADD
COLUMN teacher_id BIGINT UNIQUE REFERENCES teachers(id);
ALTER TABLE
users
ADD
COLUMN student_id BIGINT UNIQUE REFERENCES students(id);
CREATE TABLE votes(
id BIGSERIAL PRIMARY KEY,
student_id BIGINT NOT NULL UNIQUE REFERENCES students(id)
);
ALTER TABLE
students
ADD
COLUMN vote_id BIGINT UNIQUE REFERENCES votes(id);
CREATE TABLE teacher_votes(
id BIGSERIAL PRIMARY KEY,
vote_id BIGINT NOT NULL REFERENCES votes(id),
teacher_id BIGINT NOT NULL REFERENCES teachers(id),
priority INT NOT NULL
);
-- +micrate Down
-- SQL section ' Down ' is executed when this migration is rolled back
DROP TABLE teacher_votes;
DROP TABLE votes;
DROP TABLE admins;
DROP TABLE teachers;
DROP TABLE students;
DROP TABLE users;
DROP TYPE user_roles;

9
docker/backend/micrate/.editorconfig Normal file
View file

@ -0,0 +1,9 @@
root = true
[*.cr]
charset = utf-8
end_of_line = lf
insert_final_newline = true
indent_style = space
indent_size = 2
trim_trailing_whitespace = true

9
docker/backend/micrate/.gitignore vendored
View file

@ -1,10 +1,5 @@
/doc/
/libs/
/docs/
/lib/
/bin/
/.shards/
# Libraries don't need dependency lock
# Dependencies will be locked in application that uses them
/shard.lock
*.dwarf

12
docker/backend/micrate/.gitrepo
View file

@ -1,12 +0,0 @@
; DO NOT EDIT (unless you know what you are doing)
;
; This subdirectory is a git "subrepo", and this file is maintained by the
; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme
;
[subrepo]
remote = https://github.com/amberframework/micrate.git
branch = master
commit = f615e55ff607f9c1a19145dcd9e2cdcee7c15fe6
parent = 454ff552c9cae3b15dfc1d4d4c765539adbaa37c
method = merge
cmdver = 0.4.3

1
docker/backend/micrate/.travis.yml
View file

@ -1 +0,0 @@
language: crystal

14
docker/backend/micrate/Dockerfile
View file

@ -1,14 +0,0 @@
FROM crystallang/crystal:1.0.0
# Install Dependencies
ARG DEBIAN_FRONTEND=noninteractive
RUN apt-get update -qq && apt-get install -y --no-install-recommends libpq-dev libsqlite3-dev libmysqlclient-dev libreadline-dev git curl vim netcat
WORKDIR /opt/micrate
# Build Amber
ENV PATH /opt/micrate/bin:$PATH
COPY . /opt/micrate
RUN shards build micrate
CMD ["micrate", "up"]

42
docker/backend/micrate/Makefile
View file

@ -1,42 +0,0 @@
PREFIX=/usr/local
INSTALL_DIR=$(PREFIX)/bin
MICRATE_SYSTEM=$(INSTALL_DIR)/micrate
OUT_DIR=$(CURDIR)/bin
MICRATE=$(OUT_DIR)/micrate
MICRATE_SOURCES=$(shell find src/ -type f -name '*.cr')
all: build
build: lib $(MICRATE)
lib:
@shards install --production
$(MICRATE): $(MICRATE_SOURCES) | $(OUT_DIR)
@echo "Building micrate in $@"
@crystal build -o $@ src/micrate-bin.cr -p --no-debug
$(OUT_DIR) $(INSTALL_DIR):
@mkdir -p $@
run:
$(MICRATE)
install: build | $(INSTALL_DIR)
@rm -f $(MICRATE_SYSTEM)
@cp $(MICRATE) $(MICRATE_SYSTEM)
link: build | $(INSTALL_DIR)
@echo "Symlinking $(MICRATE) to $(MICRATE_SYSTEM)"
@ln -s $(MICRATE) $(MICRATE_SYSTEM)
force_link: build | $(INSTALL_DIR)
@echo "Symlinking $(MICRATE) to $(MICRATE_SYSTEM)"
@ln -sf $(MICRATE) $(MICRATE_SYSTEM)
clean:
rm -rf $(MICRATE)
distclean:
rm -rf $(MICRATE) .crystal .shards libs lib

125
docker/backend/micrate/README.md
View file

@ -1,125 +0,0 @@
# micrate
Micrate is a database migration tool written in Crystal.
It is inspired by [goose](https://bitbucket.org/liamstask/goose/). Some code was ported from there too, so check it out.
Micrate currently supports migrations for Postgres, Mysql and SQLite3, but it should be easy to add support for any other database engine with an existing [crystal-db API](https://github.com/crystal-lang/crystal-db) driver.
## Command line
To install the standalone binary tool check out the releases page, or use homebrew:
```
$ brew tap amberframework/micrate
$ brew install micrate
```
Execute `micrate help` for usage instructions. Micrate will connect to the database specified by the `DATABASE_URL` environment variable.
To create a new migration use the `scaffold` subcommand. For example, `micrate scaffold add_users_table` will create a new SQL migration file with a name such as `db/migrations/20160524162446_add_users_table.sql` that looks like this:
```sql
-- +micrate Up
-- SQL in section 'Up' is executed when this migration is applied
-- +micrate Down
-- SQL section 'Down' is executed when this migration is rolled back
```
Comments that start with `+micrate` are interpreted by micrate when running your migrations. In this case, the `Up` and `Down` directives are used to indicate which SQL statements must be run when applying or reverting a migration. You can now go along and write your migration like this:
```sql
-- +micrate Up
CREATE TABLE users(id INT PRIMARY KEY, email VARCHAR NOT NULL);
-- +micrate Down
DROP TABLE users;
```
Now run it using `micrate up`. This command will execute all pending migrations:
```
$ micrate up
Migrating db, current version: 0, target: 20160524162947
OK 20160524162446_add_users_table.sql
$ micrate dbversion # at any time you can find out the current version of the database
20160524162446
```
If you ever need to roll back the last migration, you can do so by executing `micrate down`. There's also `micrate redo` which rolls back the last migration and applies it again. Last but not least: use `micrate status` to find out the state of each migration:
```
$ micrate status
Applied At Migration
=======================================
2016-05-24 16:31:07 UTC -- 20160524162446_add_users_table.sql
Pending -- 20160524163425_add_address_to_users.sql
```
If using complex statements that might contain semicolons, you must give micrate a hint on how to split the script into separate statements. You can do this with `StatementBegin` and `StatementEnd` directives: (thanks [goose](https://bitbucket.org/liamstask/goose/) for this!)
```
-- +micrate Up
-- +micrate StatementBegin
CREATE OR REPLACE FUNCTION histories_partition_creation( DATE, DATE )
returns void AS $$
DECLARE
create_query text;
BEGIN
FOR create_query IN SELECT
'CREATE TABLE IF NOT EXISTS histories_'
|| TO_CHAR( d, 'YYYY_MM' )
|| ' ( CHECK( created_at >= timestamp '''
|| TO_CHAR( d, 'YYYY-MM-DD 00:00:00' )
|| ''' AND created_at < timestamp '''
|| TO_CHAR( d + INTERVAL '1 month', 'YYYY-MM-DD 00:00:00' )
|| ''' ) ) inherits ( histories );'
FROM generate_series( $1, $2, '1 month' ) AS d
LOOP
EXECUTE create_query;
END LOOP; -- LOOP END
END; -- FUNCTION END
$$
language plpgsql;
-- +micrate StatementEnd
```
## API
To use the Crystal API, add this to your application's `shard.yml`:
```yaml
dependencies:
micrate:
github: amberframework/micrate
```
This allows you to programatically use micrate's features. You'll see the `Micrate` module has an equivalent for every CLI command. If you need to use micrate's CLI without installing the tool (which could be convenient in a CI environment), you can write a runner script as follows:
```crystal
#! /usr/bin/env crystal
#
# To build a standalone command line client, require the
# driver you wish to use and use `Micrate::Cli`.
#
require "micrate"
require "pg"
Micrate::DB.connection_url = "postgresql://..."
Micrate::Cli.run
```
## Contributing
1. Fork it ( https://github.com/amberframework/micrate/fork )
2. Create your feature branch (git checkout -b my-new-feature)
3. Commit your changes (git commit -am 'Add some feature')
4. Push to the branch (git push origin my-new-feature)
5. Create a new Pull Request
## Contributors
- [juanedi](https://github.com/juanedi) - creator, maintainer

12
docker/backend/micrate/examples/micrate
View file

@ -1,12 +0,0 @@
#! /usr/bin/env crystal
#
# To build a standalone command line client, require the
# driver you wish to use and use `Micrate::Cli`.
#
require "../src/micrate"
require "pg"
Micrate::DB.connection_url = "postgresql://..."
Micrate::Cli.run

14
docker/backend/micrate/shard.lock Normal file
View file

@ -0,0 +1,14 @@
version: 2.0
shards:
db:
git: https://github.com/crystal-lang/crystal-db.git
version: 0.10.1
micrate:
git: https://github.com/amberframework/micrate.git
version: 0.12.0+git.commit.f615e55ff607f9c1a19145dcd9e2cdcee7c15fe6
pg:
git: https://github.com/will/crystal-pg.git
version: 0.25.0

30
docker/backend/micrate/shard.yml
View file

@ -1,33 +1,17 @@
name: micrate
version: 0.12.0
crystal: ">= 0.36.1, < 2.0.0"
version: 0.1.0
authors:
- Juan Edi <jedi11235@gmail.com>
maintainers:
- Isaac Sloan <isaac@isaacsloan.com>
- Dru Jensen <drujensen@gmail.com>
- Dominic Grimm <dominic.grimm@gmail.com>
targets:
micrate:
main: src/micrate-bin.cr
main: src/micrate.cr
crystal: 1.4.0
dependencies:
db:
github: crystal-lang/crystal-db
version: ~> 0.10.0
development_dependencies:
spectator:
gitlab: arctic-fox/spectator
branch: master
micrate:
github: amberframework/micrate
pg:
github: will/crystal-pg
version: ~> 0.24.0
mysql:
github: crystal-lang/crystal-mysql
version: ~> 0.13.0
sqlite3:
github: crystal-lang/crystal-sqlite3
version: ~> 0.18.0

53
docker/backend/micrate/spec/cli_spec.cr
View file

@ -1,53 +0,0 @@
require "./spec_helper"
Spectator.describe Micrate::Cli do
mock File do
stub self.delete(path : Path | String) { nil }
end
double :fake_db do
stub exec(query) { DB::ExecResult.new(0, 0) }
end
before_each { FAKE_DB.clear; FAKE_DB << double(:fake_db) }
mock Micrate::DB do
stub self.connect() { yield FAKE_DB.first }
end
describe "#drop_database" do
context "sqlite3" do
it "deletes the file" do
Micrate::DB.connection_url = "sqlite3:myfile"
Micrate::Cli.drop_database
expect(File).to have_received(:delete).with("myfile")
end
end
context "postgres" do
it "calls drop database" do
Micrate::DB.connection_url = "postgres://user:pswd@host:5432/database"
Micrate::Cli.drop_database
expect(Micrate::DB).to have_received(:connect)
end
end
end
describe "#create_database" do
context "sqlite3" do
it "doesn't call connect" do
Micrate::DB.connection_url = "sqlite3:myfile"
Micrate::Cli.create_database
expect(Micrate::DB).not_to have_received(:connect)
end
end
context "postgres" do
it "calls connect" do
Micrate::DB.connection_url = "postgres://user:pswd@host:5432/database"
Micrate::Cli.create_database
expect(Micrate::DB).to have_received(:connect)
end
end
end
end

74
docker/backend/micrate/spec/micrate_spec.cr
View file

@ -1,74 +0,0 @@
require "./spec_helper"
Spectator.describe Micrate do
describe "dbversion" do
it "returns 0 if table is empty" do
rows = [] of {Int64, Bool}
Micrate.extract_dbversion(rows).should eq(0)
end
it "returns last applied migration" do
# expect rows to be order by id asc
rows = [
{20160101140000, true},
{20160101130000, true},
{20160101120000, true},
] of {Int64, Bool}
Micrate.extract_dbversion(rows).should eq(20160101140000)
end
it "ignores rolled back versions" do
rows = [
{20160101140000, false},
{20160101140000, true},
{20160101120000, true},
] of {Int64, Bool}
Micrate.extract_dbversion(rows).should eq(20160101120000)
end
end
describe "up" do
context "going forward" do
it "runs all migrations if starting from clean db" do
plan = Micrate.migration_plan(sample_migrations, 0, 20160523142316, :forward)
plan.should eq([20160523142308, 20160523142313, 20160523142316])
end
it "skips already performed migrations" do
plan = Micrate.migration_plan(sample_migrations, 20160523142308, 20160523142316, :forward)
plan.should eq([20160523142313, 20160523142316])
end
end
context "going backwards" do
it "skips already performed migrations" do
plan = Micrate.migration_plan(sample_migrations, 20160523142316, 20160523142308, :backwards)
plan.should eq([20160523142316, 20160523142313])
end
end
describe "detecting unordered migrations" do
it "fails if there are unapplied migrations with older timestamp than current version" do
migrations = {
20160523142308 => false,
20160523142313 => true,
20160523142316 => false,
}
expect_raises(Micrate::UnorderedMigrationsException) do
Micrate.migration_plan(migrations, 20160523142313, 20160523142316, :forward)
end
end
end
end
end
def sample_migrations
{
20160523142308 => true,
20160523142313 => true,
20160523142316 => true,
}
end

105
docker/backend/micrate/spec/migration_spec.cr
View file

@ -1,105 +0,0 @@
require "./spec_helper"
Spectator.describe Micrate do
describe "splitting in statements" do
it "split simple statements" do
migration = Micrate::Migration.new(20160101120000, "foo.sql", "\
-- +micrate Up
CREATE TABLE foo(id INT PRIMARY KEY, name VARCHAR NOT NULL);
-- +micrate Down
DROP TABLE foo;")
statements(migration, :forward).should eq([
"-- +micrate Up\nCREATE TABLE foo(id INT PRIMARY KEY, name VARCHAR NOT NULL);",
])
statements(migration, :backwards).should eq([
"-- +micrate Down\nDROP TABLE foo;",
])
end
it "splits mixed Up and Down statements" do
migration = Micrate::Migration.new(20160101120000, "foo.sql", "\
-- +micrate Up
CREATE TABLE foo(id INT PRIMARY KEY, name VARCHAR NOT NULL);
-- +micrate Down
DROP TABLE foo;
-- +micrate Up
CREATE TABLE bar(id INT PRIMARY KEY);
-- +micrate Down
DROP TABLE bar;")
statements(migration, :forward).should eq([
"-- +micrate Up\nCREATE TABLE foo(id INT PRIMARY KEY, name VARCHAR NOT NULL);",
"-- +micrate Up\nCREATE TABLE bar(id INT PRIMARY KEY);",
])
statements(migration, :backwards).should eq([
"-- +micrate Down\nDROP TABLE foo;",
"-- +micrate Down\nDROP TABLE bar;",
])
end
# Some complex PL/psql may have semicolons within them
# To understand these we need StatementBegin/StatementEnd hints
it "splits complex statements with user hints" do
migration = Micrate::Migration.new(20160101120000, "foo.sql", "\
-- +micrate Up
-- +micrate StatementBegin
CREATE OR REPLACE FUNCTION histories_partition_creation( DATE, DATE )
returns void AS $$
DECLARE
create_query text;
BEGIN
FOR create_query IN SELECT
'CREATE TABLE IF NOT EXISTS histories_'
|| TO_CHAR( d, 'YYYY_MM' )
|| ' ( CHECK( created_at >= timestamp '''
|| TO_CHAR( d, 'YYYY-MM-DD 00:00:00' )
|| ''' AND created_at < timestamp '''
|| TO_CHAR( d + INTERVAL '1 month', 'YYYY-MM-DD 00:00:00' )
|| ''' ) ) inherits ( histories );'
FROM generate_series( $1, $2, '1 month' ) AS d
LOOP
EXECUTE create_query;
END LOOP; -- LOOP END
END; -- FUNCTION END
$$
language plpgsql;
-- +micrate StatementEnd")
ret = statements(migration, :forward)
ret.size.should eq(1)
ret[0].should eq(migration.source)
end
it "allows up and down sections with complex scripts" do
migration = Micrate::Migration.new(20160101120000, "foo.sql", "\
-- +micrate Up
-- +micrate StatementBegin
foo;
bar;
-- +micrate StatementEnd
-- +micrate Down
baz;")
statements(migration, :forward).should eq([
"-- +micrate Up\n-- +micrate StatementBegin\nfoo;\nbar;\n-- +micrate StatementEnd",
])
statements(migration, :backwards).should eq([
"-- +micrate Down\nbaz;",
])
end
end
end
def statements(migration, direction)
migration.statements(direction).map { |stmt| stmt.strip }
end

9
docker/backend/micrate/spec/spec_helper.cr
View file

@ -1,9 +0,0 @@
require "log"
require "spectator"
require "spectator/should"
require "../src/micrate"
Log.setup(:error)
# Hack to address issue #54 https://gitlab.com/arctic-fox/spectator/-/issues/54
FAKE_DB = [] of Spectator::Mocks::Double # Must be in top-level scope.

12
docker/backend/micrate/src/micrate-bin.cr
View file

@ -1,12 +0,0 @@
require "log"
require "pg"
require "mysql"
require "sqlite3"
require "./micrate"
Log.define_formatter Micrate::CliFormat, "#{message}" \
"#{data(before: " -- ")}#{context(before: " -- ")}#{exception}"
Log.setup(:info, Log::IOBackend.new(formatter: Micrate::CliFormat))
Micrate::Cli.run

221
docker/backend/micrate/src/micrate.cr
View file

@ -1,217 +1,10 @@
require "log"
require "micrate"
require "pg"
require "./micrate/*"
Log.define_formatter Micrate::CliFormat, "#{message}" \
"#{data(before: " -- ")}#{context(before: " -- ")}#{exception}"
Log.setup(:info, Log::IOBackend.new(formatter: Micrate::CliFormat))
module Micrate
Log = ::Log.for(self)
def self.db_dir
"db"
end
def self.migrations_dir
File.join(db_dir, "migrations")
end
def self.dbversion(db)
begin
rows = DB.get_versions_last_first_order(db)
return extract_dbversion(rows)
rescue Exception
DB.create_migrations_table(db)
return 0
end
end
def self.up(db)
all_migrations = migrations_by_version
if all_migrations.size == 0
Log.warn { "No migrations found!" }
return
end
current = dbversion(db)
target = all_migrations.keys.sort.last
migrate(all_migrations, current, target, db)
end
def self.down(db)
all_migrations = migrations_by_version
current = dbversion(db)
target = previous_version(current, all_migrations.keys)
migrate(all_migrations, current, target, db)
end
def self.redo(db)
all_migrations = migrations_by_version
current = dbversion(db)
previous = previous_version(current, all_migrations.keys)
if migrate(all_migrations, current, previous, db) == :success
migrate(all_migrations, previous, current, db)
end
end
def self.migration_status(db) : Hash(Migration, Time?)
# ensure that migration table exists
dbversion(db)
migration_status(migrations_by_version.values, db)
end
def self.migration_status(migrations : Array(Migration), db) : Hash(Migration, Time?)
({} of Migration => Time?).tap do |ret|
migrations.each do |m|
ret[m] = DB.get_migration_status(m, db)
end
end
end
def self.create(name, dir, time)
timestamp = time.to_s("%Y%m%d%H%M%S")
filename = File.join(dir, "#{timestamp}_#{name}.sql")
migration_template = "\
-- +micrate Up
-- SQL in section 'Up' is executed when this migration is applied
-- +micrate Down
-- SQL section 'Down' is executed when this migration is rolled back
"
Dir.mkdir_p dir
File.write(filename, migration_template)
return filename
end
def self.connection_url=(connection_url)
DB.connection_url = connection_url
end
# ---------------------------------
# Private
# ---------------------------------
private def self.migrate(all_migrations : Hash(Int, Migration), current : Int, target : Int, db)
direction = current < target ? :forward : :backwards
status = migration_status(all_migrations.values, db)
plan = migration_plan(status, current, target, direction)
if plan.empty?
Log.info { "No migrations to run. current version: #{current}" }
return :nop
end
Log.info { "Migrating db, current version: #{current}, target: #{target}" }
plan.each do |version|
migration = all_migrations[version]
# Wrap migration in a transaction
db.transaction do |tx|
migration.statements(direction).each do |stmt|
db.exec(stmt)
end
DB.record_migration(migration, direction, db)
tx.commit
Log.info { "OK #{migration.name}" }
rescue e : Exception
tx.rollback
Log.error(exception: e) { "An error occurred executing migration #{migration.version}." }
return :error
end
end
:success
end
private def self.verify_unordered_migrations(current, status : Hash(Int, Bool))
migrations = status.select { |version, is_applied| !is_applied && version < current }
.keys
if !migrations.empty?
raise UnorderedMigrationsException.new(migrations)
end
end
private def self.previous_version(current, all_versions)
all_previous = all_versions.select { |version| version < current }
if !all_previous.empty?
return all_previous.max
end
if all_versions.includes? current
# the given version is (likely) valid but we didn't find
# anything before it.
# return value must reflect that no migrations have been applied.
return 0
else
raise "no previous version found"
end
end
private def self.migrations_by_version
Dir.entries(migrations_dir)
.select { |name| File.file? File.join(migrations_dir, name) }
.select { |name| /^\d+.+\.sql$/ =~ name }
.map { |name| Migration.from_file(name) }
.index_by { |migration| migration.version }
end
def self.migration_plan(status : Hash(Migration, Time?), current : Int, target : Int, direction)
status = ({} of Int64 => Bool).tap do |h|
status.each { |migration, migrated_at| h[migration.version] = !migrated_at.nil? }
end
migration_plan(status, current, target, direction)
end
def self.migration_plan(all_versions : Hash(Int, Bool), current : Int, target : Int, direction)
verify_unordered_migrations(current, all_versions)
if direction == :forward
all_versions.keys
.sort
.select { |v| v > current && v <= target }
else
all_versions.keys
.sort
.reverse
.select { |v| v <= current && v > target }
end
end
# The most recent record for each migration specifies
# whether it has been applied or rolled back.
# The first version we find that has been applied is the current version.
def self.extract_dbversion(rows)
to_skip = [] of Int64
rows.each do |r|
version, is_applied = r
next if to_skip.includes? version
if is_applied
return version
else
to_skip.push version
end
end
return 0
end
class UnorderedMigrationsException < Exception
getter :versions
def initialize(@versions : Array(Int64))
super()
end
end
end
Micrate::DB.connection_url = ENV["BACKEND_DB_URL"]?
Micrate::Cli.run

157
docker/backend/micrate/src/micrate/cli.cr
View file

@ -1,157 +0,0 @@
require "log"
module Micrate
module Cli
Log = ::Log.for(self)
def self.drop_database
url = Micrate::DB.connection_url.to_s
if url.starts_with? "sqlite3:"
path = url.gsub("sqlite3:", "")
File.delete(path)
Log.info { "Deleted file #{path}" }
else
name = set_database_to_schema url
Micrate::DB.connect do |db|
db.exec "DROP DATABASE IF EXISTS #{name};"
end
Log.info { "Dropped database #{name}" }
end
end
def self.create_database
url = Micrate::DB.connection_url.to_s
if url.starts_with? "sqlite3:"
Log.info { "For sqlite3, the database will be created during the first migration." }
else
name = set_database_to_schema url
Micrate::DB.connect do |db|
db.exec "CREATE DATABASE #{name};"
end
Log.info { "Created database #{name}" }
end
end
def self.set_database_to_schema(url)
uri = URI.parse(url)
if path = uri.path
Micrate::DB.connection_url = url.gsub(path, "/#{uri.scheme}")
path.gsub("/", "")
else
Log.error { "Could not determine database name" }
end
end
def self.run_up
Micrate::DB.connect do |db|
Micrate.up(db)
end
end
def self.run_down
Micrate::DB.connect do |db|
Micrate.down(db)
end
end
def self.run_redo
Micrate::DB.connect do |db|
Micrate.redo(db)
end
end
def self.run_status
Micrate::DB.connect do |db|
Log.info { "Applied At Migration" }
Log.info { "=======================================" }
Micrate.migration_status(db).each do |migration, migrated_at|
ts = migrated_at.nil? ? "Pending" : migrated_at.to_s
Log.info { "%-24s -- %s\n" % [ts, migration.name] }
end
end
end
def self.run_scaffold
if ARGV.size < 1
raise "Migration name required"
end
migration_file = Micrate.create(ARGV.shift, Micrate.migrations_dir, Time.local)
Log.info { "Created #{migration_file}" }
end
def self.run_dbversion
Micrate::DB.connect do |db|
begin
Log.info { Micrate.dbversion(db) }
rescue
raise "Could not read dbversion. Please make sure the database exists and verify the connection URL."
end
end
end
def self.report_unordered_migrations(conflicting)
Log.info { "The following migrations haven't been applied but have a timestamp older then the current version:" }
conflicting.each do |version|
Log.info { " #{Migration.from_version(version).name}" }
end
Log.info { "
Micrate will not run these migrations because they may have been written with an older database model in mind.
You should probably check if they need to be updated and rename them so they are considered a newer version." }
end
def self.print_help
Log.info { "micrate is a database migration management system for Crystal projects, *heavily* inspired by Goose (https://bitbucket.org/liamstask/goose/).
Usage:
set DATABASE_URL environment variable i.e. export DATABASE_URL=postgres://user:pswd@host:port/database
micrate [options] <subcommand> [subcommand options]
Commands:
create Create the database (permissions required)
drop Drop the database (permissions required)
up Migrate the DB to the most recent version available
down Roll back the version by 1
redo Re-run the latest migration
status Dump the migration status for the current DB
scaffold Create the scaffolding for a new migration
dbversion Print the current version of the database" }
end
def self.run
if ARGV.empty?
print_help
return
end
begin
case ARGV.shift
when "create"
create_database
when "drop"
drop_database
when "up"
run_up
when "down"
run_down
when "redo"
run_redo
when "status"
run_status
when "scaffold"
run_scaffold
when "dbversion"
run_dbversion
else
print_help
end
rescue e : UnorderedMigrationsException
report_unordered_migrations(e.versions)
exit 1
rescue e : Exception
Log.error(exception: e) { "Micrate failed!" }
exit 1
end
end
end
end

67
docker/backend/micrate/src/micrate/db.cr
View file

@ -1,67 +0,0 @@
require "db"
require "./db/*"
module Micrate
module DB
@@connection_url = ENV["DATABASE_URL"]?
def self.connection_url
@@connection_url
end
def self.connection_url=(connection_url)
@@dialect = nil
@@connection_url = connection_url
end
def self.connect
validate_connection_url
::DB.connect(@@connection_url.not_nil!)
end
def self.connect(&block)
validate_connection_url
::DB.open @@connection_url.not_nil! do |db|
yield db
end
end
def self.get_versions_last_first_order(db)
db.query_all "SELECT version_id, is_applied from micrate_db_version ORDER BY id DESC", as: {Int64, Bool}
end
def self.create_migrations_table(db)
dialect.query_create_migrations_table(db)
end
def self.record_migration(migration, direction, db)
is_applied = direction == :forward
dialect.query_record_migration(migration, is_applied, db)
end
def self.exec(statement, db)
db.exec(statement)
end
def self.get_migration_status(migration, db) : Time?
rows = dialect.query_migration_status(migration, db)
if !rows.empty? && rows[0][1]
rows[0][0]
else
nil
end
end
private def self.dialect
validate_connection_url
@@dialect ||= Dialect.from_connection_url(@@connection_url.not_nil!)
end
private def self.validate_connection_url
if !@@connection_url
raise "No database connection URL is configured. Please set the DATABASE_URL environment variable."
end
end
end
end

21
docker/backend/micrate/src/micrate/db/dialect.cr
View file

@ -1,21 +0,0 @@
module Micrate::DB
abstract class Dialect
abstract def query_create_migrations_table(db)
abstract def query_migration_status(migration, db)
abstract def query_record_migration(migration, is_applied, db)
def self.from_connection_url(connection_url : String)
uri = URI.parse(connection_url)
case uri.scheme
when "postgresql", "postgres"
Postgres.new
when "mysql"
Mysql.new
when "sqlite3"
Sqlite3.new
else
raise "Could not infer SQL dialect from connection url #{connection_url}"
end
end
end
end

21
docker/backend/micrate/src/micrate/db/mysql.cr
View file

@ -1,21 +0,0 @@
module Micrate::DB
class Mysql < Dialect
def query_create_migrations_table(db)
db.exec("CREATE TABLE micrate_db_version (
id serial NOT NULL,
version_id bigint NOT NULL,
is_applied boolean NOT NULL,
tstamp timestamp NULL default now(),
PRIMARY KEY(id)
);")
end
def query_migration_status(migration, db)
db.query_all "SELECT tstamp, is_applied FROM micrate_db_version WHERE version_id=? ORDER BY tstamp DESC LIMIT 1", migration.version, as: {Time, Bool}
end
def query_record_migration(migration, is_applied, db)
db.exec("INSERT INTO micrate_db_version (version_id, is_applied) VALUES (?, ?);", migration.version, is_applied)
end
end
end

21
docker/backend/micrate/src/micrate/db/postgres.cr
View file

@ -1,21 +0,0 @@
module Micrate::DB
class Postgres < Dialect
def query_create_migrations_table(db)
db.exec("CREATE TABLE micrate_db_version (
id serial NOT NULL,
version_id bigint NOT NULL,
is_applied boolean NOT NULL,
tstamp timestamp NULL default now(),
PRIMARY KEY(id)
);")
end
def query_migration_status(migration, db)
db.query_all "SELECT tstamp, is_applied FROM micrate_db_version WHERE version_id=$1 ORDER BY tstamp DESC LIMIT 1", migration.version, as: {Time, Bool}
end
def query_record_migration(migration, is_applied, db)
db.exec("INSERT INTO micrate_db_version (version_id, is_applied) VALUES ($1, $2);", migration.version, is_applied)
end
end
end

25
docker/backend/micrate/src/micrate/db/sqlite3.cr
View file

@ -1,25 +0,0 @@
module Micrate::DB
class Sqlite3 < Dialect
def query_create_migrations_table(db)
# The current sqlite drive implementation does not store timestamps in the same
# format as the ones autogenerated by sqlite.
#
# As a workaround, we create timestamps locally so that the driver decides timestamp
# formats when writing and reading.
db.exec("CREATE TABLE micrate_db_version (
id INTEGER PRIMARY KEY AUTOINCREMENT,
version_id INTEGER NOT NULL,
is_applied INTEGER NOT NULL,
tstamp TIMESTAMP
);")
end
def query_migration_status(migration, db)
db.query_all "SELECT tstamp, is_applied FROM micrate_db_version WHERE version_id=? ORDER BY tstamp DESC LIMIT 1", migration.version, as: {Time, Bool}
end
def query_record_migration(migration, is_applied, db)
db.exec("INSERT INTO micrate_db_version (version_id, is_applied, tstamp) VALUES (?, ?, ?);", migration.version, is_applied, Time.local)
end
end
end

85
docker/backend/micrate/src/micrate/migration.cr
View file

@ -1,85 +0,0 @@
module Micrate
class Migration
SQL_CMD_PREFIX = "-- +micrate "
getter version
getter name
getter source
def initialize(@version : Int64, @name : String, @source : String)
end
# Algorithm ported from Goose
#
# Complex statements cannot be resolved by just splitting the script by semicolons.
# In this cases we allow using StatementBegin and StatementEnd directives as hints.
def statements(direction)
statements = [] of String
# track the count of each section
# so we can diagnose scripts with no annotations
up_sections = 0
down_sections = 0
buffer = Micrate::StatementBuilder.new
statement_ended = false
ignore_semicolons = false
direction_is_active = false
source.split("\n").each do |line|
if line.starts_with? SQL_CMD_PREFIX
cmd = line[SQL_CMD_PREFIX.size..-1].strip
case cmd
when "Up"
direction_is_active = direction == :forward
up_sections += 1
when "Down"
direction_is_active = direction == :backwards
down_sections += 1
when "StatementBegin"
if direction_is_active
ignore_semicolons = true
end
when "StatementEnd"
if direction_is_active
statement_ended = ignore_semicolons == true
ignore_semicolons = false
end
else
# TODO? invalid command
end
end
next unless direction_is_active
buffer.write(line + "\n")
if (!ignore_semicolons && ends_with_semicolon(line)) || statement_ended
statement_ended = false
statements.push buffer.to_s
buffer.reset
end
end
statements
end
def ends_with_semicolon(s)
s.split("--")[0].strip.ends_with? ";"
end
def self.from_file(file_name)
full_path = File.join(Micrate.migrations_dir, file_name)
version = file_name.split("_")[0].to_i64
new(version, file_name, File.read(full_path))
end
def self.from_version(version)
file_name = Dir.entries(Micrate.migrations_dir)
.find { |name| name.starts_with? version.to_s }
.not_nil!
self.from_file(file_name)
end
end
end

21
docker/backend/micrate/src/micrate/statement_builder.cr
View file

@ -1,21 +0,0 @@
module Micrate
class StatementBuilder
@buffer : String::Builder
def initialize
@buffer = String::Builder.new
end
def write(s)
@buffer.write(s.to_slice)
end
def reset
@buffer = String::Builder.new
end
def to_s
@buffer.to_s
end
end
end

3
docker/backend/micrate/src/micrate/version.cr
View file

@ -1,3 +0,0 @@
module Micrate
VERSION = "0.10.0"
end

2
docker/backend/shard.yml
View file

@ -25,8 +25,6 @@ license: GNU GPLv3
targets:
backend:
main: src/cli/backend.cr
micrate:
main: src/cli/micrate.cr
crystal: 1.4.0

10
docker/backend/src/backend/db.cr
View file

@ -27,7 +27,7 @@ module Backend
extend self
# Migration UIDs
MIGRATIONS = {{ run("./macros/migrations.cr", "#{__DIR__}/db/migrations/*.cr").stringify.split("\n") }}
MIGRATIONS = {{ run("./macros/migrations.cr", "db/migrations/*.sql").stringify.split("\n") }}
def init(severity = {% if flag?(:development) %} ::Log::Severity::Debug {% else %} ::Log::Severity::Info {% end %}) : Nil
::Log.builder.bind "clear.*", severity, ::Log::IOBackend.new
@ -37,13 +37,7 @@ module Backend
end
def schema_up_to_date? : Bool
last_migration = ClearMetadata.query.last!.value
if last_migration == "-1"
false
else
last_migration == MIGRATIONS.last
end
MicrateDbVersion.query.last!.version_id == MIGRATIONS.last.to_i64
end
end
end

11
docker/backend/src/backend/db/clear_metadata.cr
View file

@ -1,11 +0,0 @@
module Backend
module Db
class ClearMetadata
include Clear::Model
self.table = :__clear_metadatas
column metatype : String, primary: true
column value : String, primary: true
end
end
end

9
docker/backend/src/backend/db/migrations.cr
View file

@ -1,9 +0,0 @@
require "./migrations/*"
module Backend
module Db
# DB SQL migrations
module Migration
end
end
end

21
docker/backend/src/backend/db/migrations/1_create_users.cr
View file

@ -1,21 +0,0 @@
module Backend
module Db
module Migrations
class CreateUsers1
include Clear::Migration
def change(dir) : Nil
create_enum :user_role, %w(teacher student)
create_table :users, id: false do |t|
t.column :id, :serial, primary: true, null: false
t.column :username, :string, unique: true, index: true, null: false
t.column :role, :user_role, null: false
t.column :skif, :bool, null: false
t.column :admin, :bool, default: false, null: false
end
end
end
end
end
end

22
docker/backend/src/backend/db/migrations/2_create_externals.cr
View file

@ -1,22 +0,0 @@
module Backend
module Db
module Migrations
class CreateExternals2
include Clear::Migration
def change(dir) : Nil
create_table :teachers, id: false do |t|
t.column :id, :serial, primary: true, null: false
t.references to: "users", on_delete: :cascade, null: false, primary: true
t.column :max_students, :int, null: false
end
create_table :students, id: false do |t|
t.column :id, :serial, primary: true, null: false
t.references to: "users", on_delete: :cascade, null: false, primary: true
end
end
end
end
end
end

23
docker/backend/src/backend/db/migrations/3_create_votes.cr
View file

@ -1,23 +0,0 @@
module Backend
module Db
module Migrations
class CreateVotes3
include Clear::Migration
def change(dir) : Nil
create_table :votes, id: false do |t|
t.column :id, :serial, primary: true, null: false
t.references to: "students", on_delete: :cascade, null: false, primary: true
end
create_table :teacher_votes, id: false do |t|
t.column :id, :serial, primary: true, null: false
t.references to: "votes", on_delete: :cascade, null: false
t.references to: "teachers", on_delete: :cascade, null: false
t.column :priority, :int, null: false
end
end
end
end
end
end

2
docker/backend/src/backend/macros/migrations.cr
View file

@ -1 +1 @@
print Dir[ARGV[0]].map { |f| File.basename(f).split("_", 2).first }.uniq!.sort!.join("\n")
print Dir[ARGV[0]].map { |f| File.basename(f).split("_", 2).first.to_i64 }.uniq!.sort!.join("\n")

4
docker/backend/src/backend/runner.cr
View file

@ -55,11 +55,11 @@ module Backend
end && ex.nil?
Log.info { "Database schema is up to date." }
else
Log.warn { "Database schema is not up to date. Please run `bash scripts/clear.sh migrate`." }
Log.warn { "Database schema is not up to date. Please run `bash scripts/micrate.sh up`." }
if ex
raise ex
else
raise Exception.new("Database schema is not up to date.") unless Backend.config.db.allow_old_schema
raise Exception.new unless Backend.config.db.allow_old_schema
end
end
end

2
scripts/clear.sh → scripts/micrate.sh
View file

@ -16,4 +16,4 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
docker-compose exec backend clear "$@"
docker-compose exec backend micrate "$@"