Add support for running integration tests using testcontainers - postgres

.circleci/config.yml

@@ -1,22 +1,29 @@
+version: 2.1
 jobs:
   build:
-    docker:
-      - image: clojure:lein-2.8.0
-    working_directory: /root/migratus
+    parameters:
+      clojure-tag:
+        type: string
+    machine:
+      image: ubuntu-2204:2022.04.2
+    working_directory: /home/circleci/migratus
     steps:
       - checkout
       - restore_cache:
           keys:
             - v1-dependency-jars-{{ checksum "project.clj" }}
             - v1-dependency-jars
-      - run: lein test
+      - run: docker run --rm -u root -v $PWD:/root/migratus -v /var/run/docker.sock:/var/run/docker.sock -v $PWD/.m2:/root/.m2 -v $PWD/.lein:/root/.lein -w=/root/migratus cimg/clojure:<< parameters.clojure-tag >> lein test
       - save_cache:
           key: v1-dependency-jars-{{ checksum "project.clj" }}
           paths:
             - /root/.m2
+            - /root/.lein
 
 workflows:
-  version: 2
   build:
     jobs:
-      - build
+      - build:
+          matrix:
+            parameters:
+              clojure-tag: ["1.10-openjdk-8.0", "1.11-openjdk-8.0", "1.11-openjdk-11.0", "1.11-openjdk-17.0"]

CHANGES.md

@@ -1,6 +1,11 @@
+### Unreleased
+
+new feature: Run basic tests against PostgreSQL using testcontainers
+
 ### 1.3.8
 
 new feature: [Provide deps.edn and kaocha test runner](https://github.com/yogthos/migratus/pull/212)
+
 new feature: [Port migratus to next.jdbc](https://github.com/yogthos/migratus/pull/214)
 
 ### 1.3.7

README.md

@@ -468,6 +468,9 @@ Run tests with kaocha:
    bin/kaocha --fail-fast
 
    bin/kaocha --fail-fast --focus migratus.test.migration.sql/test-run-sql-migrations
+
+   # Run only integration tests - defined in tests.edn
+   bin/kaocha testcontainers
 ```
 
 ## License

deps.edn

@@ -14,19 +14,15 @@
                  :extra-deps
                  {jar-migrations/jar-migrations {:mvn/version "1.0.0"}
                   ch.qos.logback/logback-classic {:mvn/version "1.2.3"}
+                  clj-test-containers/clj-test-containers {:mvn/version "0.7.1"}
                   com.h2database/h2 {:mvn/version "2.1.214"}
                   hikari-cp/hikari-cp {:mvn/version "2.13.0"}
-                  org.clojure/tools.trace {:mvn/version "0.7.11"}}}
+                  org.clojure/tools.trace {:mvn/version "0.7.11"}
+                  org.postgresql/postgresql {:mvn/version "42.2.5"}}}
            :test-runner {:extra-paths ["test"]
                          :extra-deps
-                         {jar-migrations/jar-migrations {:mvn/version "1.0.0"}
-                          ch.qos.logback/logback-classic {:mvn/version "1.2.3"}
-                          com.h2database/h2 {:mvn/version "2.1.214"}
-                          hikari-cp/hikari-cp {:mvn/version "2.13.0"}
-                          lambdaisland/kaocha           {:mvn/version "1.66.1034"}
+                         {lambdaisland/kaocha           {:mvn/version "1.66.1034"}
                           lambdaisland/kaocha-cloverage {:mvn/version "1.0.75"}
                           lambdaisland/kaocha-junit-xml {:mvn/version "0.0.76"}
-                          orchestra/orchestra           {:mvn/version "2021.01.01-1"}
-                          org.clojure/test.check        {:mvn/version "1.1.1"}
-                          org.testcontainers/postgresql {:mvn/version "1.17.2"}}
+                          orchestra/orchestra           {:mvn/version "2021.01.01-1"}}
                          :main-opts   ["-m" "kaocha.runner" "--reporter" "kaocha.report/documentation"]}}}

project.clj

@@ -10,6 +10,8 @@
                  [org.clojure/tools.logging "1.1.0"]]
   :profiles {:dev {:dependencies [[jar-migrations "1.0.0"]
                                   [ch.qos.logback/logback-classic "1.2.3"]
+                                  [clj-test-containers/clj-test-containers "0.7.1"]
                                   [com.h2database/h2 "2.1.214"]
                                   [hikari-cp/hikari-cp "2.13.0"]
-                                  [org.clojure/tools.trace "0.7.11"]]}})
+                                  [org.clojure/tools.trace "0.7.11"]
+                                  [org.postgresql/postgresql "42.2.5"]]}})

test/migrations-postgres/20220820030400-create-table-quux.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS quux;

test/migrations-postgres/20220820030400-create-table-quux.up.sql

@@ -0,0 +1,3 @@
+CREATE TABLE IF NOT EXISTS quux(id bigint, name varchar(255));
+--;;
+CREATE INDEX quux_name on quux(name);

test/migrations-postgres/init.sql

@@ -0,0 +1,2 @@
+CREATE SCHEMA foo;
+CREATE TABLE IF NOT EXISTS foo(id bigint);

test/migratus/test/migration/sql.clj

@@ -3,7 +3,9 @@
             [clojure.test :refer :all]
             [migratus.core :as core]
             [migratus.database :as db]
-            [migratus.migration.sql :refer :all]))
+            [migratus.migration.sql :refer :all]
+            [next.jdbc :as jdbc]
+            [next.jdbc.result-set :as rs]))
 
 (def db-store (str (.getName (io/file ".")) "/site.db"))
 
@@ -13,6 +15,15 @@
 (def test-config {:migration-dir "migrations/"
                   :db            db-spec})
 
+(defn db-tables-and-views
+  "Fetch tables and views (database metadata) from DB.
+   Returns a collection of table metadata."
+  [datasource]
+  (with-open [con (jdbc/get-connection datasource)]
+    (-> (.getMetaData con)
+        (.getTables nil nil nil (into-array ["TABLE" "VIEW"]))
+        (rs/datafiable-result-set datasource))))
+
 (defn reset-db []
   (letfn [(delete [f]
             (when (.exists f)

test/migratus/testcontainers/postgres.clj

@@ -0,0 +1,65 @@
+(ns migratus.testcontainers.postgres
+  "Integration tests for postgresql using testcontainers.org"
+  {:authors ["Eugen Stan"]}
+  (:require [clj-test-containers.core :as tc]
+            [clojure.tools.logging :as log]
+            [clojure.set :as set]
+            [clojure.test :refer [deftest is testing]]
+            [migratus.test.migration.sql :as test-sql]
+            [migratus.core :as migratus]
+            [next.jdbc :as jdbc]
+            [next.jdbc.result-set :as rs]))
+
+(def postgres-image (or (System/getenv "MIGRATUS_TESTCONTAINERS_POSTGRES_IMAGE") 
+                        "postgres:14"))
+
+(def pg-container-spec {:image-name    postgres-image
+                        :exposed-ports [5432]
+                        :env-vars      {"POSTGRES_PASSWORD" "pw"}
+                        :wait-for      {:wait-strategy :port}})
+
+
+(deftest postgres-migrations-test
+
+  (testing "Migrations are applied succesfully in PostgreSQL."
+    (let [pg-container (tc/create pg-container-spec)
+          initialized-pg-container (tc/start! pg-container)
+          meta->table-names #(into #{} (map :pg_class/table_name %))]
+      (Thread/sleep 1000)
+      (let [ds (jdbc/get-datasource {:dbtype   "postgresql"
+                                     :dbname   "postgres"
+                                     :user     "postgres"
+                                     :password "pw"
+                                     :host     (:host initialized-pg-container)
+                                     :port     (get (:mapped-ports initialized-pg-container) 5432)})
+            config {:store :database
+                    :migration-dir "migrations-postgres"
+                    :init-script "init.sql"
+                    :migration-table-name "foo_bar"
+                    :db {:dbtype   "postgresql"
+                         :dbname   "postgres"
+                         :user     "postgres"
+                         :password "pw"
+                         :host     (:host initialized-pg-container)
+                         :port     (get (:mapped-ports initialized-pg-container) 5432)}}]
+        (is (= [] (test-sql/db-tables-and-views ds)) "db is empty before migrations")
+
+        ;; init 
+        (migratus/init config)
+        (let [db-meta (test-sql/db-tables-and-views ds)
+              table-names (meta->table-names db-meta)]
+          (is (= #{"foo"} table-names) "db is initialized"))
+
+        ;; migrate
+        (migratus/migrate config)
+        (let [db-meta (test-sql/db-tables-and-views ds)
+              table-names (meta->table-names db-meta) 
+              expected-tables #{"quux" "foo" "foo_bar"}]
+          (log/info "Tables are" table-names)
+          (is (= (count expected-tables) (count db-meta))
+              (str "expected table count is ok."))
+
+          (is (set/subset? expected-tables table-names)
+              "contains some tables that we expect")))
+
+      (tc/stop! initialized-pg-container))))

tests.edn

@@ -1,7 +1,10 @@
 #kaocha/v1
  {:tests [{:id :unit
            :test-paths ["test"]
-           :ns-patterns ["migratus\\.test.*"]}]
+           :ns-patterns ["migratus\\.test\\..*"]}
+          {:id :testcontainers
+           :test-paths ["test"]
+           :ns-patterns ["migratus\\.testcontainers\\..*"]}]
   :plugins [:kaocha.plugin/junit-xml
             :kaocha.plugin/cloverage
             :kaocha.plugin.alpha/spec-test-check]