Migrate from MongoDB to Postgres

.github/workflows/ci-build.yml

@@ -18,9 +18,6 @@ jobs:
           ref: ${{ github.event.pull_request.head.ref }}
           repository: ${{ github.event.pull_request.head.repo.full_name }}
 
-      - name: MongoDB in GitHub Actions
-        uses: supercharge/mongodb-github-action@1.3.0
-
       - name: Restore dependency cache
         uses: actions/cache@v2.1.4
         with:

.mergify.yml

@@ -1,21 +1,18 @@
-pull_request_rules:
-  - name: For dependabot, automatic merge on approval and when when GitHub branch protection passes on main
+queue_rules:
+  - name: default
     conditions:
-      - "#approved-reviews-by>=1"
+      - -draft # not a draft
       - base=main
-      - label=dependencies
-    actions:
-      merge:
-        method: merge
-        strict: false
-
+
 pull_request_rules:
   - name: Automatic merge on approval and when when GitHub branch protection passes on main
     conditions:
       - "#approved-reviews-by>=1"
+      - "#changes-requested-reviews-by<1"
+      - -draft
       - base=main
       - label!=dependencies
     actions:
-      merge:
+      queue:
         method: merge
-        strict: smart
+        name: default

CONTRIBUTING.md

@@ -78,8 +78,8 @@ Java annotation based.
 Templates for the view use [Thymeleaf](http://www.thymeleaf.org/), 
 which allows them to be displayed in browsers as static prototypes.
 
-MongoDB is used to store information about `Workflow` and `QueuedWorkflow` 
-objects using [Spring Data JPA](https://docs.spring.io/spring-data/mongodb/docs/current/reference/html/).
+PostgreSQL is used to store information about `Workflow` and `QueuedWorkflow` 
+objects using [Spring Data JPA](https://docs.spring.io/spring-data/jpa/docs/current/reference/html/).
 
 The application also uses a triple store to keep the RDF representing 
 workflows (gathered from [cwltool](https://github.com/common-workflow-language/cwltool)'s 

Dockerfile

@@ -45,7 +45,7 @@ ADD pom.xml LICENSE.md NOTICE.md README.md /usr/src/app/
 
 # add src/ (which often change)
 ADD src /usr/src/app/src
-# Skip tests while building as that requires a local mongodb
+# Skip tests while building as that requires a local postgres
 RUN mvn clean package -DskipTests && cp target/cwlviewer-*.jar /usr/lib/cwlviewer.jar && rm -rf target
 
 # NOTE: ~/.m2/repository is a VOLUME and so will be deleted anyway

README.md

@@ -59,16 +59,16 @@ If you have modified the source code, then you may want to build the docker imag
 
     docker build -t commonworkflowlanguage/cwlviewer .
 
-## Running Spring Boot locally for development, with MongoDB and Jena Fuseki in Docker
+## Running Spring Boot locally for development, with PostgreSQL and Jena Fuseki in Docker
 
 Create `docker-compose.override.yml`:
 
 ```
 version: '3.2'
 services:
-  mongo:
+  postgres:
     ports:
-     - "27017:27017"
+     - "5432:5432"
   sparql:
     ports:
      - "3030:3030"
@@ -94,21 +94,21 @@ To completely reset the state, you must delete the data volumes:
 
 ```
 docker-compose down
-docker volume rm  cwlviewer_bundle cwlviewer_git cwlviewer_graphviz cwlviewer_mongo cwlviewer_sparql
+docker volume rm  cwlviewer_bundle cwlviewer_git cwlviewer_graphviz cwlviewer_postgres cwlviewer_sparql
 ```
 
 ## Running without Docker
 
 ### Requirements
 
-#### MongoDB
+#### PostgreSQL
 
-You will need to have [MongoDB](https://www.mongodb.com/) running,
-by default on `localhost:27017`
+You will need to have [PostgreSQL](https://www.postgresql.org/) running,
+by default on `localhost:5432`
 
 If you are running from the command line, you can override this by supplying
-system properties like `-Dspring.data.mongodb.host=mongo.example.org` and
-`-Dspring.data.mongodb.port=1337`
+system properties like `-Dspring.datasource.url=jdbc:postgresql://localhost:5432/cwlviewer` and
+`-Dspring.datasource.password=sa`
 
 #### Apache Jena Fuseki (or alternative SPARQL server)
 

docker-compose.yml

@@ -3,7 +3,7 @@ volumes:
   bundle:
   git:
   graphviz:
-  mongo:
+  postgres:
   sparql:
 services:
   spring:
@@ -15,7 +15,7 @@ services:
     ports:
      - "8080:8080"
     links:
-     - mongo
+     - postgres
      - sparql
     volumes:
      - type: volume
@@ -29,19 +29,26 @@ services:
        target: /data/graphviz
     environment:
      - SPARQL_ENDPOINT=http://sparql:3030/cwlviewer/
-     - SPRING_DATA_MONGODB_HOST=mongo
-     - SPRING_DATA_MONGODB_PORT=27017
+     - SPRING_DATASOURCE_URL=jdbc:postgresql://postgres:5432/cwlviewer
+     - SPRING_DATASOURCE_USERNAME=sa
+     - SPRING_DATASOURCE_PASSWORD=sa
      - BUNDLESTORAGE=/data/bundle
      - GITSTORAGE=/data/git
      - GRAPHVIZSTORAGE=/data/graphviz
-  mongo:
+  postgres:
     restart: always
-    image: mongo:3.4
-    command: mongod --setParameter="internalQueryExecMaxBlockingSortBytes=335544320"
+    image: postgres:14-alpine
+    command: postgres -c listen_addresses='*' -c log_statement=all
+    environment:
+      - POSTGRES_USER=sa
+      - POSTGRES_PASSWORD=sa
+      - POSTGRES_DB=cwlviewer
+      - PGDATA=/var/lib/postgresql/data/cwlviewer/
     volumes:
       - type: volume
-        source: mongo
-        target: /data/db
+        source: postgres
+        target: /var/lib/postgresql/data
+      - ./pg_hba.conf/:/var/lib/postgresql/data/pg_hba.conf
   sparql:
     restart: always
 ## For debugging, expose the Fuseki port by enabling:

pga_hba.conf

@@ -0,0 +1 @@
+host all all 0.0.0.0/0 trust

pom.xml

@@ -52,10 +52,6 @@
 
   <dependencies>
     <!-- SpringBoot -->
-    <dependency>
-      <groupId>org.springframework.boot</groupId>
-      <artifactId>spring-boot-starter-data-mongodb</artifactId>
-    </dependency>
     <dependency>
       <groupId>org.springframework.boot</groupId>
       <artifactId>spring-boot-starter-thymeleaf</artifactId>
@@ -65,11 +61,36 @@
       <artifactId>spring-boot-starter-web</artifactId>
     </dependency>
     <!-- Spring Data -->
+    <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-data-jpa</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.springframework.data</groupId>
       <artifactId>spring-data-commons</artifactId>
       <version>2.6.2</version>
     </dependency>
+    <dependency>
+      <groupId>com.vladmihalcea</groupId>
+      <artifactId>hibernate-types-55</artifactId>
+      <version>2.14.0</version>
+    </dependency>
+    <!-- Liquibase; note that Flyway, while probably having more documentation and users,
+         changed its license model. Similarly to mongo, we prefer an open license that
+         won't lock users or developers from updating or modifying the code if necessary.
+         Hence our choice for liquibase. Feel free to suggest alternatives, but keep in
+         mind the license.
+    -->
+    <dependency>
+      <groupId>org.liquibase</groupId>
+      <artifactId>liquibase-core</artifactId>
+    </dependency>
+    <!-- Postgres -->
+    <dependency>
+      <groupId>org.postgresql</groupId>
+      <artifactId>postgresql</artifactId>
+      <scope>runtime</scope>
+    </dependency>
     <!-- YAML -->
     <dependency>
       <groupId>org.yaml</groupId>
@@ -125,9 +146,8 @@
     </dependency>
     <!-- For JSR-303, javax.validation -->
     <dependency>
-      <groupId>org.hibernate.validator</groupId>
-      <artifactId>hibernate-validator</artifactId>
-      <version>7.0.2.Final</version>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-validation</artifactId>
     </dependency>
     <!-- Test dependencies -->
     <dependency>
@@ -136,14 +156,21 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-all</artifactId>
-      <version>1.10.19</version>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>junit-jupiter</artifactId>
+      <version>1.16.2</version>
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>de.flapdoodle.embed</groupId>
-      <artifactId>de.flapdoodle.embed.mongo</artifactId>
+      <groupId>org.testcontainers</groupId>
+      <artifactId>postgresql</artifactId>
+      <version>1.16.2</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <version>1.10.19</version>
       <scope>test</scope>
     </dependency>
   </dependencies>

src/main/java/org/commonwl/view/CwlViewerApplication.java

@@ -21,14 +21,14 @@
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.data.mongodb.repository.config.EnableMongoRepositories;
 import org.springframework.scheduling.annotation.EnableAsync;
 import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.transaction.annotation.EnableTransactionManagement;
 
 @SpringBootApplication
-@EnableMongoRepositories
 @EnableAsync
 @EnableScheduling
+@EnableTransactionManagement
 public class CwlViewerApplication {
 
 	public static void main(String[] args) {

src/main/java/org/commonwl/view/WebConfig.java

@@ -83,6 +83,6 @@ public void configureContentNegotiation(ContentNegotiationConfigurer configurer)
 
     @Override
     public void addCorsMappings(CorsRegistry registry) {
-	registry.addMapping("/**").exposedHeaders("Location");  // .setMaxAge(Long.MAX_VALUE)
+        registry.addMapping("/**").exposedHeaders("Location");  // .setMaxAge(Long.MAX_VALUE)
     }
 }

src/main/java/org/commonwl/view/git/GitDetails.java

@@ -24,11 +24,12 @@
 import java.io.Serializable;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.util.Objects;
 
 /**
  * Represents all the parameters necessary to access a file/directory with Git
  */
-@JsonIgnoreProperties(value = {"internalUrl"})
+@JsonIgnoreProperties(value = {"internalUrl"}, ignoreUnknown = true)
 public class GitDetails implements Serializable {
 
     private String repoUrl;
@@ -226,4 +227,19 @@ public static String normaliseUrl(String url) {
                   .replace("www.", "");
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        GitDetails that = (GitDetails) o;
+        return Objects.equals(repoUrl, that.repoUrl) &&
+                Objects.equals(branch, that.branch) &&
+                Objects.equals(path, that.path) &&
+                Objects.equals(packedId, that.packedId);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(repoUrl, branch, path, packedId);
+    }
 }

src/main/java/org/commonwl/view/util/BaseEntity.java

@@ -0,0 +1,14 @@
+package org.commonwl.view.util;
+
+import com.vladmihalcea.hibernate.type.json.JsonBinaryType;
+import org.hibernate.annotations.TypeDef;
+import org.hibernate.annotations.TypeDefs;
+
+import javax.persistence.MappedSuperclass;
+
+@TypeDefs({
+        @TypeDef(name = "json", typeClass = JsonBinaryType.class)
+})
+@MappedSuperclass
+public class BaseEntity {
+}