Supporting postgres in-memory db translation for HSQL

CHANGELOG.md

@@ -5,10 +5,13 @@
 ### Functionality Improvements
 #182: Adding Hibernate reverse-engineering API. See [ORM Integration docs](https://goldmansachs.github.io/obevo/orm-integration.html) for more details.
 #221 #223 #225: Oracle reverse-engineering improvements - unicode characters, nested tables, types, comments
+#227: PostgreSQL improvements for kata - reverse-engineering, in-memory databases
 
 ### Technical Improvements
+#227: Upgrading checkstyle version to avoid security prompts at Github site
 
 ### Bug Fixes
+#227: Fixed drop behavior of PostgreSQL for multiple views that depended on each other
 
 
 ## 7.0.2

obevo-db-impls/obevo-db-postgresql/obevo-db-postgresql.iml

@@ -78,6 +78,10 @@
     <orderEntry type="library" scope="RUNTIME" name="Maven: com.google.protobuf:protobuf-java:2.6.0" level="project" />
     <orderEntry type="library" scope="RUNTIME" name="Maven: us.fatehi:schemacrawler-db2:14.16.04-java7" level="project" />
     <orderEntry type="library" name="Maven: org.postgresql:postgresql:42.1.1.jre7" level="project" />
+    <orderEntry type="module" module-name="obevo-db-h2" />
+    <orderEntry type="library" name="Maven: com.h2database:h2:1.4.187" level="project" />
+    <orderEntry type="module" module-name="obevo-db-hsql" />
+    <orderEntry type="library" name="Maven: org.hsqldb:hsqldb:2.3.4" level="project" />
     <orderEntry type="module" module-name="obevo-db-internal-test-helper" scope="TEST" />
     <orderEntry type="module" module-name="obevo-db" scope="TEST" production-on-test="" />
     <orderEntry type="module" module-name="obevo-db-unittest-util" scope="TEST" />

obevo-db-impls/obevo-db-postgresql/pom.xml

@@ -49,6 +49,21 @@
             <artifactId>postgresql</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>com.goldmansachs.obevo</groupId>
+            <artifactId>obevo-db-h2</artifactId>
+            <version>${project.version}</version>
+            <optional>true</optional>
+            <!-- optional for the translation code -->
+        </dependency>
+        <dependency>
+            <groupId>com.goldmansachs.obevo</groupId>
+            <artifactId>obevo-db-hsql</artifactId>
+            <version>${project.version}</version>
+            <optional>true</optional>
+            <!-- optional for the translation code -->
+        </dependency>
+
         <dependency>
             <groupId>com.goldmansachs.obevo</groupId>
             <artifactId>obevo-db-internal-test-helper</artifactId>

obevo-db-impls/obevo-db-postgresql/src/main/java/com/gs/obevo/db/impl/platforms/postgresql/PostgreSqlDbPlatform.java

@@ -20,6 +20,8 @@
 import com.gs.obevo.db.api.appdata.GrantTargetType;
 import com.gs.obevo.db.api.platform.DbChangeType;
 import com.gs.obevo.db.api.platform.DbChangeTypeImpl;
+import com.gs.obevo.db.api.platform.DbPlatform;
+import com.gs.obevo.db.api.platform.DbTranslationDialect;
 import com.gs.obevo.db.apps.reveng.AbstractDdlReveng;
 import com.gs.obevo.db.impl.platforms.AbstractDbPlatform;
 import org.eclipse.collections.api.block.function.Function;
@@ -116,4 +118,13 @@ public ImmutableSet<String> getRequiredValidationObjectTypes() {
     public AbstractDdlReveng getDdlReveng() {
         return new PostgreSqlPgDumpReveng();
     }
+
+    @Override
+    public DbTranslationDialect getDbTranslationDialect(DbPlatform targetDialect) {
+        if (targetDialect.getClass().getName().equals("com.gs.obevo.db.impl.platforms.hsql.HsqlDbPlatform")) {
+            return new PostgreSqlToHsqlTranslationDialect();
+        } else {
+            return super.getDbTranslationDialect(targetDialect);
+        }
+    }
 }

obevo-db-impls/obevo-db-postgresql/src/main/java/com/gs/obevo/db/impl/platforms/postgresql/PostgreSqlPgDumpReveng.java

@@ -105,7 +105,8 @@ protected boolean doRevengOrInstructions(PrintStream out, AquaRevengArgs args, F
 
     private String getCommandWithDefaults(boolean container, AquaRevengArgs args, File interimDir, String username, String password, String dbHost, String dbPort, String dbName, String dbSchema, String outputDirectory) {
         String prefix = container ? "docker exec $CONTAINER_NAME " : "";
-        String fileSuffix = container ? "> " + interimDir.getAbsolutePath() : " -f " + interimDir.getAbsolutePath();
+        String fileSuffix = container ? "> " : " -f ";
+        fileSuffix += new File(interimDir, "revengoutput.txt").getAbsolutePath();
         return prefix + "pg_dump -O -s --no-privileges" +
                 " -h " + ObjectUtils.defaultIfNull(args.getDbHost(), dbHost) +
                 " -p " + ObjectUtils.defaultIfNull(args.getDbPort(), dbPort) +

obevo-db-impls/obevo-db-postgresql/src/main/java/com/gs/obevo/db/impl/platforms/postgresql/PostgreSqlToH2TranslationDialect.java

@@ -0,0 +1,27 @@
+/**
+ * Copyright 2017 Goldman Sachs.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.gs.obevo.db.impl.platforms.postgresql;
+
+import com.gs.obevo.db.impl.platforms.DefaultDbTranslationDialect;
+import org.eclipse.collections.api.list.ImmutableList;
+import org.eclipse.collections.impl.factory.Lists;
+
+public class PostgreSqlToH2TranslationDialect extends DefaultDbTranslationDialect {
+    @Override
+    public ImmutableList<String> getInitSqls() {
+        return Lists.immutable.with("SET MODE PostgreSQL");
+    }
+}

obevo-db-impls/obevo-db-postgresql/src/main/java/com/gs/obevo/db/impl/platforms/postgresql/PostgreSqlToHsqlTranslationDialect.java

@@ -0,0 +1,92 @@
+/**
+ * Copyright 2017 Goldman Sachs.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.gs.obevo.db.impl.platforms.postgresql;
+
+import java.sql.Connection;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.gs.obevo.api.platform.ChangeType;
+import com.gs.obevo.db.impl.core.jdbc.JdbcHelper;
+import com.gs.obevo.db.impl.platforms.DefaultDbTranslationDialect;
+import com.gs.obevo.db.impl.platforms.sqltranslator.InMemoryTranslator;
+import com.gs.obevo.db.impl.platforms.sqltranslator.PostColumnSqlTranslator;
+import com.gs.obevo.db.impl.platforms.sqltranslator.PreParsedSqlTranslator;
+import com.gs.obevo.db.impl.platforms.sqltranslator.SqlTranslatorConfigHelper;
+import com.gs.obevo.db.sqlparser.syntaxparser.CreateTable;
+import com.gs.obevo.db.sqlparser.syntaxparser.CreateTableColumn;
+import com.gs.obevo.impl.PrepareDbChange;
+import org.eclipse.collections.api.list.ImmutableList;
+import org.eclipse.collections.api.set.ImmutableSet;
+import org.eclipse.collections.impl.factory.Lists;
+import org.eclipse.collections.impl.factory.Sets;
+
+public class PostgreSqlToHsqlTranslationDialect extends DefaultDbTranslationDialect {
+    private final PostColumnSqlTranslator replaceNextvalWithIdentity = new PostColumnSqlTranslator() {
+        private final Pattern defaultPattern = Pattern.compile("(?i)default\\s+nextval.*", Pattern.DOTALL);
+
+        @Override
+        public String handlePostColumnText(String postColumnText, CreateTableColumn column, CreateTable table) {
+            Matcher defaultMatcher = defaultPattern.matcher(postColumnText);
+            if (defaultMatcher.find()) {
+                postColumnText = defaultMatcher.replaceFirst("IDENTITY");
+            }
+
+            return postColumnText;
+        }
+    };
+
+    private final PreParsedSqlTranslator substituteCreateOrReplace = new PreParsedSqlTranslator() {
+        Pattern pattern = Pattern.compile("(?i)^\\s*create\\s+or\\s+replace\\s+", Pattern.DOTALL);
+        @Override
+        public String preprocessSql(String sql) {
+            Matcher matcher = pattern.matcher(sql);
+            if (matcher.find()) {
+                sql = matcher.replaceFirst("create ");
+            }
+            return sql;
+        }
+    };
+
+    @Override
+    public ImmutableList<String> getInitSqls() {
+        return Lists.immutable.with(
+                "SET DATABASE SQL SYNTAX PGS TRUE"
+                , "SET DATABASE TRANSACTION CONTROL MVCC"
+        );
+    }
+
+    @Override
+    public ImmutableList<PrepareDbChange> getAdditionalTranslators() {
+        SqlTranslatorConfigHelper configHelper = SqlTranslatorConfigHelper.createInMemoryDefault();
+
+        configHelper.getPreParsedSqlTranslators()
+                .with(substituteCreateOrReplace);
+        configHelper.getPostColumnSqlTranslators()
+                .with(replaceNextvalWithIdentity);
+        return Lists.immutable.<PrepareDbChange>with(new InMemoryTranslator(configHelper));
+    }
+
+    @Override
+    public void initSchema(JdbcHelper jdbc, Connection conn) {
+        updateAndIgnoreException(conn, jdbc, "create type int4 as integer");
+    }
+
+    @Override
+    public ImmutableSet<String> getDisabledChangeTypeNames() {
+        return Sets.immutable.of(ChangeType.FUNCTION_STR, ChangeType.SP_STR);
+    }
+}

obevo-db-impls/obevo-db-postgresql/src/test/java/com/gs/obevo/db/impl/platforms/postgresql/PostgreSqlInMemConversionTest.java

@@ -0,0 +1,53 @@
+/**
+ * Copyright 2017 Goldman Sachs.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.gs.obevo.db.impl.platforms.postgresql;
+
+import com.gs.obevo.db.api.platform.DbDeployerAppContext;
+import com.gs.obevo.db.impl.platforms.h2.H2DbPlatform;
+import com.gs.obevo.db.impl.platforms.hsql.HsqlDbPlatform;
+import com.gs.obevo.db.unittest.UnitTestDbBuilder;
+import org.junit.Test;
+
+public class PostgreSqlInMemConversionTest {
+    @Test
+    public void testInMemoryHsql() {
+        DbDeployerAppContext context = UnitTestDbBuilder.newBuilder()
+                .setSourcePath("platforms/postgresql/example1/step1/system-config-inmem.xml")
+                .setReferenceEnvName("unittestrefhsql")
+                .setDbPlatform(new HsqlDbPlatform())
+                .setDbServer("mydb2testHsql")
+                .buildContext();
+        context.setupEnvInfra();
+        context.cleanAndDeploy();
+
+        // TODO add assertions
+    }
+
+    // Implementation for H2 TBD
+//    @Test
+//    public void testInMemoryH2() {
+//        DbDeployerAppContext context = UnitTestDbBuilder.newBuilder()
+//                .setSourcePath("platforms/postgresql/example1/step1/system-config-inmem.xml")
+//                .setReferenceEnvName("unittestrefh2")
+//                .setDbPlatform(new H2DbPlatform())
+//                .setDbServer("mydb2testH2")
+//                .buildContext();
+//        context.setupEnvInfra();
+//        context.cleanAndDeploy();
+//
+//        // TODO add assertions
+//    }
+}

obevo-db-impls/obevo-db-postgresql/src/test/resources/platforms/postgresql/example1/step1/schema1/table/TABLE_A.ddl

@@ -1,29 +1,29 @@
-//// CHANGE name=chng1
-CREATE TABLE TABLE_A (
-	A_ID    INT	NOT NULL,
-	B_ID    INT	NOT NULL,
-	STRING_FIELD	VARCHAR(30)	NULL,
-	TIMESTAMP_FIELD	TIMESTAMP	NULL,
-    PRIMARY KEY (A_ID)
-)
-GO
-
-//// CHANGE name=emptychange
-//// CHANGE name=emptychange2
-
-
-//// CHANGE FK name=chng2
-ALTER TABLE TABLE_A ADD FOREIGN KEY (B_ID) REFERENCES TABLE_B(B_ID)
-GO
-//// CHANGE name=chng3
-ALTER TABLE TABLE_A ADD COLUMN C_ID INT NULL
-GO
-//// CHANGE name=extra1
-ALTER TABLE TABLE_A ADD COLUMN EXTRA1 INT NULL
-GO
-//// CHANGE name=extra2
-ALTER TABLE TABLE_A ADD COLUMN EXTRA2 INT NULL
-GO
-//// CHANGE name=extra3
-ALTER TABLE TABLE_A ADD COLUMN EXTRA3 INT NULL
-GO
+//// CHANGE name=chng1
+CREATE TABLE TABLE_A (
+	A_ID    INT	NOT NULL,
+	B_ID    INT	NOT NULL,
+	STRING_FIELD	VARCHAR(30)	NULL,
+	TIMESTAMP_FIELD	TIMESTAMP	NULL,
+    PRIMARY KEY (A_ID)
+) WITHOUT OIDS
+GO
+
+//// CHANGE name=emptychange
+//// CHANGE name=emptychange2
+
+
+//// CHANGE FK name=chng2
+ALTER TABLE TABLE_A ADD FOREIGN KEY (B_ID) REFERENCES TABLE_B(B_ID)
+GO
+//// CHANGE name=chng3
+ALTER TABLE TABLE_A ADD COLUMN C_ID INT NULL
+GO
+//// CHANGE name=extra1
+ALTER TABLE TABLE_A ADD COLUMN EXTRA1 INT NULL
+GO
+//// CHANGE name=extra2
+ALTER TABLE TABLE_A ADD COLUMN EXTRA2 INT NULL
+GO
+//// CHANGE name=extra3
+ALTER TABLE TABLE_A ADD COLUMN EXTRA3 INT NULL
+GO

obevo-db-impls/obevo-db-postgresql/src/test/resources/platforms/postgresql/example1/step1/schema1/table/TABLE_B.ddl

@@ -1,5 +1,5 @@
-//// CHANGE name=chng1
-CREATE TABLE TABLE_B (
-	B_ID    INT	NOT NULL,
-    PRIMARY KEY (B_ID)
-)
+//// CHANGE name=chng1
+CREATE TABLE TABLE_B (
+	B_ID    INT	NOT NULL,
+    PRIMARY KEY (B_ID)
+) WITHOUT OIDS

obevo-db-impls/obevo-db-postgresql/src/test/resources/platforms/postgresql/example1/step1/schema1/table/TABLE_SPECIAL_COL.ddl

@@ -3,5 +3,5 @@ CREATE TABLE TABLE_SPECIAL_COL (
     UUID_PK UUID NOT NULL,
     STR1 VARCHAR(30) NULL,
     PRIMARY KEY (UUID_PK)
-)
+) WITHOUT OIDS
 GO

obevo-db-impls/obevo-db-postgresql/src/test/resources/platforms/postgresql/example1/step1/schema1/table/TAB_WITH_SEQ.ddl

@@ -1,8 +1,8 @@
-//// METADATA excludePlatforms=REDSHIFT
-//// CHANGE name=chng1
-CREATE TABLE TAB_WITH_SEQ (
-	ID           	int4 NOT NULL DEFAULT nextval('MYSEQ1'::regclass),
-	FIELD1    INT	NULL,
-    PRIMARY KEY (ID)
-)
-GO
+//// METADATA excludePlatforms=REDSHIFT
+//// CHANGE name=chng1
+CREATE TABLE TAB_WITH_SEQ (
+	ID           	int4 NOT NULL DEFAULT nextval('MYSEQ1'::regclass),
+	FIELD1    INT	NULL,
+    PRIMARY KEY (ID)
+) WITHOUT OIDS
+GO

obevo-db-impls/obevo-db-postgresql/src/test/resources/platforms/postgresql/example1/step1/schema1/usertype/usertype0.hsql.sql

@@ -0,0 +1,3 @@
+//// METADATA includePlatforms="HSQL"
+CREATE TYPE usertype0 AS VARCHAR(1);
+GO

obevo-db-impls/obevo-db-postgresql/src/test/resources/platforms/postgresql/example1/step1/schema1/usertype/usertype0.sql

@@ -1,3 +1,4 @@
+//// METADATA excludePlatforms="HSQL"
 CREATE TYPE usertype0 AS ENUM (
     '1',
     '2',

obevo-db-impls/obevo-db-postgresql/src/test/resources/platforms/postgresql/example1/step1/system-config-inmem.xml

@@ -0,0 +1,27 @@
+<!--
+
+    Copyright 2017 Goldman Sachs.
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+-->
+<dbSystemConfig type="POSTGRESQL" sourceDirs="platforms/postgresql/example1/step1">
+    <schemas>
+        <schema name="schema1" />
+        <schema name="schema2" />
+    </schemas>
+    <environments>
+        <dbEnvironment name="unittestrefh2" cleanBuildAllowed="true" />
+        <dbEnvironment name="unittestrefhsql" cleanBuildAllowed="true" />
+    </environments>
+</dbSystemConfig>