CLDR-14213 Drop Derby Support: next phase (#2778)

* CLDR-14213 Drop Derby Support: next phase

- remove all infrastructure support for Derby
- does not remove Derby workarounds yet

* CLDR-14213 Drop Derby Support: skip in unit tests

tools/cldr-apps/pom.xml

@@ -130,18 +130,6 @@
 			<artifactId>commons-io</artifactId>
 		</dependency>
 
-		<!-- db -->
-		<dependency>
-			<groupId>org.apache.derby</groupId>
-			<artifactId>derby</artifactId>
-			<scope>test</scope>
-		</dependency>
-		<dependency>
-			<groupId>org.apache.derby</groupId>
-			<artifactId>derbytools</artifactId>
-			<scope>test</scope>
-		</dependency>
-
 		<dependency>
 			<groupId>mysql</groupId>
 			<artifactId>mysql-connector-java</artifactId>

tools/cldr-apps/src/main/java/org/unicode/cldr/web/DBUtils.java

@@ -63,6 +63,14 @@
  * Singleton utility class for simple(r) DB access.
  */
 public class DBUtils {
+    /**
+     * Throw an error if Derby is attempted to be used. Eventually this code will be removed entirely.
+     * @deprecated
+     */
+    @Deprecated
+    public static final void derbyNotSupported() {
+        throw new RuntimeException("Derby DB is not supported, see CLDR-14213");
+    }
     /**
      * Set property org.unicode.cldr.web.DBUtils.level=FINEST to trace SQL,
      * org.unicode.cldr.web.DBUtils.level=FINE to trace connection opens
@@ -73,22 +81,18 @@ public class DBUtils {
     private static final String JDBC_SURVEYTOOL = ("jdbc/SurveyTool");
     private static DataSource datasource = null;
     private String connectionUrl = null;
-    public static boolean db_Derby = false;
-    public static boolean db_Mysql = false;
+    public static final boolean db_Derby = false;
+    public static final boolean db_Mysql = true;
 
     /**
      * Return a string as to which SQL flavor is in use.
      *
-     * @return
+     * @return "MySql"
+     * @deprecated Derby is going away
      */
+    @Deprecated
     public static String getDBKind() {
-        if (db_Derby) {
-            return "Derby";
-        } else if (db_Mysql) {
-            return "MySql";
-        } else {
-            return "Unknown";
-        }
+        return "MySql";
     }
 
     public String getDBInfo() {
@@ -630,12 +634,13 @@ private void setupSqlForServerType() {
         SurveyLog.debug("setting up SQL for database type " + dbInfo);
         logger.info("setting up SQL for database type " + dbInfo);
         if (dbInfo.contains("Derby")) {
-            db_Derby = true;
-            logger.info("Note: Derby (embedded) mode. ** some features may not work as expected **");
-            db_UnicodeType = java.sql.Types.VARCHAR;
+            derbyNotSupported();
+            // db_Derby = true;
+            // logger.info("Note: Derby (embedded) mode. ** some features may not work as expected **");
+            // db_UnicodeType = java.sql.Types.VARCHAR;
         } else if (dbInfo.contains("MySQL")) {
             logger.info("Note: MySQL mode");
-            db_Mysql = true;
+            // db_Mysql = true;
             DB_SQL_IDENTITY = "AUTO_INCREMENT PRIMARY KEY";
             DB_SQL_BINCOLLATE = " COLLATE latin1_bin ";
             DB_SQL_MB4 = " CHARACTER SET utf8mb4 COLLATE utf8mb4_bin";

tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestAll.java

@@ -11,9 +11,6 @@
 import java.sql.SQLException;
 import java.util.logging.Logger;
 
-import javax.sql.DataSource;
-
-import org.apache.derby.jdbc.EmbeddedDataSource;
 import org.unicode.cldr.test.CheckCLDR;
 import org.unicode.cldr.util.CLDRConfig;
 import org.unicode.cldr.util.CLDRConfig.Environment;
@@ -28,10 +25,8 @@
 import org.unicode.cldr.web.DBUtils;
 import org.unicode.cldr.web.SurveyLog;
 
-import com.ibm.icu.dev.test.TestFmwk;
 import com.ibm.icu.dev.test.TestFmwk.TestGroup;
 import com.ibm.icu.dev.test.TestLog;
-import com.ibm.icu.dev.util.ElapsedTimer;
 import com.ibm.icu.text.Collator;
 import com.ibm.icu.text.RuleBasedCollator;
 
@@ -41,13 +36,15 @@
 public class TestAll extends TestGroup {
     private static final Logger logger = SurveyLog.forClass(TestAll.class);
 
-    private static final String DB_SUBDIR = "db";
     private static final String CLDR_TEST_JDBC = TestAll.class.getPackage().getName() + ".jdbcurl";
-    private static final String CLDR_TEST_KEEP_DB = TestAll.class.getPackage().getName() + ".KeepDb";
-    private static final String CLDR_TEST_DISK_DB = TestAll.class.getPackage().getName() + ".DiskDb";
     private static boolean sane = false;
     private static final boolean DEBUG = CldrUtility.getProperty("DEBUG", false);
 
+    /**
+     * True if the Test DB is setup properly.
+     */
+    public static final boolean HAVE_TEST_DB = !(CldrUtility.getProperty(CLDR_TEST_JDBC, "").isEmpty());
+
     /**
      * Verify some setup things
      */
@@ -86,9 +83,6 @@ private static void pleaseSet(String var, String err, String sugg) {
         throw new IllegalArgumentException("Error: variable " + var + " " + err + ", please set -D" + var + sugg);
     }
 
-    private static final String DERBY_DRIVER = "org.apache.derby.jdbc.EmbeddedDriver";
-    public static final String DERBY_PREFIX = "jdbc:derby:";
-
     public static void main(String[] args) {
         main(args, null);
         /* NOTREACHED */
@@ -112,18 +106,6 @@ public static String[] doResetDb(String[] args) {
             throw new InternalError(
                 "Error: the CLDRConfig Environment is not UNITTEST. Please set -DCLDR_ENVIRONMENT=UNITTESTS");
         }
-        if (CldrUtility.getProperty(CLDR_TEST_KEEP_DB, false)) {
-            if (DEBUG)
-                logger.warning("Keeping database..");
-        } else {
-            if (DEBUG)
-                logger.warning("Removing old test database..  set -D" + CLDR_TEST_KEEP_DB
-                    + "=true if you want to keep it..");
-            File f = getEmptyDir(DB_SUBDIR);
-            f.delete();
-            if (DEBUG)
-                logger.warning("Erased: " + f.getAbsolutePath() + " - now exists=" + f.isDirectory());
-        }
         return args;
     }
 
@@ -299,63 +281,16 @@ public static File emptyDir(File dir) {
         return dir;
     }
 
-    static void initDerby() {
-        long start = System.currentTimeMillis();
-        try {
-            Class.forName(DERBY_DRIVER);
-        } catch (ClassNotFoundException e) {
-            throw new RuntimeException(e);
-        } finally {
-            if (DEBUG)
-                System.err.println("Load " + DERBY_DRIVER + " - " + ElapsedTimer.elapsedTime(start));
-        }
-    }
-
     static void makeDataSource() {
         final String jdbcUrl = CldrUtility.getProperty(CLDR_TEST_JDBC, "");
         System.err.println(CLDR_TEST_JDBC +"="+jdbcUrl);
         if (!jdbcUrl.isEmpty()) {
             DBUtils.makeInstanceFrom(null, jdbcUrl);
-        } else if (CldrUtility.getProperty(CLDR_TEST_DISK_DB, false)) {
-            initDerby();
-            DBUtils.makeInstanceFrom(setupDerbyDataSource(getDir(DB_SUBDIR)), null);
         } else {
-            initDerby();
-            DBUtils.makeInstanceFrom(setupDerbyDataSource(null), null);
+            throw new RuntimeException("Error: set the -DCLDR_TEST_JDBC property to a valid MySQL URL.");
         }
     }
 
-    // from
-    // http://svn.apache.org/viewvc/commons/proper/dbcp/trunk/doc/ManualPoolingDataSourceExample.java?view=co
-
-    private static boolean isSetup = false;
-
-    /**
-     * null = inmemory.
-     *
-     * @param theDir
-     * @return
-     */
-    public static DataSource setupDerbyDataSource(File theDir) {
-        org.apache.derby.jdbc.EmbeddedDataSource ds = new EmbeddedDataSource();
-        if (theDir != null) {
-            ds.setDatabaseName(theDir.getAbsolutePath());
-        } else {
-            ds.setDatabaseName("memory:sttest");
-        }
-        if (isSetup == false || (theDir != null && !theDir.exists())) {
-            isSetup = true;
-            if (theDir != null) {
-                if (DEBUG)
-                    logger.warning("Using new: " + theDir.getAbsolutePath() + " baseDir = "
-                        + getBaseDir().getAbsolutePath());
-            }
-
-            ds.setCreateDatabase("create");
-        }
-        return ds;
-    }
-
     public static CLDRProgressIndicator getProgressIndicator(TestLog t) {
         final TestLog test = t;
         return new CLDRProgressIndicator() {
@@ -410,17 +345,11 @@ static public BufferedReader getUTF8Data(String name) throws java.io.IOException
         return FileReaders.openFile(TestAll.class, "data/" + name);
     }
 
-    /**
-     * Print a warning and skip if the test is derby-sensitive.
-     * Usage:   if(skipIfDerby(this)) return;
-     * @return true if skip
-     */
-    public static boolean skipIfDerby(TestFmwk t) {
-        if (DBUtils.getDBKind().equals("Derby")
-            && t.logKnownIssue("CLDR-14213", "May not work with Derby")) {
+    public static final boolean skipIfNoDb() {
+        if (!HAVE_TEST_DB) {
+            System.err.println("DB tests skipped because -D"+CLDR_TEST_JDBC+" was not set to a MySQL URL.");
             return true;
-        } else {
-            return false;
         }
+        return false;
     }
 }

tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestSTFactory.java

@@ -54,7 +54,7 @@ public static void main(String[] args) {
     }
 
     public void TestBasicFactory() throws SQLException {
-        if (TestAll.skipIfDerby(this)) return;
+        if (TestAll.skipIfNoDb()) return;
         CLDRLocale locale = CLDRLocale.getInstance("aa");
         STFactory fac = getFactory();
         CLDRFile mt = fac.make(locale, false);
@@ -65,7 +65,7 @@ public void TestBasicFactory() throws SQLException {
     }
 
     public void TestReadonlyLocales() throws SQLException {
-        if (TestAll.skipIfDerby(this)) return;
+        if (TestAll.skipIfNoDb()) return;
         STFactory fac = getFactory();
 
         verifyReadOnly(fac.make("root", false));
@@ -108,7 +108,7 @@ private String votedToString(boolean didVote) {
     }
 
     public void TestBasicVote() throws SQLException, IOException, InvalidXPathException, VoteNotAcceptedException {
-        if (TestAll.skipIfDerby(this)) return;
+        if (TestAll.skipIfNoDb()) return;
         STFactory fac = getFactory();
 
         final String somePath = "//ldml/localeDisplayNames/keys/key[@type=\"collation\"]";
@@ -247,7 +247,7 @@ public void TestBasicVote() throws SQLException, IOException, InvalidXPathExcept
     }
 
     public void TestDenyVote() throws SQLException, IOException {
-        if (TestAll.skipIfDerby(this)) return;
+        if (TestAll.skipIfNoDb()) return;
         STFactory fac = getFactory();
         final String somePath2 = "//ldml/localeDisplayNames/keys/key[@type=\"numbers\"]";
         //String originalValue2 = null;
@@ -293,7 +293,7 @@ public void TestDenyVote() throws SQLException, IOException {
     }
 
     public void TestSparseVote() throws SQLException, IOException, InvalidXPathException, SurveyException {
-        if (TestAll.skipIfDerby(this)) return;
+        if (TestAll.skipIfNoDb()) return;
         STFactory fac = getFactory();
 
         final String somePath2 = "//ldml/localeDisplayNames/keys/key[@type=\"calendar\"]";
@@ -374,18 +374,20 @@ public void TestSparseVote() throws SQLException, IOException, InvalidXPathExcep
     }
 
     public void TestVettingDataDriven() throws SQLException, IOException {
+        if (TestAll.skipIfNoDb()) return;
         runDataDrivenTest(TestSTFactory.class.getSimpleName()); // TestSTFactory.xml
     }
 
     public void TestUserRegistry() throws SQLException, IOException {
+        if (TestAll.skipIfNoDb()) return;
         runDataDrivenTest("TestUserRegistry");
     }
 
     /*
      * TODO: shorten this function, over 300 lines; use subroutines
      */
     private void runDataDrivenTest(final String fileBasename) throws SQLException, IOException {
-        if (TestAll.skipIfDerby(this)) return;
+        if (TestAll.skipIfNoDb()) return;
         final STFactory fac = getFactory();
         final File targDir = TestAll.getEmptyDir(TestSTFactory.class.getName() + "_output");
 
@@ -713,7 +715,7 @@ public UserRegistry.User getUserFromAttrs(final Map<String, String> attrs, Strin
     }
 
     public void TestVettingWithNonDistinguishing() throws SQLException, IOException, InvalidXPathException, SurveyException {
-        if (TestAll.skipIfDerby(this)) return;
+        if (TestAll.skipIfNoDb()) return;
         STFactory fac = getFactory();
 
         final String somePath2 = "//ldml/dates/calendars/calendar[@type=\"hebrew\"]/dateFormats/dateFormatLength[@type=\"full\"]/dateFormat[@type=\"standard\"]/pattern[@type=\"standard\"]";
@@ -894,4 +896,11 @@ public static STFactory createFactory() throws SQLException {
     static {
         noDtdPlease.put("DTD_DIR", CLDRPaths.COMMON_DIRECTORY + File.separator + "dtd" + File.separator);
     }
+
+    // This is just here to note a log known issue if you ran without setting the DB URL
+    public void TestLogDbIssue() {
+        if (TestAll.skipIfNoDb()) {
+            logKnownIssue("CLDR-14213", "Skipping DB tests because no MySQL URL has been set.");
+        }
+    }
 }

tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestUserRegistry.java

@@ -14,6 +14,7 @@ public static void main(String[] args) {
     }
 
     public TestUserRegistry() {
+        if (TestAll.skipIfNoDb()) return;
         TestAll.setupTestDb();
     }
 
@@ -24,7 +25,7 @@ public TestUserRegistry() {
      * org.unicode.cldr.unittest.TestUtilities.TestCanCreateOrSetLevelTo()
      */
     public void TestCanSetUserLevel() {
-        if (TestAll.skipIfDerby(this)) return;
+        if (TestAll.skipIfNoDb()) return;
         UserRegistry reg = CookieSession.sm.reg;
         int id = 2468;
 
@@ -81,7 +82,7 @@ public void TestOrgList() {
      * Test the ability of a user to vote in a locale
      */
     public void TestUserLocaleAuthorization() {
-        if (TestAll.skipIfDerby(this)) {
+        if (TestAll.skipIfNoDb()) {
             return;
         }
         UserRegistry reg = CookieSession.sm.reg;

tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestUserSettingsData.java

@@ -11,6 +11,7 @@
 public class TestUserSettingsData extends TestFmwk {
 
     private void setupDB() {
+        if (TestAll.skipIfNoDb()) return;
         long start = System.currentTimeMillis();
         TestAll.setupTestDb();
         logln("Set up test DB: " + ElapsedTimer.elapsedTime(start));
@@ -38,7 +39,7 @@ private String expect(String expectString, String key, String defaultValue, User
 
     public void TestSeparate() throws SQLException {
         {
-            if (TestAll.skipIfDerby(this)) return;
+            if (TestAll.skipIfNoDb()) return;
             setupDB();
             UserSettingsData d = getData();
 

tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestXPathTable.java

@@ -25,12 +25,14 @@ public static void main(String[] args) {
     }
 
     public TestXPathTable() {
+        if (TestAll.skipIfNoDb()) return;
         TestAll.setupTestDb();
     }
 
     public static final int TEST_COUNT = 200;
 
     public void TestPutGet() throws SQLException {
+        if (TestAll.skipIfNoDb()) return;
         logln("Testing " + TEST_COUNT + " xpaths");
         Connection conn = DBUtils.getInstance().getDBConnection();
         XPathTable xpt = XPathTable.createTable(conn);
@@ -116,6 +118,7 @@ public void TestRemoveDraftAltProposed() {
     }
 
     public void TestNonDistinguishing() throws SQLException {
+        if (TestAll.skipIfNoDb()) return;
         Connection conn = DBUtils.getInstance().getDBConnection();
         XPathTable xpt = XPathTable.createTable(conn);
         DBUtils.closeDBConnection(conn);