From 23f49aeae99c7b4a05deffb8f218c5568240583f Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 13 Mar 2023 18:02:20 -0500 Subject: [PATCH] CLDR-14213 Drop Derby Support: next phase (#2778) * CLDR-14213 Drop Derby Support: next phase - remove all infrastructure support for Derby - does not remove Derby workarounds yet * CLDR-14213 Drop Derby Support: skip in unit tests --- tools/cldr-apps/pom.xml | 12 --- .../java/org/unicode/cldr/web/DBUtils.java | 33 ++++--- .../unicode/cldr/unittest/web/TestAll.java | 91 ++----------------- .../cldr/unittest/web/TestSTFactory.java | 23 +++-- .../cldr/unittest/web/TestUserRegistry.java | 5 +- .../unittest/web/TestUserSettingsData.java | 3 +- .../cldr/unittest/web/TestXPathTable.java | 3 + tools/pom.xml | 12 --- 8 files changed, 53 insertions(+), 129 deletions(-) diff --git a/tools/cldr-apps/pom.xml b/tools/cldr-apps/pom.xml index 60e69cd87ff..8236e1cb7b0 100644 --- a/tools/cldr-apps/pom.xml +++ b/tools/cldr-apps/pom.xml @@ -130,18 +130,6 @@ commons-io - - - org.apache.derby - derby - test - - - org.apache.derby - derbytools - test - - mysql mysql-connector-java diff --git a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DBUtils.java b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DBUtils.java index 044442f4c13..10de006419a 100644 --- a/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DBUtils.java +++ b/tools/cldr-apps/src/main/java/org/unicode/cldr/web/DBUtils.java @@ -63,6 +63,14 @@ * Singleton utility class for simple(r) DB access. */ public class DBUtils { + /** + * Throw an error if Derby is attempted to be used. Eventually this code will be removed entirely. + * @deprecated + */ + @Deprecated + public static final void derbyNotSupported() { + throw new RuntimeException("Derby DB is not supported, see CLDR-14213"); + } /** * Set property org.unicode.cldr.web.DBUtils.level=FINEST to trace SQL, * org.unicode.cldr.web.DBUtils.level=FINE to trace connection opens @@ -73,22 +81,18 @@ public class DBUtils { private static final String JDBC_SURVEYTOOL = ("jdbc/SurveyTool"); private static DataSource datasource = null; private String connectionUrl = null; - public static boolean db_Derby = false; - public static boolean db_Mysql = false; + public static final boolean db_Derby = false; + public static final boolean db_Mysql = true; /** * Return a string as to which SQL flavor is in use. * - * @return + * @return "MySql" + * @deprecated Derby is going away */ + @Deprecated public static String getDBKind() { - if (db_Derby) { - return "Derby"; - } else if (db_Mysql) { - return "MySql"; - } else { - return "Unknown"; - } + return "MySql"; } public String getDBInfo() { @@ -630,12 +634,13 @@ private void setupSqlForServerType() { SurveyLog.debug("setting up SQL for database type " + dbInfo); logger.info("setting up SQL for database type " + dbInfo); if (dbInfo.contains("Derby")) { - db_Derby = true; - logger.info("Note: Derby (embedded) mode. ** some features may not work as expected **"); - db_UnicodeType = java.sql.Types.VARCHAR; + derbyNotSupported(); + // db_Derby = true; + // logger.info("Note: Derby (embedded) mode. ** some features may not work as expected **"); + // db_UnicodeType = java.sql.Types.VARCHAR; } else if (dbInfo.contains("MySQL")) { logger.info("Note: MySQL mode"); - db_Mysql = true; + // db_Mysql = true; DB_SQL_IDENTITY = "AUTO_INCREMENT PRIMARY KEY"; DB_SQL_BINCOLLATE = " COLLATE latin1_bin "; DB_SQL_MB4 = " CHARACTER SET utf8mb4 COLLATE utf8mb4_bin"; diff --git a/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestAll.java b/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestAll.java index 9f69da37441..606e12e8a44 100644 --- a/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestAll.java +++ b/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestAll.java @@ -11,9 +11,6 @@ import java.sql.SQLException; import java.util.logging.Logger; -import javax.sql.DataSource; - -import org.apache.derby.jdbc.EmbeddedDataSource; import org.unicode.cldr.test.CheckCLDR; import org.unicode.cldr.util.CLDRConfig; import org.unicode.cldr.util.CLDRConfig.Environment; @@ -28,10 +25,8 @@ import org.unicode.cldr.web.DBUtils; import org.unicode.cldr.web.SurveyLog; -import com.ibm.icu.dev.test.TestFmwk; import com.ibm.icu.dev.test.TestFmwk.TestGroup; import com.ibm.icu.dev.test.TestLog; -import com.ibm.icu.dev.util.ElapsedTimer; import com.ibm.icu.text.Collator; import com.ibm.icu.text.RuleBasedCollator; @@ -41,13 +36,15 @@ public class TestAll extends TestGroup { private static final Logger logger = SurveyLog.forClass(TestAll.class); - private static final String DB_SUBDIR = "db"; private static final String CLDR_TEST_JDBC = TestAll.class.getPackage().getName() + ".jdbcurl"; - private static final String CLDR_TEST_KEEP_DB = TestAll.class.getPackage().getName() + ".KeepDb"; - private static final String CLDR_TEST_DISK_DB = TestAll.class.getPackage().getName() + ".DiskDb"; private static boolean sane = false; private static final boolean DEBUG = CldrUtility.getProperty("DEBUG", false); + /** + * True if the Test DB is setup properly. + */ + public static final boolean HAVE_TEST_DB = !(CldrUtility.getProperty(CLDR_TEST_JDBC, "").isEmpty()); + /** * Verify some setup things */ @@ -86,9 +83,6 @@ private static void pleaseSet(String var, String err, String sugg) { throw new IllegalArgumentException("Error: variable " + var + " " + err + ", please set -D" + var + sugg); } - private static final String DERBY_DRIVER = "org.apache.derby.jdbc.EmbeddedDriver"; - public static final String DERBY_PREFIX = "jdbc:derby:"; - public static void main(String[] args) { main(args, null); /* NOTREACHED */ @@ -112,18 +106,6 @@ public static String[] doResetDb(String[] args) { throw new InternalError( "Error: the CLDRConfig Environment is not UNITTEST. Please set -DCLDR_ENVIRONMENT=UNITTESTS"); } - if (CldrUtility.getProperty(CLDR_TEST_KEEP_DB, false)) { - if (DEBUG) - logger.warning("Keeping database.."); - } else { - if (DEBUG) - logger.warning("Removing old test database.. set -D" + CLDR_TEST_KEEP_DB - + "=true if you want to keep it.."); - File f = getEmptyDir(DB_SUBDIR); - f.delete(); - if (DEBUG) - logger.warning("Erased: " + f.getAbsolutePath() + " - now exists=" + f.isDirectory()); - } return args; } @@ -299,63 +281,16 @@ public static File emptyDir(File dir) { return dir; } - static void initDerby() { - long start = System.currentTimeMillis(); - try { - Class.forName(DERBY_DRIVER); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } finally { - if (DEBUG) - System.err.println("Load " + DERBY_DRIVER + " - " + ElapsedTimer.elapsedTime(start)); - } - } - static void makeDataSource() { final String jdbcUrl = CldrUtility.getProperty(CLDR_TEST_JDBC, ""); System.err.println(CLDR_TEST_JDBC +"="+jdbcUrl); if (!jdbcUrl.isEmpty()) { DBUtils.makeInstanceFrom(null, jdbcUrl); - } else if (CldrUtility.getProperty(CLDR_TEST_DISK_DB, false)) { - initDerby(); - DBUtils.makeInstanceFrom(setupDerbyDataSource(getDir(DB_SUBDIR)), null); } else { - initDerby(); - DBUtils.makeInstanceFrom(setupDerbyDataSource(null), null); + throw new RuntimeException("Error: set the -DCLDR_TEST_JDBC property to a valid MySQL URL."); } } - // from - // http://svn.apache.org/viewvc/commons/proper/dbcp/trunk/doc/ManualPoolingDataSourceExample.java?view=co - - private static boolean isSetup = false; - - /** - * null = inmemory. - * - * @param theDir - * @return - */ - public static DataSource setupDerbyDataSource(File theDir) { - org.apache.derby.jdbc.EmbeddedDataSource ds = new EmbeddedDataSource(); - if (theDir != null) { - ds.setDatabaseName(theDir.getAbsolutePath()); - } else { - ds.setDatabaseName("memory:sttest"); - } - if (isSetup == false || (theDir != null && !theDir.exists())) { - isSetup = true; - if (theDir != null) { - if (DEBUG) - logger.warning("Using new: " + theDir.getAbsolutePath() + " baseDir = " - + getBaseDir().getAbsolutePath()); - } - - ds.setCreateDatabase("create"); - } - return ds; - } - public static CLDRProgressIndicator getProgressIndicator(TestLog t) { final TestLog test = t; return new CLDRProgressIndicator() { @@ -410,17 +345,11 @@ static public BufferedReader getUTF8Data(String name) throws java.io.IOException return FileReaders.openFile(TestAll.class, "data/" + name); } - /** - * Print a warning and skip if the test is derby-sensitive. - * Usage: if(skipIfDerby(this)) return; - * @return true if skip - */ - public static boolean skipIfDerby(TestFmwk t) { - if (DBUtils.getDBKind().equals("Derby") - && t.logKnownIssue("CLDR-14213", "May not work with Derby")) { + public static final boolean skipIfNoDb() { + if (!HAVE_TEST_DB) { + System.err.println("DB tests skipped because -D"+CLDR_TEST_JDBC+" was not set to a MySQL URL."); return true; - } else { - return false; } + return false; } } diff --git a/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestSTFactory.java b/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestSTFactory.java index 347971c3153..bba40ae05f6 100644 --- a/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestSTFactory.java +++ b/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestSTFactory.java @@ -54,7 +54,7 @@ public static void main(String[] args) { } public void TestBasicFactory() throws SQLException { - if (TestAll.skipIfDerby(this)) return; + if (TestAll.skipIfNoDb()) return; CLDRLocale locale = CLDRLocale.getInstance("aa"); STFactory fac = getFactory(); CLDRFile mt = fac.make(locale, false); @@ -65,7 +65,7 @@ public void TestBasicFactory() throws SQLException { } public void TestReadonlyLocales() throws SQLException { - if (TestAll.skipIfDerby(this)) return; + if (TestAll.skipIfNoDb()) return; STFactory fac = getFactory(); verifyReadOnly(fac.make("root", false)); @@ -108,7 +108,7 @@ private String votedToString(boolean didVote) { } public void TestBasicVote() throws SQLException, IOException, InvalidXPathException, VoteNotAcceptedException { - if (TestAll.skipIfDerby(this)) return; + if (TestAll.skipIfNoDb()) return; STFactory fac = getFactory(); final String somePath = "//ldml/localeDisplayNames/keys/key[@type=\"collation\"]"; @@ -247,7 +247,7 @@ public void TestBasicVote() throws SQLException, IOException, InvalidXPathExcept } public void TestDenyVote() throws SQLException, IOException { - if (TestAll.skipIfDerby(this)) return; + if (TestAll.skipIfNoDb()) return; STFactory fac = getFactory(); final String somePath2 = "//ldml/localeDisplayNames/keys/key[@type=\"numbers\"]"; //String originalValue2 = null; @@ -293,7 +293,7 @@ public void TestDenyVote() throws SQLException, IOException { } public void TestSparseVote() throws SQLException, IOException, InvalidXPathException, SurveyException { - if (TestAll.skipIfDerby(this)) return; + if (TestAll.skipIfNoDb()) return; STFactory fac = getFactory(); final String somePath2 = "//ldml/localeDisplayNames/keys/key[@type=\"calendar\"]"; @@ -374,10 +374,12 @@ public void TestSparseVote() throws SQLException, IOException, InvalidXPathExcep } public void TestVettingDataDriven() throws SQLException, IOException { + if (TestAll.skipIfNoDb()) return; runDataDrivenTest(TestSTFactory.class.getSimpleName()); // TestSTFactory.xml } public void TestUserRegistry() throws SQLException, IOException { + if (TestAll.skipIfNoDb()) return; runDataDrivenTest("TestUserRegistry"); } @@ -385,7 +387,7 @@ public void TestUserRegistry() throws SQLException, IOException { * TODO: shorten this function, over 300 lines; use subroutines */ private void runDataDrivenTest(final String fileBasename) throws SQLException, IOException { - if (TestAll.skipIfDerby(this)) return; + if (TestAll.skipIfNoDb()) return; final STFactory fac = getFactory(); final File targDir = TestAll.getEmptyDir(TestSTFactory.class.getName() + "_output"); @@ -713,7 +715,7 @@ public UserRegistry.User getUserFromAttrs(final Map attrs, Strin } public void TestVettingWithNonDistinguishing() throws SQLException, IOException, InvalidXPathException, SurveyException { - if (TestAll.skipIfDerby(this)) return; + if (TestAll.skipIfNoDb()) return; STFactory fac = getFactory(); final String somePath2 = "//ldml/dates/calendars/calendar[@type=\"hebrew\"]/dateFormats/dateFormatLength[@type=\"full\"]/dateFormat[@type=\"standard\"]/pattern[@type=\"standard\"]"; @@ -894,4 +896,11 @@ public static STFactory createFactory() throws SQLException { static { noDtdPlease.put("DTD_DIR", CLDRPaths.COMMON_DIRECTORY + File.separator + "dtd" + File.separator); } + + // This is just here to note a log known issue if you ran without setting the DB URL + public void TestLogDbIssue() { + if (TestAll.skipIfNoDb()) { + logKnownIssue("CLDR-14213", "Skipping DB tests because no MySQL URL has been set."); + } + } } diff --git a/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestUserRegistry.java b/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestUserRegistry.java index c22f13cb393..1e1ecec673b 100644 --- a/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestUserRegistry.java +++ b/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestUserRegistry.java @@ -14,6 +14,7 @@ public static void main(String[] args) { } public TestUserRegistry() { + if (TestAll.skipIfNoDb()) return; TestAll.setupTestDb(); } @@ -24,7 +25,7 @@ public TestUserRegistry() { * org.unicode.cldr.unittest.TestUtilities.TestCanCreateOrSetLevelTo() */ public void TestCanSetUserLevel() { - if (TestAll.skipIfDerby(this)) return; + if (TestAll.skipIfNoDb()) return; UserRegistry reg = CookieSession.sm.reg; int id = 2468; @@ -81,7 +82,7 @@ public void TestOrgList() { * Test the ability of a user to vote in a locale */ public void TestUserLocaleAuthorization() { - if (TestAll.skipIfDerby(this)) { + if (TestAll.skipIfNoDb()) { return; } UserRegistry reg = CookieSession.sm.reg; diff --git a/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestUserSettingsData.java b/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestUserSettingsData.java index 9dfd246dadd..0f7cea1fa54 100644 --- a/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestUserSettingsData.java +++ b/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestUserSettingsData.java @@ -11,6 +11,7 @@ public class TestUserSettingsData extends TestFmwk { private void setupDB() { + if (TestAll.skipIfNoDb()) return; long start = System.currentTimeMillis(); TestAll.setupTestDb(); logln("Set up test DB: " + ElapsedTimer.elapsedTime(start)); @@ -38,7 +39,7 @@ private String expect(String expectString, String key, String defaultValue, User public void TestSeparate() throws SQLException { { - if (TestAll.skipIfDerby(this)) return; + if (TestAll.skipIfNoDb()) return; setupDB(); UserSettingsData d = getData(); diff --git a/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestXPathTable.java b/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestXPathTable.java index cd1a586f6d7..b64ec7e952e 100644 --- a/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestXPathTable.java +++ b/tools/cldr-apps/src/test/java/org/unicode/cldr/unittest/web/TestXPathTable.java @@ -25,12 +25,14 @@ public static void main(String[] args) { } public TestXPathTable() { + if (TestAll.skipIfNoDb()) return; TestAll.setupTestDb(); } public static final int TEST_COUNT = 200; public void TestPutGet() throws SQLException { + if (TestAll.skipIfNoDb()) return; logln("Testing " + TEST_COUNT + " xpaths"); Connection conn = DBUtils.getInstance().getDBConnection(); XPathTable xpt = XPathTable.createTable(conn); @@ -116,6 +118,7 @@ public void TestRemoveDraftAltProposed() { } public void TestNonDistinguishing() throws SQLException { + if (TestAll.skipIfNoDb()) return; Connection conn = DBUtils.getInstance().getDBConnection(); XPathTable xpt = XPathTable.createTable(conn); DBUtils.closeDBConnection(conn); diff --git a/tools/pom.xml b/tools/pom.xml index 18bfacfa71b..19070cadec0 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -29,7 +29,6 @@ 5.0.3 8.0.30 - 10.15.2.0 @@ -139,17 +138,6 @@ - - org.apache.derby - derby - ${derby.version} - - - org.apache.derby - derbytools - ${derby.version} - - mysql mysql-connector-java