diff --git a/build.gradle b/build.gradle index 15fd1a9..c0a8cb4 100644 --- a/build.gradle +++ b/build.gradle @@ -233,17 +233,6 @@ dependencies { // needed for syslog4j implementation 'net.java.dev.jna:jna:3.4.0' - // ### Blobstore / Shock client and dependencies ### - // TODO BUILD can the code that uses these be removed? I don't think the NMS - // talks to Shock any more - - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/shock/shock-client-0.0.15.jar', - 'shock-client-0.0.15' - ) - implementation 'org.apache.httpcomponents:httpclient:4.3.1' - implementation 'org.apache.httpcomponents:httpmime:4.3.1' - // ### Test ### testImplementation 'junit:junit:4.12' diff --git a/deploy.cfg b/deploy.cfg index 0f3b299..490d1b4 100644 --- a/deploy.cfg +++ b/deploy.cfg @@ -55,10 +55,6 @@ method-spec-mongo-dbname = method_store_repo_db # method-spec-mongo-user = mongouser # method-spec-mongo-password = ***** method-spec-admin-users = kbaseadmin,kbaseadmin2 -# method-spec-shock-url = -# method-spec-shock-user = -# method-spec-shock-password = -# method-spec-shock-token = endpoint-host=https://ci.kbase.us endpoint-base=/services diff --git a/deployment/conf/.templates/deployment.cfg.templ b/deployment/conf/.templates/deployment.cfg.templ index 4641111..ee28e18 100644 --- a/deployment/conf/.templates/deployment.cfg.templ +++ b/deployment/conf/.templates/deployment.cfg.templ @@ -36,10 +36,6 @@ method-spec-mongo-dbname = {{ default .Env.method_spec_mongo_dbname "method_stor method-spec-mongo-user = {{ default .Env.method_spec_mongo_user "" }} method-spec-mongo-password = {{ default .Env.method_spec_mongo_password "" }} method-spec-admin-users = {{ default .Env.method_spec_admin_users "kbaseadmin,kbaseadmin2" }} -# method-spec-shock-url = {{ default .Env.method_spec_shock_url "" }} -# method-spec-shock-user = {{ default .Env.method_spec_shock_user "" }} -# method-spec-shock-password = {{ default .Env.method_spec_shock_password "" }} -# method-spec-shock-token = {{ default .Env.method_spec_shock_token "" }} endpoint-host={{ default .Env.endpoint_host "https://ci.kbase.us" }} endpoint-base={{ default .Env.endpoint_base "/services" }} diff --git a/src/us/kbase/narrativemethodstore/NarrativeMethodStoreServer.java b/src/us/kbase/narrativemethodstore/NarrativeMethodStoreServer.java index 5b07819..ebd34d2 100644 --- a/src/us/kbase/narrativemethodstore/NarrativeMethodStoreServer.java +++ b/src/us/kbase/narrativemethodstore/NarrativeMethodStoreServer.java @@ -20,8 +20,6 @@ import org.ini4j.Ini; -import us.kbase.auth.AuthConfig; -import us.kbase.auth.ConfigurableAuthService; import us.kbase.narrativemethodstore.db.NarrativeCategoriesIndex; import us.kbase.narrativemethodstore.db.ServiceUrlTemplateEvaluater; import us.kbase.narrativemethodstore.db.Validator; @@ -57,10 +55,6 @@ public class NarrativeMethodStoreServer extends JsonServerServlet { public static final String CFG_PROP_MONGO_PASSWORD = "method-spec-mongo-password"; public static final String CFG_PROP_MONGO_READONLY = "method-spec-mongo-readonly"; public static final String CFG_PROP_ADMIN_USERS = "method-spec-admin-users"; - public static final String CFG_PROP_SHOCK_URL = "method-spec-shock-url"; - public static final String CFG_PROP_SHOCK_USER = "method-spec-shock-user"; - public static final String CFG_PROP_SHOCK_PASSWORD = "method-spec-shock-password"; - public static final String CFG_PROP_SHOCK_TOKEN = "method-spec-shock-token"; public static final String CFG_PROP_ENDPOINT_BASE = "endpoint-base"; public static final String CFG_PROP_ENDPOINT_HOST = "endpoint-host"; public static final String CFG_PROP_DEFAULT_TAG = "method-spec-default-tag"; @@ -202,20 +196,6 @@ public static synchronized LocalGitDB getLocalGitDB() throws Exception { System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_MONGO_READONLY +" = " + mongoRO); System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_ADMIN_USERS +" = " + getAdminUsers()); List adminUsers = Arrays.asList(getAdminUsers().trim().split(Pattern.quote(","))); - String shockUrl = config().get(CFG_PROP_SHOCK_URL); - System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_SHOCK_URL +" = " + (shockUrl == null ? "" : shockUrl)); - String shockUser = config().get(CFG_PROP_SHOCK_USER); - if (shockUser != null && shockUser.trim().isEmpty()) { - shockUser = null; - } - String shockPwd = config().get(CFG_PROP_SHOCK_PASSWORD); - String shockTokenText = config().get(CFG_PROP_SHOCK_TOKEN); - if (shockTokenText != null && shockTokenText.trim().isEmpty()) { - shockTokenText = null; - } - System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_SHOCK_USER +" = " + (shockUser == null ? "" : shockUser)); - System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_SHOCK_PASSWORD +" = " + (shockPwd == null ? "" : "[*****]")); - System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_SHOCK_TOKEN +" = " + (shockTokenText == null ? "" : "[*****]")); String endpointHost = config().get(CFG_PROP_ENDPOINT_HOST); System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_ENDPOINT_HOST +" = " + (endpointHost == null ? "" : endpointHost)); String endpointBase = config().get(CFG_PROP_ENDPOINT_BASE); @@ -232,21 +212,10 @@ public static synchronized LocalGitDB getLocalGitDB() throws Exception { String authAllowInsecure = config().get(CFG_PROP_AUTH_INSECURE); System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_AUTH_INSECURE +" = " + (authAllowInsecure == null ? " ('false' will be used)" : authAllowInsecure)); - AuthToken shockToken = null; - if (shockUser != null || shockTokenText != null) { - ConfigurableAuthService authService = new ConfigurableAuthService( - new AuthConfig().withKBaseAuthServerURL(new URL(authServiceUrl)) - .withAllowInsecureURLs("true".equals(authAllowInsecure))); - if (shockTokenText == null) { - shockToken = authService.login(shockUser, shockPwd == null ? "" : shockPwd).getToken(); - } else { - shockToken = authService.validateToken(shockTokenText); - } - } localGitDB = new LocalGitDB(new URL(getGitRepo()), getGitBranch(), new File(getGitLocalDir()), getGitRefreshRate(), getCacheSize(), - new MongoDynamicRepoDB(getMongoHost(), getMongoDbname(), dbUser, dbPwd, adminUsers, mongoRO, - shockUrl == null ? null : new URL(shockUrl), shockToken), new File(getTempDir()), - new ServiceUrlTemplateEvaluater(endpointHost, endpointBase), RepoTag.valueOf(defaultTag)); + new MongoDynamicRepoDB(getMongoHost(), getMongoDbname(), dbUser, dbPwd, adminUsers, mongoRO), + new File(getTempDir()), + new ServiceUrlTemplateEvaluater(endpointHost, endpointBase), RepoTag.valueOf(defaultTag)); } return localGitDB; } diff --git a/src/us/kbase/narrativemethodstore/db/mongo/MongoDynamicRepoDB.java b/src/us/kbase/narrativemethodstore/db/mongo/MongoDynamicRepoDB.java index 4aa19d8..182f0ca 100644 --- a/src/us/kbase/narrativemethodstore/db/mongo/MongoDynamicRepoDB.java +++ b/src/us/kbase/narrativemethodstore/db/mongo/MongoDynamicRepoDB.java @@ -9,7 +9,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.URL; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -29,7 +28,6 @@ import com.mongodb.MongoCredential; import com.mongodb.ServerAddress; -import us.kbase.auth.AuthToken; import us.kbase.narrativemethodstore.db.DynamicRepoDB; import us.kbase.narrativemethodstore.db.FileId; import us.kbase.narrativemethodstore.db.FilePointer; @@ -38,18 +36,11 @@ import us.kbase.narrativemethodstore.db.JsonRepoProvider.RepoData; import us.kbase.narrativemethodstore.db.github.RepoTag; import us.kbase.narrativemethodstore.exceptions.NarrativeMethodStoreException; -import us.kbase.shock.client.BasicShockClient; -import us.kbase.shock.client.ShockNodeId; -import us.kbase.shock.client.exceptions.InvalidShockUrlException; -import us.kbase.shock.client.exceptions.ShockHttpException; public class MongoDynamicRepoDB implements DynamicRepoDB { private final DB db; private final Set globalAdmins; private final boolean isReadOnly; - private final URL shockUrl; - private final AuthToken serviceToken; - public static final long MAX_MONGO_FILE_LENGTH = 1024 * 1024; //////////////////////////////////////////////////////////////////// private static final String TABLE_REPO_INFO = "repo_info"; private static final String FIELD_RI_MODULE_NAME = "module_name"; @@ -75,12 +66,15 @@ public class MongoDynamicRepoDB implements DynamicRepoDB { private static final String FIELD_RF_HEX_DATA = "hex_data"; private static final String FIELD_RF_SHOCK_NODE_ID = "shock_node_id"; - public MongoDynamicRepoDB(String host, String database, String dbUser, String dbPwd, - List globalAdminUserIds, boolean isReadOnly, URL shockUrl, - AuthToken serviceToken) throws NarrativeMethodStoreException { + public MongoDynamicRepoDB( + final String host, + final String database, + final String dbUser, + final String dbPwd, + final List globalAdminUserIds, + final boolean isReadOnly) + throws NarrativeMethodStoreException { this.isReadOnly = isReadOnly; - this.shockUrl = shockUrl; - this.serviceToken = serviceToken; try { db = getDB(host, database, dbUser, dbPwd); if (!isReadOnly) @@ -158,7 +152,7 @@ private void throwRepoWasntRegistered(String repoModuleName) private void throwChangeOperation() throws NarrativeMethodStoreException { throw new NarrativeMethodStoreException("Change operation couldn't be performed in " + - "read-only mode"); + "read-only mode"); } @Override @@ -512,16 +506,9 @@ public FileId saveFile(String moduleName, FileProvider file) String shockNodeId = null; is = file.openStream(); try { - if (length <= MAX_MONGO_FILE_LENGTH || shockUrl == null) { - hexData = MongoUtils.streamToHex(is); - } else { - try { - BasicShockClient cl = new BasicShockClient(shockUrl, serviceToken); - shockNodeId = cl.addNode(is, fileName, "JSON").getId().getId(); - } catch (Exception ex) { - throw new NarrativeMethodStoreException(ex); - } - } + // note that mongo can take ~16MB of data this way tops or it'll throw an error + // it's been running in production for years this way so I guess it works...? + hexData = MongoUtils.streamToHex(is); } finally { try { is.close(); @@ -585,21 +572,9 @@ private void loadFile(Map obj, OutputStream target) throws NarrativeMethodStoreException { try { String hexData = (String)obj.get(FIELD_RF_HEX_DATA); - if (hexData == null) { - String shockNodeId = (String)obj.get(FIELD_RF_SHOCK_NODE_ID); - BasicShockClient cl = new BasicShockClient(shockUrl, serviceToken); - cl.getFile(new ShockNodeId(shockNodeId), target); - } else { - target.write(MongoUtils.hexToBytes(hexData)); - } + target.write(MongoUtils.hexToBytes(hexData)); } catch (IOException ex) { throw new NarrativeMethodStoreException(ex); - } catch (InvalidShockUrlException ex) { - throw new NarrativeMethodStoreException(ex); - } catch (ShockHttpException ex) { - throw new NarrativeMethodStoreException(ex); - } catch (IllegalArgumentException ex) { - throw new NarrativeMethodStoreException(ex); } } diff --git a/src/us/kbase/narrativemethodstore/db/mongo/test/MongoDynamicRepoDB2Test.java b/src/us/kbase/narrativemethodstore/db/mongo/test/MongoDynamicRepoDB2Test.java index 73af02b..3e1760c 100644 --- a/src/us/kbase/narrativemethodstore/db/mongo/test/MongoDynamicRepoDB2Test.java +++ b/src/us/kbase/narrativemethodstore/db/mongo/test/MongoDynamicRepoDB2Test.java @@ -303,9 +303,7 @@ private MongoDynamicRepoDB getDB(final List admins) null, null, admins, - false, - null, - null); + false); } } diff --git a/src/us/kbase/narrativemethodstore/db/mongo/test/MongoDynamicRepoDBTest.java b/src/us/kbase/narrativemethodstore/db/mongo/test/MongoDynamicRepoDBTest.java index 93face6..a47e31f 100644 --- a/src/us/kbase/narrativemethodstore/db/mongo/test/MongoDynamicRepoDBTest.java +++ b/src/us/kbase/narrativemethodstore/db/mongo/test/MongoDynamicRepoDBTest.java @@ -1,7 +1,5 @@ package us.kbase.narrativemethodstore.db.mongo.test; -import static us.kbase.narrativemethodstore.db.mongo.MongoUtils.toMap; - import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; @@ -25,12 +23,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.mongodb.DB; -import com.mongodb.DBCollection; -import com.mongodb.DBCursor; -import com.mongodb.DBObject; import com.mongodb.MongoClient; -import us.kbase.auth.AuthToken; import us.kbase.common.service.UObject; import us.kbase.narrativemethodstore.MethodBriefInfo; import us.kbase.narrativemethodstore.MethodParameter; @@ -50,8 +44,6 @@ import us.kbase.narrativemethodstore.exceptions.NarrativeMethodStoreException; import us.kbase.narrativemethodstore.util.FileUtils; import us.kbase.narrativemethodstore.util.TextUtils; -import us.kbase.shock.client.BasicShockClient; -import us.kbase.shock.client.ShockNodeId; // It'd be nice to know why this is ignored @Ignore @@ -59,9 +51,6 @@ public class MongoDynamicRepoDBTest { private static final String dbName = "test_repo_registry_mongo"; private static final MongoDBHelper dbHelper = new MongoDBHelper("registry"); - private static URL shockUrl = null; - private static AuthToken shockToken = null; - @BeforeClass public static void beforeClass() throws Exception { dbHelper.startup(System.getProperty("mongod.path")); @@ -90,7 +79,7 @@ private void testRepo(boolean localFiles) throws Exception { String host = "localhost:" + dbHelper.getMongoPort(); MongoDynamicRepoDB db = new MongoDynamicRepoDB(host, dbName, null, null, - Arrays.asList(globalAdmin), false, shockUrl, shockToken); + Arrays.asList(globalAdmin), false); Assert.assertEquals(0, db.listRepoModuleNames().size()); RepoProvider pvd = localFiles ? new FileRepoProvider(new File(localPath)) : new GitHubRepoProvider(new URL(gitUrl), null, dbHelper.getWorkDir()); @@ -220,7 +209,7 @@ public void testPy() throws Exception { String globalAdmin = "admin"; String host = "localhost:" + dbHelper.getMongoPort(); MongoDynamicRepoDB db = new MongoDynamicRepoDB(host, dbName, null, null, - Arrays.asList(globalAdmin), false, shockUrl, shockToken); + Arrays.asList(globalAdmin), false); Assert.assertEquals(0, db.listRepoModuleNames().size()); RepoProvider pvd = new FileRepoProvider(repoDir); db.registerRepo(userId, pvd); @@ -286,24 +275,6 @@ public void cleanup() throws Exception { String host = "localhost:" + dbHelper.getMongoPort(); final MongoClient mc = new MongoClient(host); final DB db = mc.getDB(dbName); - if (shockUrl != null) { - try { - final DBCollection files = db.getCollection("repo_files"); - final DBCursor it = files.find(); - for (final DBObject dbo: it) { - Map obj = toMap(dbo); - String fileName = (String)obj.get("file_name"); - String shockNodeId = (String)obj.get("shock_node_id"); - if (shockNodeId != null) { - System.out.println("Deleting shock node for " + fileName); - BasicShockClient cl = new BasicShockClient(shockUrl, shockToken); - cl.deleteNode(new ShockNodeId(shockNodeId)); - } - } - } catch (Exception ex) { - ex.printStackTrace(); - } - } db.dropDatabase(); mc.close(); }