Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove shock client #96

Merged
merged 1 commit into from
Apr 30, 2024
Merged

Remove shock client #96

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 0 additions & 11 deletions build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,17 +233,6 @@ dependencies {
// needed for syslog4j
implementation 'net.java.dev.jna:jna:3.4.0'

// ### Blobstore / Shock client and dependencies ###
// TODO BUILD can the code that uses these be removed? I don't think the NMS
// talks to Shock any more

implementation fromURL(
'https://github.com/kbase/jars/raw/master/lib/jars/kbase/shock/shock-client-0.0.15.jar',
'shock-client-0.0.15'
)
implementation 'org.apache.httpcomponents:httpclient:4.3.1'
implementation 'org.apache.httpcomponents:httpmime:4.3.1'

// ### Test ###

testImplementation 'junit:junit:4.12'
Expand Down
4 changes: 0 additions & 4 deletions deploy.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,6 @@ method-spec-mongo-dbname = method_store_repo_db
# method-spec-mongo-user = mongouser
# method-spec-mongo-password = *****
method-spec-admin-users = kbaseadmin,kbaseadmin2
# method-spec-shock-url =
# method-spec-shock-user =
# method-spec-shock-password =
# method-spec-shock-token =
endpoint-host=https://ci.kbase.us
endpoint-base=/services

Expand Down
4 changes: 0 additions & 4 deletions deployment/conf/.templates/deployment.cfg.templ
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,6 @@ method-spec-mongo-dbname = {{ default .Env.method_spec_mongo_dbname "method_stor
method-spec-mongo-user = {{ default .Env.method_spec_mongo_user "" }}
method-spec-mongo-password = {{ default .Env.method_spec_mongo_password "" }}
method-spec-admin-users = {{ default .Env.method_spec_admin_users "kbaseadmin,kbaseadmin2" }}
# method-spec-shock-url = {{ default .Env.method_spec_shock_url "" }}
# method-spec-shock-user = {{ default .Env.method_spec_shock_user "" }}
# method-spec-shock-password = {{ default .Env.method_spec_shock_password "" }}
# method-spec-shock-token = {{ default .Env.method_spec_shock_token "" }}
endpoint-host={{ default .Env.endpoint_host "https://ci.kbase.us" }}
endpoint-base={{ default .Env.endpoint_base "/services" }}

Expand Down
37 changes: 3 additions & 34 deletions src/us/kbase/narrativemethodstore/NarrativeMethodStoreServer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@

import org.ini4j.Ini;

import us.kbase.auth.AuthConfig;
import us.kbase.auth.ConfigurableAuthService;
import us.kbase.narrativemethodstore.db.NarrativeCategoriesIndex;
import us.kbase.narrativemethodstore.db.ServiceUrlTemplateEvaluater;
import us.kbase.narrativemethodstore.db.Validator;
Expand Down Expand Up @@ -57,10 +55,6 @@ public class NarrativeMethodStoreServer extends JsonServerServlet {
public static final String CFG_PROP_MONGO_PASSWORD = "method-spec-mongo-password";
public static final String CFG_PROP_MONGO_READONLY = "method-spec-mongo-readonly";
public static final String CFG_PROP_ADMIN_USERS = "method-spec-admin-users";
public static final String CFG_PROP_SHOCK_URL = "method-spec-shock-url";
public static final String CFG_PROP_SHOCK_USER = "method-spec-shock-user";
public static final String CFG_PROP_SHOCK_PASSWORD = "method-spec-shock-password";
public static final String CFG_PROP_SHOCK_TOKEN = "method-spec-shock-token";
public static final String CFG_PROP_ENDPOINT_BASE = "endpoint-base";
public static final String CFG_PROP_ENDPOINT_HOST = "endpoint-host";
public static final String CFG_PROP_DEFAULT_TAG = "method-spec-default-tag";
Expand Down Expand Up @@ -202,20 +196,6 @@ public static synchronized LocalGitDB getLocalGitDB() throws Exception {
System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_MONGO_READONLY +" = " + mongoRO);
System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_ADMIN_USERS +" = " + getAdminUsers());
List<String> adminUsers = Arrays.asList(getAdminUsers().trim().split(Pattern.quote(",")));
String shockUrl = config().get(CFG_PROP_SHOCK_URL);
System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_SHOCK_URL +" = " + (shockUrl == null ? "<not-set>" : shockUrl));
String shockUser = config().get(CFG_PROP_SHOCK_USER);
if (shockUser != null && shockUser.trim().isEmpty()) {
shockUser = null;
}
String shockPwd = config().get(CFG_PROP_SHOCK_PASSWORD);
String shockTokenText = config().get(CFG_PROP_SHOCK_TOKEN);
if (shockTokenText != null && shockTokenText.trim().isEmpty()) {
shockTokenText = null;
}
System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_SHOCK_USER +" = " + (shockUser == null ? "<not-set>" : shockUser));
System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_SHOCK_PASSWORD +" = " + (shockPwd == null ? "<not-set>" : "[*****]"));
System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_SHOCK_TOKEN +" = " + (shockTokenText == null ? "<not-set>" : "[*****]"));
String endpointHost = config().get(CFG_PROP_ENDPOINT_HOST);
System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_ENDPOINT_HOST +" = " + (endpointHost == null ? "<not-set>" : endpointHost));
String endpointBase = config().get(CFG_PROP_ENDPOINT_BASE);
Expand All @@ -232,21 +212,10 @@ public static synchronized LocalGitDB getLocalGitDB() throws Exception {
String authAllowInsecure = config().get(CFG_PROP_AUTH_INSECURE);
System.out.println(NarrativeMethodStoreServer.class.getName() + ": " + CFG_PROP_AUTH_INSECURE +" = " +
(authAllowInsecure == null ? "<not-set> ('false' will be used)" : authAllowInsecure));
AuthToken shockToken = null;
if (shockUser != null || shockTokenText != null) {
ConfigurableAuthService authService = new ConfigurableAuthService(
new AuthConfig().withKBaseAuthServerURL(new URL(authServiceUrl))
.withAllowInsecureURLs("true".equals(authAllowInsecure)));
if (shockTokenText == null) {
shockToken = authService.login(shockUser, shockPwd == null ? "" : shockPwd).getToken();
} else {
shockToken = authService.validateToken(shockTokenText);
}
}
localGitDB = new LocalGitDB(new URL(getGitRepo()), getGitBranch(), new File(getGitLocalDir()), getGitRefreshRate(), getCacheSize(),
new MongoDynamicRepoDB(getMongoHost(), getMongoDbname(), dbUser, dbPwd, adminUsers, mongoRO,
shockUrl == null ? null : new URL(shockUrl), shockToken), new File(getTempDir()),
new ServiceUrlTemplateEvaluater(endpointHost, endpointBase), RepoTag.valueOf(defaultTag));
new MongoDynamicRepoDB(getMongoHost(), getMongoDbname(), dbUser, dbPwd, adminUsers, mongoRO),
new File(getTempDir()),
new ServiceUrlTemplateEvaluater(endpointHost, endpointBase), RepoTag.valueOf(defaultTag));
}
return localGitDB;
}
Expand Down
51 changes: 13 additions & 38 deletions src/us/kbase/narrativemethodstore/db/mongo/MongoDynamicRepoDB.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
Expand All @@ -29,7 +28,6 @@
import com.mongodb.MongoCredential;
import com.mongodb.ServerAddress;

import us.kbase.auth.AuthToken;
import us.kbase.narrativemethodstore.db.DynamicRepoDB;
import us.kbase.narrativemethodstore.db.FileId;
import us.kbase.narrativemethodstore.db.FilePointer;
Expand All @@ -38,18 +36,11 @@
import us.kbase.narrativemethodstore.db.JsonRepoProvider.RepoData;
import us.kbase.narrativemethodstore.db.github.RepoTag;
import us.kbase.narrativemethodstore.exceptions.NarrativeMethodStoreException;
import us.kbase.shock.client.BasicShockClient;
import us.kbase.shock.client.ShockNodeId;
import us.kbase.shock.client.exceptions.InvalidShockUrlException;
import us.kbase.shock.client.exceptions.ShockHttpException;

public class MongoDynamicRepoDB implements DynamicRepoDB {
private final DB db;
private final Set<String> globalAdmins;
private final boolean isReadOnly;
private final URL shockUrl;
private final AuthToken serviceToken;
public static final long MAX_MONGO_FILE_LENGTH = 1024 * 1024;
////////////////////////////////////////////////////////////////////
private static final String TABLE_REPO_INFO = "repo_info";
private static final String FIELD_RI_MODULE_NAME = "module_name";
Expand All @@ -75,12 +66,15 @@ public class MongoDynamicRepoDB implements DynamicRepoDB {
private static final String FIELD_RF_HEX_DATA = "hex_data";
private static final String FIELD_RF_SHOCK_NODE_ID = "shock_node_id";

public MongoDynamicRepoDB(String host, String database, String dbUser, String dbPwd,
List<String> globalAdminUserIds, boolean isReadOnly, URL shockUrl,
AuthToken serviceToken) throws NarrativeMethodStoreException {
public MongoDynamicRepoDB(
final String host,
final String database,
final String dbUser,
final String dbPwd,
final List<String> globalAdminUserIds,
final boolean isReadOnly)
throws NarrativeMethodStoreException {
this.isReadOnly = isReadOnly;
this.shockUrl = shockUrl;
this.serviceToken = serviceToken;
try {
db = getDB(host, database, dbUser, dbPwd);
if (!isReadOnly)
Expand Down Expand Up @@ -158,7 +152,7 @@ private void throwRepoWasntRegistered(String repoModuleName)
private void throwChangeOperation()
throws NarrativeMethodStoreException {
throw new NarrativeMethodStoreException("Change operation couldn't be performed in " +
"read-only mode");
"read-only mode");
}

@Override
Expand Down Expand Up @@ -512,16 +506,9 @@ public FileId saveFile(String moduleName, FileProvider file)
String shockNodeId = null;
is = file.openStream();
try {
if (length <= MAX_MONGO_FILE_LENGTH || shockUrl == null) {
hexData = MongoUtils.streamToHex(is);
} else {
try {
BasicShockClient cl = new BasicShockClient(shockUrl, serviceToken);
shockNodeId = cl.addNode(is, fileName, "JSON").getId().getId();
} catch (Exception ex) {
throw new NarrativeMethodStoreException(ex);
}
}
// note that mongo can take ~16MB of data this way tops or it'll throw an error
// it's been running in production for years this way so I guess it works...?
hexData = MongoUtils.streamToHex(is);
} finally {
try {
is.close();
Expand Down Expand Up @@ -585,21 +572,9 @@ private void loadFile(Map<String, Object> obj, OutputStream target)
throws NarrativeMethodStoreException {
try {
String hexData = (String)obj.get(FIELD_RF_HEX_DATA);
if (hexData == null) {
String shockNodeId = (String)obj.get(FIELD_RF_SHOCK_NODE_ID);
BasicShockClient cl = new BasicShockClient(shockUrl, serviceToken);
cl.getFile(new ShockNodeId(shockNodeId), target);
} else {
target.write(MongoUtils.hexToBytes(hexData));
}
target.write(MongoUtils.hexToBytes(hexData));
} catch (IOException ex) {
throw new NarrativeMethodStoreException(ex);
} catch (InvalidShockUrlException ex) {
throw new NarrativeMethodStoreException(ex);
} catch (ShockHttpException ex) {
throw new NarrativeMethodStoreException(ex);
} catch (IllegalArgumentException ex) {
throw new NarrativeMethodStoreException(ex);
}
}

Expand Down
4 changes: 1 addition & 3 deletions src/us/kbase/narrativemethodstore/db/mongo/test/MongoDynamicRepoDB2Test.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,9 +303,7 @@ private MongoDynamicRepoDB getDB(final List<String> admins)
null,
null,
admins,
false,
null,
null);
false);
}

}
33 changes: 2 additions & 31 deletions src/us/kbase/narrativemethodstore/db/mongo/test/MongoDynamicRepoDBTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package us.kbase.narrativemethodstore.db.mongo.test;

import static us.kbase.narrativemethodstore.db.mongo.MongoUtils.toMap;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
Expand All @@ -25,12 +23,8 @@

import com.fasterxml.jackson.databind.JsonNode;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.MongoClient;

import us.kbase.auth.AuthToken;
import us.kbase.common.service.UObject;
import us.kbase.narrativemethodstore.MethodBriefInfo;
import us.kbase.narrativemethodstore.MethodParameter;
Expand All @@ -50,18 +44,13 @@
import us.kbase.narrativemethodstore.exceptions.NarrativeMethodStoreException;
import us.kbase.narrativemethodstore.util.FileUtils;
import us.kbase.narrativemethodstore.util.TextUtils;
import us.kbase.shock.client.BasicShockClient;
import us.kbase.shock.client.ShockNodeId;

// It'd be nice to know why this is ignored
@Ignore
public class MongoDynamicRepoDBTest {
private static final String dbName = "test_repo_registry_mongo";
private static final MongoDBHelper dbHelper = new MongoDBHelper("registry");

private static URL shockUrl = null;
private static AuthToken shockToken = null;

@BeforeClass
public static void beforeClass() throws Exception {
dbHelper.startup(System.getProperty("mongod.path"));
Expand Down Expand Up @@ -90,7 +79,7 @@ private void testRepo(boolean localFiles) throws Exception {

String host = "localhost:" + dbHelper.getMongoPort();
MongoDynamicRepoDB db = new MongoDynamicRepoDB(host, dbName, null, null,
Arrays.asList(globalAdmin), false, shockUrl, shockToken);
Arrays.asList(globalAdmin), false);
Assert.assertEquals(0, db.listRepoModuleNames().size());
RepoProvider pvd = localFiles ? new FileRepoProvider(new File(localPath)) :
new GitHubRepoProvider(new URL(gitUrl), null, dbHelper.getWorkDir());
Expand Down Expand Up @@ -220,7 +209,7 @@ public void testPy() throws Exception {
String globalAdmin = "admin";
String host = "localhost:" + dbHelper.getMongoPort();
MongoDynamicRepoDB db = new MongoDynamicRepoDB(host, dbName, null, null,
Arrays.asList(globalAdmin), false, shockUrl, shockToken);
Arrays.asList(globalAdmin), false);
Assert.assertEquals(0, db.listRepoModuleNames().size());
RepoProvider pvd = new FileRepoProvider(repoDir);
db.registerRepo(userId, pvd);
Expand Down Expand Up @@ -286,24 +275,6 @@ public void cleanup() throws Exception {
String host = "localhost:" + dbHelper.getMongoPort();
final MongoClient mc = new MongoClient(host);
final DB db = mc.getDB(dbName);
if (shockUrl != null) {
try {
final DBCollection files = db.getCollection("repo_files");
final DBCursor it = files.find();
for (final DBObject dbo: it) {
Map<String, Object> obj = toMap(dbo);
String fileName = (String)obj.get("file_name");
String shockNodeId = (String)obj.get("shock_node_id");
if (shockNodeId != null) {
System.out.println("Deleting shock node for " + fileName);
BasicShockClient cl = new BasicShockClient(shockUrl, shockToken);
cl.deleteNode(new ShockNodeId(shockNodeId));
}
}
} catch (Exception ex) {
ex.printStackTrace();
}
}
db.dropDatabase();
mc.close();
}
Expand Down
Loading