Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove fields that would cause caching failures in various tasks #4511

Merged
merged 2 commits into from
Aug 21, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ public ResponseEntity<TaskResponse> createConfigureModelTask(
// stripping the metadata from the model before its sent since it can cause
// gollm to fail with massive inputs
model.get().setMetadata(null);
input.setAmr(model.get());
input.setAmr(model.get().serializeWithoutTerariumFields());

// Create the task
final TaskRequest req = new TaskRequest();
Expand Down Expand Up @@ -372,7 +372,7 @@ public ResponseEntity<TaskResponse> createConfigFromDatasetTask(
// stripping the metadata from the model before its sent since it can cause
// gollm to fail with massive inputs
model.get().setMetadata(null);
input.setAmr(model.get());
input.setAmr(model.get().serializeWithoutTerariumFields());

// set matrix string if provided
if (body != null && !body.getMatrixStr().isEmpty()) {
Expand Down Expand Up @@ -464,12 +464,7 @@ public ResponseEntity<TaskResponse> createCompareModelsTask(
throw new ResponseStatusException(HttpStatus.NOT_FOUND, messages.get("model.not-found"));
}

try {
amrs.add(objectMapper.writeValueAsString(model.get()));
} catch (final JsonProcessingException e) {
log.error("Unable to serialize model card", e);
throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, messages.get("task.gollm.json-processing"));
}
amrs.add(model.get().serializeWithoutTerariumFields());
}

// if the number of models is less than 2, return an error
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package software.uncharted.terarium.hmiserver.models;

import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.databind.MapperFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.hypersistence.utils.hibernate.type.json.JsonType;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.persistence.Column;
Expand Down Expand Up @@ -72,4 +75,21 @@ protected TerariumAsset cloneSuperFields(final TerariumAsset asset) {

return asset;
}

public String serializeWithoutTerariumFields() {
final ObjectMapper mapper = new ObjectMapper();
mapper.setConfig(mapper.getSerializationConfig().with(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY));
final ObjectNode objectNode = mapper.convertValue(this, ObjectNode.class);
objectNode.remove("id");
objectNode.remove("createdOn");
objectNode.remove("updatedOn");
objectNode.remove("deletedOn");
objectNode.remove("name");
objectNode.remove("description");
objectNode.remove("temporary");
objectNode.remove("publicAsset");
objectNode.remove("fileNames");
objectNode.remove("userId");
return objectNode.toString();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public static class Input {
List<String> datasets;

@JsonProperty("amr")
Model amr;
String amr;

@JsonProperty("matrix_str")
String matrixStr;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ public static class Input {
String researchPaper;

@JsonProperty("amr")
Model amr;
String amr;
}

@Data
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package software.uncharted.terarium.hmiserver.models;

import java.sql.Timestamp;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import software.uncharted.terarium.hmiserver.TerariumApplicationTests;
import software.uncharted.terarium.hmiserver.models.dataservice.model.Model;
import software.uncharted.terarium.hmiserver.models.dataservice.modelparts.ModelHeader;

@Slf4j
public class TerariumAssetTests extends TerariumApplicationTests {

@Test
void testSerializeWithoutTerariumFields() throws Exception {
final Model model1 = new Model();
model1.setName("myname");
model1.setDescription("mydescription");
model1.setFileNames(null);
model1.setDeletedOn(null);
model1.setTemporary(false);
model1.setPublicAsset(false);
model1.setUserId("my user");
model1.setCreatedOn(new Timestamp(System.currentTimeMillis()));
model1.setUpdatedOn(new Timestamp(System.currentTimeMillis()));
model1.setHeader(
new ModelHeader()
.setName("test-name")
.setModelSchema("test-schema")
.setModelVersion("0.1.2")
.setDescription("test-description")
.setSchemaName("petrinet")
);

final String json1 = model1.serializeWithoutTerariumFields();

final Model model2 = new Model();
model2.setName("myname 2");
model2.setDescription("mydescription 2");
model2.setFileNames(null);
model2.setDeletedOn(null);
model2.setTemporary(false);
model2.setPublicAsset(false);
model2.setUserId("my user 2");
model2.setCreatedOn(new Timestamp(System.currentTimeMillis()));
model2.setUpdatedOn(new Timestamp(System.currentTimeMillis()));
model2.setHeader(
new ModelHeader()
.setName("test-name")
.setModelSchema("test-schema")
.setModelVersion("0.1.2")
.setDescription("test-description")
.setSchemaName("petrinet")
);

final String json2 = model2.serializeWithoutTerariumFields();

log.info("json1: {}", json1);

Assertions.assertEquals(json1, json2);
}
}
Loading