BXC-4304 map coll nums to pids (#69)

* add solrj to pom.xml

* add solr server config

* add generateCollectionNumbersToPidMapping method and tests

* fix import

* downgrade wiremock to 2.32.0 to avoid problems with jetty

pom.xml

@@ -14,14 +14,15 @@
     <description>Utilities for migrating content from CDM to Box-c</description>
     <properties>
         <picocli.version>4.6.1</picocli.version>
-        <wiremock.version>2.35.0</wiremock.version>
+        <wiremock.version>2.32.0</wiremock.version>
         <sqlite-jdbc.version>3.36.0.1</sqlite-jdbc.version>
         <poi.version>5.1.0</poi.version>
         <log4j-to-slf4j.version>2.17.1</log4j-to-slf4j.version>
         <build-tools.path></build-tools.path>
         <mysql.version>8.0.28</mysql.version>
         <csv.version>1.9.0</csv.version>
         <junit.jupiter.version>5.9.2</junit.jupiter.version>
+        <solr.solrj.version>8.10.1</solr.solrj.version>
     </properties>
 
     <build>
@@ -160,6 +161,11 @@
             <artifactId>mysql-connector-java</artifactId>
             <version>${mysql.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.solr</groupId>
+            <artifactId>solr-solrj</artifactId>
+            <version>${solr.solrj.version}</version>
+        </dependency>
         <!-- Logging -->
         <dependency>
             <groupId>org.slf4j</groupId>

src/main/java/edu/unc/lib/boxc/migration/cdm/model/BxcEnvironment.java

@@ -7,6 +7,7 @@
  */
 public class BxcEnvironment {
     private String httpBaseUrl;
+    private String solrServerUrl;
 
     public String getHttpBaseUrl() {
         return httpBaseUrl;
@@ -15,4 +16,12 @@ public String getHttpBaseUrl() {
     public void setHttpBaseUrl(String httpBaseUrl) {
         this.httpBaseUrl = httpBaseUrl;
     }
+
+    public String getSolrServerUrl() {
+        return solrServerUrl;
+    }
+
+    public void setSolrServerUrl(String solrServerUrl) {
+        this.solrServerUrl = solrServerUrl;
+    }
 }

src/main/java/edu/unc/lib/boxc/migration/cdm/services/ArchivalDestinationsService.java

@@ -3,14 +3,22 @@
 import edu.unc.lib.boxc.migration.cdm.exceptions.MigrationException;
 import edu.unc.lib.boxc.migration.cdm.model.MigrationProject;
 import edu.unc.lib.boxc.migration.cdm.options.DestinationMappingOptions;
+import edu.unc.lib.boxc.search.api.exceptions.SolrRuntimeException;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocumentList;
 import org.slf4j.Logger;
 
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import static org.slf4j.LoggerFactory.getLogger;
 
@@ -23,6 +31,12 @@ public class ArchivalDestinationsService {
 
     private MigrationProject project;
     private CdmIndexService indexService;
+    private String solrServerUrl;
+    private HttpSolrClient solr;
+
+    public void initialize() {
+        solr = new HttpSolrClient.Builder(solrServerUrl).build();
+    }
 
     /**
      * Generates a unique list of values in the accepted field name
@@ -53,11 +67,50 @@ public List<String> generateCollectionNumbersList(DestinationMappingOptions opti
         }
     }
 
+    /**
+     * Generates a map of archival collection numbers to boxc pids
+     * @param options destination mapping options
+     * @return A map
+     */
+    public Map<String, String> generateCollectionNumbersToPidMapping(DestinationMappingOptions options) throws Exception {
+        Map<String, String> mapCollNumToPid = new HashMap<>();
+
+        List<String> listCollectionNumbers = generateCollectionNumbersList(options);
+
+        for (String collNum : listCollectionNumbers) {
+            String collNumQuery = "collectionId:" + collNum;
+            SolrQuery query = new SolrQuery();
+            query.set("q", collNumQuery);
+            query.setFilterQueries("resourceType:Collection");
+
+            try {
+                QueryResponse response = solr.query(query);
+                SolrDocumentList results = response.getResults();
+                if (results.isEmpty()) {
+                    mapCollNumToPid.put(collNum, null);
+                } else {
+                    mapCollNumToPid.put(collNum, results.get(0).getFieldValue("pid").toString());
+                }
+            } catch (SolrServerException e) {
+                throw new SolrRuntimeException(e);
+            }
+        }
+        return mapCollNumToPid;
+    }
+
     public void setProject(MigrationProject project) {
         this.project = project;
     }
 
     public void setIndexService(CdmIndexService indexService) {
         this.indexService = indexService;
     }
+
+    public void setSolrServerUrl(String solrServerUrl) {
+        this.solrServerUrl = solrServerUrl;
+    }
+
+    public void setSolr(HttpSolrClient solr) {
+        this.solr = solr;
+    }
 }

src/main/java/edu/unc/lib/boxc/migration/cdm/services/IndexFilteringService.java

@@ -4,7 +4,7 @@
 import edu.unc.lib.boxc.migration.cdm.model.MigrationProject;
 import edu.unc.lib.boxc.migration.cdm.options.IndexFilteringOptions;
 import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 
 import java.sql.Connection;

src/test/java/edu/unc/lib/boxc/migration/cdm/services/ArchivalDestinationsServiceTest.java

@@ -5,34 +5,65 @@
 import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper;
 import edu.unc.lib.boxc.migration.cdm.test.SipServiceHelper;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.util.NamedList;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
 
 import java.nio.file.Path;
 import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertIterableEquals;
+import static org.mockito.MockitoAnnotations.openMocks;
+import static org.mockito.Mockito.when;
 
 public class ArchivalDestinationsServiceTest {
     private static final String PROJECT_NAME = "proj";
+    private static final String SOLR_URL = "http://example.com:88/solr";
+
+    @Captor
+    private ArgumentCaptor<SolrQuery> solrQueryCaptor;
 
     @TempDir
     public Path tmpFolder;
 
     private SipServiceHelper testHelper;
     private MigrationProject project;
     private ArchivalDestinationsService service;
+    private AutoCloseable closeable;
+
+    @Mock
+    private HttpSolrClient solrClient;
 
     @BeforeEach
     public void setup() throws Exception {
+        closeable = openMocks(this);
         project = MigrationProjectFactory.createMigrationProject(
                 tmpFolder, PROJECT_NAME, null, "user",
                 CdmEnvironmentHelper.DEFAULT_ENV_ID, BxcEnvironmentHelper.DEFAULT_ENV_ID);
         testHelper = new SipServiceHelper(project, tmpFolder);
         service = new ArchivalDestinationsService();
         service.setProject(project);
         service.setIndexService(testHelper.getIndexService());
+        service.setSolrServerUrl(SOLR_URL);
+        service.setSolr(solrClient);
+    }
+
+    @AfterEach
+    void closeService() throws Exception {
+        closeable.close();
     }
 
     @Test
@@ -45,4 +76,66 @@ public void archivalCollectionNumberTest() throws Exception {
         var result = service.generateCollectionNumbersList(options);
         assertIterableEquals(Arrays.asList("216", "604", "607"), result);
     }
+
+    @Test
+    public void collectionNumberToNullPidTest() throws Exception {
+        testHelper.indexExportData("mini_keepsakes");
+        QueryResponse testResponse = new QueryResponse();
+        testResponse.setResponse(new NamedList<>(Map.of("response", new SolrDocumentList())));
+        when(solrClient.query(solrQueryCaptor.capture())).thenReturn(testResponse);
+
+        Map<String, String> expected = new HashMap<>();
+        expected.put("216", null);
+        expected.put("604", null);
+        expected.put("607", null);
+
+        var options = new DestinationMappingOptions();
+        options.setFieldName("dmrecord");
+
+        var result = service.generateCollectionNumbersToPidMapping(options);
+        assertEquals(expected, result);
+        var solrValues = solrQueryCaptor.getAllValues();
+        assertEquals(3, solrValues.size());
+        assertEquals("collectionId:216", solrValues.get(0).getQuery());
+        assertEquals("collectionId:604", solrValues.get(1).getQuery());
+        assertEquals("collectionId:607", solrValues.get(2).getQuery());
+        assertEquals("resourceType:Collection",
+                Arrays.stream(solrValues.get(0).getFilterQueries()).findFirst().get());
+    }
+
+    @Test
+    public void collectionNumberToPidTest() throws Exception {
+        testHelper.indexExportData("mini_keepsakes");
+
+        QueryResponse testResponseA = new QueryResponse();
+        SolrDocument testDocumentA = new SolrDocument();
+        testDocumentA.addField("pid", "40147");
+        SolrDocumentList testListA = new SolrDocumentList();
+        testListA.add(testDocumentA);
+        testResponseA.setResponse(new NamedList<>(Map.of("response", testListA)));
+
+        QueryResponse testResponseB = new QueryResponse();
+        testResponseB.setResponse(new NamedList<>(Map.of("response", new SolrDocumentList())));
+
+        when(solrClient.query(solrQueryCaptor.capture())).thenReturn(testResponseA).thenReturn(testResponseB)
+                .thenReturn(testResponseA);
+
+        Map<String, String> expected = new HashMap<>();
+        expected.put("216", "40147");
+        expected.put("604", null);
+        expected.put("607", "40147");
+
+        var options = new DestinationMappingOptions();
+        options.setFieldName("dmrecord");
+
+        var result = service.generateCollectionNumbersToPidMapping(options);
+        assertEquals(expected, result);
+        var solrValues = solrQueryCaptor.getAllValues();
+        assertEquals(3, solrValues.size());
+        assertEquals("collectionId:216", solrValues.get(0).getQuery());
+        assertEquals("collectionId:604", solrValues.get(1).getQuery());
+        assertEquals("collectionId:607", solrValues.get(2).getQuery());
+        assertEquals("resourceType:Collection",
+                Arrays.stream(solrValues.get(0).getFilterQueries()).findFirst().get());
+    }
 }

src/test/java/edu/unc/lib/boxc/migration/cdm/test/BxcEnvironmentHelper.java

@@ -13,6 +13,8 @@ public class BxcEnvironmentHelper {
     public static final String DEFAULT_ENV_ID = "test";
     public static final int TEST_HTTP_PORT = 46887;
     public static final String TEST_BASE_URL = "http://localhost:" + TEST_HTTP_PORT + "/bxc/";
+    public static final int TEST_SOLR_PORT = 46887;
+    public static final String TEST_SOLR_URL = "http://localhost:" + TEST_SOLR_PORT + "/solr";
 
     /**
      * @return environment mapping containing test environment
@@ -27,6 +29,7 @@ public static Map<String, BxcEnvironment> getTestMapping() {
     public static BxcEnvironment getTestEnv() {
         var testEnv = new BxcEnvironment();
         testEnv.setHttpBaseUrl(TEST_BASE_URL);
+        testEnv.setSolrServerUrl(TEST_SOLR_URL);
         return testEnv;
     }
 }