diff --git a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar b/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar deleted file mode 100644 index a23530b895c..00000000000 Binary files a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar and /dev/null differ diff --git a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar.md5 b/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar.md5 deleted file mode 100644 index d8b1ce2fa75..00000000000 --- a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -f578d8ec91811d5d72981355cb7a1f0f diff --git a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar.sha1 b/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar.sha1 deleted file mode 100644 index 4c7d114634b..00000000000 --- a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -523abaf48b4423eb874dbc086b876aa917930a04 diff --git a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom b/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom deleted file mode 100644 index 2915745c27d..00000000000 --- a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom +++ /dev/null @@ -1,77 +0,0 @@ - - - - xoai - com.lyncode - 4.1.0 - - 4.0.0 - - XOAI Commons - xoai-common - 4.1.0-header-patch - - - - com.lyncode - xml-io - - - com.lyncode - test-support - - - - commons-codec - commons-codec - - - - commons-io - commons-io - - - - com.google.guava - guava - - - - xml-apis - xml-apis - - - - org.hamcrest - hamcrest-all - - - - org.codehaus.woodstox - stax2-api - - - - javax.xml.stream - stax-api - - - - org.apache.commons - commons-lang3 - - - - stax - stax-api - - - - junit - junit - test - - - - - diff --git a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom.md5 b/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom.md5 deleted file mode 100644 index 15f47f4140a..00000000000 --- a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom.md5 +++ /dev/null @@ -1 +0,0 @@ -346e9f235523e52256006bbe8eba60bb diff --git a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom.sha1 b/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom.sha1 deleted file mode 100644 index 88668a4d49c..00000000000 --- a/local_lib/com/lyncode/xoai-common/4.1.0-header-patch/xoai-common-4.1.0-header-patch.pom.sha1 +++ /dev/null @@ -1 +0,0 @@ -cd83d08c097d6aa1b27b20ef4742c7e4fa47e6b5 diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar deleted file mode 100644 index 28e5da7b0d6..00000000000 Binary files a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar and /dev/null differ diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar.md5 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar.md5 deleted file mode 100644 index 67dda34a6c9..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -546f1ab3f3f654280f88e429ba3471ae diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar.sha1 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar.sha1 deleted file mode 100644 index 50e8f2f42cd..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-javadoc.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f4da7ebc3fda69e1e7db12bda6d7b5fb4aecc7a4 diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar deleted file mode 100644 index bdec990e2c6..00000000000 Binary files a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar and /dev/null differ diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar.md5 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar.md5 deleted file mode 100644 index 77416d80f87..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -ec87ba7cb8e7396fc903acdbacd31ff6 diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar.sha1 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar.sha1 deleted file mode 100644 index a6157b4dc0b..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch-sources.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -370c2955550a42b11fe7b9007771c506f5769639 diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar deleted file mode 100644 index 331c9a80cd1..00000000000 Binary files a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar and /dev/null differ diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar.md5 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar.md5 deleted file mode 100644 index 27381662d09..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -544e9b97062d054370695b9b09d4bb1c diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar.sha1 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar.sha1 deleted file mode 100644 index 37dd4e47c2c..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -67c505461f3c190894bb036cc866eb640c2f6a48 diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom deleted file mode 100644 index 87d67b8c4a7..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom +++ /dev/null @@ -1,54 +0,0 @@ - - - - xoai - com.lyncode - 4.1.0-header-patch - - - 4.0.0 - - XOAI Data Provider - xoai-data-provider - 4.1.0-header-patch - - - - com.lyncode - xoai-common - ${project.version} - - - - log4j - log4j - - - - com.google.guava - guava - - - - com.lyncode - builder-commons - - - - org.apache.commons - commons-lang3 - - - - org.mockito - mockito-all - test - - - - junit - junit - test - - - diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom.md5 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom.md5 deleted file mode 100644 index 5959ea476c7..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom.md5 +++ /dev/null @@ -1 +0,0 @@ -a1b49c13fcf448de9628798f8682fcaa diff --git a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom.sha1 b/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom.sha1 deleted file mode 100644 index 87fd86c23e0..00000000000 --- a/local_lib/com/lyncode/xoai-data-provider/4.1.0-header-patch/xoai-data-provider-4.1.0-header-patch.pom.sha1 +++ /dev/null @@ -1 +0,0 @@ -41be98af31f8d17d83ab6c38bd7939ba212eab8d diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar deleted file mode 100644 index 4382b3ded5d..00000000000 Binary files a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar and /dev/null differ diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar.md5 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar.md5 deleted file mode 100644 index c9e720e6039..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -21bc45a29b715720f4b77f51bf9f1754 diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar.sha1 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar.sha1 deleted file mode 100644 index 756955d2840..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-javadoc.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b544162e82d322116b87d99f2fbb6ddd4c4745e1 diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar deleted file mode 100644 index 314dad81872..00000000000 Binary files a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar and /dev/null differ diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar.md5 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar.md5 deleted file mode 100644 index 27c55f9af58..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -88dc05805672ebe01ded1197a582cd60 diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar.sha1 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar.sha1 deleted file mode 100644 index 1098e506b93..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch-sources.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fe41289cb74c56e9282dd09c22df2eda47c68a0d diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar deleted file mode 100644 index 781fc1ce1e2..00000000000 Binary files a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar and /dev/null differ diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar.md5 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar.md5 deleted file mode 100644 index 84891040047..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar.md5 +++ /dev/null @@ -1 +0,0 @@ -52f8b446f78009757d593312778f428c diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar.sha1 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar.sha1 deleted file mode 100644 index dbded3dd83f..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -feb6903ad32d4b42461b7ca1b3fae6146740bb31 diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom deleted file mode 100644 index c45e15a91f9..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom +++ /dev/null @@ -1,67 +0,0 @@ - - - - xoai - com.lyncode - 4.1.0-header-patch - - 4.0.0 - - XOAI Service Provider - xoai-service-provider - 4.1.0-header-patch - - - - com.lyncode - xoai-common - ${project.version} - - - - com.lyncode - xml-io - - - - log4j - log4j - - - - org.apache.commons - commons-lang3 - - - - org.apache.httpcomponents - httpclient - - - - org.codehaus.woodstox - wstx-asl - - - - - com.lyncode - xoai-data-provider - ${project.version} - test - - - - org.mockito - mockito-all - test - - - - junit - junit - test - - - - diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom.md5 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom.md5 deleted file mode 100644 index 5e51f198572..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom.md5 +++ /dev/null @@ -1 +0,0 @@ -b97b8ee92daa5fc4fd87004465f9ad2b diff --git a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom.sha1 b/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom.sha1 deleted file mode 100644 index 2c6dc74f02b..00000000000 --- a/local_lib/com/lyncode/xoai-service-provider/4.1.0-header-patch/xoai-service-provider-4.1.0-header-patch.pom.sha1 +++ /dev/null @@ -1 +0,0 @@ -f772583549263bd72ea4d5268d9db0a84c27cb9f diff --git a/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom b/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom deleted file mode 100644 index 89a14d88c51..00000000000 --- a/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom +++ /dev/null @@ -1,273 +0,0 @@ - - 4.0.0 - pom - - - xoai-common - xoai-data-provider - xoai-service-provider - - - - org.sonatype.oss - oss-parent - 7 - - - com.lyncode - xoai - 4.1.0-header-patch - - XOAI : OAI-PMH Java Toolkit - http://www.lyncode.com - - - 1.9.5 - 15.0 - 3.1 - 1.2.14 - 4.2.1 - 4.0.0 - - 1.0.2 - 1.0.3 - 1.0.4 - - - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - - scm:git:git@github.com:lyncode/xoai.git - scm:git:git@github.com:lyncode/xoai.git - git@github.com:lyncode/xoai.git - xoai-4.1.0 - - - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.5 - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.8.1 - - - org.apache.maven.plugins - maven-source-plugin - 2.2.1 - - - org.apache.maven.plugins - maven-release-plugin - 2.5 - - - org.apache.maven.plugins - maven-compiler-plugin - 3.1 - - - - - - org.apache.maven.plugins - maven-release-plugin - - true - false - release - deploy - - - - org.apache.maven.plugins - maven-compiler-plugin - - 1.6 - 1.6 - false - false - true - - - - org.apache.maven.plugins - maven-javadoc-plugin - true - - - attach-javadocs - - jar - - - - - - org.apache.maven.plugins - maven-source-plugin - true - - - attach-sources - - jar - - - - - - - - - - - com.lyncode - xml-io - ${lyncode.xml-io} - - - - com.lyncode - test-support - ${lyncode.test-support} - - - - - log4j - log4j - ${log4j.version} - - - - org.apache.commons - commons-lang3 - ${commons.lang3.version} - - - - org.apache.httpcomponents - httpclient - ${http-commons.version} - - - - org.codehaus.woodstox - wstx-asl - ${woodstox.version} - - - - org.codehaus.woodstox - stax2-api - 3.0.4 - - - - commons-codec - commons-codec - 1.3 - - - org.hamcrest - hamcrest-all - 1.3 - - - xalan - xalan - 2.7.2 - - - dom4j - dom4j - 1.6.1 - - - - javax.xml.stream - stax-api - 1.0-2 - - - jaxen - jaxen - 1.1.4 - - - junit - junit - 4.11 - - - commons-io - commons-io - 2.4 - - - - xml-apis - xml-apis - 1.0.b2 - - - - stax - stax-api - 1.0.1 - - - - org.mockito - mockito-all - ${mockito.version} - - - - com.google.guava - guava - ${guava.version} - - - - com.lyncode - builder-commons - ${lyncode.builder-commons} - - - - - - - - DSpace @ Lyncode - dspace@lyncode.com - Lyncode - http://www.lyncode.com - - - - diff --git a/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom.md5 b/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom.md5 deleted file mode 100644 index d2fdadd114f..00000000000 --- a/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom.md5 +++ /dev/null @@ -1 +0,0 @@ -b50966bebe8cfdcb58478cf029b08aa3 diff --git a/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom.sha1 b/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom.sha1 deleted file mode 100644 index b142cd649e8..00000000000 --- a/local_lib/com/lyncode/xoai/4.1.0-header-patch/xoai-4.1.0-header-patch.pom.sha1 +++ /dev/null @@ -1 +0,0 @@ -28a5d65399cbc25b29b270caebbb86e292c5ba18 diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 8fe611d7716..ccc0a9a7f60 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -161,6 +161,9 @@ 1.21 4.5.13 4.4.14 + + + 5.0.0-RC1 1.15.0 @@ -301,7 +304,7 @@ Local repository for hosting jars not available from network repositories. file://${project.basedir}/local_lib - - - - - - - - - - - - - com.lyncode - xoai-common - 4.1.0-header-patch - - - com.lyncode + + + io.gdcc xoai-data-provider - 4.1.0-header-patch - - - log4j - log4j - - + ${gdcc.xoai.version} - com.lyncode + io.gdcc xoai-service-provider - 4.1.0-header-patch - - - log4j - log4j - - - - - - - - ch.qos.reload4j - reload4j - ${reload4j.version} - runtime + ${gdcc.xoai.version} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java index 60abc97bccd..5b3e4df331d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/FastGetRecord.java @@ -72,7 +72,6 @@ public class FastGetRecord { - private static final String DATAVERSE_EXTENDED_METADATA = "dataverse_json"; private static final String XML_METADATA_TAG = "metadata"; private static final String XML_METADATA_TAG_OPEN = "<"+XML_METADATA_TAG+">"; private static final String XML_METADATA_TAG_CLOSE = ""; @@ -222,13 +221,7 @@ public void harvestRecord(String baseURL, String identifier, String metadataPref //metadataOut.println(""); /* ? */ metadataFlag = true; - } else if (line.matches(".*<"+XML_METADATA_TAG+" [^>]*>.*")) { - if (metadataPrefix.equals(DATAVERSE_EXTENDED_METADATA)) { - oaiResponseHeader = oaiResponseHeader.concat(line); - metadataWritten = true; - metadataFlag = true; - } - } + } } //System.out.println(line); @@ -380,19 +373,12 @@ public void harvestRecord(String baseURL, String identifier, String metadataPref try { StringReader reader = new StringReader(oaiResponseHeader); xmlr = xmlInputFactory.createXMLStreamReader(reader); - processOAIheader(xmlr, metadataPrefix.equals(DATAVERSE_EXTENDED_METADATA)); + processOAIheader(xmlr); } catch (XMLStreamException ex) { - //Logger.getLogger("global").log(Level.SEVERE, null, ex); if (this.errorMessage == null) { this.errorMessage = "Malformed GetRecord response; baseURL=" + baseURL + ", identifier=" + identifier + ", metadataPrefix=" + metadataPrefix; } - - // delete the temp metadata file; we won't need it: - if (savedMetadataFile != null) { - //savedMetadataFile.delete(); - } - } try { @@ -414,14 +400,8 @@ public void harvestRecord(String baseURL, String identifier, String metadataPref if (!(metadataWritten) && !(this.isDeleted())) { this.errorMessage = "Failed to parse GetRecord response; baseURL=" + baseURL + ", identifier=" + identifier + ", metadataPrefix=" + metadataPrefix; - //savedMetadataFile.delete(); - } - - if (this.isDeleted()) { - //savedMetadataFile.delete(); } - } else { this.errorMessage = "GetRecord request failed. HTTP error code "+responseCode; } @@ -445,16 +425,16 @@ private static String getRequestURL(String baseURL, return requestURL.toString(); } - private void processOAIheader (XMLStreamReader xmlr, boolean extensionMode) throws XMLStreamException, IOException { + private void processOAIheader (XMLStreamReader xmlr) throws XMLStreamException, IOException { // is this really a GetRecord response? xmlr.nextTag(); xmlr.require(XMLStreamConstants.START_ELEMENT, null, "OAI-PMH"); - processOAIPMH(xmlr, extensionMode); + processOAIPMH(xmlr); } - private void processOAIPMH (XMLStreamReader xmlr, boolean extensionMode) throws XMLStreamException, IOException { + private void processOAIPMH (XMLStreamReader xmlr) throws XMLStreamException, IOException { for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { @@ -477,7 +457,7 @@ else if (xmlr.getLocalName().equals("error")) { } else if (xmlr.getLocalName().equals("GetRecord")) { - processGetRecordSection(xmlr, extensionMode); + processGetRecordSection(xmlr); } } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("OAI-PMH")) return; @@ -485,11 +465,11 @@ else if (xmlr.getLocalName().equals("GetRecord")) { } } - private void processGetRecordSection (XMLStreamReader xmlr, boolean extensionMode) throws XMLStreamException, IOException { + private void processGetRecordSection (XMLStreamReader xmlr) throws XMLStreamException, IOException { for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("record")) { - processRecord(xmlr, extensionMode); + processRecord(xmlr); } } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("GetRecord")) return; @@ -498,7 +478,7 @@ private void processGetRecordSection (XMLStreamReader xmlr, boolean extensionMod } - private void processRecord (XMLStreamReader xmlr, boolean extensionMode) throws XMLStreamException, IOException { + private void processRecord (XMLStreamReader xmlr) throws XMLStreamException, IOException { for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("header")) { @@ -506,11 +486,6 @@ private void processRecord (XMLStreamReader xmlr, boolean extensionMode) throws this.recordDeleted = true; } processHeader(xmlr); - } else if (xmlr.getLocalName().equals("metadata")) { - if (extensionMode) { - String extendedMetadataApiUrl = xmlr.getAttributeValue(null, "directApiCall"); - processMetadataExtended(extendedMetadataApiUrl); - } } } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("record")) return; @@ -532,67 +507,6 @@ else if (xmlr.getLocalName().equals("setSpec")) {/*do nothing*/} } } - private void processMetadataExtended (String extendedApiUrl) throws IOException { - InputStream in = null; - int responseCode = 0; - HttpURLConnection con = null; - - - - try { - URL url = new URL(extendedApiUrl.replaceAll("&", "&")); // is this necessary? - - con = (HttpURLConnection) url.openConnection(); - con.setRequestProperty("User-Agent", "DataverseHarvester/3.0"); - responseCode = con.getResponseCode(); - } catch (MalformedURLException mue) { - throw new IOException ("Bad API URL: "+extendedApiUrl); - } catch (FileNotFoundException e) { - responseCode = HttpURLConnection.HTTP_UNAVAILABLE; - } - - - - - if (responseCode == 200) { - in = con.getInputStream(); - // TODO: - /* we should probably still support gzip/compress encoding here - ? - String contentEncoding = con.getHeaderField("Content-Encoding"); - - // support for the standard compress/gzip/deflate compression - // schemes: - - if ("compress".equals(contentEncoding)) { - ZipInputStream zis = new ZipInputStream(con.getInputStream()); - zis.getNextEntry(); - in = zis; - } else if ("gzip".equals(contentEncoding)) { - in = new GZIPInputStream(con.getInputStream()); - } else if ("deflate".equals(contentEncoding)) { - in = new InflaterInputStream(con.getInputStream()); - } ... - */ - FileOutputStream tempOut = new FileOutputStream(savedMetadataFile); - - int bufsize; - byte[] buffer = new byte[4 * 8192]; - - while ((bufsize = in.read(buffer)) != -1) { - tempOut.write(buffer, 0, bufsize); - tempOut.flush(); - } - - in.close(); - tempOut.close(); - return; - } - - throw new IOException("Failed to download extended metadata."); - - } - - // (from Gustavo's ddiServiceBean -- L.A.) // /* We had to add this method because the ref getElementText has a bug where it diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java index 71cc23e242b..e7156dfe9aa 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvesterServiceBean.java @@ -5,6 +5,7 @@ */ package edu.harvard.iq.dataverse.harvest.client; +import static java.net.HttpURLConnection.HTTP_OK; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.Dataverse; @@ -17,7 +18,6 @@ import java.util.Date; import java.util.Iterator; import java.util.List; -//import java.net.URLEncoder; import java.util.logging.FileHandler; import java.util.logging.Level; import java.util.logging.Logger; @@ -28,22 +28,26 @@ import javax.ejb.Stateless; import javax.ejb.Timer; import javax.inject.Named; -//import javax.xml.bind.Unmarshaller; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; import org.apache.commons.lang3.mutable.MutableBoolean; -import org.apache.commons.lang3.mutable.MutableLong; import org.xml.sax.SAXException; -import com.lyncode.xoai.model.oaipmh.Header; +import io.gdcc.xoai.model.oaipmh.results.record.Header; import edu.harvard.iq.dataverse.EjbDataverseEngine; import edu.harvard.iq.dataverse.api.imports.ImportServiceBean; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.harvest.client.oai.OaiHandler; import edu.harvard.iq.dataverse.harvest.client.oai.OaiHandlerException; import edu.harvard.iq.dataverse.search.IndexServiceBean; +import java.io.FileOutputStream; import java.io.FileWriter; +import java.io.InputStream; import java.io.PrintWriter; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; @@ -75,13 +79,12 @@ public class HarvesterServiceBean { IndexServiceBean indexService; private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean"); - private static final SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd"); private static final SimpleDateFormat logFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH-mm-ss"); public static final String HARVEST_RESULT_SUCCESS="success"; public static final String HARVEST_RESULT_FAILED="failed"; - private static final Long INDEXING_CONTENT_BATCH_SIZE = 10000000L; - + public static final String DATAVERSE_PROPRIETARY_METADATA_FORMAT="dataverse_json"; + public static final String DATAVERSE_PROPRIETARY_METADATA_API="/api/datasets/export?exporter="+DATAVERSE_PROPRIETARY_METADATA_FORMAT+"&persistentId="; public HarvesterServiceBean() { @@ -183,24 +186,7 @@ public void doHarvest(DataverseRequest dataverseRequest, Long harvestingClientId hdLogger.log(Level.INFO, "COMPLETED HARVEST, server=" + harvestingClientConfig.getArchiveUrl() + ", metadataPrefix=" + harvestingClientConfig.getMetadataPrefix()); hdLogger.log(Level.INFO, "Datasets created/updated: " + harvestedDatasetIds.size() + ", datasets deleted: " + deletedIdentifiers.size() + ", datasets failed: " + failedIdentifiers.size()); - // now index all the datasets we have harvested - created, modified or deleted: - /* (TODO: may not be needed at all. In Dataverse4, we may be able to get away with the normal - reindexing after every import. See the rest of the comments about batch indexing throughout - this service bean) - if (this.processedSizeThisBatch > 0) { - hdLogger.log(Level.INFO, "POST HARVEST, reindexing the remaining studies."); - if (this.harvestedDatasetIdsThisBatch != null) { - hdLogger.log(Level.INFO, this.harvestedDatasetIdsThisBatch.size()+" studies in the batch"); - } - hdLogger.log(Level.INFO, this.processedSizeThisBatch + " bytes of content"); - indexService.updateIndexList(this.harvestedDatasetIdsThisBatch); - hdLogger.log(Level.INFO, "POST HARVEST, calls to index finished."); - } else { - hdLogger.log(Level.INFO, "(All harvested content already reindexed)"); - } - */ } - //mailService.sendHarvestNotification(...getSystemEmail(), harvestingDataverse.getName(), logFileName, logTimestamp, harvestErrorOccurred.booleanValue(), harvestedDatasetIds.size(), failedIdentifiers); } catch (Throwable e) { harvestErrorOccurred.setValue(true); String message = "Exception processing harvest, server= " + harvestingClientConfig.getHarvestingUrl() + ",format=" + harvestingClientConfig.getMetadataPrefix() + " " + e.getClass().getName() + " " + e.getMessage(); @@ -235,8 +221,8 @@ private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClien logBeginOaiHarvest(hdLogger, harvestingClient); List harvestedDatasetIds = new ArrayList(); - MutableLong processedSizeThisBatch = new MutableLong(0L); OaiHandler oaiHandler; + HttpClient httpClient = null; try { oaiHandler = new OaiHandler(harvestingClient); @@ -248,22 +234,35 @@ private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClien hdLogger.log(Level.SEVERE, errorMessage); throw new IOException(errorMessage); } - + + if (DATAVERSE_PROPRIETARY_METADATA_FORMAT.equals(oaiHandler.getMetadataPrefix())) { + // If we are harvesting native Dataverse json, we'll also need this + // jdk http client to make direct calls to the remote Dataverse API: + httpClient = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.ALWAYS).build(); + } + try { for (Iterator
idIter = oaiHandler.runListIdentifiers(); idIter.hasNext();) { Header h = idIter.next(); String identifier = h.getIdentifier(); - Date dateStamp = h.getDatestamp(); + Date dateStamp = Date.from(h.getDatestamp()); hdLogger.info("processing identifier: " + identifier + ", date: " + dateStamp); + + if (h.isDeleted()) { + hdLogger.info("Deleting harvesting dataset for " + identifier + ", per ListIdentifiers."); + + deleteHarvestedDatasetIfExists(identifier, oaiHandler.getHarvestingClient().getDataverse(), dataverseRequest, deletedIdentifiers, hdLogger); + continue; + } MutableBoolean getRecordErrorOccurred = new MutableBoolean(false); // Retrieve and process this record with a separate GetRecord call: - Long datasetId = processRecord(dataverseRequest, hdLogger, importCleanupLog, oaiHandler, identifier, getRecordErrorOccurred, processedSizeThisBatch, deletedIdentifiers, dateStamp); - hdLogger.info("Total content processed in this batch so far: "+processedSizeThisBatch); + Long datasetId = processRecord(dataverseRequest, hdLogger, importCleanupLog, oaiHandler, identifier, getRecordErrorOccurred, deletedIdentifiers, dateStamp, httpClient); + if (datasetId != null) { harvestedDatasetIds.add(datasetId); @@ -280,20 +279,6 @@ private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClien //temporary: //throw new IOException("Exception occured, stopping harvest"); } - - // reindexing in batches? - this is from DVN 3; - // we may not need it anymore. - if ( processedSizeThisBatch.longValue() > INDEXING_CONTENT_BATCH_SIZE ) { - - hdLogger.log(Level.INFO, "REACHED CONTENT BATCH SIZE LIMIT; calling index ("+ harvestedDatasetIdsThisBatch.size()+" datasets in the batch)."); - //indexService.updateIndexList(this.harvestedDatasetIdsThisBatch); - hdLogger.log(Level.INFO, "REINDEX DONE."); - - - processedSizeThisBatch.setValue(0L); - harvestedDatasetIdsThisBatch = null; - } - } } catch (OaiHandlerException e) { throw new IOException("Failed to run ListIdentifiers: " + e.getMessage()); @@ -303,57 +288,56 @@ private List harvestOAI(DataverseRequest dataverseRequest, HarvestingClien return harvestedDatasetIds; - } - + } - - private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, PrintWriter importCleanupLog, OaiHandler oaiHandler, String identifier, MutableBoolean recordErrorOccurred, MutableLong processedSizeThisBatch, List deletedIdentifiers, Date dateStamp) { + private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, PrintWriter importCleanupLog, OaiHandler oaiHandler, String identifier, MutableBoolean recordErrorOccurred, List deletedIdentifiers, Date dateStamp, HttpClient httpClient) { String errMessage = null; Dataset harvestedDataset = null; logGetRecord(hdLogger, oaiHandler, identifier); File tempFile = null; - try { - FastGetRecord record = oaiHandler.runGetRecord(identifier); - errMessage = record.getErrorMessage(); + try { + boolean deleted = false; + + if (DATAVERSE_PROPRIETARY_METADATA_FORMAT.equals(oaiHandler.getMetadataPrefix())) { + // Make direct call to obtain the proprietary Dataverse metadata + // in JSON from the remote Dataverse server: + String metadataApiUrl = oaiHandler.getProprietaryDataverseMetadataURL(identifier); + logger.info("calling "+metadataApiUrl); + tempFile = retrieveProprietaryDataverseMetadata(httpClient, metadataApiUrl); + + } else { + FastGetRecord record = oaiHandler.runGetRecord(identifier); + errMessage = record.getErrorMessage(); + deleted = record.isDeleted(); + tempFile = record.getMetadataFile(); + } if (errMessage != null) { hdLogger.log(Level.SEVERE, "Error calling GetRecord - " + errMessage); - } else if (record.isDeleted()) { - hdLogger.info("Deleting harvesting dataset for "+identifier+", per the OAI server's instructions."); - Dataset dataset = datasetService.getDatasetByHarvestInfo(oaiHandler.getHarvestingClient().getDataverse(), identifier); - if (dataset != null) { - hdLogger.info("Deleting dataset " + dataset.getGlobalIdString()); - datasetService.deleteHarvestedDataset(dataset, dataverseRequest, hdLogger); - // TODO: - // check the status of that Delete - see if it actually succeeded - deletedIdentifiers.add(identifier); - } else { - hdLogger.info("No dataset found for "+identifier+", skipping delete. "); - } - + } else if (deleted) { + hdLogger.info("Deleting harvesting dataset for "+identifier+", per GetRecord."); + + deleteHarvestedDatasetIfExists(identifier, oaiHandler.getHarvestingClient().getDataverse(), dataverseRequest, deletedIdentifiers, hdLogger); } else { hdLogger.info("Successfully retrieved GetRecord response."); - tempFile = record.getMetadataFile(); PrintWriter cleanupLog; harvestedDataset = importService.doImportHarvestedDataset(dataverseRequest, oaiHandler.getHarvestingClient(), identifier, oaiHandler.getMetadataPrefix(), - record.getMetadataFile(), + tempFile, dateStamp, importCleanupLog); hdLogger.fine("Harvest Successful for identifier " + identifier); - hdLogger.fine("Size of this record: " + record.getMetadataFile().length()); - processedSizeThisBatch.add(record.getMetadataFile().length()); + hdLogger.fine("Size of this record: " + tempFile.length()); } } catch (Throwable e) { logGetRecordException(hdLogger, oaiHandler, identifier, e); errMessage = "Caught exception while executing GetRecord on "+identifier; - //logException(e, hdLogger); } finally { if (tempFile != null) { @@ -364,14 +348,12 @@ private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, P } } - // TODO: the message below is taken from DVN3; - figure out what it means... - // // If we got an Error from the OAI server or an exception happened during import, then // set recordErrorOccurred to true (if recordErrorOccurred is being used) // otherwise throw an exception (if recordErrorOccurred is not used, i.e null) if (errMessage != null) { - if (recordErrorOccurred != null) { + if (recordErrorOccurred != null) { recordErrorOccurred.setValue(true); } else { throw new EJBException(errMessage); @@ -380,7 +362,55 @@ private Long processRecord(DataverseRequest dataverseRequest, Logger hdLogger, P return harvestedDataset != null ? harvestedDataset.getId() : null; } - + + File retrieveProprietaryDataverseMetadata (HttpClient client, String remoteApiUrl) throws IOException { + + if (client == null) { + throw new IOException("Null Http Client, cannot make a call to obtain native metadata."); + } + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(remoteApiUrl)) + .GET() + .header("User-Agent", "DataverseHarvester/6.0") + .build(); + + HttpResponse response; + + try { + response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new IOException("Failed to connect to the remote dataverse server to obtain native metadata"); + } + + int responseCode = response.statusCode(); + + if (responseCode == HTTP_OK) { + File tempMetadataFile = File.createTempFile("meta", ".tmp"); + + try (InputStream inputStream = response.body(); + FileOutputStream outputStream = new FileOutputStream(tempMetadataFile);) { + inputStream.transferTo(outputStream); + return tempMetadataFile; + } + } + + throw new IOException("Failed to download native metadata from the remote dataverse server."); + } + + private void deleteHarvestedDatasetIfExists(String persistentIdentifier, Dataverse harvestingDataverse, DataverseRequest dataverseRequest, List deletedIdentifiers, Logger hdLogger) { + Dataset dataset = datasetService.getDatasetByHarvestInfo(harvestingDataverse, persistentIdentifier); + if (dataset != null) { + datasetService.deleteHarvestedDataset(dataset, dataverseRequest, hdLogger); + // TODO: + // check the status of that Delete - see if it actually succeeded + deletedIdentifiers.add(persistentIdentifier); + return; + } + hdLogger.info("No dataset found for " + persistentIdentifier + ", skipping delete. "); + } + private void logBeginOaiHarvest(Logger hdLogger, HarvestingClient harvestingClient) { hdLogger.log(Level.INFO, "BEGIN HARVEST, oaiUrl=" +harvestingClient.getHarvestingUrl() @@ -448,47 +478,5 @@ private void logException(Throwable e, Logger logger) { } while ((e = e.getCause()) != null); logger.severe(fullMessage); } - - /* - some dead code below: - this functionality has been moved into OaiHandler. - TODO: test that harvesting is still working and remove. - - private ServiceProvider getServiceProvider(String baseOaiUrl, Granularity oaiGranularity) { - Context context = new Context(); - - context.withBaseUrl(baseOaiUrl); - context.withGranularity(oaiGranularity); - context.withOAIClient(new HttpOAIClient(baseOaiUrl)); - - ServiceProvider serviceProvider = new ServiceProvider(context); - return serviceProvider; - } - */ - - /** - * Creates an XOAI parameters object for the ListIdentifiers call - * - * @param metadataPrefix - * @param set - * @param from - * @return ListIdentifiersParameters - */ - /* - private ListIdentifiersParameters buildParams(String metadataPrefix, String set, Date from) { - ListIdentifiersParameters mip = ListIdentifiersParameters.request(); - mip.withMetadataPrefix(metadataPrefix); - - if (from != null) { - mip.withFrom(from); - } - - if (set != null) { - mip.withSetSpec(set); - } - return mip; - } - */ - - + } diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java index d1aaea50793..c0a039e2d2b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java @@ -1,33 +1,27 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package edu.harvard.iq.dataverse.harvest.client.oai; -import com.lyncode.xoai.model.oaipmh.Description; -import com.lyncode.xoai.model.oaipmh.Granularity; -import com.lyncode.xoai.model.oaipmh.Header; -import com.lyncode.xoai.model.oaipmh.MetadataFormat; -import com.lyncode.xoai.model.oaipmh.Set; -import com.lyncode.xoai.serviceprovider.ServiceProvider; -import com.lyncode.xoai.serviceprovider.client.HttpOAIClient; -import com.lyncode.xoai.serviceprovider.exceptions.BadArgumentException; -import com.lyncode.xoai.serviceprovider.exceptions.InvalidOAIResponse; -import com.lyncode.xoai.serviceprovider.exceptions.NoSetHierarchyException; -import com.lyncode.xoai.serviceprovider.model.Context; -import com.lyncode.xoai.serviceprovider.parameters.ListIdentifiersParameters; +import io.gdcc.xoai.model.oaipmh.Granularity; +import io.gdcc.xoai.model.oaipmh.results.record.Header; +import io.gdcc.xoai.model.oaipmh.results.MetadataFormat; +import io.gdcc.xoai.model.oaipmh.results.Set; +import io.gdcc.xoai.serviceprovider.ServiceProvider; +import io.gdcc.xoai.serviceprovider.client.JdkHttpOaiClient; +import io.gdcc.xoai.serviceprovider.exceptions.BadArgumentException; +import io.gdcc.xoai.serviceprovider.exceptions.InvalidOAIResponse; +import io.gdcc.xoai.serviceprovider.exceptions.NoSetHierarchyException; +import io.gdcc.xoai.serviceprovider.exceptions.IdDoesNotExistException; +import io.gdcc.xoai.serviceprovider.model.Context; +import io.gdcc.xoai.serviceprovider.parameters.ListIdentifiersParameters; import edu.harvard.iq.dataverse.harvest.client.FastGetRecord; +import static edu.harvard.iq.dataverse.harvest.client.HarvesterServiceBean.DATAVERSE_PROPRIETARY_METADATA_API; import edu.harvard.iq.dataverse.harvest.client.HarvestingClient; import java.io.IOException; import java.io.Serializable; -import java.io.UnsupportedEncodingException; import javax.xml.parsers.ParserConfigurationException; import org.apache.commons.lang3.StringUtils; import org.xml.sax.SAXException; import javax.xml.transform.TransformerException; -import java.net.URLEncoder; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; @@ -74,8 +68,9 @@ public OaiHandler(HarvestingClient harvestingClient) throws OaiHandlerException this.harvestingClient = harvestingClient; } - private String baseOaiUrl; //= harvestingClient.getHarvestingUrl(); - private String metadataPrefix; // = harvestingClient.getMetadataPrefix(); + private String baseOaiUrl; + private String dataverseApiUrl; // if the remote server is a Dataverse and we access its native metadata + private String metadataPrefix; private String setName; private Date fromDate; private Boolean setListTruncated = false; @@ -124,7 +119,7 @@ public boolean isSetListTruncated() { return setListTruncated; } - private ServiceProvider getServiceProvider() throws OaiHandlerException { + public ServiceProvider getServiceProvider() throws OaiHandlerException { if (serviceProvider == null) { if (baseOaiUrl == null) { throw new OaiHandlerException("Could not instantiate Service Provider, missing OAI server URL."); @@ -133,8 +128,8 @@ private ServiceProvider getServiceProvider() throws OaiHandlerException { context.withBaseUrl(baseOaiUrl); context.withGranularity(Granularity.Second); - context.withOAIClient(new HttpOAIClient(baseOaiUrl)); - + // builds the client with the default parameters and the JDK http client: + context.withOAIClient(JdkHttpOaiClient.newBuilder().withBaseUrl(baseOaiUrl).build()); serviceProvider = new ServiceProvider(context); } @@ -199,6 +194,16 @@ public List runListMetadataFormats() throws OaiHandlerException { try { mfIter = sp.listMetadataFormats(); + } catch (IdDoesNotExistException idnee) { + // TODO: + // not sure why this exception is now thrown by List Metadata Formats (?) + // but looks like it was added in xoai 4.2. + // It appears that the answer is, they added it because you can + // call ListMetadataFormats on a specific identifier, optionally, + // and therefore it is possible to get back that response. Of course + // it will never be the case when calling it on an entire repository. + // But it's ok. + throw new OaiHandlerException("Id does not exist exception"); } catch (InvalidOAIResponse ior) { throw new OaiHandlerException("No valid response received from the OAI server."); } @@ -261,7 +266,7 @@ private ListIdentifiersParameters buildListIdentifiersParams() throws OaiHandler mip.withMetadataPrefix(metadataPrefix); if (this.fromDate != null) { - mip.withFrom(this.fromDate); + mip.withFrom(this.fromDate.toInstant()); } if (!StringUtils.isEmpty(this.setName)) { @@ -271,6 +276,18 @@ private ListIdentifiersParameters buildListIdentifiersParams() throws OaiHandler return mip; } + public String getProprietaryDataverseMetadataURL(String identifier) { + + if (dataverseApiUrl == null) { + dataverseApiUrl = baseOaiUrl.replaceFirst("/oai", ""); + } + + StringBuilder requestURL = new StringBuilder(dataverseApiUrl); + requestURL.append(DATAVERSE_PROPRIETARY_METADATA_API).append(identifier); + + return requestURL.toString(); + } + public void runIdentify() { // not implemented yet // (we will need it, both for validating the remote server, diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java index 057903d506a..6cdc4e5c277 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAIRecordServiceBean.java @@ -12,6 +12,7 @@ import edu.harvard.iq.dataverse.export.ExportService; import edu.harvard.iq.dataverse.search.IndexServiceBean; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; +import java.time.Instant; import java.io.File; import java.io.IOException; import java.sql.Timestamp; @@ -286,15 +287,15 @@ public List findOaiRecordsBySetName(String setName) { return findOaiRecordsBySetName(setName, null, null); } - public List findOaiRecordsBySetName(String setName, Date from, Date until) { + public List findOaiRecordsBySetName(String setName, Instant from, Instant until) { return findOaiRecordsBySetName(setName, from, until, false); } - public List findOaiRecordsNotInThisSet(String setName, Date from, Date until) { + public List findOaiRecordsNotInThisSet(String setName, Instant from, Instant until) { return findOaiRecordsBySetName(setName, from, until, true); } - public List findOaiRecordsBySetName(String setName, Date from, Date until, boolean excludeSet) { + public List findOaiRecordsBySetName(String setName, Instant from, Instant until, boolean excludeSet) { if (setName == null) { setName = ""; @@ -314,35 +315,18 @@ public List findOaiRecordsBySetName(String setName, Date from, Date u logger.fine("Query: "+queryString); TypedQuery query = em.createQuery(queryString, OAIRecord.class); - if (setName != null) { query.setParameter("setName",setName); } - if (from != null) { query.setParameter("from",from,TemporalType.TIMESTAMP); } - // In order to achieve inclusivity on the "until" matching, we need to do - // the following (if the "until" parameter is supplied): - // 1) if the supplied "until" parameter has the time portion (and is not just - // a date), we'll increment it by one second. This is because the time stamps we - // keep in the database also have fractional thousands of a second. - // So, a record may be shown as "T17:35:45", but in the database it is - // actually "17:35:45.356", so "<= 17:35:45" isn't going to work on this - // time stamp! - So we want to try "<= 17:35:45" instead. - // 2) if it's just a date, we'll increment it by a *full day*. Otherwise - // our database time stamp of 2016-10-23T17:35:45.123Z is NOT going to - // match " <= 2016-10-23" - which is really going to be interpreted as - // "2016-10-23T00:00:00.000". - // -- L.A. 4.6 + if (setName != null) { + query.setParameter("setName",setName); + } + // TODO: review and phase out the use of java.util.Date throughout this service. + + if (from != null) { + query.setParameter("from",Date.from(from),TemporalType.TIMESTAMP); + } if (until != null) { - // 24 * 3600 * 1000 = number of milliseconds in a day. - - if (until.getTime() % (24 * 3600 * 1000) == 0) { - // The supplied "until" parameter is a date, with no time - // portion. - logger.fine("plain date. incrementing by one day"); - until.setTime(until.getTime()+(24 * 3600 * 1000)); - } else { - logger.fine("date and time. incrementing by one second"); - until.setTime(until.getTime()+1000); - } - query.setParameter("until",until,TemporalType.TIMESTAMP); + Date untilDate = Date.from(until); + query.setParameter("until",untilDate,TemporalType.TIMESTAMP); } try { diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java index f300f02f70c..6b28c8808a0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/OAISetServiceBean.java @@ -67,7 +67,7 @@ public OAISet find(Object pk) { return em.find(OAISet.class, pk); } - public boolean specExists(String spec) { + public boolean setExists(String spec) { boolean specExists = false; OAISet set = findBySpec(spec); diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java index d8619c42dfa..5eacb1addb6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/web/servlet/OAIServlet.java @@ -5,26 +5,19 @@ */ package edu.harvard.iq.dataverse.harvest.server.web.servlet; -import com.lyncode.xml.exceptions.XmlWriteException; -import com.lyncode.xoai.dataprovider.builder.OAIRequestParametersBuilder; -import com.lyncode.xoai.dataprovider.exceptions.OAIException; -import com.lyncode.xoai.dataprovider.repository.Repository; -import com.lyncode.xoai.dataprovider.repository.RepositoryConfiguration; -import com.lyncode.xoai.dataprovider.model.Context; -import com.lyncode.xoai.dataprovider.model.MetadataFormat; -import com.lyncode.xoai.services.impl.SimpleResumptionTokenFormat; -import com.lyncode.xoai.dataprovider.repository.ItemRepository; -import com.lyncode.xoai.dataprovider.repository.SetRepository; -import com.lyncode.xoai.model.oaipmh.DeletedRecord; -import com.lyncode.xoai.model.oaipmh.Granularity; -import com.lyncode.xoai.model.oaipmh.OAIPMH; -import static com.lyncode.xoai.model.oaipmh.OAIPMH.NAMESPACE_URI; -import static com.lyncode.xoai.model.oaipmh.OAIPMH.SCHEMA_LOCATION; -import com.lyncode.xoai.model.oaipmh.Verb; -import com.lyncode.xoai.xml.XSISchema; - -import com.lyncode.xoai.xml.XmlWriter; -import static com.lyncode.xoai.xml.XmlWriter.defaultContext; +import io.gdcc.xoai.dataprovider.DataProvider; +import io.gdcc.xoai.dataprovider.repository.Repository; +import io.gdcc.xoai.dataprovider.repository.RepositoryConfiguration; +import io.gdcc.xoai.dataprovider.model.Context; +import io.gdcc.xoai.dataprovider.model.MetadataFormat; +import io.gdcc.xoai.dataprovider.request.RequestBuilder; +import io.gdcc.xoai.dataprovider.request.RequestBuilder.RawRequest; +import io.gdcc.xoai.dataprovider.repository.ItemRepository; +import io.gdcc.xoai.dataprovider.repository.SetRepository; +import io.gdcc.xoai.model.oaipmh.DeletedRecord; +import io.gdcc.xoai.model.oaipmh.OAIPMH; + +import io.gdcc.xoai.xml.XmlWriter; import edu.harvard.iq.dataverse.DatasetServiceBean; import edu.harvard.iq.dataverse.DataverseServiceBean; import edu.harvard.iq.dataverse.export.ExportException; @@ -32,24 +25,20 @@ import edu.harvard.iq.dataverse.export.spi.Exporter; import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean; import edu.harvard.iq.dataverse.harvest.server.OAISetServiceBean; -import edu.harvard.iq.dataverse.harvest.server.xoai.XdataProvider; -import edu.harvard.iq.dataverse.harvest.server.xoai.XgetRecord; -import edu.harvard.iq.dataverse.harvest.server.xoai.XitemRepository; -import edu.harvard.iq.dataverse.harvest.server.xoai.XsetRepository; -import edu.harvard.iq.dataverse.harvest.server.xoai.XlistRecords; +import edu.harvard.iq.dataverse.harvest.server.xoai.DataverseXoaiItemRepository; +import edu.harvard.iq.dataverse.harvest.server.xoai.DataverseXoaiSetRepository; import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.MailUtil; import edu.harvard.iq.dataverse.util.SystemConfig; +import io.gdcc.xoai.exceptions.OAIException; import org.apache.commons.lang3.StringUtils; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.OutputStream; -import java.util.Date; -import java.util.HashMap; import java.util.logging.Logger; import javax.ejb.EJB; +import javax.inject.Inject; +import org.eclipse.microprofile.config.inject.ConfigProperty; import javax.mail.internet.InternetAddress; import javax.servlet.ServletConfig; import javax.servlet.ServletException; @@ -57,12 +46,14 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.xml.stream.XMLStreamException; +import org.eclipse.microprofile.config.Config; +import org.eclipse.microprofile.config.ConfigProvider; /** * * @author Leonid Andreev * Dedicated servlet for handling OAI-PMH requests. - * Uses lyncode XOAI data provider implementation for serving content. + * Uses lyncode/Dspace/gdcc XOAI data provider implementation for serving content. * The servlet itself is somewhat influenced by the older OCLC OAIcat implementation. */ public class OAIServlet extends HttpServlet { @@ -79,23 +70,42 @@ public class OAIServlet extends HttpServlet { @EJB SystemConfig systemConfig; + + @Inject + @ConfigProperty(name = "dataverse.oai.server.maxidentifiers", defaultValue="100") + private Integer maxListIdentifiers; + + @Inject + @ConfigProperty(name = "dataverse.oai.server.maxsets", defaultValue="100") + private Integer maxListSets; + + @Inject + @ConfigProperty(name = "dataverse.oai.server.maxrecords", defaultValue="10") + private Integer maxListRecords; private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.web.servlet.OAIServlet"); - protected HashMap attributesMap = new HashMap(); - private static final String OAI_PMH = "OAI-PMH"; - private static final String RESPONSEDATE_FIELD = "responseDate"; - private static final String REQUEST_FIELD = "request"; + // If we are going to stick with this solution - of providing a minimalist + // xml record containing a link to the proprietary json metadata API for + // "dataverse json harvesting", we'll probably want to create minimalistic, + // but valid schemas for this format as well. + // UPDATE: we are keeping this hack on the server side purely for backward + // compatibility with older (pre v6) Dataverses who may still be using the + // format. Once v6 has been around for a while, we will get rid of it completely. + // Starting this version, harvesting clients will not be making GetRecord + // calls at all when using harvesting dataverse_json; instead they will only + // be calling ListIdentifiers, and then making direct calls to the export + // API of the remote Dataverse, to obtain the records in native json. This + // is how we should have implemented this in the first place, really. private static final String DATAVERSE_EXTENDED_METADATA_FORMAT = "dataverse_json"; - private static final String DATAVERSE_EXTENDED_METADATA_INFO = "Custom Dataverse metadata in JSON format (Dataverse4 to Dataverse4 harvesting only)"; - private static final String DATAVERSE_EXTENDED_METADATA_SCHEMA = "JSON schema pending"; - + private static final String DATAVERSE_EXTENDED_METADATA_NAMESPACE = "Custom Dataverse metadata in JSON format (Dataverse4 to Dataverse4 harvesting only)"; + private static final String DATAVERSE_EXTENDED_METADATA_SCHEMA = "JSON schema pending"; private Context xoaiContext; private SetRepository setRepository; private ItemRepository itemRepository; private RepositoryConfiguration repositoryConfiguration; private Repository xoaiRepository; - private XdataProvider dataProvider; + private DataProvider dataProvider; public void init(ServletConfig config) throws ServletException { super.init(config); @@ -106,18 +116,17 @@ public void init(ServletConfig config) throws ServletException { xoaiContext = addDataverseJsonMetadataFormat(xoaiContext); } - setRepository = new XsetRepository(setService); - itemRepository = new XitemRepository(recordService, datasetService); + setRepository = new DataverseXoaiSetRepository(setService); + itemRepository = new DataverseXoaiItemRepository(recordService, datasetService, systemConfig.getDataverseSiteUrl()); repositoryConfiguration = createRepositoryConfiguration(); - + xoaiRepository = new Repository() .withSetRepository(setRepository) .withItemRepository(itemRepository) - .withResumptionTokenFormatter(new SimpleResumptionTokenFormat()) .withConfiguration(repositoryConfiguration); - dataProvider = new XdataProvider(getXoaiContext(), getXoaiRepository()); + dataProvider = new DataProvider(getXoaiContext(), getXoaiRepository()); } private Context createContext() { @@ -145,6 +154,7 @@ private void addSupportedMetadataFormats(Context context) { metadataFormat = MetadataFormat.metadataFormat(formatName); metadataFormat.withNamespace(exporter.getXMLNameSpace()); metadataFormat.withSchemaLocation(exporter.getXMLSchemaLocation()); + } catch (ExportException ex) { metadataFormat = null; } @@ -153,12 +163,11 @@ private void addSupportedMetadataFormats(Context context) { } } } - //return context; } private Context addDataverseJsonMetadataFormat(Context context) { MetadataFormat metadataFormat = MetadataFormat.metadataFormat(DATAVERSE_EXTENDED_METADATA_FORMAT); - metadataFormat.withNamespace(DATAVERSE_EXTENDED_METADATA_INFO); + metadataFormat.withNamespace(DATAVERSE_EXTENDED_METADATA_NAMESPACE); metadataFormat.withSchemaLocation(DATAVERSE_EXTENDED_METADATA_SCHEMA); context.withMetadataFormat(metadataFormat); return context; @@ -169,26 +178,30 @@ private boolean isDataverseOaiExtensionsSupported() { } private RepositoryConfiguration createRepositoryConfiguration() { - // TODO: - // some of the settings below - such as the max list numbers - - // need to be configurable! + Config config = ConfigProvider.getConfig(); + String repositoryName = config.getOptionalValue("dataverse.oai.server.repositoryname", String.class).orElse(""); - String dataverseName = dataverseService.getRootDataverseName(); - String repositoryName = StringUtils.isEmpty(dataverseName) || "Root".equals(dataverseName) ? "Test Dataverse OAI Archive" : dataverseName + " Dataverse OAI Archive"; - InternetAddress internetAddress = MailUtil.parseSystemAddress(settingsService.getValueForKey(SettingsServiceBean.Key.SystemEmail)); - - RepositoryConfiguration repositoryConfiguration = new RepositoryConfiguration() + if (repositoryName == null || repositoryName.isEmpty()) { + String dataverseName = dataverseService.getRootDataverseName(); + repositoryName = StringUtils.isEmpty(dataverseName) || "Root".equals(dataverseName) ? "Test Dataverse OAI Archive" : dataverseName + " Dataverse OAI Archive"; + } + // The admin email address associated with this installation: + // (Note: if the setting does not exist, we are going to assume that they + // have a reason not to want to advertise their email address, so no + // email will be shown in the output of Identify. + InternetAddress systemEmailAddress = MailUtil.parseSystemAddress(settingsService.getValueForKey(SettingsServiceBean.Key.SystemEmail)); + + RepositoryConfiguration repositoryConfiguration = RepositoryConfiguration.defaults() + .withEnableMetadataAttributes(true) .withRepositoryName(repositoryName) .withBaseUrl(systemConfig.getDataverseSiteUrl()+"/oai") - .withCompression("gzip") // ? - .withCompression("deflate") // ? - .withAdminEmail(internetAddress != null ? internetAddress.getAddress() : null) + .withCompression("gzip") + .withCompression("deflate") + .withAdminEmail(systemEmailAddress != null ? systemEmailAddress.getAddress() : null) .withDeleteMethod(DeletedRecord.TRANSIENT) - .withGranularity(Granularity.Second) - .withMaxListIdentifiers(100) - .withMaxListRecords(100) - .withMaxListSets(100) - .withEarliestDate(new Date()); + .withMaxListIdentifiers(maxListIdentifiers) + .withMaxListRecords(maxListRecords) + .withMaxListSets(maxListSets); return repositoryConfiguration; } @@ -221,9 +234,9 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { processRequest(request, response); } - + - private void processRequest(HttpServletRequest request, HttpServletResponse response) + private void processRequest(HttpServletRequest httpServletRequest, HttpServletResponse response) throws ServletException, IOException { try { @@ -233,150 +246,22 @@ private void processRequest(HttpServletRequest request, HttpServletResponse resp "Sorry. OAI Service is disabled on this Dataverse node."); return; } + + RawRequest rawRequest = RequestBuilder.buildRawRequest(httpServletRequest.getParameterMap()); - OAIRequestParametersBuilder parametersBuilder = newXoaiRequest(); - - for (Object p : request.getParameterMap().keySet()) { - String parameterName = (String)p; - String parameterValue = request.getParameter(parameterName); - parametersBuilder = parametersBuilder.with(parameterName, parameterValue); - - } - - OAIPMH handle = dataProvider.handle(parametersBuilder); + OAIPMH handle = dataProvider.handle(rawRequest); response.setContentType("text/xml;charset=UTF-8"); - - if (isGetRecord(request) && !handle.hasErrors()) { - writeGetRecord(response, handle); - } else if (isListRecords(request) && !handle.hasErrors()) { - writeListRecords(response, handle); - } else { - XmlWriter xmlWriter = new XmlWriter(response.getOutputStream()); + + try (XmlWriter xmlWriter = new XmlWriter(response.getOutputStream(), repositoryConfiguration);) { xmlWriter.write(handle); - xmlWriter.flush(); - xmlWriter.close(); } - } catch (IOException ex) { - logger.warning("IO exception in Get; "+ex.getMessage()); - throw new ServletException ("IO Exception in Get", ex); - } catch (OAIException oex) { - logger.warning("OAI exception in Get; "+oex.getMessage()); - throw new ServletException ("OAI Exception in Get", oex); - } catch (XMLStreamException xse) { - logger.warning("XML Stream exception in Get; "+xse.getMessage()); - throw new ServletException ("XML Stream Exception in Get", xse); - } catch (XmlWriteException xwe) { - logger.warning("XML Write exception in Get; "+xwe.getMessage()); - throw new ServletException ("XML Write Exception in Get", xwe); - } catch (Exception e) { - logger.warning("Unknown exception in Get; "+e.getMessage()); - throw new ServletException ("Unknown servlet exception in Get.", e); + } catch (XMLStreamException | OAIException e) { + throw new ServletException (e); } } - // Custom methods for the potentially expensive GetRecord and ListRecords requests: - - private void writeListRecords(HttpServletResponse response, OAIPMH handle) throws IOException { - OutputStream outputStream = response.getOutputStream(); - - outputStream.write(oaiPmhResponseToString(handle).getBytes()); - - Verb verb = handle.getVerb(); - - if (verb == null) { - throw new IOException("An error or a valid response must be set"); - } - - if (!verb.getType().equals(Verb.Type.ListRecords)) { - throw new IOException("writeListRecords() called on a non-ListRecords verb"); - } - - outputStream.write(("<" + verb.getType().displayName() + ">").getBytes()); - - outputStream.flush(); - - ((XlistRecords) verb).writeToStream(outputStream); - - outputStream.write(("").getBytes()); - outputStream.write(("\n").getBytes()); - - outputStream.flush(); - outputStream.close(); - - } - - private void writeGetRecord(HttpServletResponse response, OAIPMH handle) throws IOException, XmlWriteException, XMLStreamException { - OutputStream outputStream = response.getOutputStream(); - - outputStream.write(oaiPmhResponseToString(handle).getBytes()); - - Verb verb = handle.getVerb(); - - if (verb == null) { - throw new IOException("An error or a valid response must be set"); - } - - if (!verb.getType().equals(Verb.Type.GetRecord)) { - throw new IOException("writeListRecords() called on a non-GetRecord verb"); - } - - outputStream.write(("<" + verb.getType().displayName() + ">").getBytes()); - - outputStream.flush(); - - ((XgetRecord) verb).writeToStream(outputStream); - - outputStream.write(("").getBytes()); - outputStream.write(("\n").getBytes()); - - outputStream.flush(); - outputStream.close(); - - } - - // This function produces the string representation of the top level, - // "service" record of an OAIPMH response (i.e., the header that precedes - // the actual "payload" record, such as , , - // , etc. - - private String oaiPmhResponseToString(OAIPMH handle) { - try { - ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(); - XmlWriter writer = new XmlWriter(byteOutputStream, defaultContext()); - - writer.writeStartElement(OAI_PMH); - writer.writeDefaultNamespace(NAMESPACE_URI); - writer.writeNamespace(XSISchema.PREFIX, XSISchema.NAMESPACE_URI); - writer.writeAttribute(XSISchema.PREFIX, XSISchema.NAMESPACE_URI, "schemaLocation", - NAMESPACE_URI + " " + SCHEMA_LOCATION); - - writer.writeElement(RESPONSEDATE_FIELD, handle.getResponseDate(), Granularity.Second); - writer.writeElement(REQUEST_FIELD, handle.getRequest()); - writer.writeEndElement(); - writer.flush(); - writer.close(); - - String ret = byteOutputStream.toString().replaceFirst("", ""); - - return ret; - } catch (Exception ex) { - logger.warning("caught exception trying to convert an OAIPMH response header to string: " + ex.getMessage()); - ex.printStackTrace(); - return null; - } - } - - private boolean isGetRecord(HttpServletRequest request) { - return "GetRecord".equals(request.getParameter("verb")); - - } - - private boolean isListRecords(HttpServletRequest request) { - return "ListRecords".equals(request.getParameter("verb")); - } - protected Context getXoaiContext () { return xoaiContext; } @@ -385,11 +270,6 @@ protected Repository getXoaiRepository() { return xoaiRepository; } - protected OAIRequestParametersBuilder newXoaiRequest() { - return new OAIRequestParametersBuilder(); - } - - public boolean isHarvestingServerEnabled() { return systemConfig.isOAIServerEnabled(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xitem.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItem.java similarity index 63% rename from src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xitem.java rename to src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItem.java index bd7a35ddb79..ecfd2f82369 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xitem.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItem.java @@ -1,18 +1,14 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package edu.harvard.iq.dataverse.harvest.server.xoai; -import com.lyncode.xoai.dataprovider.model.Item; -import com.lyncode.xoai.dataprovider.model.Set; -import com.lyncode.xoai.model.oaipmh.About; +import io.gdcc.xoai.dataprovider.model.Item; +import io.gdcc.xoai.dataprovider.model.Set; +import io.gdcc.xoai.model.oaipmh.results.record.Metadata; +import io.gdcc.xoai.model.oaipmh.results.record.About; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.harvest.server.OAIRecord; import edu.harvard.iq.dataverse.util.StringUtil; +import java.time.Instant; import java.util.ArrayList; -import java.util.Date; import java.util.List; @@ -20,13 +16,13 @@ * * @author Leonid Andreev * - * This is an implemention of an Lyncode XOAI Item; + * This is an implemention of a Lyncode/DSpace/gdcc XOAI Item. * You can think of it as an XOAI Item wrapper around the * Dataverse OAIRecord entity. */ -public class Xitem implements Item { +public final class DataverseXoaiItem implements Item { - public Xitem(OAIRecord oaiRecord) { + public DataverseXoaiItem(OAIRecord oaiRecord) { super(); this.oaiRecord = oaiRecord; oaisets = new ArrayList<>(); @@ -34,7 +30,7 @@ public Xitem(OAIRecord oaiRecord) { oaisets.add(new Set(oaiRecord.getSetName())); } } - + private OAIRecord oaiRecord; public OAIRecord getOaiRecord() { @@ -51,19 +47,21 @@ public Dataset getDataset() { return dataset; } - public Xitem withDataset(Dataset dataset) { + public DataverseXoaiItem withDataset(Dataset dataset) { this.dataset = dataset; return this; } + private Metadata metadata; + @Override - public List getAbout() { - return null; + public Metadata getMetadata() { + return metadata; } - - @Override - public Xmetadata getMetadata() { - return new Xmetadata((String)null); + + public DataverseXoaiItem withMetadata(Metadata metadata) { + this.metadata = metadata; + return this; } @Override @@ -72,8 +70,8 @@ public String getIdentifier() { } @Override - public Date getDatestamp() { - return oaiRecord.getLastUpdateTime(); + public Instant getDatestamp() { + return oaiRecord.getLastUpdateTime().toInstant(); } private List oaisets; @@ -82,12 +80,10 @@ public Date getDatestamp() { public List getSets() { return oaisets; - } @Override public boolean isDeleted() { return oaiRecord.isRemoved(); } - } diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java new file mode 100644 index 00000000000..faf3cf9ddc4 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiItemRepository.java @@ -0,0 +1,270 @@ +package edu.harvard.iq.dataverse.harvest.server.xoai; + +import io.gdcc.xoai.dataprovider.exceptions.handler.IdDoesNotExistException; +import io.gdcc.xoai.dataprovider.filter.ScopedFilter; +import io.gdcc.xoai.dataprovider.model.Item; +import io.gdcc.xoai.dataprovider.model.ItemIdentifier; +import io.gdcc.xoai.dataprovider.model.Set; +import io.gdcc.xoai.dataprovider.model.MetadataFormat; +import io.gdcc.xoai.dataprovider.repository.ItemRepository; +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetServiceBean; +import edu.harvard.iq.dataverse.export.ExportException; +import edu.harvard.iq.dataverse.export.ExportService; +import edu.harvard.iq.dataverse.harvest.server.OAIRecord; +import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean; +import edu.harvard.iq.dataverse.util.StringUtil; +import io.gdcc.xoai.dataprovider.exceptions.handler.HandlerException; +import io.gdcc.xoai.dataprovider.exceptions.handler.NoMetadataFormatsException; +import io.gdcc.xoai.dataprovider.repository.ResultsPage; +import io.gdcc.xoai.model.oaipmh.ResumptionToken; +import io.gdcc.xoai.model.oaipmh.results.record.Metadata; +import io.gdcc.xoai.xml.EchoElement; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.time.Instant; +import java.util.List; +import java.util.logging.Logger; + +/** + * + * @author Leonid Andreev + * Implements an XOAI "Item Repository". Retrieves Dataverse "OAIRecords" + * representing harvestable local datasets and translates them into + * XOAI "items". + */ + +public class DataverseXoaiItemRepository implements ItemRepository { + private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.xoai.DataverseXoaiItemRepository"); + + private final OAIRecordServiceBean recordService; + private final DatasetServiceBean datasetService; + private final String serverUrl; + + public DataverseXoaiItemRepository (OAIRecordServiceBean recordService, DatasetServiceBean datasetService, String serverUrl) { + this.recordService = recordService; + this.datasetService = datasetService; + this.serverUrl = serverUrl; + } + + @Override + public ItemIdentifier getItem(String identifier) throws IdDoesNotExistException { + // This method is called when ListMetadataFormats request specifies + // the identifier, requesting the formats available for this specific record. + // In our case, under the current implementation, we need to simply look + // up if the record exists; if it does, all the OAI formats that we serve + // should be available for this record. + + List oaiRecords = recordService.findOaiRecordsByGlobalId(identifier); + if (oaiRecords != null && !oaiRecords.isEmpty()) { + for (OAIRecord oaiRecord : oaiRecords) { + // We can return the first *active* record we find for this identifier. + if (!oaiRecord.isRemoved()) { + return new DataverseXoaiItem(oaiRecord); + } + } + } + + throw new IdDoesNotExistException(); + } + + @Override + public Item getItem(String identifier, MetadataFormat metadataFormat) throws HandlerException { + logger.fine("getItem; calling findOaiRecordsByGlobalId, identifier " + identifier); + + if (metadataFormat == null) { + throw new NoMetadataFormatsException("Metadata Format is Required"); + } + + List oaiRecords = recordService.findOaiRecordsByGlobalId(identifier); + if (oaiRecords != null && !oaiRecords.isEmpty()) { + DataverseXoaiItem xoaiItem = null; + for (OAIRecord oaiRecord : oaiRecords) { + if (xoaiItem == null) { + xoaiItem = new DataverseXoaiItem(oaiRecord); + xoaiItem = addMetadata(xoaiItem, metadataFormat); + } else { + // Adding extra set specs to the XOAI Item, if this oaiRecord + // is part of multiple sets: + if (!StringUtil.isEmpty(oaiRecord.getSetName())) { + xoaiItem.getSets().add(new Set(oaiRecord.getSetName())); + } + } + } + if (xoaiItem != null) { + return xoaiItem; + } + } + + throw new IdDoesNotExistException(); + } + + @Override + public ResultsPage getItemIdentifiers(List filters, MetadataFormat metadataFormat, int maxResponseLength, ResumptionToken.Value resumptionToken) throws HandlerException { + + return (ResultsPage)getRepositoryRecords(metadataFormat, maxResponseLength, resumptionToken, false); + + } + + @Override + public ResultsPage getItems(List filters, MetadataFormat metadataFormat, int maxResponseLength, ResumptionToken.Value resumptionToken) throws HandlerException { + + return (ResultsPage)getRepositoryRecords(metadataFormat, maxResponseLength, resumptionToken, true); + } + + private ResultsPage getRepositoryRecords ( + MetadataFormat metadataFormat, + int maxResponseLength, + ResumptionToken.Value resumptionToken, + boolean fullItems) throws HandlerException { + + int offset = Long.valueOf(resumptionToken.getOffset()).intValue(); + String setSpec = resumptionToken.getSetSpec(); + Instant from = resumptionToken.getFrom(); + Instant until = resumptionToken.getUntil(); + + boolean hasMore = false; + + logger.fine("calling " + (fullItems ? "getItems" : "getItemIdentifiers") + + "; offset=" + offset + + ", length=" + maxResponseLength + + ", setSpec=" + setSpec + + ", from=" + from + + ", until=" + until); + + List oaiRecords = recordService.findOaiRecordsBySetName(setSpec, from, until); + + List xoaiItems = new ArrayList<>(); + + if (oaiRecords != null && !oaiRecords.isEmpty()) { + logger.fine("total " + oaiRecords.size() + " records returned"); + + for (int i = offset; i < offset + maxResponseLength && i < oaiRecords.size(); i++) { + OAIRecord record = oaiRecords.get(i); + DataverseXoaiItem xoaiItem = new DataverseXoaiItem(record); + + if (fullItems) { + // If we are cooking "full" Items (for the ListRecords verb), + // add the metadata to the item object (if not a deleted + // record, if available, etc.): + xoaiItem = addMetadata(xoaiItem, metadataFormat); + } + + xoaiItems.add(xoaiItem); + } + + // Run a second pass, looking for records in this set that occur + // in *other* sets. Then we'll add these multiple sets to the + // formatted output in the header: + addExtraSets(xoaiItems, setSpec, from, until); + + hasMore = offset + maxResponseLength < oaiRecords.size(); + + ResultsPage result = new ResultsPage(resumptionToken, hasMore, xoaiItems, oaiRecords.size()); + logger.fine("returning result with " + xoaiItems.size() + " items."); + return result; + } + + return new ResultsPage(resumptionToken, false, xoaiItems, 0); + } + + private void addExtraSets(Object xoaiItemsList, String setSpec, Instant from, Instant until) { + + List xoaiItems = (List)xoaiItemsList; + + List oaiRecords = recordService.findOaiRecordsNotInThisSet(setSpec, from, until); + + if (oaiRecords == null || oaiRecords.isEmpty()) { + return; + } + + // Make a second pass through the list of xoaiItems already found for this set, + // and add any other sets in which this item occurs: + + int j = 0; + for (int i = 0; i < xoaiItems.size(); i++) { + // fast-forward the second list, until we find a oaiRecord with this identifier, + // or until we are past this oaiRecord (both lists are sorted alphabetically by + // the identifier: + DataverseXoaiItem xitem = xoaiItems.get(i); + + while (j < oaiRecords.size() && xitem.getIdentifier().compareTo(oaiRecords.get(j).getGlobalId()) > 0) { + j++; + } + + while (j < oaiRecords.size() && xitem.getIdentifier().equals(oaiRecords.get(j).getGlobalId())) { + xoaiItems.get(i).getSets().add(new Set(oaiRecords.get(j).getSetName())); + j++; + } + } + } + + private DataverseXoaiItem addMetadata(DataverseXoaiItem xoaiItem, MetadataFormat metadataFormat) { + // This may be a "deleted" record - i.e., a oaiRecord kept in + // the OAI set for a dataset that's no longer in this Dataverse. + // (it serves to tell the remote client to delete it from their + // holdings too). + // If this is the case here, there's nothing we need to do for this item. + // If not, if it's a live record, let's try to look up the dataset and + // open the pre-generated metadata stream. + + if (!xoaiItem.isDeleted()) { + Dataset dataset = datasetService.findByGlobalId(xoaiItem.getIdentifier()); + if (dataset != null) { + try { + Metadata metadata = getDatasetMetadata(dataset, metadataFormat.getPrefix()); + xoaiItem.withDataset(dataset).withMetadata(metadata); + } catch (ExportException | IOException ex) { + // This is not supposed to happen in normal operations; + // since by design only the datasets for which the metadata + // records have been pre-generated ("exported") should be + // served as "OAI Record". But, things happen. If for one + // reason or another that cached metadata file is no longer there, + // we are not going to serve any metadata for this oaiRecord, + // BUT we are going to include it marked as "deleted" + // (because skipping it could potentially mess up the + // counts and offsets, in a resumption token scenario. + xoaiItem.getOaiRecord().setRemoved(true); + } + } else { + // If dataset (somehow) no longer exists (again, this is + // not supposed to happen), we will serve the oaiRecord, + // marked as "deleted" and without any metadata. + // We can't just skip it, because that could mess up the + // counts and offsets, in a resumption token scenario. + xoaiItem.getOaiRecord().setRemoved(true); + } + } + return xoaiItem; + } + + private Metadata getDatasetMetadata(Dataset dataset, String metadataPrefix) throws ExportException, IOException { + Metadata metadata; + + if ("dataverse_json".equals(metadataPrefix)) { + // Solely for backward compatibility, for older Dataverse harvesting clients + // that may still be relying on harvesting "dataverse_json"; + // we will want to eventually get rid of this hack! + // @Deprecated(since = "5.0") + metadata = new Metadata( + new EchoElement("custom metadata")) + .withAttribute("directApiCall", customDataverseJsonApiUri(dataset.getGlobalId().asString())); + + } else { + InputStream pregeneratedMetadataStream; + pregeneratedMetadataStream = ExportService.getInstance().getExport(dataset, metadataPrefix); + + metadata = Metadata.copyFromStream(pregeneratedMetadataStream); + } + return metadata; + } + + private String customDataverseJsonApiUri(String identifier) { + String ret = serverUrl + + "/api/datasets/export?exporter=dataverse_json&persistentId=" + + identifier; + + return ret; + } +} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XsetRepository.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiSetRepository.java similarity index 56% rename from src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XsetRepository.java rename to src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiSetRepository.java index 8e58e1bbf9a..b4e275b6059 100644 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XsetRepository.java +++ b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/DataverseXoaiSetRepository.java @@ -1,15 +1,9 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ package edu.harvard.iq.dataverse.harvest.server.xoai; -import com.lyncode.xoai.model.xoai.Element; -import com.lyncode.xoai.dataprovider.repository.SetRepository; -import com.lyncode.xoai.dataprovider.handlers.results.ListSetsResult; -import com.lyncode.xoai.dataprovider.model.Set; -import com.lyncode.xoai.model.xoai.XOAIMetadata; +import io.gdcc.xoai.model.xoai.Element; +import io.gdcc.xoai.dataprovider.repository.SetRepository; +import io.gdcc.xoai.dataprovider.model.Set; +import io.gdcc.xoai.model.xoai.XOAIMetadata; import edu.harvard.iq.dataverse.harvest.server.OAISet; import edu.harvard.iq.dataverse.harvest.server.OAISetServiceBean; @@ -21,13 +15,12 @@ * * @author Leonid Andreev */ -public class XsetRepository implements SetRepository { - private static Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.xoai.XsetRepository"); +public class DataverseXoaiSetRepository implements SetRepository { + private static final Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.xoai.DataverseXoaiSetRepository"); private OAISetServiceBean setService; - public XsetRepository (OAISetServiceBean setService) { - super(); + public DataverseXoaiSetRepository (OAISetServiceBean setService) { this.setService = setService; } @@ -42,7 +35,6 @@ public void setSetService(OAISetServiceBean setService) { @Override public boolean supportSets() { - logger.fine("calling supportSets()"); List dataverseOAISets = setService.findAllNamedSets(); if (dataverseOAISets == null || dataverseOAISets.isEmpty()) { @@ -52,7 +44,7 @@ public boolean supportSets() { } @Override - public ListSetsResult retrieveSets(int offset, int length) { + public List getSets() { logger.fine("calling retrieveSets()"); List dataverseOAISets = setService.findAllNamedSets(); List XOAISets = new ArrayList(); @@ -62,25 +54,21 @@ public ListSetsResult retrieveSets(int offset, int length) { OAISet dataverseSet = dataverseOAISets.get(i); Set xoaiSet = new Set(dataverseSet.getSpec()); xoaiSet.withName(dataverseSet.getName()); - XOAIMetadata xMetadata = new XOAIMetadata(); + XOAIMetadata xoaiMetadata = new XOAIMetadata(); Element element = new Element("description"); element.withField("description", dataverseSet.getDescription()); - xMetadata.getElements().add(element); - xoaiSet.withDescription(xMetadata); + xoaiMetadata.getElements().add(element); + xoaiSet.withDescription(xoaiMetadata); XOAISets.add(xoaiSet); } } - return new ListSetsResult(offset + length < XOAISets.size(), XOAISets.subList(offset, Math.min(offset + length, XOAISets.size()))); + return XOAISets; } @Override public boolean exists(String setSpec) { - //for (Set s : this.sets) - // if (s.getSpec().equals(setSpec)) - // return true; - - return false; + return setService.setExists(setSpec); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XdataProvider.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XdataProvider.java deleted file mode 100644 index 8ba8fe96bec..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XdataProvider.java +++ /dev/null @@ -1,116 +0,0 @@ -package edu.harvard.iq.dataverse.harvest.server.xoai; - - -import com.lyncode.builder.Builder; -import com.lyncode.xoai.dataprovider.exceptions.*; -import com.lyncode.xoai.dataprovider.handlers.*; -import com.lyncode.xoai.exceptions.InvalidResumptionTokenException; -import com.lyncode.xoai.dataprovider.model.Context; -import com.lyncode.xoai.model.oaipmh.Identify; -import com.lyncode.xoai.model.oaipmh.OAIPMH; -import com.lyncode.xoai.model.oaipmh.Request; -import com.lyncode.xoai.dataprovider.parameters.OAICompiledRequest; -import com.lyncode.xoai.dataprovider.parameters.OAIRequest; -import com.lyncode.xoai.dataprovider.repository.Repository; -import com.lyncode.xoai.services.api.DateProvider; -import com.lyncode.xoai.services.impl.UTCDateProvider; -import static com.lyncode.xoai.dataprovider.parameters.OAIRequest.Parameter.*; - -import java.util.logging.Logger; - -/** - * - * @author Leonid Andreev - */ -public class XdataProvider { - private static Logger log = Logger.getLogger(XdataProvider.class.getCanonicalName()); - - public static XdataProvider dataProvider (Context context, Repository repository) { - return new XdataProvider(context, repository); - } - - private Repository repository; - private DateProvider dateProvider; - - private final IdentifyHandler identifyHandler; - private final XgetRecordHandler getRecordHandler; - private final ListSetsHandler listSetsHandler; - private final XlistRecordsHandler listRecordsHandler; - private final ListIdentifiersHandler listIdentifiersHandler; - private final ListMetadataFormatsHandler listMetadataFormatsHandler; - private final ErrorHandler errorsHandler; - - public XdataProvider (Context context, Repository repository) { - this.repository = repository; - this.dateProvider = new UTCDateProvider(); - - this.identifyHandler = new IdentifyHandler(context, repository); - this.listSetsHandler = new ListSetsHandler(context, repository); - this.listMetadataFormatsHandler = new ListMetadataFormatsHandler(context, repository); - this.listRecordsHandler = new XlistRecordsHandler(context, repository); - this.listIdentifiersHandler = new ListIdentifiersHandler(context, repository); - //this.getRecordHandler = new GetRecordHandler(context, repository); - this.getRecordHandler = new XgetRecordHandler(context, repository); - this.errorsHandler = new ErrorHandler(); - } - - public OAIPMH handle (Builder builder) throws OAIException { - return handle(builder.build()); - } - - public OAIPMH handle (OAIRequest requestParameters) throws OAIException { - log.fine("Handling OAI request"); - Request request = new Request(repository.getConfiguration().getBaseUrl()) - .withVerbType(requestParameters.get(Verb)) - .withResumptionToken(requestParameters.get(ResumptionToken)) - .withIdentifier(requestParameters.get(Identifier)) - .withMetadataPrefix(requestParameters.get(MetadataPrefix)) - .withSet(requestParameters.get(Set)) - .withFrom(requestParameters.get(From)) - .withUntil(requestParameters.get(Until)); - - OAIPMH response = new OAIPMH() - .withRequest(request) - .withResponseDate(dateProvider.now()); - try { - OAICompiledRequest parameters = compileParameters(requestParameters); - - switch (request.getVerbType()) { - case Identify: - Identify identify = identifyHandler.handle(parameters); - identify.getDescriptions().clear(); // We don't want to use the default description - response.withVerb(identify); - break; - case ListSets: - response.withVerb(listSetsHandler.handle(parameters)); - break; - case ListMetadataFormats: - response.withVerb(listMetadataFormatsHandler.handle(parameters)); - break; - case GetRecord: - response.withVerb(getRecordHandler.handle(parameters)); - break; - case ListIdentifiers: - response.withVerb(listIdentifiersHandler.handle(parameters)); - break; - case ListRecords: - response.withVerb(listRecordsHandler.handle(parameters)); - break; - } - } catch (HandlerException e) { - log.fine("HandlerException when executing "+request.getVerbType()+": " + e.getMessage()); - response.withError(errorsHandler.handle(e)); - } - - return response; - } - - private OAICompiledRequest compileParameters(OAIRequest requestParameters) throws IllegalVerbException, UnknownParameterException, BadArgumentException, DuplicateDefinitionException, BadResumptionToken { - try { - return requestParameters.compile(); - } catch (InvalidResumptionTokenException e) { - throw new BadResumptionToken("The resumption token is invalid"); - } - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XgetRecord.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XgetRecord.java deleted file mode 100644 index d86f555d105..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XgetRecord.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xoai.model.oaipmh.GetRecord; -import com.lyncode.xoai.model.oaipmh.Record; -import java.io.IOException; -import java.io.OutputStream; - -/** - * - * @author Leonid Andreev - * - * This is the Dataverse extension of XOAI GetRecord, - * optimized to stream individual records to the output directly - */ - -public class XgetRecord extends GetRecord { - private static final String RECORD_FIELD = "record"; - private static final String RECORD_START_ELEMENT = "<"+RECORD_FIELD+">"; - private static final String RECORD_CLOSE_ELEMENT = ""; - private static final String RESUMPTION_TOKEN_FIELD = "resumptionToken"; - private static final String EXPIRATION_DATE_ATTRIBUTE = "expirationDate"; - private static final String COMPLETE_LIST_SIZE_ATTRIBUTE = "completeListSize"; - private static final String CURSOR_ATTRIBUTE = "cursor"; - - - public XgetRecord(Xrecord record) { - super(record); - } - - public void writeToStream(OutputStream outputStream) throws IOException { - - if (this.getRecord() == null) { - throw new IOException("XgetRecord: null Record"); - } - Xrecord xrecord = (Xrecord) this.getRecord(); - - outputStream.write(RECORD_START_ELEMENT.getBytes()); - outputStream.flush(); - - xrecord.writeToStream(outputStream); - - outputStream.write(RECORD_CLOSE_ELEMENT.getBytes()); - outputStream.flush(); - - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XgetRecordHandler.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XgetRecordHandler.java deleted file mode 100644 index ba28894482a..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XgetRecordHandler.java +++ /dev/null @@ -1,92 +0,0 @@ -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xml.exceptions.XmlWriteException; -import com.lyncode.xoai.dataprovider.exceptions.BadArgumentException; -import com.lyncode.xoai.dataprovider.exceptions.CannotDisseminateFormatException; -import com.lyncode.xoai.dataprovider.parameters.OAICompiledRequest; -import com.lyncode.xoai.dataprovider.exceptions.CannotDisseminateRecordException; -import com.lyncode.xoai.dataprovider.exceptions.HandlerException; -import com.lyncode.xoai.dataprovider.exceptions.IdDoesNotExistException; -import com.lyncode.xoai.dataprovider.exceptions.NoMetadataFormatsException; -import com.lyncode.xoai.dataprovider.exceptions.OAIException; -import com.lyncode.xoai.dataprovider.handlers.VerbHandler; -import com.lyncode.xoai.dataprovider.handlers.helpers.ItemHelper; -import com.lyncode.xoai.dataprovider.model.Context; -import com.lyncode.xoai.dataprovider.model.Item; -import com.lyncode.xoai.dataprovider.model.MetadataFormat; -import com.lyncode.xoai.dataprovider.model.Set; -import com.lyncode.xoai.model.oaipmh.*; -import com.lyncode.xoai.dataprovider.repository.Repository; -import com.lyncode.xoai.xml.XSLPipeline; -import com.lyncode.xoai.xml.XmlWriter; -import edu.harvard.iq.dataverse.Dataset; - -import javax.xml.stream.XMLStreamException; -import javax.xml.transform.TransformerException; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.logging.Logger; - -/* - * @author Leonid Andreev -*/ -public class XgetRecordHandler extends VerbHandler { - private static Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.xoai.XgetRecordHandler"); - public XgetRecordHandler(Context context, Repository repository) { - super(context, repository); - } - - @Override - public GetRecord handle(OAICompiledRequest parameters) throws OAIException, HandlerException { - - MetadataFormat format = getContext().formatForPrefix(parameters.getMetadataPrefix()); - Item item = getRepository().getItemRepository().getItem(parameters.getIdentifier()); - - if (getContext().hasCondition() && - !getContext().getCondition().getFilter(getRepository().getFilterResolver()).isItemShown(item)) - throw new IdDoesNotExistException("This context does not include this item"); - - if (format.hasCondition() && - !format.getCondition().getFilter(getRepository().getFilterResolver()).isItemShown(item)) - throw new CannotDisseminateRecordException("Format not applicable to this item"); - - - Xrecord record = this.createRecord(parameters, item); - GetRecord result = new XgetRecord(record); - - return result; - } - - private Xrecord createRecord(OAICompiledRequest parameters, Item item) - throws BadArgumentException, CannotDisseminateRecordException, - OAIException, NoMetadataFormatsException, CannotDisseminateFormatException { - MetadataFormat format = getContext().formatForPrefix(parameters.getMetadataPrefix()); - Header header = new Header(); - - Dataset dataset = ((Xitem)item).getDataset(); - Xrecord xrecord = new Xrecord().withFormatName(parameters.getMetadataPrefix()).withDataset(dataset); - header.withIdentifier(item.getIdentifier()); - - ItemHelper itemHelperWrap = new ItemHelper(item); - header.withDatestamp(item.getDatestamp()); - for (Set set : itemHelperWrap.getSets(getContext(), getRepository().getFilterResolver())) - header.withSetSpec(set.getSpec()); - if (item.isDeleted()) - header.withStatus(Header.Status.DELETED); - - xrecord.withHeader(header); - xrecord.withMetadata(item.getMetadata()); - - return xrecord; - } - - private XSLPipeline toPipeline(Item item) throws XmlWriteException, XMLStreamException { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - XmlWriter writer = new XmlWriter(output); - Metadata metadata = item.getMetadata(); - metadata.write(writer); - writer.close(); - return new XSLPipeline(new ByteArrayInputStream(output.toByteArray()), true); - } -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XitemRepository.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XitemRepository.java deleted file mode 100644 index b4c60a3171d..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XitemRepository.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xoai.dataprovider.exceptions.IdDoesNotExistException; -import com.lyncode.xoai.dataprovider.exceptions.OAIException; -import com.lyncode.xoai.dataprovider.filter.ScopedFilter; -import com.lyncode.xoai.dataprovider.handlers.results.ListItemIdentifiersResult; -import com.lyncode.xoai.dataprovider.handlers.results.ListItemsResults; -import com.lyncode.xoai.dataprovider.model.Item; -import com.lyncode.xoai.dataprovider.model.ItemIdentifier; -import com.lyncode.xoai.dataprovider.model.Set; -import com.lyncode.xoai.dataprovider.repository.ItemRepository; -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.DatasetServiceBean; -import edu.harvard.iq.dataverse.harvest.server.OAIRecord; -import edu.harvard.iq.dataverse.harvest.server.OAIRecordServiceBean; -import edu.harvard.iq.dataverse.util.StringUtil; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.logging.Logger; - -/** - * - * @author Leonid Andreev - * Implements an XOAI "Item Repository". Retrieves Dataverse "OAIRecords" - * representing harvestable local datasets and translates them into - * XOAI "items". - */ - -public class XitemRepository implements ItemRepository { - private static Logger logger = Logger.getLogger("edu.harvard.iq.dataverse.harvest.server.xoai.XitemRepository"); - - private OAIRecordServiceBean recordService; - private DatasetServiceBean datasetService; - - public XitemRepository (OAIRecordServiceBean recordService, DatasetServiceBean datasetService) { - super(); - this.recordService = recordService; - this.datasetService = datasetService; - } - - private List list = new ArrayList(); - - - @Override - public Item getItem(String identifier) throws IdDoesNotExistException, OAIException { - logger.fine("getItem; calling findOaiRecordsByGlobalId, identifier " + identifier); - List oaiRecords = recordService.findOaiRecordsByGlobalId(identifier); - if (oaiRecords != null && !oaiRecords.isEmpty()) { - Xitem xoaiItem = null; - for (OAIRecord record : oaiRecords) { - if (xoaiItem == null) { - Dataset dataset = datasetService.findByGlobalId(record.getGlobalId()); - if (dataset != null) { - xoaiItem = new Xitem(record).withDataset(dataset); - } - } else { - // Adding extra set specs to the XOAI Item, if this record - // is part of multiple sets: - if (!StringUtil.isEmpty(record.getSetName())) { - xoaiItem.getSets().add(new Set(record.getSetName())); - } - } - } - if (xoaiItem != null) { - return xoaiItem; - } - } - - throw new IdDoesNotExistException(); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiers(List filters, int offset, int length) throws OAIException { - return getItemIdentifiers(filters, offset, length, null, null, null); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiers(List filters, int offset, int length, Date from) throws OAIException { - return getItemIdentifiers(filters, offset, length, null, from, null); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiersUntil(List filters, int offset, int length, Date until) throws OAIException { - return getItemIdentifiers(filters, offset, length, null, null, until); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiers(List filters, int offset, int length, Date from, Date until) throws OAIException { - return getItemIdentifiers(filters, offset, length, null, from, until); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiers(List filters, int offset, int length, String setSpec) throws OAIException { - return getItemIdentifiers(filters, offset, length, setSpec, null, null); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiers(List filters, int offset, int length, String setSpec, Date from) throws OAIException { - return getItemIdentifiers(filters, offset, length, setSpec, from, null); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiersUntil(List filters, int offset, int length, String setSpec, Date until) throws OAIException { - return getItemIdentifiers(filters, offset, length, setSpec, null, until); - } - - @Override - public ListItemIdentifiersResult getItemIdentifiers(List filters, int offset, int length, String setSpec, Date from, Date until) throws OAIException { - logger.fine("calling getItemIdentifiers; offset=" + offset - + ", length=" + length - + ", setSpec=" + setSpec - + ", from=" + from - + ", until=" + until); - - List oaiRecords = recordService.findOaiRecordsBySetName(setSpec, from, until); - - logger.fine("total " + oaiRecords.size() + " returned"); - - List xoaiItems = new ArrayList<>(); - if (oaiRecords != null && !oaiRecords.isEmpty()) { - - for (int i = offset; i < offset + length && i < oaiRecords.size(); i++) { - OAIRecord record = oaiRecords.get(i); - xoaiItems.add(new Xitem(record)); - } - - // Run a second pass, looking for records in this set that occur - // in *other* sets. Then we'll add these multiple sets to the - // formatted output in the header: - addExtraSets(xoaiItems, setSpec, from, until); - - boolean hasMore = offset + length < oaiRecords.size(); - ListItemIdentifiersResult result = new ListItemIdentifiersResult(hasMore, xoaiItems); - logger.fine("returning result with " + xoaiItems.size() + " items."); - return result; - } - - return new ListItemIdentifiersResult(false, xoaiItems); - } - - @Override - public ListItemsResults getItems(List filters, int offset, int length) throws OAIException { - return getItems(filters, offset, length, null, null, null); - } - - @Override - public ListItemsResults getItems(List filters, int offset, int length, Date from) throws OAIException { - return getItems(filters, offset, length, null, from, null); - } - - @Override - public ListItemsResults getItemsUntil(List filters, int offset, int length, Date until) throws OAIException { - return getItems(filters, offset, length, null, null, until); - } - - @Override - public ListItemsResults getItems(List filters, int offset, int length, Date from, Date until) throws OAIException { - return getItems(filters, offset, length, null, from, until); - } - - @Override - public ListItemsResults getItems(List filters, int offset, int length, String setSpec) throws OAIException { - return getItems(filters, offset, length, setSpec, null, null); - } - - @Override - public ListItemsResults getItems(List filters, int offset, int length, String setSpec, Date from) throws OAIException { - return getItems(filters, offset, length, setSpec, from, null); - } - - @Override - public ListItemsResults getItemsUntil(List filters, int offset, int length, String setSpec, Date until) throws OAIException { - return getItems(filters, offset, length, setSpec, null, until); - } - - @Override - public ListItemsResults getItems(List filters, int offset, int length, String setSpec, Date from, Date until) throws OAIException { - logger.fine("calling getItems; offset=" + offset - + ", length=" + length - + ", setSpec=" + setSpec - + ", from=" + from - + ", until=" + until); - - List oaiRecords = recordService.findOaiRecordsBySetName(setSpec, from, until); - - logger.fine("total " + oaiRecords.size() + " returned"); - - List xoaiItems = new ArrayList<>(); - if (oaiRecords != null && !oaiRecords.isEmpty()) { - - for (int i = offset; i < offset + length && i < oaiRecords.size(); i++) { - OAIRecord oaiRecord = oaiRecords.get(i); - Dataset dataset = datasetService.findByGlobalId(oaiRecord.getGlobalId()); - if (dataset != null) { - Xitem xItem = new Xitem(oaiRecord).withDataset(dataset); - xoaiItems.add(xItem); - } - } - - addExtraSets(xoaiItems, setSpec, from, until); - - boolean hasMore = offset + length < oaiRecords.size(); - ListItemsResults result = new ListItemsResults(hasMore, xoaiItems); - logger.fine("returning result with " + xoaiItems.size() + " items."); - return result; - } - - return new ListItemsResults(false, xoaiItems); - } - - private void addExtraSets(Object xoaiItemsList, String setSpec, Date from, Date until) { - - List xoaiItems = (List)xoaiItemsList; - - List oaiRecords = recordService.findOaiRecordsNotInThisSet(setSpec, from, until); - - if (oaiRecords == null || oaiRecords.isEmpty()) { - return; - } - - // Make a second pass through the list of xoaiItems already found for this set, - // and add any other sets in which this item occurs: - - int j = 0; - for (int i = 0; i < xoaiItems.size(); i++) { - // fast-forward the second list, until we find a record with this identifier, - // or until we are past this record (both lists are sorted alphabetically by - // the identifier: - Xitem xitem = xoaiItems.get(i); - - while (j < oaiRecords.size() && xitem.getIdentifier().compareTo(oaiRecords.get(j).getGlobalId()) > 0) { - j++; - } - - while (j < oaiRecords.size() && xitem.getIdentifier().equals(oaiRecords.get(j).getGlobalId())) { - xoaiItems.get(i).getSets().add(new Set(oaiRecords.get(j).getSetName())); - j++; - } - } - - } -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecords.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecords.java deleted file mode 100644 index 15bd005cacf..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecords.java +++ /dev/null @@ -1,86 +0,0 @@ -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xml.exceptions.XmlWriteException; -import static com.lyncode.xoai.model.oaipmh.Granularity.Second; -import com.lyncode.xoai.model.oaipmh.ListRecords; -import com.lyncode.xoai.model.oaipmh.Record; -import com.lyncode.xoai.model.oaipmh.ResumptionToken; -import com.lyncode.xoai.xml.XmlWriter; -import static com.lyncode.xoai.xml.XmlWriter.defaultContext; -import java.io.ByteArrayOutputStream; - -import java.io.IOException; -import java.io.OutputStream; -import javax.xml.stream.XMLStreamException; - -/** - * - * @author Leonid Andreev - * - * This is the Dataverse extension of XOAI ListRecords, - * optimized to stream individual records using fast dumping - * of pre-exported metadata fragments (and by-passing expensive - * XML parsing and writing). - */ -public class XlistRecords extends ListRecords { - private static final String RECORD_FIELD = "record"; - private static final String RECORD_START_ELEMENT = "<"+RECORD_FIELD+">"; - private static final String RECORD_CLOSE_ELEMENT = ""; - private static final String RESUMPTION_TOKEN_FIELD = "resumptionToken"; - private static final String EXPIRATION_DATE_ATTRIBUTE = "expirationDate"; - private static final String COMPLETE_LIST_SIZE_ATTRIBUTE = "completeListSize"; - private static final String CURSOR_ATTRIBUTE = "cursor"; - - public void writeToStream(OutputStream outputStream) throws IOException { - if (!this.records.isEmpty()) { - for (Record record : this.records) { - outputStream.write(RECORD_START_ELEMENT.getBytes()); - outputStream.flush(); - - ((Xrecord)record).writeToStream(outputStream); - - outputStream.write(RECORD_CLOSE_ELEMENT.getBytes()); - outputStream.flush(); - } - } - - if (resumptionToken != null) { - - String resumptionTokenString = resumptionTokenToString(resumptionToken); - if (resumptionTokenString == null) { - throw new IOException("XlistRecords: failed to output resumption token"); - } - outputStream.write(resumptionTokenString.getBytes()); - outputStream.flush(); - } - } - - private String resumptionTokenToString(ResumptionToken token) { - try { - ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(); - XmlWriter writer = new XmlWriter(byteOutputStream, defaultContext()); - - writer.writeStartElement(RESUMPTION_TOKEN_FIELD); - - if (token.getExpirationDate() != null) - writer.writeAttribute(EXPIRATION_DATE_ATTRIBUTE, token.getExpirationDate(), Second); - if (token.getCompleteListSize() != null) - writer.writeAttribute(COMPLETE_LIST_SIZE_ATTRIBUTE, "" + token.getCompleteListSize()); - if (token.getCursor() != null) - writer.writeAttribute(CURSOR_ATTRIBUTE, "" + token.getCursor()); - if (token.getValue() != null) - writer.write(token.getValue()); - - writer.writeEndElement(); // resumptionToken; - writer.flush(); - writer.close(); - - String ret = byteOutputStream.toString(); - - return ret; - } catch (XMLStreamException | XmlWriteException e) { - return null; - } - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecordsHandler.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecordsHandler.java deleted file mode 100644 index 8fe13bc4044..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XlistRecordsHandler.java +++ /dev/null @@ -1,168 +0,0 @@ -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xml.exceptions.XmlWriteException; -import com.lyncode.xoai.dataprovider.exceptions.BadArgumentException; -import com.lyncode.xoai.dataprovider.exceptions.CannotDisseminateFormatException; -import com.lyncode.xoai.dataprovider.exceptions.CannotDisseminateRecordException; -import com.lyncode.xoai.dataprovider.exceptions.DoesNotSupportSetsException; -import com.lyncode.xoai.dataprovider.exceptions.HandlerException; -import com.lyncode.xoai.dataprovider.exceptions.NoMatchesException; -import com.lyncode.xoai.dataprovider.exceptions.NoMetadataFormatsException; -import com.lyncode.xoai.dataprovider.exceptions.OAIException; -import com.lyncode.xoai.dataprovider.handlers.VerbHandler; -import com.lyncode.xoai.dataprovider.handlers.results.ListItemsResults; -import com.lyncode.xoai.dataprovider.handlers.helpers.ItemHelper; -import com.lyncode.xoai.dataprovider.handlers.helpers.ItemRepositoryHelper; -import com.lyncode.xoai.dataprovider.handlers.helpers.SetRepositoryHelper; -import com.lyncode.xoai.dataprovider.model.Context; -import com.lyncode.xoai.dataprovider.model.Item; -import com.lyncode.xoai.dataprovider.model.MetadataFormat; -import com.lyncode.xoai.dataprovider.model.Set; -import com.lyncode.xoai.dataprovider.parameters.OAICompiledRequest; -import com.lyncode.xoai.dataprovider.repository.Repository; -import com.lyncode.xoai.model.oaipmh.Header; -import com.lyncode.xoai.model.oaipmh.ListRecords; -import com.lyncode.xoai.model.oaipmh.Metadata; -import com.lyncode.xoai.model.oaipmh.Record; -import com.lyncode.xoai.model.oaipmh.ResumptionToken; -import com.lyncode.xoai.xml.XSLPipeline; -import com.lyncode.xoai.xml.XmlWriter; -import edu.harvard.iq.dataverse.Dataset; - -import javax.xml.stream.XMLStreamException; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.util.List; - -/** - * - * @author Leonid Andreev - * - * This is Dataverse's own implementation of ListRecords Verb Handler - * (used instead of the ListRecordsHandler provided by XOAI). - * It is customized to support the optimizations that allows - * Dataverse to directly output pre-exported metadata records to the output - * stream, bypassing expensive XML parsing and writing. - */ -public class XlistRecordsHandler extends VerbHandler { - private static java.util.logging.Logger logger = java.util.logging.Logger.getLogger("XlistRecordsHandler"); - private final ItemRepositoryHelper itemRepositoryHelper; - private final SetRepositoryHelper setRepositoryHelper; - - public XlistRecordsHandler(Context context, Repository repository) { - super(context, repository); - this.itemRepositoryHelper = new ItemRepositoryHelper(getRepository().getItemRepository()); - this.setRepositoryHelper = new SetRepositoryHelper(getRepository().getSetRepository()); - } - - @Override - public ListRecords handle(OAICompiledRequest parameters) throws OAIException, HandlerException { - XlistRecords res = new XlistRecords(); - int length = getRepository().getConfiguration().getMaxListRecords(); - - if (parameters.hasSet() && !getRepository().getSetRepository().supportSets()) - throw new DoesNotSupportSetsException(); - - int offset = getOffset(parameters); - ListItemsResults result; - if (!parameters.hasSet()) { - if (parameters.hasFrom() && !parameters.hasUntil()) - result = itemRepositoryHelper.getItems(getContext(), offset, - length, parameters.getMetadataPrefix(), - parameters.getFrom()); - else if (!parameters.hasFrom() && parameters.hasUntil()) - result = itemRepositoryHelper.getItemsUntil(getContext(), offset, - length, parameters.getMetadataPrefix(), - parameters.getUntil()); - else if (parameters.hasFrom() && parameters.hasUntil()) - result = itemRepositoryHelper.getItems(getContext(), offset, - length, parameters.getMetadataPrefix(), - parameters.getFrom(), parameters.getUntil()); - else - result = itemRepositoryHelper.getItems(getContext(), offset, - length, parameters.getMetadataPrefix()); - } else { - if (!setRepositoryHelper.exists(getContext(), parameters.getSet())) { - // throw new NoMatchesException(); - } - if (parameters.hasFrom() && !parameters.hasUntil()) - result = itemRepositoryHelper.getItems(getContext(), offset, - length, parameters.getMetadataPrefix(), - parameters.getSet(), parameters.getFrom()); - else if (!parameters.hasFrom() && parameters.hasUntil()) - result = itemRepositoryHelper.getItemsUntil(getContext(), offset, - length, parameters.getMetadataPrefix(), - parameters.getSet(), parameters.getUntil()); - else if (parameters.hasFrom() && parameters.hasUntil()) - result = itemRepositoryHelper.getItems(getContext(), offset, - length, parameters.getMetadataPrefix(), - parameters.getSet(), parameters.getFrom(), - parameters.getUntil()); - else - result = itemRepositoryHelper.getItems(getContext(), offset, - length, parameters.getMetadataPrefix(), - parameters.getSet()); - } - - List results = result.getResults(); - if (results.isEmpty()) throw new NoMatchesException(); - for (Item i : results) - res.withRecord(this.createRecord(parameters, i)); - - - ResumptionToken.Value currentResumptionToken = new ResumptionToken.Value(); - if (parameters.hasResumptionToken()) { - currentResumptionToken = parameters.getResumptionToken(); - } else if (result.hasMore()) { - currentResumptionToken = parameters.extractResumptionToken(); - } - - XresumptionTokenHelper resumptionTokenHelper = new XresumptionTokenHelper(currentResumptionToken, - getRepository().getConfiguration().getMaxListRecords()); - res.withResumptionToken(resumptionTokenHelper.resolve(result.hasMore())); - - return res; - } - - - private int getOffset(OAICompiledRequest parameters) { - if (!parameters.hasResumptionToken()) - return 0; - if (parameters.getResumptionToken().getOffset() == null) - return 0; - return parameters.getResumptionToken().getOffset().intValue(); - } - - private Record createRecord(OAICompiledRequest parameters, Item item) - throws BadArgumentException, CannotDisseminateRecordException, - OAIException, NoMetadataFormatsException, CannotDisseminateFormatException { - MetadataFormat format = getContext().formatForPrefix(parameters.getMetadataPrefix()); - Header header = new Header(); - - Dataset dataset = ((Xitem)item).getDataset(); - Xrecord xrecord = new Xrecord().withFormatName(parameters.getMetadataPrefix()).withDataset(dataset); - header.withIdentifier(item.getIdentifier()); - - ItemHelper itemHelperWrap = new ItemHelper(item); - header.withDatestamp(item.getDatestamp()); - for (Set set : itemHelperWrap.getSets(getContext(), getRepository().getFilterResolver())) - header.withSetSpec(set.getSpec()); - if (item.isDeleted()) - header.withStatus(Header.Status.DELETED); - - xrecord.withHeader(header); - xrecord.withMetadata(item.getMetadata()); - - return xrecord; - } - - - private XSLPipeline toPipeline(Item item) throws XmlWriteException, XMLStreamException { - ByteArrayOutputStream output = new ByteArrayOutputStream(); - XmlWriter writer = new XmlWriter(output); - Metadata metadata = item.getMetadata(); - metadata.write(writer); - writer.close(); - return new XSLPipeline(new ByteArrayInputStream(output.toByteArray()), true); - } -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xmetadata.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xmetadata.java deleted file mode 100644 index 225b9b13777..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xmetadata.java +++ /dev/null @@ -1,27 +0,0 @@ - -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xml.exceptions.XmlWriteException; -import com.lyncode.xoai.model.oaipmh.Metadata; -import com.lyncode.xoai.xml.XmlWriter; - -/** - * - * @author Leonid Andreev - */ -public class Xmetadata extends Metadata { - - - public Xmetadata(String value) { - super(value); - } - - - @Override - public void write(XmlWriter writer) throws XmlWriteException { - // Do nothing! - // - rather than writing Metadata as an XML writer stram, we will write - // the pre-exported *and pre-validated* content as a byte stream, directly. - } - -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xrecord.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xrecord.java deleted file mode 100644 index 7e115c78f06..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/Xrecord.java +++ /dev/null @@ -1,184 +0,0 @@ -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xoai.model.oaipmh.Header; -import com.lyncode.xoai.model.oaipmh.Record; -import com.lyncode.xoai.xml.XmlWriter; -import static com.lyncode.xoai.xml.XmlWriter.defaultContext; - -import edu.harvard.iq.dataverse.Dataset; -import edu.harvard.iq.dataverse.export.ExportException; -import edu.harvard.iq.dataverse.export.ExportService; -import static edu.harvard.iq.dataverse.util.SystemConfig.FQDN; -import static edu.harvard.iq.dataverse.util.SystemConfig.SITE_URL; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetAddress; -import java.net.UnknownHostException; -import org.apache.poi.util.ReplacingInputStream; - -/** - * - * @author Leonid Andreev - * - * This is the Dataverse extension of XOAI Record, - * optimized to directly output a pre-exported metadata record to the - * output stream, thus by-passing expensive parsing and writing by - * an XML writer, as in the original XOAI implementation. - */ - -public class Xrecord extends Record { - private static final String METADATA_FIELD = "metadata"; - private static final String METADATA_START_ELEMENT = "<"+METADATA_FIELD+">"; - private static final String METADATA_END_ELEMENT = ""; - private static final String HEADER_FIELD = "header"; - private static final String STATUS_ATTRIBUTE = "status"; - private static final String IDENTIFIER_FIELD = "identifier"; - private static final String DATESTAMP_FIELD = "datestamp"; - private static final String SETSPEC_FIELD = "setSpec"; - private static final String DATAVERSE_EXTENDED_METADATA_FORMAT = "dataverse_json"; - private static final String DATAVERSE_EXTENDED_METADATA_API = "/api/datasets/export"; - - protected Dataset dataset; - protected String formatName; - - - public Dataset getDataset() { - return dataset; - } - - public Xrecord withDataset(Dataset dataset) { - this.dataset = dataset; - return this; - } - - - public String getFormatName() { - return formatName; - } - - - public Xrecord withFormatName(String formatName) { - this.formatName = formatName; - return this; - } - - public void writeToStream(OutputStream outputStream) throws IOException { - outputStream.flush(); - - String headerString = itemHeaderToString(this.header); - - if (headerString == null) { - throw new IOException("Xrecord: failed to stream item header."); - } - - outputStream.write(headerString.getBytes()); - - // header.getStatus() is only non-null when it's indicating "deleted". - if (header.getStatus() == null) { // Deleted records should not show metadata - if (!isExtendedDataverseMetadataMode(formatName)) { - outputStream.write(METADATA_START_ELEMENT.getBytes()); - - outputStream.flush(); - - if (dataset != null && formatName != null) { - InputStream inputStream = null; - try { - inputStream = new ReplacingInputStream( - ExportService.getInstance().getExport(dataset, formatName), - "", - "" - ); - } catch (ExportException ex) { - inputStream = null; - } - - if (inputStream == null) { - throw new IOException("Xrecord: failed to open metadata stream."); - } - writeMetadataStream(inputStream, outputStream); - } - outputStream.write(METADATA_END_ELEMENT.getBytes()); - } else { - outputStream.write(customMetadataExtensionRef(this.dataset.getGlobalIdString()).getBytes()); - } - } - outputStream.flush(); - } - - private String itemHeaderToString(Header header) { - try { - ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream(); - XmlWriter writer = new XmlWriter(byteOutputStream, defaultContext()); - - writer.writeStartElement(HEADER_FIELD); - - if (header.getStatus() != null) { - writer.writeAttribute(STATUS_ATTRIBUTE, header.getStatus().value()); - } - writer.writeElement(IDENTIFIER_FIELD, header.getIdentifier()); - writer.writeElement(DATESTAMP_FIELD, header.getDatestamp()); - for (String setSpec : header.getSetSpecs()) { - writer.writeElement(SETSPEC_FIELD, setSpec); - } - writer.writeEndElement(); // header - writer.flush(); - writer.close(); - - String ret = byteOutputStream.toString(); - - return ret; - } catch (Exception ex) { - return null; - } - } - - private void writeMetadataStream(InputStream inputStream, OutputStream outputStream) throws IOException { - int bufsize; - byte[] buffer = new byte[4 * 8192]; - - while ((bufsize = inputStream.read(buffer)) != -1) { - outputStream.write(buffer, 0, bufsize); - outputStream.flush(); - } - - inputStream.close(); - } - - private String customMetadataExtensionRef(String identifier) { - String ret = "<" + METADATA_FIELD - + " directApiCall=\"" - + getDataverseSiteUrl() - + DATAVERSE_EXTENDED_METADATA_API - + "?exporter=" - + DATAVERSE_EXTENDED_METADATA_FORMAT - + "&persistentId=" - + identifier - + "\"" - + "/>"; - - return ret; - } - - private boolean isExtendedDataverseMetadataMode(String formatName) { - return DATAVERSE_EXTENDED_METADATA_FORMAT.equals(formatName); - } - - private String getDataverseSiteUrl() { - String hostUrl = System.getProperty(SITE_URL); - if (hostUrl != null && !"".equals(hostUrl)) { - return hostUrl; - } - String hostName = System.getProperty(FQDN); - if (hostName == null) { - try { - hostName = InetAddress.getLocalHost().getCanonicalHostName(); - } catch (UnknownHostException e) { - return null; - } - } - hostUrl = "https://" + hostName; - return hostUrl; - } -} diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XresumptionTokenHelper.java b/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XresumptionTokenHelper.java deleted file mode 100644 index 7f9eac2cbe8..00000000000 --- a/src/main/java/edu/harvard/iq/dataverse/harvest/server/xoai/XresumptionTokenHelper.java +++ /dev/null @@ -1,61 +0,0 @@ - -package edu.harvard.iq.dataverse.harvest.server.xoai; - -import com.lyncode.xoai.dataprovider.handlers.helpers.ResumptionTokenHelper; -import com.lyncode.xoai.model.oaipmh.ResumptionToken; -import static java.lang.Math.round; -import static com.google.common.base.Predicates.isNull; - -/** - * - * @author Leonid Andreev - * Dataverse's own version of the XOAI ResumptionTokenHelper - * Fixes the issue with the offset cursor: the OAI validation spec - * insists that it starts with 0, while the XOAI implementation uses 1 - * as the initial offset. - */ -public class XresumptionTokenHelper { - - private ResumptionToken.Value current; - private long maxPerPage; - private Long totalResults; - - public XresumptionTokenHelper(ResumptionToken.Value current, long maxPerPage) { - this.current = current; - this.maxPerPage = maxPerPage; - } - - public XresumptionTokenHelper withTotalResults(long totalResults) { - this.totalResults = totalResults; - return this; - } - - public ResumptionToken resolve (boolean hasMoreResults) { - if (isInitialOffset() && !hasMoreResults) return null; - else { - if (hasMoreResults) { - ResumptionToken.Value next = current.next(maxPerPage); - return populate(new ResumptionToken(next)); - } else { - ResumptionToken resumptionToken = new ResumptionToken(); - resumptionToken.withCursor(round((current.getOffset()) / maxPerPage)); - if (totalResults != null) - resumptionToken.withCompleteListSize(totalResults); - return resumptionToken; - } - } - } - - private boolean isInitialOffset() { - return isNull().apply(current.getOffset()) || current.getOffset() == 0; - } - - private ResumptionToken populate(ResumptionToken resumptionToken) { - if (totalResults != null) - resumptionToken.withCompleteListSize(totalResults); - resumptionToken.withCursor(round((resumptionToken.getValue().getOffset() - maxPerPage)/ maxPerPage)); - return resumptionToken; - } - - -} diff --git a/src/main/resources/META-INF/microprofile-config.properties b/src/main/resources/META-INF/microprofile-config.properties index 16298d83118..be02bb1b090 100644 --- a/src/main/resources/META-INF/microprofile-config.properties +++ b/src/main/resources/META-INF/microprofile-config.properties @@ -8,3 +8,10 @@ dataverse.db.host=localhost dataverse.db.port=5432 dataverse.db.user=dataverse dataverse.db.name=dataverse +# OAI SERVER +dataverse.oai.server.maxidentifiers=100 +dataverse.oai.server.maxrecords=10 +dataverse.oai.server.maxsets=100 +# the OAI repository name, as shown by the Identify verb, +# can be customized via the setting below: +#dataverse.oai.server.repositoryname=