diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 8abb73faff2..a70dcd4e8db 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1081,7 +1081,9 @@ These archival Bags include all of the files and metadata in a given dataset ver The Dataverse Software offers an internal archive workflow which may be configured as a PostPublication workflow via an admin API call to manually submit previously published Datasets and prior versions to a configured archive such as Chronopolis. The workflow creates a `JSON-LD `_ serialized `OAI-ORE `_ map file, which is also available as a metadata export format in the Dataverse Software web interface. -At present, the DPNSubmitToArchiveCommand, LocalSubmitToArchiveCommand, and GoogleCloudSubmitToArchive are the only implementations extending the AbstractSubmitToArchiveCommand and using the configurable mechanisms discussed below. +At present, archiving classes include the DuraCloudSubmitToArchiveCommand, LocalSubmitToArchiveCommand, GoogleCloudSubmitToArchive, and S3SubmitToArchiveCommand , which all extend the AbstractSubmitToArchiveCommand and use the configurable mechanisms discussed below. + +All current options support the archival status APIs and the same status is available in the dataset page version table (for contributors/those who could view the unpublished dataset, with more detail available to superusers). .. _Duracloud Configuration: @@ -1144,7 +1146,7 @@ ArchiverClassName - the fully qualified class to be used for archiving. For exam Google Cloud Configuration ++++++++++++++++++++++++++ -The Google Cloud Archiver can send archival Bags to a bucket in Google's cloud, including those in the 'Coldline' storage class (cheaper, with slower access) +The Google Cloud Archiver can send Dataverse Archival Bags to a bucket in Google's cloud, including those in the 'Coldline' storage class (cheaper, with slower access) ``curl http://localhost:8080/api/admin/settings/:ArchiverClassName -X PUT -d "edu.harvard.iq.dataverse.engine.command.impl.GoogleCloudSubmitToArchiveCommand"`` @@ -1168,6 +1170,31 @@ For example: ``cp /usr/local/payara5/glassfish/domains/domain1/files/googlecloudkey.json`` +.. _S3 Archiver Configuration: + +S3 Configuration +++++++++++++++++ + +The S3 Archiver can send Dataverse Archival Bag to a bucket at any S3 endpoint. The configuration for the S3 Archiver is independent of any S3 store that may be configured in Dataverse and may, for example, leverage colder (cheaper, slower access) storage. + +``curl http://localhost:8080/api/admin/settings/:ArchiverClassName -X PUT -d "edu.harvard.iq.dataverse.engine.command.impl.S3SubmitToArchiveCommand"`` + +``curl http://localhost:8080/api/admin/settings/:ArchiverSettings -X PUT -d ":S3ArchiverConfig, :BagGeneratorThreads"`` + +The S3 Archiver defines one custom setting, a required :S3ArchiverConfig. It can also use the :BagGeneratorThreads setting as described in the DuraCloud Configuration section above. + +The credentials for your S3 account, can be stored in a profile in a standard credentials file (e.g. ~/.aws/credentials) referenced via "profile" key in the :S3ArchiverConfig setting (will default to the default entry), or can via MicroProfile settings as described for S3 stores (dataverse.s3archiver.access-key and dataverse.s3archiver.secret-key) + +The :S3ArchiverConfig setting is a json object that must include an "s3_bucket_name" and may include additional S3-related parameters as described for S3 Stores, including "profile", "connection-pool-size","custom-endpoint-url", "custom-endpoint-region", "path-style-access", "payload-signing", and "chunked-encoding". + +\:S3ArchiverConfig - minimally includes the name of the bucket to use. For example: + +``curl http://localhost:8080/api/admin/settings/:S3ArchiverConfig -X PUT -d '{"s3_bucket_name":"archival-bucket"}'`` + +\:S3ArchiverConfig - example to also set the name of an S3 profile to use. For example: + +``curl http://localhost:8080/api/admin/settings/:S3ArchiverConfig -X PUT -d '{"s3_bucket_name":"archival-bucket", "profile":"archiver"}'`` + .. _Archiving API Call: API Calls @@ -2665,6 +2692,12 @@ This is the local file system path to be used with the LocalSubmitToArchiveComma These are the bucket and project names to be used with the GoogleCloudSubmitToArchiveCommand class. Further information is in the :ref:`Google Cloud Configuration` section above. +:S3ArchiverConfig ++++++++++++++++++ + +This is the JSON configuration object setting to be used with the S3SubmitToArchiveCommand class. Further information is in the :ref:`S3 Archiver Configuration` section above. + + .. _:InstallationName: :InstallationName diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 719af592fcb..2236bdc24ba 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -5599,7 +5599,7 @@ public boolean isArchivable() { archivable = ((Boolean) m.invoke(null, params) == true); } catch (ClassNotFoundException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { - logger.warning("Failed to call is Archivable on configured archiver class: " + className); + logger.warning("Failed to call isArchivable on configured archiver class: " + className); e.printStackTrace(); } } @@ -5635,7 +5635,7 @@ public boolean isVersionArchivable() { } } catch (ClassNotFoundException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { - logger.warning("Failed to call is Archivable on configured archiver class: " + className); + logger.warning("Failed to call isSingleVersion on configured archiver class: " + className); e.printStackTrace(); } } @@ -5646,12 +5646,7 @@ public boolean isVersionArchivable() { public boolean isSomeVersionArchived() { if (someVersionArchived == null) { - someVersionArchived = false; - for (DatasetVersion dv : dataset.getVersions()) { - if (dv.getArchivalCopyLocation() != null) { - someVersionArchived = true; - } - } + someVersionArchived = ArchiverUtil.isSomeVersionArchived(dataset); } return someVersionArchived; } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java index 0e8c3bb65a5..b988fd05f03 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractSubmitToArchiveCommand.java @@ -1,5 +1,7 @@ package edu.harvard.iq.dataverse.engine.command.impl; +import edu.harvard.iq.dataverse.DOIDataCiteRegisterService; +import edu.harvard.iq.dataverse.DataCitation; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DvObject; @@ -94,6 +96,13 @@ public String describe() { return super.describe() + "DatasetVersion: [" + version.getId() + " (v" + version.getFriendlyVersionNumber()+")]"; } + + String getDataCiteXml(DatasetVersion dv) { + DataCitation dc = new DataCitation(dv); + Map metadata = dc.getDataCiteMetadata(); + return DOIDataCiteRegisterService.getMetadataFromDvObject(dv.getDataset().getGlobalId().asString(), metadata, + dv.getDataset()); + } public Thread startBagThread(DatasetVersion dv, PipedInputStream in, DigestInputStream digestInputStream2, String dataciteXml, ApiToken token) throws IOException, InterruptedException { @@ -160,7 +169,7 @@ public void run() { } return bagThread; } - + public static boolean isArchivable(Dataset dataset, SettingsWrapper settingsWrapper) { return true; } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java index c7da2247a31..2ca73af3b3c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/DuraCloudSubmitToArchiveCommand.java @@ -1,7 +1,5 @@ package edu.harvard.iq.dataverse.engine.command.impl; -import edu.harvard.iq.dataverse.DOIDataCiteRegisterService; -import edu.harvard.iq.dataverse.DataCitation; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DatasetLock.Reason; @@ -108,10 +106,7 @@ public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken t if (!store.spaceExists(spaceName)) { store.createSpace(spaceName); } - DataCitation dc = new DataCitation(dv); - Map metadata = dc.getDataCiteMetadata(); - String dataciteXml = DOIDataCiteRegisterService.getMetadataFromDvObject( - dv.getDataset().getGlobalId().asString(), metadata, dv.getDataset()); + String dataciteXml = getDataCiteXml(dv); MessageDigest messageDigest = MessageDigest.getInstance("MD5"); try (PipedInputStream dataciteIn = new PipedInputStream(); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GoogleCloudSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GoogleCloudSubmitToArchiveCommand.java index d93dcc9156a..5d017173685 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GoogleCloudSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/GoogleCloudSubmitToArchiveCommand.java @@ -1,7 +1,5 @@ package edu.harvard.iq.dataverse.engine.command.impl; -import edu.harvard.iq.dataverse.DOIDataCiteRegisterService; -import edu.harvard.iq.dataverse.DataCitation; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DatasetLock.Reason; @@ -73,10 +71,7 @@ public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken t String spaceName = dataset.getGlobalId().asString().replace(':', '-').replace('/', '-') .replace('.', '-').toLowerCase(); - DataCitation dc = new DataCitation(dv); - Map metadata = dc.getDataCiteMetadata(); - String dataciteXml = DOIDataCiteRegisterService.getMetadataFromDvObject( - dv.getDataset().getGlobalId().asString(), metadata, dv.getDataset()); + String dataciteXml = getDataCiteXml(dv); MessageDigest messageDigest = MessageDigest.getInstance("MD5"); try (PipedInputStream dataciteIn = new PipedInputStream(); DigestInputStream digestInputStream = new DigestInputStream(dataciteIn, messageDigest)) { diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java index c12bdc63981..c7e91b2967b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/LocalSubmitToArchiveCommand.java @@ -1,7 +1,5 @@ package edu.harvard.iq.dataverse.engine.command.impl; -import edu.harvard.iq.dataverse.DOIDataCiteRegisterService; -import edu.harvard.iq.dataverse.DataCitation; import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.DatasetLock.Reason; @@ -58,11 +56,8 @@ public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken t String spaceName = dataset.getGlobalId().asString().replace(':', '-').replace('/', '-') .replace('.', '-').toLowerCase(); - DataCitation dc = new DataCitation(dv); - Map metadata = dc.getDataCiteMetadata(); - String dataciteXml = DOIDataCiteRegisterService - .getMetadataFromDvObject(dv.getDataset().getGlobalId().asString(), metadata, dv.getDataset()); - + String dataciteXml = getDataCiteXml(dv); + FileUtils.writeStringToFile( new File(localPath + "/" + spaceName + "-datacite.v" + dv.getFriendlyVersionNumber() + ".xml"), dataciteXml, StandardCharsets.UTF_8); @@ -70,6 +65,7 @@ public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken t bagger.setNumConnections(getNumberOfBagGeneratorThreads()); bagger.setAuthenticationKey(token.getTokenString()); zipName = localPath + "/" + spaceName + "v" + dv.getFriendlyVersionNumber() + ".zip"; + //ToDo: generateBag(File f, true) seems to do the same thing (with a .tmp extension) - since we don't have to use a stream here, could probably just reuse the existing code? bagger.generateBag(new FileOutputStream(zipName + ".partial")); File srcFile = new File(zipName + ".partial"); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/S3SubmitToArchiveCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/S3SubmitToArchiveCommand.java new file mode 100644 index 00000000000..f24d956e9d7 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/S3SubmitToArchiveCommand.java @@ -0,0 +1,269 @@ +package edu.harvard.iq.dataverse.engine.command.impl; + +import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.DatasetVersion; +import edu.harvard.iq.dataverse.DatasetLock.Reason; +import edu.harvard.iq.dataverse.authorization.Permission; +import edu.harvard.iq.dataverse.authorization.users.ApiToken; +import edu.harvard.iq.dataverse.engine.command.Command; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; +import edu.harvard.iq.dataverse.engine.command.RequiredPermissions; +import edu.harvard.iq.dataverse.util.bagit.BagGenerator; +import edu.harvard.iq.dataverse.util.bagit.OREMap; +import edu.harvard.iq.dataverse.util.json.JsonUtil; +import edu.harvard.iq.dataverse.workflow.step.Failure; +import edu.harvard.iq.dataverse.workflow.step.WorkflowStepResult; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.util.Map; +import java.util.logging.Logger; + +import javax.json.Json; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; + +import org.eclipse.microprofile.config.Config; +import org.eclipse.microprofile.config.ConfigProvider; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.auth.AWSCredentialsProviderChain; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.profile.ProfileCredentialsProvider; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.PutObjectRequest; +import com.amazonaws.services.s3.transfer.TransferManager; +import com.amazonaws.services.s3.transfer.TransferManagerBuilder; + +@RequiredPermissions(Permission.PublishDataset) +public class S3SubmitToArchiveCommand extends AbstractSubmitToArchiveCommand implements Command { + + private static final Logger logger = Logger.getLogger(S3SubmitToArchiveCommand.class.getName()); + private static final String S3_CONFIG = ":S3ArchiverConfig"; + + private static final Config config = ConfigProvider.getConfig(); + protected AmazonS3 s3 = null; + protected TransferManager tm = null; + private String spaceName = null; + protected String bucketName = null; + + public S3SubmitToArchiveCommand(DataverseRequest aRequest, DatasetVersion version) { + super(aRequest, version); + } + + @Override + public WorkflowStepResult performArchiveSubmission(DatasetVersion dv, ApiToken token, + Map requestedSettings) { + logger.fine("In S3SubmitToArchiveCommand..."); + JsonObject configObject = null; + + try { + configObject = JsonUtil.getJsonObject(requestedSettings.get(S3_CONFIG)); + logger.fine("Config: " + configObject); + bucketName = configObject.getString("s3_bucket_name", null); + } catch (Exception e) { + logger.warning("Unable to parse " + S3_CONFIG + " setting as a Json object"); + } + if (configObject != null && bucketName != null) { + + s3 = createClient(configObject); + tm = TransferManagerBuilder.standard().withS3Client(s3).build(); + + //Set a failure status that will be updated if we succeed + JsonObjectBuilder statusObject = Json.createObjectBuilder(); + statusObject.add(DatasetVersion.ARCHIVAL_STATUS, DatasetVersion.ARCHIVAL_STATUS_FAILURE); + statusObject.add(DatasetVersion.ARCHIVAL_STATUS_MESSAGE, "Bag not transferred"); + + try { + + Dataset dataset = dv.getDataset(); + if (dataset.getLockFor(Reason.finalizePublication) == null) { + + spaceName = getSpaceName(dataset); + String dataciteXml = getDataCiteXml(dv); + try (ByteArrayInputStream dataciteIn = new ByteArrayInputStream(dataciteXml.getBytes("UTF-8"))) { + // Add datacite.xml file + ObjectMetadata om = new ObjectMetadata(); + om.setContentLength(dataciteIn.available()); + String dcKey = spaceName + "/" + getDataCiteFileName(spaceName, dv) + ".xml"; + tm.upload(new PutObjectRequest(bucketName, dcKey, dataciteIn, om)).waitForCompletion(); + om = s3.getObjectMetadata(bucketName, dcKey); + if (om == null) { + logger.warning("Could not write datacite xml to S3"); + return new Failure("S3 Archiver failed writing datacite xml file"); + } + + // Store BagIt file + String fileName = getFileName(spaceName, dv); + + String bagKey = spaceName + "/" + fileName + ".zip"; + // Add BagIt ZIP file + // Google uses MD5 as one way to verify the + // transfer + + // Generate bag + BagGenerator bagger = new BagGenerator(new OREMap(dv, false), dataciteXml); + bagger.setAuthenticationKey(token.getTokenString()); + if (bagger.generateBag(fileName, false)) { + File bagFile = bagger.getBagFile(fileName); + + try (FileInputStream in = new FileInputStream(bagFile)) { + om = new ObjectMetadata(); + om.setContentLength(bagFile.length()); + + tm.upload(new PutObjectRequest(bucketName, bagKey, in, om)).waitForCompletion(); + om = s3.getObjectMetadata(bucketName, bagKey); + + if (om == null) { + logger.severe("Error sending file to S3: " + fileName); + return new Failure("Error in transferring Bag file to S3", + "S3 Submission Failure: incomplete transfer"); + } + } catch (RuntimeException rte) { + logger.severe("Error creating Bag during S3 archiving: " + rte.getMessage()); + return new Failure("Error in generating Bag", + "S3 Submission Failure: archive file not created"); + } + + logger.fine("S3 Submission step: Content Transferred"); + + // Document the location of dataset archival copy location (actually the URL + // where you can + // view it as an admin) + + // Unsigned URL - gives location but not access without creds + statusObject.add(DatasetVersion.ARCHIVAL_STATUS, DatasetVersion.ARCHIVAL_STATUS_SUCCESS); + statusObject.add(DatasetVersion.ARCHIVAL_STATUS_MESSAGE, s3.getUrl(bucketName, bagKey).toString()); + } else { + logger.warning("Could not write local Bag file " + fileName); + return new Failure("S3 Archiver fail writing temp local bag"); + } + + } + } else { + logger.warning( + "S3 Archiver Submision Workflow aborted: Dataset locked for publication/pidRegister"); + return new Failure("Dataset locked"); + } + } catch (Exception e) { + logger.warning(e.getLocalizedMessage()); + e.printStackTrace(); + return new Failure("S3 Archiver Submission Failure", + e.getLocalizedMessage() + ": check log for details"); + + } finally { + dv.setArchivalCopyLocation(statusObject.build().toString()); + } + return WorkflowStepResult.OK; + } else { + return new Failure( + "S3 Submission not configured - no \":S3ArchivalProfile\" and/or \":S3ArchivalConfig\" or no bucket-name defined in config."); + } + } + + protected String getDataCiteFileName(String spaceName, DatasetVersion dv) { + return spaceName + "_datacite.v" + dv.getFriendlyVersionNumber(); + } + + protected String getFileName(String spaceName, DatasetVersion dv) { + return spaceName + ".v" + dv.getFriendlyVersionNumber(); + } + + protected String getSpaceName(Dataset dataset) { + if (spaceName == null) { + spaceName = dataset.getGlobalId().asString().replace(':', '-').replace('/', '-').replace('.', '-') + .toLowerCase(); + } + return spaceName; + } + + private AmazonS3 createClient(JsonObject configObject) { + // get a standard client, using the standard way of configuration the + // credentials, etc. + AmazonS3ClientBuilder s3CB = AmazonS3ClientBuilder.standard(); + + ClientConfiguration cc = new ClientConfiguration(); + Integer poolSize = configObject.getInt("connection-pool-size", 256); + cc.setMaxConnections(poolSize); + s3CB.setClientConfiguration(cc); + + /** + * Pass in a URL pointing to your S3 compatible storage. For possible values see + * https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/client/builder/AwsClientBuilder.EndpointConfiguration.html + */ + String s3CEUrl = configObject.getString("custom-endpoint-url", ""); + /** + * Pass in a region to use for SigV4 signing of requests. Defaults to + * "dataverse" as it is not relevant for custom S3 implementations. + */ + String s3CERegion = configObject.getString("custom-endpoint-region", "dataverse"); + + // if the admin has set a system property (see below) we use this endpoint URL + // instead of the standard ones. + if (!s3CEUrl.isEmpty()) { + s3CB.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(s3CEUrl, s3CERegion)); + } + /** + * Pass in a boolean value if path style access should be used within the S3 + * client. Anything but case-insensitive "true" will lead to value of false, + * which is default value, too. + */ + Boolean s3pathStyleAccess = configObject.getBoolean("path-style-access", false); + // some custom S3 implementations require "PathStyleAccess" as they us a path, + // not a subdomain. default = false + s3CB.withPathStyleAccessEnabled(s3pathStyleAccess); + + /** + * Pass in a boolean value if payload signing should be used within the S3 + * client. Anything but case-insensitive "true" will lead to value of false, + * which is default value, too. + */ + Boolean s3payloadSigning = configObject.getBoolean("payload-signing", false); + /** + * Pass in a boolean value if chunked encoding should not be used within the S3 + * client. Anything but case-insensitive "false" will lead to value of true, + * which is default value, too. + */ + Boolean s3chunkedEncoding = configObject.getBoolean("chunked-encoding", true); + // Openstack SWIFT S3 implementations require "PayloadSigning" set to true. + // default = false + s3CB.setPayloadSigningEnabled(s3payloadSigning); + // Openstack SWIFT S3 implementations require "ChunkedEncoding" set to false. + // default = true + // Boolean is inverted, otherwise setting + // dataverse.files..chunked-encoding=false would result in leaving Chunked + // Encoding enabled + s3CB.setChunkedEncodingDisabled(!s3chunkedEncoding); + + /** + * Pass in a string value if this archiver should use a non-default AWS S3 + * profile. The default is "default" which should work when only one profile + * exists. + */ + ProfileCredentialsProvider profileCredentials = new ProfileCredentialsProvider(configObject.getString("profile", "default")); + + // Try to retrieve credentials via Microprofile Config API, too. For production + // use, you should not use env + // vars or system properties to provide these, but use the secrets config source + // provided by Payara. + AWSStaticCredentialsProvider staticCredentials = new AWSStaticCredentialsProvider(new BasicAWSCredentials( + config.getOptionalValue("dataverse.s3archiver.access-key", String.class).orElse(""), + config.getOptionalValue("dataverse.s3archiver.secret-key", String.class).orElse(""))); + + // Add both providers to chain - the first working provider will be used (so + // static credentials are the fallback) + AWSCredentialsProviderChain providerChain = new AWSCredentialsProviderChain(profileCredentials, + staticCredentials); + s3CB.setCredentials(providerChain); + + // let's build the client :-) + AmazonS3 client = s3CB.build(); + return client; + } + +} diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java index b21fc807574..18ec6243d5a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/ArchiverUtil.java @@ -5,6 +5,7 @@ import java.lang.reflect.Method; import java.util.logging.Logger; +import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.engine.command.impl.AbstractSubmitToArchiveCommand; @@ -58,4 +59,17 @@ public static boolean onlySingleVersionArchiving(Class