Merge pull request #10 from silverstripe-terraformers/feature/list-re…

…cords-filters

Add support for ListRecords filters. Some refactor

.github/workflows/main.yml

@@ -6,14 +6,4 @@ on:
 
 jobs:
   ci:
-    uses: silverstripe/github-actions-ci-cd/.github/workflows/ci.yml@0.1.15
-    with:
-      default_jobs: |
-        - php: 7.4
-          phpcoverage: true
-        - php: 7.4
-          phplinting: true
-        - php: 7.4
-          phpunit: true
-        - php: 8.0
-          phpunit: true
+    uses: silverstripe/github-actions-ci-cd/.github/workflows/ci.yml@v0.1

README.md

@@ -39,7 +39,7 @@ The goal of this module is to provide you with an easy entry point to start serv
 This module does **not** include anything to help with becoming an
 [Open Archive Initiative Harvester](http://www.openarchives.org/OAI/openarchivesprotocol.html#harvester).
 
-I am still new to the OAI spec, so we will be doing our absolute best to get everything right. Test coverage for what
+We are still new to the OAI spec, so we will be doing our absolute best to get everything right. Test coverage for what
 has been built is very high, but that doesn't help if we've just gotten something incorrect in the spec, so please be
 prepared to contribute your thoughts and/or code to help improve this module.
 
@@ -52,7 +52,8 @@ allow you to easily supplement this module with additional Metadata format suppo
 
 ### Identify
 
-Recommended environment variable: `OAI_API_ADMIN_EMAIL`
+Repository name: The default is simply using the Site Title that you have set in the CMS
+Admin email: Set using the environment variable `OAI_API_ADMIN_EMAIL`
 
 Please check out `OaiController::Identify()`. There are many options there for how you can configure different values
 for this verb.
@@ -61,10 +62,6 @@ for this verb.
 
 The response for this endpoint is generated based on the config you specify for `OaiController::$supported_formats`.
 
-### List Sets
-
-TBA
-
 ### List Identifiers
 
 TBA
@@ -76,8 +73,16 @@ that you have specified for `OaiController::$supported_formats`.
 
 The output of this endpoint is based on your current OAI Records
 
-Filter support: TBA
-Resumption tokens: TBA
+Filter support:
+
+* `from`: specifies a lower bound for datestamp-based selective harvesting. UTC+0 datetimes must be provided.
+* `until`: specifies an upper bound for datestamp-based selective harvesting. UTC+0 datetimes must be provided.
+* `set`: TBA
+* `resumptionToken`: TBA
+
+### List Sets
+
+TBA
 
 ### Get Record
 
@@ -114,8 +119,8 @@ All updates for `OaiRecords` are performed through Queued Jobs. Please see the d
 This modules makes no assumptions about how you wish to populate OAI Record data. As such, you need to specify how your
 `DataObjects` are going to map to the expected OAI fields.
 
-You can have a look in `OaiRecord` for a list of MANAGED_FIELDS fields. All of which support you adding CSV values for
-when you need to have multiple of one field.
+You can have a look in `OaiRecord` for a list of MANAGED_FIELDS. All of which support you adding CSV values for when
+you need to have multiple of one field.
 
 A note on CSV parsing: If you're anticipating that some of your properties could contain commas, then you might instead
 need to map to a method that appropriately wraps your property value in quotes. See OaiRecord for the supported

composer.json

@@ -11,8 +11,8 @@
     "license": "BSD-3-Clause",
     "require": {
         "php": ">=7.4",
-        "silverstripe/framework": "^4.10",
-        "symbiote/silverstripe-queuedjobs": "^4.8"
+        "silverstripe/framework": "^4.7",
+        "symbiote/silverstripe-queuedjobs": "^4"
     },
     "require-dev": {
         "phpunit/phpunit": "^9.5",

src/Controllers/OaiController.php

@@ -10,7 +10,6 @@
 use SilverStripe\Core\Environment;
 use SilverStripe\Core\Injector\Injector;
 use SilverStripe\ORM\DataList;
-use SilverStripe\ORM\FieldType\DBDatetime;
 use SilverStripe\SiteConfig\SiteConfig;
 use Terraformers\OpenArchive\Documents\Errors\BadVerbDocument;
 use Terraformers\OpenArchive\Documents\Errors\CannotDisseminateFormatDocument;
@@ -20,7 +19,9 @@
 use Terraformers\OpenArchive\Documents\OaiDocument;
 use Terraformers\OpenArchive\Formatters\OaiDcFormatter;
 use Terraformers\OpenArchive\Formatters\OaiRecordFormatter;
+use Terraformers\OpenArchive\Helpers\DateTimeHelper;
 use Terraformers\OpenArchive\Models\OaiRecord;
+use Throwable;
 
 class OaiController extends Controller
 {
@@ -59,6 +60,13 @@ class OaiController extends Controller
 
     private static string $supportedDeletedRecord = self::DELETED_SUPPORT_PERSISTENT;
 
+    /**
+     * All dates provided by the OAI repository must be ISO8601, and with an additional requirement that only "zulu" is
+     * supported by the OAI spec (IE: YYYY-MM-DD, or YYYY-MM-DDTHH:MM:SSZ). The "Z" indicator means that we are using
+     * "zulu" or "UTC+0" as the timezone
+     *
+     * @see http://www.openarchives.org/OAI/openarchivesprotocol.html#Dates
+     */
     private static string $supportedGranularity = 'YYYY-MM-DDThh:mm:ssZ';
 
     public function index(HTTPRequest $request): HTTPResponse
@@ -86,9 +94,7 @@ protected function BadVerbResponse(HTTPRequest $request): HTTPResponse
         $xmlDocument->setResponseDate();
         $xmlDocument->setRequestUrl($requestUrl);
 
-        $this->getResponse()->setBody($xmlDocument->getDocumentBody());
-
-        return $this->getResponse();
+        return $this->getResponseWithDocumentBody($xmlDocument);
     }
 
     protected function CannotDisseminateFormatResponse(HTTPRequest $request): HTTPResponse
@@ -99,9 +105,7 @@ protected function CannotDisseminateFormatResponse(HTTPRequest $request): HTTPRe
         $xmlDocument->setResponseDate();
         $xmlDocument->setRequestUrl($requestUrl);
 
-        $this->getResponse()->setBody($xmlDocument->getDocumentBody());
-
-        return $this->getResponse();
+        return $this->getResponseWithDocumentBody($xmlDocument);
     }
 
     /**
@@ -112,7 +116,7 @@ protected function Identify(HTTPRequest $request): HTTPResponse
         $xmlDocument = IdentifyDocument::create();
 
         // Response Date defaults to the time of the Request. Extension point is provided in this method
-        $xmlDocument->setResponseDate($this->getResponseDate());
+        $xmlDocument->setResponseDate();
         // Request URL defaults to the current URL. Extension point is provided in this method
         $xmlDocument->setRequestUrl($this->getRequestUrl($request));
         // Base URL defaults to the current URL. Extension point is provided in this method
@@ -132,9 +136,7 @@ protected function Identify(HTTPRequest $request): HTTPResponse
         // Domain can be edited through extension points provided. IDs are always just a number
         $xmlDocument->setOaiIdentifier(Director::host(), 1);
 
-        $this->getResponse()->setBody($xmlDocument->getDocumentBody());
-
-        return $this->getResponse();
+        return $this->getResponseWithDocumentBody($xmlDocument);
     }
 
     /**
@@ -154,9 +156,7 @@ protected function ListMetadataFormats(HTTPRequest $request): HTTPResponse
             $xmlDocument->addSupportedFormatter($formatter);
         }
 
-        $this->getResponse()->setBody($xmlDocument->getDocumentBody());
-
-        return $this->getResponse();
+        return $this->getResponseWithDocumentBody($xmlDocument);
     }
 
     /**
@@ -178,6 +178,13 @@ protected function ListRecords(HTTPRequest $request): HTTPResponse
             return $this->CannotDisseminateFormatResponse($request);
         }
 
+        // The OaiRecord formatter that we're going to use
+        $xmlDocument = ListRecordsDocument::create($this->getOaiRecordFormatter($metadataPrefix));
+        // Response Date defaults to the time of the Request. Extension point is provided in this method
+        $xmlDocument->setResponseDate();
+        // Request URL defaults to the current URL. Extension point is provided in this method
+        $xmlDocument->setRequestUrl($this->getRequestUrl($request));
+
         // The lower bound for selective harvesting
         $from = $request->getVar('from');
         // The upper bound for selective harvesting
@@ -187,17 +194,42 @@ protected function ListRecords(HTTPRequest $request): HTTPResponse
         // An encoded string containing pagination requirements for selective harvesting
         $resumptionToken = $request->getVar('resumptionToken');
 
+        if ($from) {
+            try {
+                $from = DateTimeHelper::getLocalStringFromUtc($from);
+            } catch (Throwable $e) {
+                $xmlDocument->addError(OaiDocument::ERROR_BAD_ARGUMENT, 'Invalid \'from\' date format provided');
+            }
+        }
+
+        if ($until) {
+            try {
+                $until = DateTimeHelper::getLocalStringFromUtc($until);
+            } catch (Throwable $e) {
+                $xmlDocument->addError(OaiDocument::ERROR_BAD_ARGUMENT, 'Invalid \'until\' date format provided');
+            }
+        }
+
+        if ($xmlDocument->hasErrors()) {
+            return $this->getResponseWithDocumentBody($xmlDocument);
+        }
+
         $oaiRecords = $this->fetchOaiRecords($from, $until, $set, $resumptionToken);
 
-        // The OaiRecord formatter that we're going to use
-        $xmlDocument = ListRecordsDocument::create($this->getOaiRecordFormatter($metadataPrefix));
-        // Response Date defaults to the time of the Request. Extension point is provided in this method
-        $xmlDocument->setResponseDate($this->getResponseDate());
-        // Request URL defaults to the current URL. Extension point is provided in this method
-        $xmlDocument->setRequestUrl($this->getRequestUrl($request));
+        if (!$oaiRecords->count()) {
+            $xmlDocument->addError(OaiDocument::ERROR_NO_RECORDS_MATCH);
+
+            return $this->getResponseWithDocumentBody($xmlDocument);
+        }
+
         // Start processing whatever OaiRecords we found
         $xmlDocument->processOaiRecords($oaiRecords);
 
+        return $this->getResponseWithDocumentBody($xmlDocument);
+    }
+
+    protected function getResponseWithDocumentBody(OaiDocument $xmlDocument): HTTPResponse
+    {
         $this->getResponse()->setBody($xmlDocument->getDocumentBody());
 
         return $this->getResponse();
@@ -230,22 +262,15 @@ protected function getBaseUrl(HTTPRequest $request): string
         return $baseUrl;
     }
 
-    protected function getResponseDate(): int
-    {
-        $timestamp = DBDatetime::now()->getTimestamp();
-
-        $this->extend('updateOaiResponseDate', $timestamp);
-
-        return $timestamp;
-    }
-
-    protected function getEarliestDatestamp(): int
+    protected function getEarliestDatestamp(): string
     {
-        $timestamp = 0;
+        // We're just going to set it to the start of the Unix timestamp (meaning, there could be any range of
+        // datestamps in our system)
+        $dateString = '1970-01-01T00:00:00Z';
 
-        $this->extend('updateOaiEarliestDatestamp', $timestamp);
+        $this->extend('updateOaiEarliestDatestamp', $dateString);
 
-        return $timestamp;
+        return $dateString;
     }
 
     protected function getRepositoryName(): string
@@ -257,6 +282,10 @@ protected function getRepositoryName(): string
         return $repositoryName;
     }
 
+    /**
+     * Regarding dates, please @see $supportedGranularity docblock. All dates passed to this method should already be
+     * adjusted to local server time
+     */
     protected function fetchOaiRecords(
         ?string $from = null,
         ?string $until = null,
@@ -265,7 +294,6 @@ protected function fetchOaiRecords(
     ): DataList {
         $filters = [];
 
-        // Filter support still to be tested
         if ($from) {
             $filters['LastEdited:GreaterThanOrEqual'] = $from;
         }