From 039fa8f099b7b93f5fc5ef7c46c7bd05aa32669a Mon Sep 17 00:00:00 2001 From: Chris Penny Date: Mon, 4 Apr 2022 13:28:45 +1200 Subject: [PATCH] Add support for expiring Resumption Tokens --- src/Controllers/OaiController.php | 11 ++ src/Documents/ListRecordsDocument.php | 9 ++ src/Helpers/DateTimeHelper.php | 1 + src/Helpers/ResumptionTokenHelper.php | 34 ++++++ tests/Helpers/ResumptionTokenHelperTest.php | 109 ++++++++++++++++---- 5 files changed, 144 insertions(+), 20 deletions(-) diff --git a/src/Controllers/OaiController.php b/src/Controllers/OaiController.php index 7570ddd..c115c1d 100644 --- a/src/Controllers/OaiController.php +++ b/src/Controllers/OaiController.php @@ -76,6 +76,13 @@ class OaiController extends Controller */ private static string $oai_records_per_page = '100'; + /** + * The expiration time (in seconds) of any resumption tokens that are generated. Default is 60 minutes + * + * Set this to null if you want an infinite duration + */ + private static ?int $resumption_token_expiry = 3600; + public function index(HTTPRequest $request): HTTPResponse { $this->getResponse()->addHeader('Content-type', 'text/xml'); @@ -271,6 +278,10 @@ protected function ListRecords(HTTPRequest $request): HTTPResponse ); $xmlDocument->setResumptionToken($newResumptionToken); + } elseif ($resumptionToken) { + // If this is the last page of a request that included a Resumption Token, then we specifically need to add + // an empty Token - indicating that the list is now complete + $xmlDocument->setResumptionToken(''); } return $this->getResponseWithDocumentBody($xmlDocument); diff --git a/src/Documents/ListRecordsDocument.php b/src/Documents/ListRecordsDocument.php index 4ddbc23..ffb47fc 100644 --- a/src/Documents/ListRecordsDocument.php +++ b/src/Documents/ListRecordsDocument.php @@ -4,6 +4,7 @@ use SilverStripe\ORM\PaginatedList; use Terraformers\OpenArchive\Formatters\OaiRecordFormatter; +use Terraformers\OpenArchive\Helpers\ResumptionTokenHelper; use Terraformers\OpenArchive\Models\OaiRecord; class ListRecordsDocument extends OaiDocument @@ -38,6 +39,14 @@ public function setResumptionToken(string $resumptionToken): void $resumptionTokenElement = $this->findOrCreateElement('resumptionToken', $listRecordsElement); $resumptionTokenElement->nodeValue = $resumptionToken; + + $tokenExpiry = ResumptionTokenHelper::getExpiryFromResumptionToken($resumptionToken); + + if (!$tokenExpiry) { + return; + } + + $resumptionTokenElement->setAttribute('expirationDate', $tokenExpiry); } } diff --git a/src/Helpers/DateTimeHelper.php b/src/Helpers/DateTimeHelper.php index 1bc013e..fc0b1d2 100644 --- a/src/Helpers/DateTimeHelper.php +++ b/src/Helpers/DateTimeHelper.php @@ -15,6 +15,7 @@ public static function getLocalStringFromUtc(string $utcDateString): string throw new Exception('Invalid UTC date format provided'); } + // Note: strtotime() already converts UTC date strings (UTC+Z) into local timestamps return date('Y-m-d H:i:s', strtotime($utcDateString)); } diff --git a/src/Helpers/ResumptionTokenHelper.php b/src/Helpers/ResumptionTokenHelper.php index bbc2b6e..7aaeeb0 100644 --- a/src/Helpers/ResumptionTokenHelper.php +++ b/src/Helpers/ResumptionTokenHelper.php @@ -3,6 +3,8 @@ namespace Terraformers\OpenArchive\Helpers; use Exception; +use SilverStripe\ORM\FieldType\DBDatetime; +use Terraformers\OpenArchive\Controllers\OaiController; /** * Resumption Tokens are a form of pagination, however, they also contain a level of validation. @@ -31,6 +33,16 @@ public static function generateResumptionToken( 'verb' => $verb, ]; + // Check to see if we want to give our Tokens an expiry date + $tokenExpiryLength = OaiController::config()->get('resumption_token_expiry'); + + if ($tokenExpiryLength) { + // Set the expiry date for a time in the future matching the expiry length + $parts['expiry'] = DateTimeHelper::getUtcStringFromLocal( + date('Y-m-d H:i:s', DBDatetime::now()->getTimestamp() + $tokenExpiryLength) + ); + } + if ($from) { $parts['from'] = $from; } @@ -61,6 +73,7 @@ public static function getPageFromResumptionToken( $resumptionFrom = $resumptionParts['from'] ?? null; $resumptionUntil = $resumptionParts['until'] ?? null; $resumptionSet = $resumptionParts['set'] ?? null; + $resumptionExpiry = $resumptionParts['expiry'] ?? null; // Every Resumption Token should include (at the very least) the active page, if it doesn't, then it's invalid if (!$resumptionPage) { @@ -76,10 +89,31 @@ public static function getPageFromResumptionToken( throw new Exception('Invalid resumption token'); } + // The duration that each Token lives (in seconds) + $tokenExpiryLength = OaiController::config()->get('resumption_token_expiry'); + + // The duration has been set to infinite, so we can return now + if (!$tokenExpiryLength) { + return $resumptionPage; + } + + // If the current time is greater than the expiry date of the Resumption Token, then this Token is invalid + // Note: strtotime() already converts UTC date strings (UTC+Z) into local timestamps + if (DBDatetime::now()->getTimestamp() > strtotime($resumptionExpiry)) { + throw new Exception('Invalid resumption token'); + } + // The Resumption Token is valid, so we can return whatever value we have for page return $resumptionPage; } + public static function getExpiryFromResumptionToken(string $resumptionToken): ?string + { + $resumptionParts = static::getResumptionTokenParts($resumptionToken); + + return $resumptionParts['expiry'] ?? null; + } + protected static function getResumptionTokenParts(string $resumptionToken): array { $decode = base64_decode($resumptionToken, true); diff --git a/tests/Helpers/ResumptionTokenHelperTest.php b/tests/Helpers/ResumptionTokenHelperTest.php index fcbed19..9068d43 100644 --- a/tests/Helpers/ResumptionTokenHelperTest.php +++ b/tests/Helpers/ResumptionTokenHelperTest.php @@ -4,29 +4,102 @@ use ReflectionClass; use SilverStripe\Dev\SapphireTest; +use SilverStripe\ORM\FieldType\DBDatetime; +use Terraformers\OpenArchive\Controllers\OaiController; use Terraformers\OpenArchive\Helpers\ResumptionTokenHelper; class ResumptionTokenHelperTest extends SapphireTest { - public function testResumptionToken(): void + public function testResumptionTokenWithExpiry(): void { + // We'll use Auckland time for our tests + date_default_timezone_set('Pacific/Auckland'); + + DBDatetime::set_mock_now('2020-01-01 13:00:00'); + + $verb = 'ListRecords'; + $page = 3; + $from = '2022-01-01T01:00:00Z'; + $until = '2022-01-01T02:00:00Z'; + $set = 2; + // -13 for UTC+0, but +1 for the duration of the expiry + $expiry = '2020-01-01T01:00:00Z'; + $expectedParts = [ - 'verb' => 'ListRecords', - 'page' => 3, - 'from' => '2022-01-01T01:00:00Z', - 'until' => '2022-01-01T02:00:00Z', - 'set' => 2, + 'verb' => $verb, + 'page' => $page, + 'from' => $from, + 'until' => $until, + 'set' => $set, + 'expiry' => $expiry, ]; // Generate our Token - $token = ResumptionTokenHelper::generateResumptionToken( - 'ListRecords', - 3, - '2022-01-01T01:00:00Z', - '2022-01-01T02:00:00Z', - 2 + $token = ResumptionTokenHelper::generateResumptionToken($verb, $page, $from, $until, $set); + + // Now decode that Token + $reflection = new ReflectionClass(ResumptionTokenHelper::class); + $method = $reflection->getMethod('getResumptionTokenParts'); + $method->setAccessible(true); + $resumptionParts = $method->invoke(null, $token); + + // And check that the Token that was encoded and decoded matches our expected values + $this->assertEquals(ksort($expectedParts), ksort($resumptionParts)); + // Check that our "get page number" method works as well + $this->assertEquals( + $page, + ResumptionTokenHelper::getPageFromResumptionToken($token, $verb, $from, $until, $set) ); + // Check that our "get expiry" method works as well + $this->assertEquals($expiry, ResumptionTokenHelper::getExpiryFromResumptionToken($token)); + } + + public function testResumptionTokenHasExpired(): void + { + // We'll use Auckland time for our tests + date_default_timezone_set('Pacific/Auckland'); + + $this->expectExceptionMessage('Invalid resumption token'); + + DBDatetime::set_mock_now('2020-01-01 13:00:00'); + + $verb = 'ListRecords'; + $page = 3; + $from = '2022-01-01T01:00:00Z'; + $until = '2022-01-01T02:00:00Z'; + $set = 2; + + // Generate our Token + $token = ResumptionTokenHelper::generateResumptionToken($verb, $page, $from, $until, $set); + + // Now set the time to a couple hours later. This should invalidate the Resumption Token + DBDatetime::set_mock_now('2020-01-01 15:00:00'); + + // This should throw an Exception + ResumptionTokenHelper::getPageFromResumptionToken($token, $verb, $from, $until, $set); + } + + public function testResumptionTokenNoExpiry(): void + { + OaiController::config()->set('resumption_token_expiry', null); + + $verb = 'ListRecords'; + $page = 3; + $from = '2022-01-01T01:00:00Z'; + $until = '2022-01-01T02:00:00Z'; + $set = 2; + + $expectedParts = [ + 'verb' => $verb, + 'page' => $page, + 'from' => $from, + 'until' => $until, + 'set' => $set, + ]; + + // Generate our Token + $token = ResumptionTokenHelper::generateResumptionToken($verb, $page, $from, $until, $set); // Now decode that Token $reflection = new ReflectionClass(ResumptionTokenHelper::class); @@ -38,15 +111,11 @@ public function testResumptionToken(): void $this->assertEquals(ksort($expectedParts), ksort($resumptionParts)); // And check that our "get page number" method works as well $this->assertEquals( - 3, - ResumptionTokenHelper::getPageFromResumptionToken( - $token, - 'ListRecords', - '2022-01-01T01:00:00Z', - '2022-01-01T02:00:00Z', - 2 - ) + $page, + ResumptionTokenHelper::getPageFromResumptionToken($token, $verb, $from, $until, $set) ); + // Check that our "get expiry" method works as well (expecting there to be no value) + $this->assertNull(ResumptionTokenHelper::getExpiryFromResumptionToken($token)); } }