From 67cafb6a182d50e0efa236e4410e60bd7fd4fc3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Fri, 10 Jul 2020 10:24:31 +0200 Subject: [PATCH 1/8] [INDEX-4] Collect FDP metadata (#11) --- .gitignore | 2 + package-lock.json | 5 + package.json | 3 +- pom.xml | 24 ++- .../index/FairDataPointIndexApplication.java | 1 + .../api/controller/ApiExceptionHandler.java | 38 ++++ .../api/controller/EntriesController.java | 5 +- .../index/api/controller/PingController.java | 19 +- .../fairdata/fdp/index/api/dto/ErrorDTO.java | 33 ++++ .../fdp/index/api/dto/IndexEntryDTO.java | 8 +- .../fdp/index/config/AsyncConfig.java | 34 ++++ .../fdp/index/config/CustomConfig.java | 50 +++++ .../fdp/index/config/ObjectMapperConfig.java | 43 ++++ .../fdp/index/config/StorageConfig.java | 2 +- .../changelogs/DatabaseChangeLog.java | 37 ++++ ...ryRepository.java => EventRepository.java} | 10 +- .../repository/IndexEntryRepository.java | 44 +++++ .../fairdata/fdp/index/entity/IndexEntry.java | 23 ++- .../fdp/index/entity/IndexEntryState.java | 30 +++ .../fdp/index/entity/RepositoryMetadata.java | 41 ++++ .../fdp/index/entity/config/EventsConfig.java | 36 ++++ .../fdp/index/entity/config/package-info.java | 25 +++ .../fdp/index/entity/events/Event.java | 103 ++++++++++ .../fdp/index/entity/events/EventType.java | 28 +++ .../fdp/index/entity/events/IncomingPing.java | 35 ++++ .../entity/events/MetadataRetrieval.java | 38 ++++ .../fdp/index/entity/events/package-info.java | 25 +++ .../fdp/index/entity/http/Exchange.java | 49 +++++ .../index/entity/http/ExchangeDirection.java | 28 +++ .../fdp/index/entity/http/ExchangeState.java | 31 +++ .../fdp/index/entity/http/Request.java | 61 ++++++ .../fdp/index/entity/http/Response.java | 50 +++++ .../fdp/index/entity/http/package-info.java | 25 +++ .../IncorrectPingFormatException.java | 32 +++ .../fdp/index/exceptions/IndexException.java | 44 +++++ .../fdp/index/exceptions/package-info.java | 25 +++ .../fdp/index/service/EventService.java | 187 ++++++++++++++++++ .../fdp/index/service/IndexEntryService.java | 86 ++++++-- .../fdp/index/utils/IncomingPingUtils.java | 45 +++++ .../index/utils/MetadataRetrievalUtils.java | 173 ++++++++++++++++ .../fdp/index/utils/package-info.java | 25 +++ .../index/web/controller/EntryController.java | 61 ++++++ .../index/web/controller/HomeController.java | 25 ++- src/main/resources/application.yml | 14 ++ src/main/resources/static/img/no_data.svg | 1 + src/main/resources/static/js/common.js | 5 + src/main/resources/templates/entry.html | 118 +++++++++++ src/main/resources/templates/home.html | 122 ++++++++---- src/main/resources/templates/layout.html | 3 +- src/main/scss/site.scss | 91 ++++++++- .../api/entries/EntriesAll_GET_Test.java | 8 +- .../api/entries/EntriesPage_GET_Test.java | 10 +- .../api/ping/ReceivePing_POST_Test.java | 14 +- .../acceptance/web/home/Entry_GET_Test.java | 182 +++++++++++++++++ .../acceptance/web/home/Home_GET_Test.java | 59 +++--- .../acceptance/web/home/Static_GET_Test.java | 80 ++++++++ .../index/fixtures/IndexEntryFixtures.java | 42 +++- src/test/resources/application-testing.yml | 14 ++ 58 files changed, 2313 insertions(+), 139 deletions(-) create mode 100644 src/main/java/solutions/fairdata/fdp/index/api/controller/ApiExceptionHandler.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/api/dto/ErrorDTO.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/config/AsyncConfig.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/config/CustomConfig.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/config/ObjectMapperConfig.java rename src/main/java/solutions/fairdata/fdp/index/database/repository/{EntryRepository.java => EventRepository.java} (79%) create mode 100644 src/main/java/solutions/fairdata/fdp/index/database/repository/IndexEntryRepository.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/IndexEntryState.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/RepositoryMetadata.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/config/EventsConfig.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/config/package-info.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/events/Event.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/events/EventType.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/events/IncomingPing.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/events/MetadataRetrieval.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/events/package-info.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/http/Exchange.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/http/ExchangeDirection.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/http/ExchangeState.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/http/Request.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/http/Response.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/entity/http/package-info.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/exceptions/IncorrectPingFormatException.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/exceptions/IndexException.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/exceptions/package-info.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/service/EventService.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/utils/IncomingPingUtils.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/utils/MetadataRetrievalUtils.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/utils/package-info.java create mode 100644 src/main/java/solutions/fairdata/fdp/index/web/controller/EntryController.java create mode 100644 src/main/resources/static/img/no_data.svg create mode 100644 src/main/resources/static/js/common.js create mode 100644 src/main/resources/templates/entry.html create mode 100644 src/test/java/solutions/fairdata/fdp/index/acceptance/web/home/Entry_GET_Test.java create mode 100644 src/test/java/solutions/fairdata/fdp/index/acceptance/web/home/Static_GET_Test.java diff --git a/.gitignore b/.gitignore index 0e13eeb..99ddcbc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +node/ +node_modules/ target/ pom.xml.tag pom.xml.releaseBackup diff --git a/package-lock.json b/package-lock.json index b9f8d43..1f2f491 100644 --- a/package-lock.json +++ b/package-lock.json @@ -974,6 +974,11 @@ "pinkie": "^2.0.0" } }, + "popper.js": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.16.1.tgz", + "integrity": "sha512-Wb4p1J4zyFTbM+u6WuO4XstYx4Ky9Cewe4DWrel7B0w6VVICvPwdOpotjzcf6eD8TsckVnIMNONQyPIUFOUbCQ==" + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", diff --git a/package.json b/package.json index 5120e81..20b60bb 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "bootstrap": "^4.5.0", "copyfiles": "^2.1.0", "jquery": "^3.5.1", - "node-sass": "4.14.1" + "node-sass": "4.14.1", + "popper.js": "^1.16.1" } } diff --git a/pom.xml b/pom.xml index 868aa2e..77f441b 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ solutions.fairdata fairdatapoint-index - 0.1.1 + 0.2.0-SNAPSHOT 2020 @@ -45,6 +45,7 @@ 3.0 0.13 v12.16.3 + 3.2.2 4.0.0 1.3.9 @@ -76,6 +77,23 @@ mongobee ${mongobee.version} + + + org.eclipse.rdf4j + rdf4j-query + ${rdf4j.version} + + + org.eclipse.rdf4j + rdf4j-rio-api + ${rdf4j.version} + + + org.eclipse.rdf4j + rdf4j-rio-turtle + ${rdf4j.version} + runtime + org.springdoc @@ -115,6 +133,10 @@ ${project.basedir}/node_modules/bootstrap/dist/js static/js/bootstrap + + ${project.basedir}/node_modules/popper.js/dist + static/js/popper + diff --git a/src/main/java/solutions/fairdata/fdp/index/FairDataPointIndexApplication.java b/src/main/java/solutions/fairdata/fdp/index/FairDataPointIndexApplication.java index e518374..af8fd1e 100644 --- a/src/main/java/solutions/fairdata/fdp/index/FairDataPointIndexApplication.java +++ b/src/main/java/solutions/fairdata/fdp/index/FairDataPointIndexApplication.java @@ -27,6 +27,7 @@ @SpringBootApplication public class FairDataPointIndexApplication { + public static void main(String[] args) { SpringApplication.run(FairDataPointIndexApplication.class, args); } diff --git a/src/main/java/solutions/fairdata/fdp/index/api/controller/ApiExceptionHandler.java b/src/main/java/solutions/fairdata/fdp/index/api/controller/ApiExceptionHandler.java new file mode 100644 index 0000000..5c2796c --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/api/controller/ApiExceptionHandler.java @@ -0,0 +1,38 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.api.controller; + +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import solutions.fairdata.fdp.index.api.dto.ErrorDTO; +import solutions.fairdata.fdp.index.exceptions.IndexException; + +@ControllerAdvice +public class ApiExceptionHandler { + + @ExceptionHandler(IndexException.class) + public ResponseEntity handleIndexException(IndexException exception) { + return new ResponseEntity<>(exception.getErrorDTO(), exception.getStatus()); + } +} diff --git a/src/main/java/solutions/fairdata/fdp/index/api/controller/EntriesController.java b/src/main/java/solutions/fairdata/fdp/index/api/controller/EntriesController.java index 0bcfdaa..fc162f4 100644 --- a/src/main/java/solutions/fairdata/fdp/index/api/controller/EntriesController.java +++ b/src/main/java/solutions/fairdata/fdp/index/api/controller/EntriesController.java @@ -28,6 +28,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import solutions.fairdata.fdp.index.api.dto.IndexEntryDTO; import solutions.fairdata.fdp.index.service.IndexEntryService; @@ -45,8 +46,8 @@ public class EntriesController { private IndexEntryService service; @GetMapping("") - public Page getEntriesPage(Pageable pageable) { - return service.getEntriesPage(pageable).map(service::toDTO); + public Page getEntriesPage(Pageable pageable, @RequestParam(defaultValue = "all") String state) { + return service.getEntriesPage(pageable, state).map(service::toDTO); } @GetMapping("/all") diff --git a/src/main/java/solutions/fairdata/fdp/index/api/controller/PingController.java b/src/main/java/solutions/fairdata/fdp/index/api/controller/PingController.java index c967297..9df2d91 100644 --- a/src/main/java/solutions/fairdata/fdp/index/api/controller/PingController.java +++ b/src/main/java/solutions/fairdata/fdp/index/api/controller/PingController.java @@ -27,12 +27,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpEntity; import org.springframework.http.HttpStatus; import org.springframework.web.bind.annotation.*; -import solutions.fairdata.fdp.index.api.dto.PingDTO; -import solutions.fairdata.fdp.index.service.IndexEntryService; +import solutions.fairdata.fdp.index.entity.events.Event; +import solutions.fairdata.fdp.index.exceptions.IncorrectPingFormatException; +import solutions.fairdata.fdp.index.service.EventService; -import javax.validation.Valid; +import javax.servlet.http.HttpServletRequest; @Tag(name = "Ping") @RestController @@ -41,14 +43,15 @@ public class PingController { private static final Logger logger = LoggerFactory.getLogger(PingController.class); @Autowired - private IndexEntryService service; + private EventService eventService; @Operation(hidden = true) @PostMapping @ResponseStatus(HttpStatus.NO_CONTENT) - public void receivePing(@RequestBody @Valid PingDTO ping) { - logger.info("Received ping from {}", ping); - - service.storeEntry(ping.getClientUrl()); + public void receivePing(HttpEntity httpEntity, HttpServletRequest request) throws IncorrectPingFormatException { + logger.info("Received ping from {}", request.getRemoteAddr()); + final Event incomingPingEvent = eventService.acceptIncomingPing(httpEntity, request); + logger.info("Triggering metadata retrieval for {}", incomingPingEvent.getRelatedTo().getClientUrl()); + eventService.triggerMetadataRetrieval(incomingPingEvent); } } diff --git a/src/main/java/solutions/fairdata/fdp/index/api/dto/ErrorDTO.java b/src/main/java/solutions/fairdata/fdp/index/api/dto/ErrorDTO.java new file mode 100644 index 0000000..2b0fe50 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/api/dto/ErrorDTO.java @@ -0,0 +1,33 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.api.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class ErrorDTO { + private int code; + private String message; +} diff --git a/src/main/java/solutions/fairdata/fdp/index/api/dto/IndexEntryDTO.java b/src/main/java/solutions/fairdata/fdp/index/api/dto/IndexEntryDTO.java index b13cf38..7081952 100644 --- a/src/main/java/solutions/fairdata/fdp/index/api/dto/IndexEntryDTO.java +++ b/src/main/java/solutions/fairdata/fdp/index/api/dto/IndexEntryDTO.java @@ -27,7 +27,6 @@ import org.hibernate.validator.constraints.URL; import javax.validation.constraints.NotNull; -import java.time.OffsetDateTime; @Data @Schema(name = "Entry") @@ -37,8 +36,11 @@ public class IndexEntryDTO { private String clientUrl; @NotNull - private OffsetDateTime registrationTime; + private String state; @NotNull - private OffsetDateTime modificationTime; + private String registrationTime; + + @NotNull + private String modificationTime; } diff --git a/src/main/java/solutions/fairdata/fdp/index/config/AsyncConfig.java b/src/main/java/solutions/fairdata/fdp/index/config/AsyncConfig.java new file mode 100644 index 0000000..359d5b4 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/config/AsyncConfig.java @@ -0,0 +1,34 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.config; + +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.scheduling.annotation.EnableAsync; +import solutions.fairdata.fdp.index.Profiles; + +@Configuration +@EnableAsync +@Profile(Profiles.NON_TESTING) +public class AsyncConfig { +} diff --git a/src/main/java/solutions/fairdata/fdp/index/config/CustomConfig.java b/src/main/java/solutions/fairdata/fdp/index/config/CustomConfig.java new file mode 100644 index 0000000..b07a5ee --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/config/CustomConfig.java @@ -0,0 +1,50 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.config; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Scope; +import solutions.fairdata.fdp.index.entity.config.EventsConfig; + +import java.time.Duration; + +@Configuration +public class CustomConfig { + + @Bean + @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON) + public EventsConfig eventsConfig( + @Value("${fdp-index.events.retrieval.rateLimitWait:PT10M}") String cfgRetrievalRateLimitWait, + @Value("${fdp-index.events.retrieval.timeout:PT1M}") String cfgRetrievalTimeout, + @Value("${fdp-index.events.ping.validDuration:P7D}") String cfgPingValidDuration + ) { + return EventsConfig.builder() + .retrievalRateLimitWait(Duration.parse(cfgRetrievalRateLimitWait)) + .retrievalTimeout(Duration.parse(cfgRetrievalTimeout)) + .pingValidDuration(Duration.parse(cfgPingValidDuration)) + .build(); + } +} diff --git a/src/main/java/solutions/fairdata/fdp/index/config/ObjectMapperConfig.java b/src/main/java/solutions/fairdata/fdp/index/config/ObjectMapperConfig.java new file mode 100644 index 0000000..3c11502 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/config/ObjectMapperConfig.java @@ -0,0 +1,43 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.config; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Scope; + +@Configuration +public class ObjectMapperConfig { + + @Bean + @Scope(ConfigurableBeanFactory.SCOPE_SINGLETON) + public ObjectMapper objectMapper() { + ObjectMapper mapper = new ObjectMapper(); + mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + return mapper; + } +} diff --git a/src/main/java/solutions/fairdata/fdp/index/config/StorageConfig.java b/src/main/java/solutions/fairdata/fdp/index/config/StorageConfig.java index 210eee1..c9f8bf9 100644 --- a/src/main/java/solutions/fairdata/fdp/index/config/StorageConfig.java +++ b/src/main/java/solutions/fairdata/fdp/index/config/StorageConfig.java @@ -45,7 +45,7 @@ public class StorageConfig { @Bean public Mongobee mongobee() throws Exception { Mongobee runner = new Mongobee(mongoUri); - runner.setChangeLogsScanPackage("solutions.fairdata.fdp.index.storage.changelogs"); + runner.setChangeLogsScanPackage("solutions.fairdata.fdp.index.database.changelogs"); runner.setSpringEnvironment(environment); runner.execute(); return runner; diff --git a/src/main/java/solutions/fairdata/fdp/index/database/changelogs/DatabaseChangeLog.java b/src/main/java/solutions/fairdata/fdp/index/database/changelogs/DatabaseChangeLog.java index d3c50a9..7cebd76 100644 --- a/src/main/java/solutions/fairdata/fdp/index/database/changelogs/DatabaseChangeLog.java +++ b/src/main/java/solutions/fairdata/fdp/index/database/changelogs/DatabaseChangeLog.java @@ -24,7 +24,16 @@ import com.github.mongobee.changeset.ChangeLog; import com.github.mongobee.changeset.ChangeSet; +import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; +import org.bson.Document; +import solutions.fairdata.fdp.index.entity.IndexEntry; +import solutions.fairdata.fdp.index.entity.IndexEntryState; + +import java.time.Instant; +import java.time.OffsetDateTime; @ChangeLog public class DatabaseChangeLog { @@ -32,4 +41,32 @@ public class DatabaseChangeLog { public void initMongoDB(MongoDatabase db) { // Nothing to DO, just "first" making the version } + + @ChangeSet(order = "001", id = "entryTimestampsToInstant", author = "MarekSuchanek") + public void entryTimestampsToInstant(MongoDatabase db) { + MongoCollection indexEntries = db.getCollection("indexEntry"); + for (Document indexEntry : indexEntries.find()) { + if (!(indexEntry.get("registrationTime") instanceof String)) continue; + String registrationTimeStr = indexEntry.getString("registrationTime"); + Instant registrationTimeDate = OffsetDateTime.parse(registrationTimeStr).toInstant(); + String modificationTimeStr = indexEntry.getString("modificationTime"); + Instant modificationTimeDate = OffsetDateTime.parse(modificationTimeStr).toInstant(); + indexEntries.updateOne( + Filters.eq("_id", indexEntry.getObjectId("_id")), + Updates.combine( + Updates.set("registrationTime", registrationTimeDate), + Updates.set("modificationTime", modificationTimeDate) + ) + ); + } + } + + @ChangeSet(order = "002", id = "addIndexEntryState", author = "MarekSuchanek") + public void addIndexEntryState(MongoDatabase db) { + MongoCollection indexEntries = db.getCollection("indexEntry"); + indexEntries.updateMany( + Filters.exists("state", false), + Updates.set("state", IndexEntryState.Unknown.toString()) + ); + } } diff --git a/src/main/java/solutions/fairdata/fdp/index/database/repository/EntryRepository.java b/src/main/java/solutions/fairdata/fdp/index/database/repository/EventRepository.java similarity index 79% rename from src/main/java/solutions/fairdata/fdp/index/database/repository/EntryRepository.java rename to src/main/java/solutions/fairdata/fdp/index/database/repository/EventRepository.java index 4dc3cc9..02b7a71 100644 --- a/src/main/java/solutions/fairdata/fdp/index/database/repository/EntryRepository.java +++ b/src/main/java/solutions/fairdata/fdp/index/database/repository/EventRepository.java @@ -22,11 +22,15 @@ */ package solutions.fairdata.fdp.index.database.repository; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.mongodb.repository.MongoRepository; import solutions.fairdata.fdp.index.entity.IndexEntry; +import solutions.fairdata.fdp.index.entity.events.Event; -import java.util.Optional; +public interface EventRepository extends MongoRepository { -public interface EntryRepository extends MongoRepository { - Optional findByClientUrl(String clientUrl); + Iterable getAllByFinishedIsNull(); + + Page getAllByRelatedTo(IndexEntry indexEntry, Pageable pageable); } diff --git a/src/main/java/solutions/fairdata/fdp/index/database/repository/IndexEntryRepository.java b/src/main/java/solutions/fairdata/fdp/index/database/repository/IndexEntryRepository.java new file mode 100644 index 0000000..20cc2e2 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/database/repository/IndexEntryRepository.java @@ -0,0 +1,44 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.database.repository; + +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.mongodb.repository.MongoRepository; +import solutions.fairdata.fdp.index.entity.IndexEntry; +import solutions.fairdata.fdp.index.entity.IndexEntryState; + +import java.time.Instant; +import java.util.Optional; + +public interface IndexEntryRepository extends MongoRepository { + Optional findByClientUrl(String clientUrl); + + Page findAllByStateEquals(Pageable pageable, IndexEntryState state); + Page findAllByStateEqualsAndLastRetrievalTimeBefore(Pageable pageable, IndexEntryState state, Instant when); + Page findAllByStateEqualsAndLastRetrievalTimeAfter(Pageable pageable, IndexEntryState state, Instant when); + + long countAllByStateEquals(IndexEntryState state); + long countAllByStateEqualsAndLastRetrievalTimeAfter(IndexEntryState state, Instant when); + long countAllByStateEqualsAndLastRetrievalTimeBefore(IndexEntryState state, Instant when); +} diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/IndexEntry.java b/src/main/java/solutions/fairdata/fdp/index/entity/IndexEntry.java index 63dc7f6..5a66b55 100644 --- a/src/main/java/solutions/fairdata/fdp/index/entity/IndexEntry.java +++ b/src/main/java/solutions/fairdata/fdp/index/entity/IndexEntry.java @@ -25,14 +25,33 @@ import lombok.Data; import org.bson.types.ObjectId; import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.format.annotation.DateTimeFormat; + +import java.time.Duration; +import java.time.Instant; @Document @Data public class IndexEntry { @Id protected ObjectId id; + @Indexed(unique=true) private String clientUrl; - private String registrationTime; - private String modificationTime; + private IndexEntryState state = IndexEntryState.Unknown; + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private Instant registrationTime; + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private Instant modificationTime; + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private Instant lastRetrievalTime; + private RepositoryMetadata currentMetadata; + + public Duration getLastRetrievalAgo() { + if (lastRetrievalTime == null) { + return null; + } + return Duration.between(lastRetrievalTime, Instant.now()); + } } diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/IndexEntryState.java b/src/main/java/solutions/fairdata/fdp/index/entity/IndexEntryState.java new file mode 100644 index 0000000..11221cb --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/IndexEntryState.java @@ -0,0 +1,30 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.entity; + +public enum IndexEntryState { + Unknown, + Valid, // Active / Inactive based on timestamps + Unreachable, + Invalid +} diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/RepositoryMetadata.java b/src/main/java/solutions/fairdata/fdp/index/entity/RepositoryMetadata.java new file mode 100644 index 0000000..53fbb90 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/RepositoryMetadata.java @@ -0,0 +1,41 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.entity; + +import lombok.Data; + +import javax.validation.constraints.NotNull; +import java.util.HashMap; + +@Data +public class RepositoryMetadata { + + public static final Integer CURRENT_VERSION = 1; + + private Integer metadataVersion = CURRENT_VERSION; + + private String repositoryUri; + + @NotNull + private HashMap metadata = new HashMap<>(); +} diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/config/EventsConfig.java b/src/main/java/solutions/fairdata/fdp/index/entity/config/EventsConfig.java new file mode 100644 index 0000000..581ec25 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/config/EventsConfig.java @@ -0,0 +1,36 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.entity.config; + +import lombok.Builder; +import lombok.Data; + +import java.time.Duration; + +@Builder +@Data +public class EventsConfig { + private final Duration retrievalRateLimitWait; + private final Duration retrievalTimeout; + private final Duration pingValidDuration; +} diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/config/package-info.java b/src/main/java/solutions/fairdata/fdp/index/entity/config/package-info.java new file mode 100644 index 0000000..cea5231 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/config/package-info.java @@ -0,0 +1,25 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +@javax.annotation.ParametersAreNonnullByDefault +package solutions.fairdata.fdp.index.entity.config; diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/events/Event.java b/src/main/java/solutions/fairdata/fdp/index/entity/events/Event.java new file mode 100644 index 0000000..5648fe5 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/events/Event.java @@ -0,0 +1,103 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.entity.events; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.bson.types.ObjectId; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.index.Indexed; +import org.springframework.data.mongodb.core.mapping.DBRef; +import org.springframework.data.mongodb.core.mapping.Document; +import org.springframework.format.annotation.DateTimeFormat; +import solutions.fairdata.fdp.index.entity.IndexEntry; + +import javax.validation.constraints.NotNull; +import java.time.Instant; +import java.util.UUID; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Document(collection = "event") +public class Event { + @Id + protected ObjectId id; + @Indexed(unique=true) + @NotNull + private UUID uuid = UUID.randomUUID(); + @NotNull + private EventType type; + @NotNull + private Integer version; + + @DBRef + private Event triggeredBy; + @DBRef + private IndexEntry relatedTo; + + // Content (one of those) + private IncomingPing incomingPing; + private MetadataRetrieval metadataRetrieval; + + @NotNull + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private Instant created = Instant.now(); + + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private Instant executed; + + @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) + private Instant finished; + + public boolean isExecuted() { + return executed != null; + } + + public void execute() { + executed = Instant.now(); + } + + public boolean isFinished() { + return finished != null; + } + + public void finish() { + finished = Instant.now(); + } + + public Event(Integer version, IncomingPing incomingPing) { + this.type = EventType.IncomingPing; + this.version = version; + this.incomingPing = incomingPing; + } + + public Event(Integer version, Event triggerEvent, IndexEntry relatedTo, MetadataRetrieval metadataRetrieval) { + this.type = EventType.MetadataRetrieval; + this.version = version; + this.triggeredBy = triggerEvent; + this.relatedTo = relatedTo; + this.metadataRetrieval = metadataRetrieval; + } +} diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/events/EventType.java b/src/main/java/solutions/fairdata/fdp/index/entity/events/EventType.java new file mode 100644 index 0000000..c9574cc --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/events/EventType.java @@ -0,0 +1,28 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.entity.events; + +public enum EventType { + MetadataRetrieval, + IncomingPing; +} diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/events/IncomingPing.java b/src/main/java/solutions/fairdata/fdp/index/entity/events/IncomingPing.java new file mode 100644 index 0000000..5dfe1e4 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/events/IncomingPing.java @@ -0,0 +1,35 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.entity.events; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import solutions.fairdata.fdp.index.entity.http.Exchange; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class IncomingPing { + private Exchange exchange; +} diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/events/MetadataRetrieval.java b/src/main/java/solutions/fairdata/fdp/index/entity/events/MetadataRetrieval.java new file mode 100644 index 0000000..655d26a --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/events/MetadataRetrieval.java @@ -0,0 +1,38 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.entity.events; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import solutions.fairdata.fdp.index.entity.RepositoryMetadata; +import solutions.fairdata.fdp.index.entity.http.Exchange; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class MetadataRetrieval { + private String error; + private Exchange exchange; + private RepositoryMetadata metadata; +} diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/events/package-info.java b/src/main/java/solutions/fairdata/fdp/index/entity/events/package-info.java new file mode 100644 index 0000000..e847034 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/events/package-info.java @@ -0,0 +1,25 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +@javax.annotation.ParametersAreNonnullByDefault +package solutions.fairdata.fdp.index.entity.events; diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/http/Exchange.java b/src/main/java/solutions/fairdata/fdp/index/entity/http/Exchange.java new file mode 100644 index 0000000..88f541c --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/http/Exchange.java @@ -0,0 +1,49 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.entity.http; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Exchange { + private ExchangeDirection direction; + private ExchangeState state = ExchangeState.Prepared; + private String remoteAddr; + private String error; + + private Request request = new Request(); + private Response response = new Response(); + + public Exchange(ExchangeDirection direction, String remoteAddr) { + this.direction = direction; + this.remoteAddr = remoteAddr; + } + + public Exchange(ExchangeDirection direction) { + this.direction = direction; + } +} diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/http/ExchangeDirection.java b/src/main/java/solutions/fairdata/fdp/index/entity/http/ExchangeDirection.java new file mode 100644 index 0000000..c909cc4 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/http/ExchangeDirection.java @@ -0,0 +1,28 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.entity.http; + +public enum ExchangeDirection { + INCOMING, + OUTGOING +} diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/http/ExchangeState.java b/src/main/java/solutions/fairdata/fdp/index/entity/http/ExchangeState.java new file mode 100644 index 0000000..67726d1 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/http/ExchangeState.java @@ -0,0 +1,31 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.entity.http; + +public enum ExchangeState { + Prepared, + Requested, + Timeout, + Failed, + Retrieved +} diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/http/Request.java b/src/main/java/solutions/fairdata/fdp/index/entity/http/Request.java new file mode 100644 index 0000000..6e28397 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/http/Request.java @@ -0,0 +1,61 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.entity.http; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.http.HttpEntity; + +import javax.servlet.http.HttpServletRequest; +import java.net.http.HttpRequest; +import java.util.List; +import java.util.Map; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Request { + private String method; + private String url; + + private Map> headers; + private String body; + + public void setFromHttpEntity(HttpEntity httpEntity) { + body = httpEntity.getBody(); + headers = httpEntity.getHeaders(); + } + + public void setFromHttpServletRequest(HttpServletRequest request) { + method = request.getMethod(); + url = request.getRequestURI(); + } + + public void setFromHttpRequest(HttpRequest request) { + method = request.method(); + url = request.uri().toString(); + body = request.bodyPublisher().map(Object::toString).orElse(null); + headers = request.headers().map(); + } +} diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/http/Response.java b/src/main/java/solutions/fairdata/fdp/index/entity/http/Response.java new file mode 100644 index 0000000..fc1f20c --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/http/Response.java @@ -0,0 +1,50 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.entity.http; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.net.http.HttpResponse; +import java.util.List; +import java.util.Map; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class Response { + private Integer code; + private String url; + private String origin; + + private Map> headers; + private String body; + + public void setFromHttpResponse(HttpResponse response) { + code = response.statusCode(); + url = response.uri().toString(); + headers = response.headers().map(); + body = response.body(); + } +} diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/http/package-info.java b/src/main/java/solutions/fairdata/fdp/index/entity/http/package-info.java new file mode 100644 index 0000000..64c8d3f --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/entity/http/package-info.java @@ -0,0 +1,25 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +@javax.annotation.ParametersAreNonnullByDefault +package solutions.fairdata.fdp.index.entity.http; diff --git a/src/main/java/solutions/fairdata/fdp/index/exceptions/IncorrectPingFormatException.java b/src/main/java/solutions/fairdata/fdp/index/exceptions/IncorrectPingFormatException.java new file mode 100644 index 0000000..f9a62c1 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/exceptions/IncorrectPingFormatException.java @@ -0,0 +1,32 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.exceptions; + +import org.springframework.http.HttpStatus; + +public class IncorrectPingFormatException extends IndexException { + + public IncorrectPingFormatException(String message) { + super(message, HttpStatus.BAD_REQUEST); + } +} diff --git a/src/main/java/solutions/fairdata/fdp/index/exceptions/IndexException.java b/src/main/java/solutions/fairdata/fdp/index/exceptions/IndexException.java new file mode 100644 index 0000000..3255e4e --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/exceptions/IndexException.java @@ -0,0 +1,44 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.exceptions; + +import org.springframework.http.HttpStatus; +import solutions.fairdata.fdp.index.api.dto.ErrorDTO; + +public abstract class IndexException extends Exception { + + protected final HttpStatus status; + + public IndexException(String message, HttpStatus status) { + super(message); + this.status = status; + } + + public HttpStatus getStatus() { + return status; + } + + public ErrorDTO getErrorDTO() { + return new ErrorDTO(getStatus().value(), getMessage()); + } +} diff --git a/src/main/java/solutions/fairdata/fdp/index/exceptions/package-info.java b/src/main/java/solutions/fairdata/fdp/index/exceptions/package-info.java new file mode 100644 index 0000000..d0d7834 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/exceptions/package-info.java @@ -0,0 +1,25 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +@javax.annotation.ParametersAreNonnullByDefault +package solutions.fairdata.fdp.index.exceptions; diff --git a/src/main/java/solutions/fairdata/fdp/index/service/EventService.java b/src/main/java/solutions/fairdata/fdp/index/service/EventService.java new file mode 100644 index 0000000..894f35e --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/service/EventService.java @@ -0,0 +1,187 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.SneakyThrows; +import org.eclipse.rdf4j.util.iterators.EmptyIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; +import org.springframework.http.HttpEntity; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.stereotype.Service; +import solutions.fairdata.fdp.index.api.dto.PingDTO; +import solutions.fairdata.fdp.index.database.repository.EventRepository; +import solutions.fairdata.fdp.index.database.repository.IndexEntryRepository; +import solutions.fairdata.fdp.index.entity.IndexEntry; +import solutions.fairdata.fdp.index.entity.IndexEntryState; +import solutions.fairdata.fdp.index.entity.config.EventsConfig; +import solutions.fairdata.fdp.index.entity.events.Event; +import solutions.fairdata.fdp.index.entity.events.EventType; +import solutions.fairdata.fdp.index.entity.http.Exchange; +import solutions.fairdata.fdp.index.entity.http.ExchangeState; +import solutions.fairdata.fdp.index.exceptions.IncorrectPingFormatException; +import solutions.fairdata.fdp.index.utils.IncomingPingUtils; +import solutions.fairdata.fdp.index.utils.MetadataRetrievalUtils; + +import javax.annotation.PostConstruct; +import javax.servlet.http.HttpServletRequest; +import java.time.Instant; + +@Service +public class EventService { + private static final Logger logger = LoggerFactory.getLogger(EventService.class); + + @Autowired + private ObjectMapper objectMapper; + + @Autowired + private ThreadPoolTaskExecutor executor; + + @Autowired + private EventRepository eventRepository; + + @Autowired + private IndexEntryRepository indexEntryRepository; + + @Autowired + private IndexEntryService indexEntryService; + + @Autowired + private EventsConfig eventsConfig; + + public Iterable getEvents(IndexEntry indexEntry) { + // TODO: make events pagination in the future + return eventRepository.getAllByRelatedTo(indexEntry, PageRequest.of(0, 10, Sort.by(Sort.Direction.DESC, "created"))); + } + + public Iterable getEvents(String clientUrl) { + return indexEntryService.findEntry(clientUrl).map(this::getEvents).orElse(EmptyIterator::new); + } + + @SneakyThrows + public Event acceptIncomingPing(HttpEntity httpEntity, HttpServletRequest request) { + var event = IncomingPingUtils.prepareEvent(httpEntity, request); + eventRepository.save(event); + event.execute(); + try { + var pingDTO = objectMapper.readValue(httpEntity.getBody(), PingDTO.class); + var indexEntry = indexEntryService.storeEntry(pingDTO); + event.getIncomingPing().getExchange().getResponse().setCode(204); + event.setRelatedTo(indexEntry); + logger.info("Accepted incoming ping as a new event"); + } catch (Exception e) { + var ex = new IncorrectPingFormatException("Could not parse PING: " + e.getMessage()); + event.getIncomingPing().getExchange().getResponse().setCode(400); + event.getIncomingPing().getExchange().getResponse().setBody(objectMapper.writeValueAsString(ex.getErrorDTO())); + event.setFinished(Instant.now()); + eventRepository.save(event); + logger.info("Incoming ping has incorrect format: " + e.getMessage()); + throw ex; + } + event.setFinished(Instant.now()); + return eventRepository.save(event); + } + + private void processMetadataRetrieval(Event event) { + String clientUrl = event.getRelatedTo().getClientUrl(); + if (MetadataRetrievalUtils.shouldRetrieve(event, eventsConfig.getRetrievalRateLimitWait())) { + indexEntryRepository.save(event.getRelatedTo()); + eventRepository.save(event); + event.execute(); + + logger.info("Retrieving metadata for " + clientUrl); + MetadataRetrievalUtils.retrieveRepositoryMetadata(event, eventsConfig.getRetrievalTimeout()); + Exchange ex = event.getMetadataRetrieval().getExchange(); + if (ex.getState() == ExchangeState.Retrieved) { + try { + logger.info("Parsing metadata for " + clientUrl); + var metadata = MetadataRetrievalUtils.parseRepositoryMetadata(ex.getResponse().getBody()); + if (metadata.isPresent()) { + event.getMetadataRetrieval().setMetadata(metadata.get()); + event.getRelatedTo().setCurrentMetadata(metadata.get()); + event.getRelatedTo().setState(IndexEntryState.Valid); + logger.info("Storing metadata for " + clientUrl); + indexEntryRepository.save(event.getRelatedTo()); + } else { + logger.info("Repository not found in metadata for " + clientUrl); + event.getRelatedTo().setState(IndexEntryState.Invalid); + event.getMetadataRetrieval().setError("Repository not found in metadata"); + } + } catch (Exception e) { + logger.info("Cannot parse metadata for " + clientUrl); + event.getRelatedTo().setState(IndexEntryState.Invalid); + event.getMetadataRetrieval().setError("Cannot parse metadata"); + } + } else { + event.getRelatedTo().setState(IndexEntryState.Unreachable); + logger.info("Cannot retrieve metadata for " + clientUrl + ": " + ex.getError()); + } + } else { + logger.info("Rate limit reached for " + clientUrl + " (skipping metadata retrieval)"); + event.getMetadataRetrieval().setError("Rate limit reached (skipping)"); + } + event.getRelatedTo().setLastRetrievalTime(Instant.now()); + event.finish(); + eventRepository.save(event); + indexEntryRepository.save(event.getRelatedTo()); + } + + @Async + public void triggerMetadataRetrieval(Event triggerEvent) { + var event = MetadataRetrievalUtils.prepareEvent(triggerEvent); + logger.info("Triggering metadata retrieval for " + triggerEvent.getRelatedTo().getClientUrl()); + try { + processMetadataRetrieval(event); + } catch (Exception e) { + logger.error("Failed to retrieve metadata: " + e.getMessage()); + } + } + + private void resumeUnfinishedEvents() { + logger.info("Resuming unfinished events"); + for (Event event : eventRepository.getAllByFinishedIsNull()) { + logger.info("Resuming event " + event.getUuid()); + + try { + if (event.getType() == EventType.MetadataRetrieval) { + processMetadataRetrieval(event); + } else { + logger.warn("Unknown event type " + event.getUuid()); + } + } catch (Exception e) { + logger.error("Failed to resume event " + event.getUuid() + ": " + e.getMessage()); + } + } + logger.info("Finished unfinished events"); + } + + @PostConstruct + public void startResumeUnfinishedEvents() { + executor.submit(this::resumeUnfinishedEvents); + } +} diff --git a/src/main/java/solutions/fairdata/fdp/index/service/IndexEntryService.java b/src/main/java/solutions/fairdata/fdp/index/service/IndexEntryService.java index ae7c4bc..888b134 100644 --- a/src/main/java/solutions/fairdata/fdp/index/service/IndexEntryService.java +++ b/src/main/java/solutions/fairdata/fdp/index/service/IndexEntryService.java @@ -27,23 +27,34 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; -import org.springframework.stereotype.Component; +import org.springframework.stereotype.Service; +import org.springframework.validation.annotation.Validated; import solutions.fairdata.fdp.index.api.dto.IndexEntryDTO; -import solutions.fairdata.fdp.index.database.repository.EntryRepository; +import solutions.fairdata.fdp.index.api.dto.PingDTO; +import solutions.fairdata.fdp.index.database.repository.IndexEntryRepository; import solutions.fairdata.fdp.index.entity.IndexEntry; +import solutions.fairdata.fdp.index.entity.IndexEntryState; +import solutions.fairdata.fdp.index.entity.config.EventsConfig; -import java.time.OffsetDateTime; +import javax.validation.Valid; +import java.time.Instant; +import java.util.Optional; -@Component +@Service +@Validated public class IndexEntryService { private static final Logger logger = LoggerFactory.getLogger(IndexEntryService.class); @Autowired - private EntryRepository repository; + private IndexEntryRepository repository; - public void storeEntry(String clientUrl) { + @Autowired + private EventsConfig eventsConfig; + + public IndexEntry storeEntry(@Valid PingDTO pingDTO) { + var clientUrl = pingDTO.getClientUrl(); var entity = repository.findByClientUrl(clientUrl); - var now = OffsetDateTime.now(); + var now = Instant.now(); final IndexEntry entry; if (entity.isPresent()) { @@ -53,26 +64,75 @@ public void storeEntry(String clientUrl) { logger.info("Storing new entry {}", clientUrl); entry = new IndexEntry(); entry.setClientUrl(clientUrl); - entry.setRegistrationTime(now.toString()); + entry.setRegistrationTime(now); } - entry.setModificationTime(now.toString()); - repository.save(entry); + entry.setModificationTime(now); + return repository.save(entry); } public Iterable getAllEntries() { return repository.findAll(); } - public Page getEntriesPage(Pageable pageable) { + public Page getEntriesPage(Pageable pageable, String state) { + if (state.equalsIgnoreCase("active")) { + return repository.findAllByStateEqualsAndLastRetrievalTimeAfter(pageable, IndexEntryState.Valid, getValidThreshold()); + } + if (state.equalsIgnoreCase("inactive")) { + return repository.findAllByStateEqualsAndLastRetrievalTimeBefore(pageable, IndexEntryState.Valid, getValidThreshold()); + } + if (state.equalsIgnoreCase("unreachable")) { + return repository.findAllByStateEquals(pageable, IndexEntryState.Unreachable); + } + if (state.equalsIgnoreCase("invalid")) { + return repository.findAllByStateEquals(pageable, IndexEntryState.Invalid); + } + if (state.equalsIgnoreCase("unknown")) { + return repository.findAllByStateEquals(pageable, IndexEntryState.Unknown); + } return repository.findAll(pageable); } + public Optional findEntry(String clientUrl) { + return repository.findByClientUrl(clientUrl); + } + public IndexEntryDTO toDTO(IndexEntry indexEntry) { IndexEntryDTO dto = new IndexEntryDTO(); dto.setClientUrl(indexEntry.getClientUrl()); - dto.setRegistrationTime(OffsetDateTime.parse(indexEntry.getRegistrationTime())); - dto.setModificationTime(OffsetDateTime.parse(indexEntry.getModificationTime())); + dto.setState(indexEntry.getState().toString()); + dto.setRegistrationTime(indexEntry.getRegistrationTime().toString()); + dto.setModificationTime(indexEntry.getModificationTime().toString()); return dto; } + + public long countAllEntries() { + return repository.count(); + } + + public long countUnreachableEntries() { + return repository.countAllByStateEquals(IndexEntryState.Unreachable); + } + + public long countActiveEntries() { + return repository.countAllByStateEqualsAndLastRetrievalTimeAfter(IndexEntryState.Valid, getValidThreshold()); + } + + public long countInactiveEntries() { + return repository.countAllByStateEqualsAndLastRetrievalTimeBefore(IndexEntryState.Valid, getValidThreshold()); + } + + public long countInvalidEntries() { + return repository.countAllByStateEquals(IndexEntryState.Invalid); + } + + public long countUnknownEntries() { + return repository.countAllByStateEquals(IndexEntryState.Unknown); + } + + private Instant getValidThreshold() { + return Instant.now().minus(eventsConfig.getPingValidDuration()); + } + } diff --git a/src/main/java/solutions/fairdata/fdp/index/utils/IncomingPingUtils.java b/src/main/java/solutions/fairdata/fdp/index/utils/IncomingPingUtils.java new file mode 100644 index 0000000..59892b9 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/utils/IncomingPingUtils.java @@ -0,0 +1,45 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.utils; + +import org.springframework.http.HttpEntity; +import solutions.fairdata.fdp.index.entity.events.Event; +import solutions.fairdata.fdp.index.entity.events.IncomingPing; +import solutions.fairdata.fdp.index.entity.http.Exchange; +import solutions.fairdata.fdp.index.entity.http.ExchangeDirection; + +import javax.servlet.http.HttpServletRequest; + +public class IncomingPingUtils { + + private static final Integer VERSION = 1; + + public static Event prepareEvent(HttpEntity httpEntity, HttpServletRequest request) { + var incomingPing = new IncomingPing(); + var ex = new Exchange(ExchangeDirection.INCOMING, request.getRemoteAddr()); + incomingPing.setExchange(ex); + ex.getRequest().setFromHttpEntity(httpEntity); + ex.getRequest().setFromHttpServletRequest(request); + return new Event(VERSION, incomingPing); + } +} diff --git a/src/main/java/solutions/fairdata/fdp/index/utils/MetadataRetrievalUtils.java b/src/main/java/solutions/fairdata/fdp/index/utils/MetadataRetrievalUtils.java new file mode 100644 index 0000000..dfdabe1 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/utils/MetadataRetrievalUtils.java @@ -0,0 +1,173 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.utils; + +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.Resource; +import org.eclipse.rdf4j.model.Statement; +import org.eclipse.rdf4j.model.Value; +import org.eclipse.rdf4j.model.impl.SimpleValueFactory; +import org.eclipse.rdf4j.model.vocabulary.DCTERMS; +import org.eclipse.rdf4j.model.vocabulary.FOAF; +import org.eclipse.rdf4j.model.vocabulary.RDF; +import org.eclipse.rdf4j.rio.RDFFormat; +import org.eclipse.rdf4j.rio.RDFParser; +import org.eclipse.rdf4j.rio.Rio; +import org.eclipse.rdf4j.rio.helpers.StatementCollector; +import org.springframework.http.HttpHeaders; +import solutions.fairdata.fdp.index.entity.RepositoryMetadata; +import solutions.fairdata.fdp.index.entity.events.Event; +import solutions.fairdata.fdp.index.entity.events.EventType; +import solutions.fairdata.fdp.index.entity.events.MetadataRetrieval; +import solutions.fairdata.fdp.index.entity.http.Exchange; +import solutions.fairdata.fdp.index.entity.http.ExchangeDirection; +import solutions.fairdata.fdp.index.entity.http.ExchangeState; + +import java.io.IOException; +import java.io.StringReader; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.util.*; + +public class MetadataRetrievalUtils { + + private static final EventType EVENT_TYPE = EventType.MetadataRetrieval; + + private static final Integer VERSION = 1; + + private static final IRI REPOSITORY = SimpleValueFactory.getInstance().createIRI("http://www.re3data.org/schema/3-0#Repository"); + + private static final IRI COUNTRY = SimpleValueFactory.getInstance().createIRI("http://www.re3data.org/schema/3-0#institutionCountry"); + + private static final Map MAPPING = Map.of( + DCTERMS.TITLE, "title", + DCTERMS.DESCRIPTION, "description", + DCTERMS.HAS_VERSION, "version", + DCTERMS.PUBLISHER, "publisher", + COUNTRY, "country" + ); + + private static final HttpClient client = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_2) + .followRedirects(HttpClient.Redirect.ALWAYS) + .build(); + + public static boolean shouldRetrieve(Event triggerEvent, Duration rateLimitWait) { + if (triggerEvent.getRelatedTo() == null) { + return false; + } + Instant lastRetrieval = triggerEvent.getRelatedTo().getLastRetrievalTime(); + if (lastRetrieval == null) { + return true; + } + return Duration.between(lastRetrieval, Instant.now()).compareTo(rateLimitWait) > 0; + } + + public static Event prepareEvent(Event triggerEvent) { + return new Event(VERSION, triggerEvent, triggerEvent.getRelatedTo(), new MetadataRetrieval()); + } + + public static void retrieveRepositoryMetadata(Event event, Duration timeout) { + if (event.getType() != EVENT_TYPE) { + throw new IllegalArgumentException("Invalid event type"); + } + var ex = new Exchange(ExchangeDirection.OUTGOING); + event.getMetadataRetrieval().setExchange(ex); + try { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(event.getRelatedTo().getClientUrl())) + .timeout(timeout) + .header(HttpHeaders.ACCEPT, RDFFormat.TURTLE.getDefaultMIMEType()) + .GET().build(); + ex.getRequest().setFromHttpRequest(request); + ex.setState(ExchangeState.Requested); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString()); + ex.getResponse().setFromHttpResponse(response); + ex.setState(ExchangeState.Retrieved); + } catch (InterruptedException e) { + ex.setState(ExchangeState.Timeout); + ex.setError("Timeout"); + } catch (IllegalArgumentException e) { + ex.setState(ExchangeState.Failed); + ex.setError("Invalid URI: " + e.getMessage()); + } catch (IOException e) { + ex.setState(ExchangeState.Failed); + ex.setError("IO error: " + e.getMessage()); + } + } + + public static Optional parseRepositoryMetadata(String metadata) throws IOException { + RDFParser parser = Rio.createParser(RDFFormat.TURTLE); + StatementCollector collector = new StatementCollector(); + parser.setRDFHandler(collector); + + parser.parse(new StringReader(metadata), String.valueOf(StandardCharsets.UTF_8)); + ArrayList statements = new ArrayList<>(collector.getStatements()); + + return findRepository(statements).map(repository -> extractRepositoryMetadata(statements, repository)); + } + + private static RepositoryMetadata extractRepositoryMetadata(ArrayList statements, Resource repository) { + var repositoryMetadata = new RepositoryMetadata(); + repositoryMetadata.setMetadataVersion(VERSION); + repositoryMetadata.setRepositoryUri(repository.toString()); + + Value publisher = null; + for (Statement st: statements) { + if (st.getSubject().equals(repository)) { + if (MAPPING.containsKey(st.getPredicate())) { + repositoryMetadata.getMetadata().put(MAPPING.get(st.getPredicate()), st.getObject().stringValue()); + } + if (st.getPredicate().equals(DCTERMS.PUBLISHER)) { + publisher = st.getObject(); + } + } + } + + if (publisher != null) { + for (Statement st: statements) { + if (st.getSubject().equals(publisher)) { + if (st.getPredicate().equals(FOAF.NAME)) { + repositoryMetadata.getMetadata().put("publisherName", st.getObject().stringValue()); + } + } + } + } + + return repositoryMetadata; + } + + private static Optional findRepository(ArrayList statements) { + for (Statement st: statements) { + if (st.getPredicate().equals(RDF.TYPE) && st.getObject().equals(REPOSITORY)) { + return Optional.of(st.getSubject()); + } + } + return Optional.empty(); + } +} diff --git a/src/main/java/solutions/fairdata/fdp/index/utils/package-info.java b/src/main/java/solutions/fairdata/fdp/index/utils/package-info.java new file mode 100644 index 0000000..e209f10 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/utils/package-info.java @@ -0,0 +1,25 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +@javax.annotation.ParametersAreNonnullByDefault +package solutions.fairdata.fdp.index.utils; diff --git a/src/main/java/solutions/fairdata/fdp/index/web/controller/EntryController.java b/src/main/java/solutions/fairdata/fdp/index/web/controller/EntryController.java new file mode 100644 index 0000000..6911b39 --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/web/controller/EntryController.java @@ -0,0 +1,61 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.web.controller; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Controller; +import org.springframework.ui.Model; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import solutions.fairdata.fdp.index.entity.IndexEntryState; +import solutions.fairdata.fdp.index.entity.config.EventsConfig; +import solutions.fairdata.fdp.index.service.EventService; +import solutions.fairdata.fdp.index.service.IndexEntryService; + +import java.util.List; + +@Controller +@RequestMapping("/entry") +public class EntryController { + @Autowired + private IndexEntryService indexEntryService; + + @Autowired + private EventService eventService; + + @Autowired + private EventsConfig eventsConfig; + + @GetMapping + public String home(Model model, @RequestParam String clientUrl) { + model.addAttribute("clientUrl", clientUrl); + model.addAttribute("entry", indexEntryService.findEntry(clientUrl)); + model.addAttribute("events", eventService.getEvents(clientUrl)); + model.addAttribute("IndexEntryState", IndexEntryState.class); + model.addAttribute("pingValidDuration", eventsConfig.getPingValidDuration()); + model.addAttribute("specialMetadata", List.of("title", "version", "publisher", "publisherName")); + model.addAttribute("uriMetadata", List.of("country")); + return "entry"; + } +} diff --git a/src/main/java/solutions/fairdata/fdp/index/web/controller/HomeController.java b/src/main/java/solutions/fairdata/fdp/index/web/controller/HomeController.java index 0bb5151..f36e490 100644 --- a/src/main/java/solutions/fairdata/fdp/index/web/controller/HomeController.java +++ b/src/main/java/solutions/fairdata/fdp/index/web/controller/HomeController.java @@ -30,25 +30,40 @@ import org.springframework.ui.Model; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import solutions.fairdata.fdp.index.entity.IndexEntryState; +import solutions.fairdata.fdp.index.entity.config.EventsConfig; import solutions.fairdata.fdp.index.service.IndexEntryService; -import java.util.Optional; - @Controller @RequestMapping("/") public class HomeController { @Autowired - private IndexEntryService service; + private IndexEntryService indexEntryService; + + @Autowired + private EventsConfig eventsConfig; @GetMapping - public String home(Model model, @SortDefault(sort = "modificationTime", direction = Sort.Direction.DESC) Pageable pageable) { + public String home(Model model, @SortDefault(sort = "modificationTime", direction = Sort.Direction.DESC) Pageable pageable, @RequestParam(defaultValue = "reachable") String state) { var sort = pageable.getSort().stream() .findFirst() .map(o -> o.getProperty() + "," + o.getDirection().name().toLowerCase()) .orElse(""); - model.addAttribute("entries", service.getEntriesPage(pageable)); + model.addAttribute("entries", indexEntryService.getEntriesPage(pageable, state)); + model.addAttribute("pingValidDuration", eventsConfig.getPingValidDuration()); + model.addAttribute("IndexEntryState", IndexEntryState.class); + + model.addAttribute("countAll", indexEntryService.countAllEntries()); + model.addAttribute("countActive", indexEntryService.countActiveEntries()); + model.addAttribute("countInactive", indexEntryService.countInactiveEntries()); + model.addAttribute("countUnreachable", indexEntryService.countUnreachableEntries()); + model.addAttribute("countInvalid", indexEntryService.countInvalidEntries()); + model.addAttribute("countUnknown", indexEntryService.countUnknownEntries()); + model.addAttribute("sort", sort); + model.addAttribute("state", state); return "home"; } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 700fc47..6eeb59b 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -2,6 +2,7 @@ spring: data: mongodb: uri: mongodb://localhost:27017/fdp-index + auto-index-creation: false web: pageable: size-parameter: size @@ -11,6 +12,13 @@ spring: max-page-size: 2000 prefix: qualifier-delimiter: _ + task: + execution: + pool: + core-size: 2 + max-size: 5 + queue-capacity: 5000 + thread-name-prefix: fdpindex-task- fdp-index: api: @@ -19,3 +27,9 @@ fdp-index: description: This is OpenAPI specification of FAIR Data Point Index REST API. contactUrl: https://github.com/FAIRDataTeam/FAIRDataPoint-index contactName: GitHub repository + events: + retrieval: + rateLimitWait: PT10M # 10 minutes (ISO 8601) + timeout: PT1M # 1 minute (ISO 8601) + ping: + validDuration: PT1M # 7 days (ISO 8601) diff --git a/src/main/resources/static/img/no_data.svg b/src/main/resources/static/img/no_data.svg new file mode 100644 index 0000000..467c4d2 --- /dev/null +++ b/src/main/resources/static/img/no_data.svg @@ -0,0 +1 @@ +no data \ No newline at end of file diff --git a/src/main/resources/static/js/common.js b/src/main/resources/static/js/common.js new file mode 100644 index 0000000..0005849 --- /dev/null +++ b/src/main/resources/static/js/common.js @@ -0,0 +1,5 @@ +jQuery(document).ready(() => { + jQuery(function () { + jQuery('[data-toggle="tooltip"]').tooltip() + }) +}) diff --git a/src/main/resources/templates/entry.html b/src/main/resources/templates/entry.html new file mode 100644 index 0000000..431827a --- /dev/null +++ b/src/main/resources/templates/entry.html @@ -0,0 +1,118 @@ + + + + +
+ + +
+ Active + Inactive + Unreachable + Invalid + Unknown +
+ +
+ Not found +
+ +

FDP

+ +
+ No data +
+

No records

+

Sorry, but we don't know anything about such FAIR Data Point...

+
+
+ +
+
+

Timestamps

+ + + + + + + + + + + + + + + + +
Registered (first contact)
Modified (last contact)
Verified (last metadata retrieval)never
+
+ +
+

Repository metadata

+ + + + + + + + + + + + + + + + + + + + + + + + +
Repository URI
Title
Version
Publisher + +
+ +
+
+ +
+

Events

+

Last 10 events related to this entry:

+ + + + + + + + + + + + + + + + +
TypeCreatedFinished
N/A
+
+
+ +
+ + diff --git a/src/main/resources/templates/home.html b/src/main/resources/templates/home.html index 3d36838..43e7222 100644 --- a/src/main/resources/templates/home.html +++ b/src/main/resources/templates/home.html @@ -6,49 +6,111 @@
+
+
Filter:
+
+ + All + + + Active + + + Inactive + + + Unreachable + + + Invalid + + + Unknown + +
+
+ - + + + + +
Endpoint - ▲ - ▼ + ▲ + ▼ Registration - ▲ - ▼ + ▲ + ▼ Modification - ▲ - ▼ + ▲ + ▼ + + Status
- + + Active + Inactive + Unreachable + Invalid + Unknown +
no entries to display
-
- entries in total +
+
+
+ +
+ +
+
+
+
+
+ +
+ +
+
+
+
+ + +
+
-
-
-
-
-
- -
- -
-
-
-
-
- -
- -
-
-
-
- - -
-
-
-
- +
+ +
diff --git a/src/main/resources/templates/layout.html b/src/main/resources/templates/layout.html index 0c6d0f6..90663a1 100644 --- a/src/main/resources/templates/layout.html +++ b/src/main/resources/templates/layout.html @@ -7,7 +7,8 @@ - + + diff --git a/src/main/scss/site.scss b/src/main/scss/site.scss index 9f8f843..0e9f626 100644 --- a/src/main/scss/site.scss +++ b/src/main/scss/site.scss @@ -51,15 +51,20 @@ footer div.container { padding: 2rem; } -table.entries td.endpoint { +table td.endpoint { overflow-wrap: anywhere; } -table.entries td.timestamp, -table.entries th.right { +table td.timestamp, +table th.right, +table td.right { text-align: right; } +td, th { + max-width: 20em; +} + a.sort-link:hover { text-decoration: none; } @@ -73,3 +78,83 @@ a.sort-link.active { color: $primary; text-decoration: none; } + +div.error { + display: flex; + align-items: center; + justify-content: center; + margin: 2em; +} + +div.error > .error-img { + vertical-align: middle; + max-width: 30%; +} + +div.error > .error-text { + margin: 1em; + max-width: 50%; +} + +h2.entry-title { + margin: 1em 0; +} +div.entry-block { + margin-top: 1em; + padding-bottom: 0.5em; + border-top: 1px solid lightgray; + padding-top: 1.5em; +} +div.entry-block .table { + margin: 0; +} +div.entry-label { + float: right; +} + + +.index-stats { + margin-bottom: 1em; +} + +a.index-stat { + margin: 1em; + padding: 1em; + background: rgba(0, 0, 0, 0.05); + border-radius: 5px; + text-align: center; + display: block; + color: black; + text-decoration: none; + border: 3px solid $secondary; +} + +a.index-stat.active { + border: 3px solid $primary; +} + +.index-stat .number { + font-weight: bold; + font-size: 140%; + padding-bottom: 0.3em; + border-bottom: 1px solid #ccc; + margin-bottom: 0.3em; +} + +.table-filter { + display: flex; + align-items: center; + margin: 0.5em 0.75rem 1em 0.75rem; +} +.table-filter .filter-name { + font-weight: bold; +} +.table-filter .filter-content { + flex-grow: 1; + text-align: right; +} + +td.empty-table-info { + text-align: center; + font-style: italic; +} diff --git a/src/test/java/solutions/fairdata/fdp/index/acceptance/api/entries/EntriesAll_GET_Test.java b/src/test/java/solutions/fairdata/fdp/index/acceptance/api/entries/EntriesAll_GET_Test.java index ee4ac2b..e0c66a3 100644 --- a/src/test/java/solutions/fairdata/fdp/index/acceptance/api/entries/EntriesAll_GET_Test.java +++ b/src/test/java/solutions/fairdata/fdp/index/acceptance/api/entries/EntriesAll_GET_Test.java @@ -33,7 +33,7 @@ import org.springframework.http.ResponseEntity; import solutions.fairdata.fdp.index.WebIntegrationTest; import solutions.fairdata.fdp.index.api.dto.IndexEntryDTO; -import solutions.fairdata.fdp.index.database.repository.EntryRepository; +import solutions.fairdata.fdp.index.database.repository.IndexEntryRepository; import solutions.fairdata.fdp.index.entity.IndexEntry; import solutions.fairdata.fdp.index.fixtures.IndexEntryFixtures; @@ -49,7 +49,7 @@ public class EntriesAll_GET_Test extends WebIntegrationTest { @Autowired - private EntryRepository entryRepository; + private IndexEntryRepository indexEntryRepository; @Autowired private MongoTemplate mongoTemplate; @@ -86,7 +86,7 @@ public void res200_listFew() { // GIVEN (prepare data) mongoTemplate.getDb().drop(); List entries = IndexEntryFixtures.entriesFew(); - entryRepository.saveAll(entries); + indexEntryRepository.saveAll(entries); // AND (prepare request) RequestEntity request = RequestEntity @@ -112,7 +112,7 @@ public void res200_listMany() { // GIVEN (prepare data) mongoTemplate.getDb().drop(); List entries = IndexEntryFixtures.entriesN(300); - entryRepository.saveAll(entries); + indexEntryRepository.saveAll(entries); // AND (prepare request) RequestEntity request = RequestEntity diff --git a/src/test/java/solutions/fairdata/fdp/index/acceptance/api/entries/EntriesPage_GET_Test.java b/src/test/java/solutions/fairdata/fdp/index/acceptance/api/entries/EntriesPage_GET_Test.java index 0331579..c9b5744 100644 --- a/src/test/java/solutions/fairdata/fdp/index/acceptance/api/entries/EntriesPage_GET_Test.java +++ b/src/test/java/solutions/fairdata/fdp/index/acceptance/api/entries/EntriesPage_GET_Test.java @@ -34,7 +34,7 @@ import org.springframework.web.util.UriComponentsBuilder; import solutions.fairdata.fdp.index.WebIntegrationTest; import solutions.fairdata.fdp.index.api.dto.IndexEntryDTO; -import solutions.fairdata.fdp.index.database.repository.EntryRepository; +import solutions.fairdata.fdp.index.database.repository.IndexEntryRepository; import solutions.fairdata.fdp.index.entity.IndexEntry; import solutions.fairdata.fdp.index.fixtures.IndexEntryFixtures; import solutions.fairdata.fdp.index.utils.CustomPageImpl; @@ -51,7 +51,7 @@ public class EntriesPage_GET_Test extends WebIntegrationTest { @Autowired - private EntryRepository entryRepository; + private IndexEntryRepository indexEntryRepository; private final ParameterizedTypeReference> responseType = new ParameterizedTypeReference<>() {}; @@ -130,7 +130,7 @@ public void res200_pageFew() { // GIVEN (prepare data) mongoTemplate.getDb().drop(); List entries = IndexEntryFixtures.entriesFew(); - entryRepository.saveAll(entries); + indexEntryRepository.saveAll(entries); // AND (prepare request) RequestEntity request = RequestEntity @@ -164,7 +164,7 @@ public void res200_pageManyMiddle() { int page = 3; mongoTemplate.getDb().drop(); List entries = IndexEntryFixtures.entriesN(items); - entryRepository.saveAll(entries); + indexEntryRepository.saveAll(entries); // AND (prepare request) RequestEntity request = RequestEntity @@ -196,7 +196,7 @@ public void res200_pageManyLast() { int page = 3; mongoTemplate.getDb().drop(); List entries = IndexEntryFixtures.entriesN(items); - entryRepository.saveAll(entries); + indexEntryRepository.saveAll(entries); // AND (prepare request) RequestEntity request = RequestEntity diff --git a/src/test/java/solutions/fairdata/fdp/index/acceptance/api/ping/ReceivePing_POST_Test.java b/src/test/java/solutions/fairdata/fdp/index/acceptance/api/ping/ReceivePing_POST_Test.java index c46fa80..791766c 100644 --- a/src/test/java/solutions/fairdata/fdp/index/acceptance/api/ping/ReceivePing_POST_Test.java +++ b/src/test/java/solutions/fairdata/fdp/index/acceptance/api/ping/ReceivePing_POST_Test.java @@ -33,7 +33,7 @@ import org.springframework.http.ResponseEntity; import solutions.fairdata.fdp.index.WebIntegrationTest; import solutions.fairdata.fdp.index.api.dto.PingDTO; -import solutions.fairdata.fdp.index.database.repository.EntryRepository; +import solutions.fairdata.fdp.index.database.repository.IndexEntryRepository; import solutions.fairdata.fdp.index.entity.IndexEntry; import solutions.fairdata.fdp.index.fixtures.IndexEntryFixtures; @@ -48,7 +48,7 @@ public class ReceivePing_POST_Test extends WebIntegrationTest { @Autowired - private EntryRepository entryRepository; + private IndexEntryRepository indexEntryRepository; @Autowired private MongoTemplate mongoTemplate; @@ -79,12 +79,12 @@ public void res204_newEnty() { .body(reqDto); // WHEN - assertThat("Entry does not exist before the ping", entryRepository.findByClientUrl(clientUrl).isPresent(), is(Boolean.FALSE)); + assertThat("Entry does not exist before the ping", indexEntryRepository.findByClientUrl(clientUrl).isPresent(), is(Boolean.FALSE)); ResponseEntity result = client.exchange(request, responseType); // THEN assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.NO_CONTENT))); - assertThat("Entry exists after the ping", entryRepository.findByClientUrl(clientUrl).isPresent(), is(Boolean.TRUE)); + assertThat("Entry exists after the ping", indexEntryRepository.findByClientUrl(clientUrl).isPresent(), is(Boolean.TRUE)); } @Test @@ -103,13 +103,13 @@ public void res204_existingEnty() { .body(reqDto); // WHEN - entryRepository.save(indexEntry); - assertThat("Entry exists before the ping", entryRepository.findByClientUrl(clientUrl).isPresent(), is(Boolean.TRUE)); + indexEntryRepository.save(indexEntry); + assertThat("Entry exists before the ping", indexEntryRepository.findByClientUrl(clientUrl).isPresent(), is(Boolean.TRUE)); ResponseEntity result = client.exchange(request, responseType); // THEN assertThat("Correct response code is received", result.getStatusCode(), is(equalTo(HttpStatus.NO_CONTENT))); - assertThat("Entry exists after the ping", entryRepository.findByClientUrl(clientUrl).isPresent(), is(Boolean.TRUE)); + assertThat("Entry exists after the ping", indexEntryRepository.findByClientUrl(clientUrl).isPresent(), is(Boolean.TRUE)); } @Test diff --git a/src/test/java/solutions/fairdata/fdp/index/acceptance/web/home/Entry_GET_Test.java b/src/test/java/solutions/fairdata/fdp/index/acceptance/web/home/Entry_GET_Test.java new file mode 100644 index 0000000..b785c1a --- /dev/null +++ b/src/test/java/solutions/fairdata/fdp/index/acceptance/web/home/Entry_GET_Test.java @@ -0,0 +1,182 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.acceptance.web.home; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.RequestBuilder; +import org.springframework.test.web.servlet.ResultActions; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.web.util.UriComponentsBuilder; +import solutions.fairdata.fdp.index.WebIntegrationTest; +import solutions.fairdata.fdp.index.database.repository.IndexEntryRepository; +import solutions.fairdata.fdp.index.entity.IndexEntry; +import solutions.fairdata.fdp.index.entity.config.EventsConfig; +import solutions.fairdata.fdp.index.fixtures.IndexEntryFixtures; + +import java.net.URI; +import java.time.Instant; + +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.xpath; + +@DisplayName("Browse /entry") +public class Entry_GET_Test extends WebIntegrationTest { + + @Autowired + private IndexEntryRepository indexEntryRepository; + + @Autowired + private EventsConfig eventsConfig; + + private URI url(String clientUrl) { + return UriComponentsBuilder.fromUriString("/entry") + .queryParam("clientUrl", clientUrl) + .build().toUri(); + } + + @Test + @DisplayName("HTTP 200: unknown entry") + public void res200_unknownEntry() throws Exception { + // GIVEN (prepare data) + mongoTemplate.getDb().drop(); + + // AND (prepare request) + RequestBuilder request = MockMvcRequestBuilders + .get(url("http://example.com")) + .accept(MediaType.TEXT_HTML); + + // WHEN + ResultActions result = mvc.perform(request); + + // THEN + result + .andExpect(status().isOk()) + .andExpect(view().name("entry")) + .andExpect(xpath("//div[@class='entry-label']/span").string("Not found")) + .andExpect(xpath("//h2[@class='entry-title']/a").string("http://example.com")) + .andExpect(xpath("//h2[@class='entry-title']/a/@href").string("http://example.com")) + .andExpect(xpath("//div[@id='entryEmpty']").exists()) + .andExpect(xpath("//div[@id='entryFound']").doesNotExist()); + } + + @Test + @DisplayName("HTTP 200: reachable entry") + public void res200_reachableEntry() throws Exception { + // GIVEN (prepare data) + mongoTemplate.getDb().drop(); + IndexEntry indexEntry = IndexEntryFixtures.activeEntry("http://example.com"); + indexEntryRepository.save(indexEntry); + + // AND (prepare request) + RequestBuilder request = MockMvcRequestBuilders + .get(url(indexEntry.getClientUrl())) + .accept(MediaType.TEXT_HTML); + + // WHEN + ResultActions result = mvc.perform(request); + + // THEN + result + .andExpect(status().isOk()) + .andExpect(view().name("entry")) + .andExpect(xpath("//div[@class='entry-label']/a").string("Active")) + .andExpect(xpath("//div[@class='entry-label']/a/@href").string(indexEntry.getClientUrl())) + .andExpect(xpath("//h2[@class='entry-title']/a").string(indexEntry.getClientUrl())) + .andExpect(xpath("//h2[@class='entry-title']/a/@href").string(indexEntry.getClientUrl())) + .andExpect(xpath("//div[@id='entryEmpty']").doesNotExist()) + .andExpect(xpath("//div[@id='entryFound']").exists()) + .andExpect(xpath("//*[@id='repository-uri']/a").string(indexEntry.getCurrentMetadata().getRepositoryUri())) + .andExpect(xpath("//*[@id='repository-uri']/a/@href").string(indexEntry.getCurrentMetadata().getRepositoryUri())) + .andExpect(xpath("//*[@id='metadata-title']").string(indexEntry.getCurrentMetadata().getMetadata().get("title"))) + .andExpect(xpath("//*[@id='metadata-version']").string(indexEntry.getCurrentMetadata().getMetadata().get("version"))) + .andExpect(xpath("//*[@id='metadata-description']").string(indexEntry.getCurrentMetadata().getMetadata().get("description"))); + } + + @Test + @DisplayName("HTTP 200: unreachable entry") + public void res200_unreachableEntry() throws Exception { + // GIVEN (prepare data) + mongoTemplate.getDb().drop(); + IndexEntry indexEntry = IndexEntryFixtures.activeEntry("http://example.com"); + Instant oldDate = Instant.now().minus(eventsConfig.getPingValidDuration()); + indexEntry.setRegistrationTime(oldDate); + indexEntry.setModificationTime(oldDate); + indexEntry.setLastRetrievalTime(oldDate); + indexEntryRepository.save(indexEntry); + + // AND (prepare request) + RequestBuilder request = MockMvcRequestBuilders + .get(url(indexEntry.getClientUrl())) + .accept(MediaType.TEXT_HTML); + + // WHEN + ResultActions result = mvc.perform(request); + + // THEN + result + .andExpect(status().isOk()) + .andExpect(view().name("entry")) + .andExpect(xpath("//div[@class='entry-label']/span").string("Inactive")) + .andExpect(xpath("//h2[@class='entry-title']/a").string(indexEntry.getClientUrl())) + .andExpect(xpath("//h2[@class='entry-title']/a/@href").string(indexEntry.getClientUrl())) + .andExpect(xpath("//div[@id='entryEmpty']").doesNotExist()) + .andExpect(xpath("//div[@id='entryFound']").exists()) + .andExpect(xpath("//*[@id='repository-uri']/a").string(indexEntry.getCurrentMetadata().getRepositoryUri())) + .andExpect(xpath("//*[@id='repository-uri']/a/@href").string(indexEntry.getCurrentMetadata().getRepositoryUri())) + .andExpect(xpath("//*[@id='metadata-title']").string(indexEntry.getCurrentMetadata().getMetadata().get("title"))) + .andExpect(xpath("//*[@id='metadata-version']").string(indexEntry.getCurrentMetadata().getMetadata().get("version"))) + .andExpect(xpath("//*[@id='metadata-description']").string(indexEntry.getCurrentMetadata().getMetadata().get("description"))); + } + + @Test + @DisplayName("HTTP 200: invalid entry") + public void res200_invalidEntry() throws Exception { + // GIVEN (prepare data) + mongoTemplate.getDb().drop(); + IndexEntry indexEntry = IndexEntryFixtures.entryExample(); + indexEntryRepository.save(indexEntry); + + // AND (prepare request) + RequestBuilder request = MockMvcRequestBuilders + .get(url(indexEntry.getClientUrl())) + .accept(MediaType.TEXT_HTML); + + // WHEN + ResultActions result = mvc.perform(request); + + // THEN + result + .andExpect(status().isOk()) + .andExpect(view().name("entry")) + .andExpect(xpath("//div[@class='entry-label']/span").string("Invalid")) + .andExpect(xpath("//h2[@class='entry-title']/a").string(indexEntry.getClientUrl())) + .andExpect(xpath("//h2[@class='entry-title']/a/@href").string(indexEntry.getClientUrl())) + .andExpect(xpath("//div[@id='entryEmpty']").doesNotExist()) + .andExpect(xpath("//div[@id='entryFound']").exists()) + .andExpect(xpath("//*[@id='repository-uri']").doesNotExist()) + .andExpect(xpath("//*[@id='metadata-title']").doesNotExist()); + } +} diff --git a/src/test/java/solutions/fairdata/fdp/index/acceptance/web/home/Home_GET_Test.java b/src/test/java/solutions/fairdata/fdp/index/acceptance/web/home/Home_GET_Test.java index c5db9c6..1e797fe 100644 --- a/src/test/java/solutions/fairdata/fdp/index/acceptance/web/home/Home_GET_Test.java +++ b/src/test/java/solutions/fairdata/fdp/index/acceptance/web/home/Home_GET_Test.java @@ -31,11 +31,12 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.web.util.UriComponentsBuilder; import solutions.fairdata.fdp.index.WebIntegrationTest; -import solutions.fairdata.fdp.index.database.repository.EntryRepository; +import solutions.fairdata.fdp.index.database.repository.IndexEntryRepository; import solutions.fairdata.fdp.index.entity.IndexEntry; import solutions.fairdata.fdp.index.fixtures.IndexEntryFixtures; import java.net.URI; +import java.util.Comparator; import java.util.List; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; @@ -44,16 +45,23 @@ public class Home_GET_Test extends WebIntegrationTest { @Autowired - private EntryRepository entryRepository; + private IndexEntryRepository indexEntryRepository; private URI url() { return URI.create("/"); } + private URI urlWithoutPagination() { + return UriComponentsBuilder.fromUri(url()) + .queryParam("state", "all") + .build().toUri(); + } + private URI urlWithPageSize(int page, int size) { return UriComponentsBuilder.fromUri(url()) .queryParam("page", page) .queryParam("size", size) + .queryParam("state", "all") .build().toUri(); } @@ -65,7 +73,7 @@ public void res200_emptyTable() throws Exception { // AND (prepare request) RequestBuilder request = MockMvcRequestBuilders - .get(url()) + .get(urlWithoutPagination()) .accept(MediaType.TEXT_HTML); // WHEN @@ -76,10 +84,9 @@ public void res200_emptyTable() throws Exception { .andExpect(status().isOk()) .andExpect(view().name("home")) .andExpect(xpath("//table[@id='entries']/thead/tr").exists()) - .andExpect(xpath("//table[@id='entries']/tbody/tr").doesNotExist()) - .andExpect(xpath("//span[@id='totalEntries']").string("0")) - .andExpect(xpath("//span[@id='currentPage']").string("1")) - .andExpect(xpath("//span[@id='totalPages']").string("1")) + .andExpect(xpath("//table[@id='entries']/tbody/tr[@class='entry-row']").doesNotExist()) + .andExpect(xpath("//*[@id='currentPage']").string("1")) + .andExpect(xpath("//*[@id='totalPages']").string("1")) .andExpect(xpath("//*[@id='firstPage']/a").doesNotExist()) .andExpect(xpath("//*[@id='previousPage']/a").doesNotExist()) .andExpect(xpath("//*[@id='nextPage']/a").doesNotExist()) @@ -92,11 +99,11 @@ public void res200_singlePage() throws Exception { // GIVEN (prepare data) mongoTemplate.getDb().drop(); List entries = IndexEntryFixtures.entriesFew(); - entryRepository.saveAll(entries); + indexEntryRepository.saveAll(entries); // AND (prepare request) RequestBuilder request = MockMvcRequestBuilders - .get(url()) + .get(urlWithoutPagination()) .accept(MediaType.TEXT_HTML); // WHEN @@ -109,10 +116,9 @@ public void res200_singlePage() throws Exception { .andExpect(xpath("//table[@id='entries']/thead/tr").exists()) .andExpect(xpath("//table[@id='entries']/tbody/tr").nodeCount(entries.size())) .andExpect(xpath("//table[@id='entries']/tbody/tr[1]/td[@class='endpoint']/a").string(entries.get(0).getClientUrl())) - .andExpect(xpath("//table[@id='entries']/tbody/tr[1]/td[@class='endpoint']/a/@href").string(entries.get(0).getClientUrl())) - .andExpect(xpath("//span[@id='totalEntries']").string(String.valueOf(entries.size()))) - .andExpect(xpath("//span[@id='currentPage']").string("1")) - .andExpect(xpath("//span[@id='totalPages']").string("1")) + .andExpect(xpath("//table[@id='entries']/tbody/tr[1]/td[@class='endpoint']/a/@href").string("/entry?clientUrl=" + entries.get(0).getClientUrl())) + .andExpect(xpath("//*[@id='currentPage']").string("1")) + .andExpect(xpath("//*[@id='totalPages']").string("1")) .andExpect(xpath("//*[@id='firstPage']/a").doesNotExist()) .andExpect(xpath("//*[@id='previousPage']/a").doesNotExist()) .andExpect(xpath("//*[@id='nextPage']/a").doesNotExist()) @@ -127,7 +133,7 @@ public void res200_firstPageOfMany() throws Exception { int size = 50; mongoTemplate.getDb().drop(); List entries = IndexEntryFixtures.entriesN(items); - entryRepository.saveAll(entries); + indexEntryRepository.saveAll(entries); // AND (prepare request) RequestBuilder request = MockMvcRequestBuilders @@ -144,10 +150,9 @@ public void res200_firstPageOfMany() throws Exception { .andExpect(xpath("//table[@id='entries']/thead/tr").exists()) .andExpect(xpath("//table[@id='entries']/tbody/tr").nodeCount(size)) .andExpect(xpath("//table[@id='entries']/tbody/tr[1]/td[@class='endpoint']/a").string(entries.get(0).getClientUrl())) - .andExpect(xpath("//table[@id='entries']/tbody/tr[1]/td[@class='endpoint']/a/@href").string(entries.get(0).getClientUrl())) - .andExpect(xpath("//span[@id='totalEntries']").string(String.valueOf(items))) - .andExpect(xpath("//span[@id='currentPage']").string("1")) - .andExpect(xpath("//span[@id='totalPages']").string("7")) + .andExpect(xpath("//table[@id='entries']/tbody/tr[1]/td[@class='endpoint']/a/@href").string("/entry?clientUrl=" + entries.get(0).getClientUrl())) + .andExpect(xpath("//*[@id='currentPage']").string("1")) + .andExpect(xpath("//*[@id='totalPages']").string("7")) .andExpect(xpath("//*[@id='firstPage']/a").doesNotExist()) .andExpect(xpath("//*[@id='previousPage']/a").doesNotExist()) .andExpect(xpath("//*[@id='nextPage']/a").exists()) @@ -163,7 +168,7 @@ public void res200_lastPageOfMany() throws Exception { int page = 7; mongoTemplate.getDb().drop(); List entries = IndexEntryFixtures.entriesN(items); - entryRepository.saveAll(entries); + indexEntryRepository.saveAll(entries); // AND (prepare request) RequestBuilder request = MockMvcRequestBuilders @@ -180,10 +185,9 @@ public void res200_lastPageOfMany() throws Exception { .andExpect(xpath("//table[@id='entries']/thead/tr").exists()) .andExpect(xpath("//table[@id='entries']/tbody/tr").nodeCount(33)) .andExpect(xpath("//table[@id='entries']/tbody/tr[1]/td[@class='endpoint']/a").string(entries.get((page - 1) * size).getClientUrl())) - .andExpect(xpath("//table[@id='entries']/tbody/tr[1]/td[@class='endpoint']/a/@href").string(entries.get((page - 1) * size).getClientUrl())) - .andExpect(xpath("//span[@id='totalEntries']").string(String.valueOf(items))) - .andExpect(xpath("//span[@id='currentPage']").string(String.valueOf(page))) - .andExpect(xpath("//span[@id='totalPages']").string(String.valueOf(page))) + .andExpect(xpath("//table[@id='entries']/tbody/tr[1]/td[@class='endpoint']/a/@href").string("/entry?clientUrl=" + entries.get((page - 1) * size).getClientUrl())) + .andExpect(xpath("//*[@id='currentPage']").string(String.valueOf(page))) + .andExpect(xpath("//*[@id='totalPages']").string(String.valueOf(page))) .andExpect(xpath("//*[@id='firstPage']/a").exists()) .andExpect(xpath("//*[@id='previousPage']/a").exists()) .andExpect(xpath("//*[@id='nextPage']/a").doesNotExist()) @@ -199,7 +203,7 @@ public void res200_middlePageOfMany() throws Exception { int page = 4; mongoTemplate.getDb().drop(); List entries = IndexEntryFixtures.entriesN(items); - entryRepository.saveAll(entries); + indexEntryRepository.saveAll(entries); // AND (prepare request) RequestBuilder request = MockMvcRequestBuilders @@ -216,10 +220,9 @@ public void res200_middlePageOfMany() throws Exception { .andExpect(xpath("//table[@id='entries']/thead/tr").exists()) .andExpect(xpath("//table[@id='entries']/tbody/tr").nodeCount(size)) .andExpect(xpath("//table[@id='entries']/tbody/tr[1]/td[@class='endpoint']/a").string(entries.get((page - 1) * size).getClientUrl())) - .andExpect(xpath("//table[@id='entries']/tbody/tr[1]/td[@class='endpoint']/a/@href").string(entries.get((page - 1) * size).getClientUrl())) - .andExpect(xpath("//span[@id='totalEntries']").string(String.valueOf(items))) - .andExpect(xpath("//span[@id='currentPage']").string(String.valueOf(page))) - .andExpect(xpath("//span[@id='totalPages']").string(String.valueOf(7))) + .andExpect(xpath("//table[@id='entries']/tbody/tr[1]/td[@class='endpoint']/a/@href").string("/entry?clientUrl=" + entries.get((page - 1) * size).getClientUrl())) + .andExpect(xpath("//*[@id='currentPage']").string(String.valueOf(page))) + .andExpect(xpath("//*[@id='totalPages']").string(String.valueOf(7))) .andExpect(xpath("//*[@id='firstPage']/a").exists()) .andExpect(xpath("//*[@id='previousPage']/a").exists()) .andExpect(xpath("//*[@id='nextPage']/a").exists()) diff --git a/src/test/java/solutions/fairdata/fdp/index/acceptance/web/home/Static_GET_Test.java b/src/test/java/solutions/fairdata/fdp/index/acceptance/web/home/Static_GET_Test.java new file mode 100644 index 0000000..acba331 --- /dev/null +++ b/src/test/java/solutions/fairdata/fdp/index/acceptance/web/home/Static_GET_Test.java @@ -0,0 +1,80 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.acceptance.web.home; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.RequestBuilder; +import org.springframework.test.web.servlet.ResultActions; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.web.util.UriComponentsBuilder; +import solutions.fairdata.fdp.index.WebIntegrationTest; +import solutions.fairdata.fdp.index.database.repository.IndexEntryRepository; +import solutions.fairdata.fdp.index.entity.IndexEntry; +import solutions.fairdata.fdp.index.fixtures.IndexEntryFixtures; + +import java.net.URI; +import java.util.List; + +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; + +@DisplayName("Retrieve static files") +public class Static_GET_Test extends WebIntegrationTest { + + private void simpleGetTest(String url) throws Exception { + // AND (prepare request) + RequestBuilder request = MockMvcRequestBuilders.get(url); + + // WHEN + ResultActions result = mvc.perform(request); + + // THEN + result.andExpect(status().isOk()); + } + + @Test + @DisplayName("HTTP 200: site.css") + public void res200_siteCss() throws Exception { + simpleGetTest("/css/site.css"); + } + + @Test + @DisplayName("HTTP 200: favicon") + public void res200_favicon() throws Exception { + simpleGetTest("/img/favicon.png"); + } + + @Test + @DisplayName("HTTP 200: jQuery (JS)") + public void res200_jqueryJs() throws Exception { + simpleGetTest("/js/jquery/jquery.slim.min.js"); + } + + @Test + @DisplayName("HTTP 200: Bootstrap (JS)") + public void res200_boostrapJs() throws Exception { + simpleGetTest("/js/bootstrap/bootstrap.min.js"); + } +} diff --git a/src/test/java/solutions/fairdata/fdp/index/fixtures/IndexEntryFixtures.java b/src/test/java/solutions/fairdata/fdp/index/fixtures/IndexEntryFixtures.java index 71485b5..449de80 100644 --- a/src/test/java/solutions/fairdata/fdp/index/fixtures/IndexEntryFixtures.java +++ b/src/test/java/solutions/fairdata/fdp/index/fixtures/IndexEntryFixtures.java @@ -23,40 +23,62 @@ package solutions.fairdata.fdp.index.fixtures; import solutions.fairdata.fdp.index.entity.IndexEntry; +import solutions.fairdata.fdp.index.entity.IndexEntryState; +import solutions.fairdata.fdp.index.entity.RepositoryMetadata; +import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; public class IndexEntryFixtures { - private static final String TIMESTAMP = "2020-01-01T00:00:00Z"; - - private static IndexEntry newIndexEntry(String clientUrl) { + private static IndexEntry newIndexEntry(String clientUrl, Instant timestamp) { IndexEntry indexEntry = new IndexEntry(); indexEntry.setClientUrl(clientUrl); - indexEntry.setModificationTime(TIMESTAMP); - indexEntry.setRegistrationTime(TIMESTAMP); + indexEntry.setModificationTime(timestamp); + indexEntry.setRegistrationTime(timestamp); + indexEntry.setState(IndexEntryState.Invalid); return indexEntry; } public static IndexEntry entryExample() { - return newIndexEntry("http://example.com"); + return newIndexEntry("http://example.com", Instant.now()); } public static List entriesFew() { + Instant ref = Instant.now(); return Arrays.asList( - newIndexEntry("http://example.com"), - newIndexEntry("http://test.com"), - newIndexEntry("http://localhost") + newIndexEntry("http://example.com", ref), + newIndexEntry("http://test.com", ref.minusSeconds(1)), + newIndexEntry("http://localhost", ref.minusSeconds(2)) ); } public static List entriesN(int n) { ArrayList entries = new ArrayList<>(); + Instant ref = Instant.now(); for (int i = 0; i < n; i++) { - entries.add(newIndexEntry("http://example" + i + ".com")); + Instant entryTime = ref.minusSeconds(i); + entries.add(newIndexEntry("http://example" + i + ".com", entryTime)); } return entries; } + + public static IndexEntry activeEntry(String clientUrl) { + IndexEntry indexEntry = new IndexEntry(); + indexEntry.setClientUrl(clientUrl); + indexEntry.setModificationTime(Instant.now()); + indexEntry.setRegistrationTime(Instant.now()); + indexEntry.setLastRetrievalTime(Instant.now()); + indexEntry.setState(IndexEntryState.Valid); + indexEntry.setCurrentMetadata(new RepositoryMetadata()); + indexEntry.getCurrentMetadata().setRepositoryUri("http://purl.org/example"); + indexEntry.getCurrentMetadata().setMetadataVersion(1); + indexEntry.getCurrentMetadata().getMetadata().put("title", "Example FDP"); + indexEntry.getCurrentMetadata().getMetadata().put("version", "1.0.0"); + indexEntry.getCurrentMetadata().getMetadata().put("description", "This is my example FAIR Data Point"); + return indexEntry; + } } diff --git a/src/test/resources/application-testing.yml b/src/test/resources/application-testing.yml index fe07ea1..aefcc21 100644 --- a/src/test/resources/application-testing.yml +++ b/src/test/resources/application-testing.yml @@ -2,3 +2,17 @@ spring: data: mongodb: uri: mongodb://localhost:27017/fdp-index-test + +fdp-index: + api: + # url: + title: FAIR Data Point Index API + description: This is OpenAPI specification of FAIR Data Point Index REST API. + contactUrl: https://github.com/FAIRDataTeam/FAIRDataPoint-index + contactName: GitHub repository + events: + retrieval: + rateLimitWait: PT10M # 10 minutes (ISO 8601) + timeout: PT1M # 1 minute (ISO 8601) + ping: + validDuration: P7D # 7 days (ISO 8601) From 2035b2f52db6882028b95ec718e324876ae47aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Fri, 10 Jul 2020 20:00:59 +0200 Subject: [PATCH 2/8] [INDEX-14] Add rate limit by IP (#12) --- .../index/api/controller/PingController.java | 38 +++++++++++++++++-- .../fdp/index/config/CustomConfig.java | 6 ++- .../database/repository/EventRepository.java | 5 +++ .../fdp/index/entity/config/EventsConfig.java | 2 + .../index/exceptions/RateLimitException.java | 32 ++++++++++++++++ .../fdp/index/service/EventService.java | 12 ++++++ src/main/resources/application.yml | 9 ++++- 7 files changed, 97 insertions(+), 7 deletions(-) create mode 100644 src/main/java/solutions/fairdata/fdp/index/exceptions/RateLimitException.java diff --git a/src/main/java/solutions/fairdata/fdp/index/api/controller/PingController.java b/src/main/java/solutions/fairdata/fdp/index/api/controller/PingController.java index 9df2d91..ccdd316 100644 --- a/src/main/java/solutions/fairdata/fdp/index/api/controller/PingController.java +++ b/src/main/java/solutions/fairdata/fdp/index/api/controller/PingController.java @@ -23,15 +23,23 @@ package solutions.fairdata.fdp.index.api.controller; import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.media.ExampleObject; +import io.swagger.v3.oas.annotations.media.Schema; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.tags.Tag; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpEntity; import org.springframework.http.HttpStatus; -import org.springframework.web.bind.annotation.*; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestController; +import solutions.fairdata.fdp.index.api.dto.PingDTO; import solutions.fairdata.fdp.index.entity.events.Event; -import solutions.fairdata.fdp.index.exceptions.IncorrectPingFormatException; import solutions.fairdata.fdp.index.service.EventService; import javax.servlet.http.HttpServletRequest; @@ -45,10 +53,32 @@ public class PingController { @Autowired private EventService eventService; - @Operation(hidden = true) + @Operation( + description = "Inform about running FAIR Data Point. It is expected to send pings regularly (at least weekly). There is a rate limit set both per single IP within a period of time and per URL in message.", + requestBody = @RequestBody( + description = "Ping payload with FAIR Data Point info", + required = true, + content = @Content( + mediaType = "application/json", + examples = { + @ExampleObject(value = "{\"clientUrl\": \"https://example.com\"}") + }, + schema = @Schema( + type = "object", + title = "Ping", + implementation = PingDTO.class + ) + ) + ), + responses = { + @ApiResponse(responseCode = "204", description = "Ping accepted (no content)"), + @ApiResponse(responseCode = "400", description = "Invalid ping format"), + @ApiResponse(responseCode = "429", description = "Rate limit exceeded") + } + ) @PostMapping @ResponseStatus(HttpStatus.NO_CONTENT) - public void receivePing(HttpEntity httpEntity, HttpServletRequest request) throws IncorrectPingFormatException { + public void receivePing(HttpEntity httpEntity, HttpServletRequest request) { logger.info("Received ping from {}", request.getRemoteAddr()); final Event incomingPingEvent = eventService.acceptIncomingPing(httpEntity, request); logger.info("Triggering metadata retrieval for {}", incomingPingEvent.getRelatedTo().getClientUrl()); diff --git a/src/main/java/solutions/fairdata/fdp/index/config/CustomConfig.java b/src/main/java/solutions/fairdata/fdp/index/config/CustomConfig.java index b07a5ee..737b73a 100644 --- a/src/main/java/solutions/fairdata/fdp/index/config/CustomConfig.java +++ b/src/main/java/solutions/fairdata/fdp/index/config/CustomConfig.java @@ -39,12 +39,16 @@ public class CustomConfig { public EventsConfig eventsConfig( @Value("${fdp-index.events.retrieval.rateLimitWait:PT10M}") String cfgRetrievalRateLimitWait, @Value("${fdp-index.events.retrieval.timeout:PT1M}") String cfgRetrievalTimeout, - @Value("${fdp-index.events.ping.validDuration:P7D}") String cfgPingValidDuration + @Value("${fdp-index.events.ping.validDuration:P7D}") String cfgPingValidDuration, + @Value("${fdp-index.events.ping.rateLimitDuration:PT6H}") String cfgPingRateLimitDuration, + @Value("${fdp-index.events.ping.rateLimitHits:10}") int cfgPingRateLimitHits ) { return EventsConfig.builder() .retrievalRateLimitWait(Duration.parse(cfgRetrievalRateLimitWait)) .retrievalTimeout(Duration.parse(cfgRetrievalTimeout)) .pingValidDuration(Duration.parse(cfgPingValidDuration)) + .pingRateLimitDuration(Duration.parse(cfgPingRateLimitDuration)) + .pingRateLimitHits(cfgPingRateLimitHits) .build(); } } diff --git a/src/main/java/solutions/fairdata/fdp/index/database/repository/EventRepository.java b/src/main/java/solutions/fairdata/fdp/index/database/repository/EventRepository.java index 02b7a71..e7c41c3 100644 --- a/src/main/java/solutions/fairdata/fdp/index/database/repository/EventRepository.java +++ b/src/main/java/solutions/fairdata/fdp/index/database/repository/EventRepository.java @@ -28,9 +28,14 @@ import solutions.fairdata.fdp.index.entity.IndexEntry; import solutions.fairdata.fdp.index.entity.events.Event; +import java.time.Instant; +import java.util.List; + public interface EventRepository extends MongoRepository { Iterable getAllByFinishedIsNull(); Page getAllByRelatedTo(IndexEntry indexEntry, Pageable pageable); + + List findAllByIncomingPingExchangeRemoteAddrAndCreatedAfter(String remoteAddr, Instant after); } diff --git a/src/main/java/solutions/fairdata/fdp/index/entity/config/EventsConfig.java b/src/main/java/solutions/fairdata/fdp/index/entity/config/EventsConfig.java index 581ec25..7a6513d 100644 --- a/src/main/java/solutions/fairdata/fdp/index/entity/config/EventsConfig.java +++ b/src/main/java/solutions/fairdata/fdp/index/entity/config/EventsConfig.java @@ -33,4 +33,6 @@ public class EventsConfig { private final Duration retrievalRateLimitWait; private final Duration retrievalTimeout; private final Duration pingValidDuration; + private final Duration pingRateLimitDuration; + private final int pingRateLimitHits; } diff --git a/src/main/java/solutions/fairdata/fdp/index/exceptions/RateLimitException.java b/src/main/java/solutions/fairdata/fdp/index/exceptions/RateLimitException.java new file mode 100644 index 0000000..4227aab --- /dev/null +++ b/src/main/java/solutions/fairdata/fdp/index/exceptions/RateLimitException.java @@ -0,0 +1,32 @@ +/** + * The MIT License + * Copyright © 2020 https://fairdata.solutions + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package solutions.fairdata.fdp.index.exceptions; + +import org.springframework.http.HttpStatus; + +public class RateLimitException extends IndexException { + + public RateLimitException(String message) { + super(message, HttpStatus.TOO_MANY_REQUESTS); + } +} diff --git a/src/main/java/solutions/fairdata/fdp/index/service/EventService.java b/src/main/java/solutions/fairdata/fdp/index/service/EventService.java index 894f35e..41e48da 100644 --- a/src/main/java/solutions/fairdata/fdp/index/service/EventService.java +++ b/src/main/java/solutions/fairdata/fdp/index/service/EventService.java @@ -45,6 +45,7 @@ import solutions.fairdata.fdp.index.entity.http.Exchange; import solutions.fairdata.fdp.index.entity.http.ExchangeState; import solutions.fairdata.fdp.index.exceptions.IncorrectPingFormatException; +import solutions.fairdata.fdp.index.exceptions.RateLimitException; import solutions.fairdata.fdp.index.utils.IncomingPingUtils; import solutions.fairdata.fdp.index.utils.MetadataRetrievalUtils; @@ -85,6 +86,17 @@ public Iterable getEvents(String clientUrl) { @SneakyThrows public Event acceptIncomingPing(HttpEntity httpEntity, HttpServletRequest request) { + var remoteAddr = request.getRemoteAddr(); + var rateLimitSince = Instant.now().minus(eventsConfig.getPingRateLimitDuration()); + var previousPings = eventRepository.findAllByIncomingPingExchangeRemoteAddrAndCreatedAfter(remoteAddr, rateLimitSince); + if (previousPings.size() > eventsConfig.getPingRateLimitHits()) { + logger.warn("Rate limit for PING reached by: " + remoteAddr); + throw new RateLimitException(String.format( + "Rate limit reached for %s (max. %d per %s) - PING ignored", + remoteAddr, eventsConfig.getPingRateLimitHits(), eventsConfig.getPingRateLimitDuration().toString()) + ); + } + var event = IncomingPingUtils.prepareEvent(httpEntity, request); eventRepository.save(event); event.execute(); diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 6eeb59b..aea966f 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -19,7 +19,10 @@ spring: max-size: 5 queue-capacity: 5000 thread-name-prefix: fdpindex-task- - +springdoc: + swagger-ui: + supportedSubmitMethods: + - "get" fdp-index: api: # url: @@ -32,4 +35,6 @@ fdp-index: rateLimitWait: PT10M # 10 minutes (ISO 8601) timeout: PT1M # 1 minute (ISO 8601) ping: - validDuration: PT1M # 7 days (ISO 8601) + validDuration: P7D # 7 days (ISO 8601) + rateLimitDuration: PT6H + rateLimitHits: 10 From 92b4cb9d6d9a724f4d837c1179acd037cf03d969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Fri, 10 Jul 2020 20:04:41 +0200 Subject: [PATCH 3/8] Prepare for v0.2.0 release --- CHANGELOG.md | 12 ++++++++++-- pom.xml | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 314239a..6da2b6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.2.0] + +### Added + +- Metadata retrieval after receiving ping as async event with possibility to resume +- Info page for each entry with details including retrieved metadata +- States of entries with possible filtering +- Configurable rate limit per IP for receiving pings ## [0.1.1] @@ -23,6 +30,7 @@ Initial version for simple list of FAIR Data Points. - REST API to retrieve entries list (both all and paged) documented using Swagger/OpenAPI - Simple webpage with table to browse entries including sorting and pagination -[Unreleased]: /../../compare/v0.1.1...develop +[Unreleased]: /../../compare/v0.2.0...develop [0.1.0]: /../../tree/v0.1.0 [0.1.1]: /../../tree/v0.1.1 +[0.2.0]: /../../tree/v0.2.0 diff --git a/pom.xml b/pom.xml index 77f441b..d72d18c 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ solutions.fairdata fairdatapoint-index - 0.2.0-SNAPSHOT + 0.2.0 2020 From 033791d2552d0614b15da95638778d21d965f821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Fri, 10 Jul 2020 21:02:01 +0200 Subject: [PATCH 4/8] Fix default entry state for home --- .../fairdata/fdp/index/web/controller/HomeController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/solutions/fairdata/fdp/index/web/controller/HomeController.java b/src/main/java/solutions/fairdata/fdp/index/web/controller/HomeController.java index f36e490..e409c5d 100644 --- a/src/main/java/solutions/fairdata/fdp/index/web/controller/HomeController.java +++ b/src/main/java/solutions/fairdata/fdp/index/web/controller/HomeController.java @@ -45,7 +45,7 @@ public class HomeController { private EventsConfig eventsConfig; @GetMapping - public String home(Model model, @SortDefault(sort = "modificationTime", direction = Sort.Direction.DESC) Pageable pageable, @RequestParam(defaultValue = "reachable") String state) { + public String home(Model model, @SortDefault(sort = "modificationTime", direction = Sort.Direction.DESC) Pageable pageable, @RequestParam(defaultValue = "active") String state) { var sort = pageable.getSort().stream() .findFirst() .map(o -> o.getProperty() + "," + o.getDirection().name().toLowerCase()) From e3daa1c6f1ec853f47c1bdbf86ef5f088be21626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Fri, 10 Jul 2020 21:18:44 +0200 Subject: [PATCH 5/8] Fix publisher link in entry detail --- src/main/resources/templates/entry.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/templates/entry.html b/src/main/resources/templates/entry.html index 431827a..2aae640 100644 --- a/src/main/resources/templates/entry.html +++ b/src/main/resources/templates/entry.html @@ -76,7 +76,7 @@

Repository metadata

Publisher - + From 0ff2030064a82cda1157d6bb46256ce98ad0c4bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Mon, 13 Jul 2020 08:57:14 +0200 Subject: [PATCH 6/8] Fix invalid state description --- src/main/resources/templates/home.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/templates/home.html b/src/main/resources/templates/home.html index 43e7222..0e407ca 100644 --- a/src/main/resources/templates/home.html +++ b/src/main/resources/templates/home.html @@ -26,7 +26,7 @@ Unreachable + data-toggle="tooltip" data-placement="bottom" title="Entries that are not verified to be FAIR Data Points"> Invalid Date: Mon, 13 Jul 2020 13:58:10 +0200 Subject: [PATCH 7/8] Unify state tooltips --- src/main/resources/templates/entry.html | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/resources/templates/entry.html b/src/main/resources/templates/entry.html index 2aae640..3dc33af 100644 --- a/src/main/resources/templates/entry.html +++ b/src/main/resources/templates/entry.html @@ -14,11 +14,11 @@
- Active - Inactive - Unreachable - Invalid - Unknown + Active + Inactive + Unreachable + Invalid + Unknown
@@ -112,6 +112,7 @@

Events

+ From d3c84b267aae1ff500806202076f3d1a863f36b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Mon, 13 Jul 2020 14:00:49 +0200 Subject: [PATCH 8/8] Remove unnecessary rdf4j-query --- pom.xml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pom.xml b/pom.xml index d72d18c..133e95c 100644 --- a/pom.xml +++ b/pom.xml @@ -78,11 +78,6 @@ ${mongobee.version} - - org.eclipse.rdf4j - rdf4j-query - ${rdf4j.version} - org.eclipse.rdf4j rdf4j-rio-api