From adad0d6d3a2420c9685f19a56a3cda82ad88b34d Mon Sep 17 00:00:00 2001 From: Zoltan Polgar Date: Wed, 3 Jul 2024 23:12:09 +0200 Subject: [PATCH] Reorganize repository structure, use only one deployable application instead of two --- .github/workflows/deploy-grabber.yaml | 70 ------- .../{deploy-api.yaml => deploy.yaml} | 3 - .github/workflows/pr.yaml | 7 - .gitignore | 4 +- Dockerfile-api => Dockerfile | 6 +- Dockerfile-grabber | 18 -- HELP.md | 21 -- api/README.md => README.md | 0 api/pom.xml | 131 ------------ api/src/main/resources/application.properties | 2 - domain/pom.xml | 91 --------- grabber/pom.xml | 186 ------------------ .../grabber/InvestmentFundsApplication.java | 19 -- .../pozo/investmentfunds/grabber/Test.kt | 14 -- .../src/main/resources/application.properties | 7 - grabber/src/main/resources/log4j.properties | 8 - pom.xml | 159 +++++++++++++-- sheets-extension/Code.gs | 6 +- .../pozo/investmentfunds/api/DateValidator.kt | 0 .../api/InvestmentFundsApplication.kt | 0 .../pozo/investmentfunds/api/ValidDate.kt | 0 .../api/error/GlobalExceptionHandler.kt | 0 .../investmentfunds/api/funds/FundsAPI.kt | 0 .../api/funds/FundsController.kt | 0 .../investmentfunds/api/funds/FundsService.kt | 0 .../investmentfunds/api/grabber/GrabberAPI.kt | 0 .../api/grabber/GrabberController.kt | 0 .../api/grabber/GrabberService.kt | 21 +- .../api}/grabber/InvestmentFundsRoutes.kt | 56 +++--- .../api}/grabber/processors/CsvProcessor.kt | 11 +- .../api}/grabber/processors/ISINProcessor.kt | 14 +- .../api}/grabber/processors/RedisProcessor.kt | 16 +- .../investmentfunds/api/rates/RatesAPI.kt | 0 .../api/rates/RatesController.kt | 2 +- .../investmentfunds/api/rates/RatesService.kt | 2 +- .../investmentfunds/api/redis/RedisService.kt | 0 .../investmentfunds/api/sheets/SheetsAPI.kt | 0 .../api/sheets/SheetsController.kt | 2 +- .../api/sheets/SheetsService.kt | 7 +- .../domain}/DataFlowConstants.kt | 3 +- .../investmentfunds/domain/FundHeaders.kt | 2 +- .../pozo/investmentfunds/domain}/ISIN.kt | 2 +- .../investmentfunds/domain/RateHeaders.kt | 2 +- .../investmentfunds/domain}/RedisHashKey.kt | 2 +- src/main/resources/application.properties | 4 + .../main/resources/static/favicon.ico | Bin .../api/funds/FundsControllerTest.kt | 0 .../api/sheets/SheetsControllerTest.kt | 0 terraform/ecr.tf | 13 -- terraform/ecs.tf | 14 -- terraform/github-roles.tf | 3 +- terraform/github.tf | 12 -- terraform/variables.tf | 3 - 53 files changed, 228 insertions(+), 715 deletions(-) delete mode 100644 .github/workflows/deploy-grabber.yaml rename .github/workflows/{deploy-api.yaml => deploy.yaml} (97%) rename Dockerfile-api => Dockerfile (72%) delete mode 100644 Dockerfile-grabber delete mode 100644 HELP.md rename api/README.md => README.md (100%) delete mode 100644 api/pom.xml delete mode 100644 api/src/main/resources/application.properties delete mode 100644 domain/pom.xml delete mode 100644 grabber/pom.xml delete mode 100644 grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/InvestmentFundsApplication.java delete mode 100644 grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/Test.kt delete mode 100644 grabber/src/main/resources/application.properties delete mode 100644 grabber/src/main/resources/log4j.properties rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/DateValidator.kt (100%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/InvestmentFundsApplication.kt (100%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/ValidDate.kt (100%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/error/GlobalExceptionHandler.kt (100%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsAPI.kt (100%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsController.kt (100%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsService.kt (100%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberAPI.kt (100%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberController.kt (100%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberService.kt (57%) rename {grabber/src/main/kotlin/com/github/pozo/investmentfunds => src/main/kotlin/com/github/pozo/investmentfunds/api}/grabber/InvestmentFundsRoutes.kt (64%) rename {grabber/src/main/kotlin/com/github/pozo/investmentfunds => src/main/kotlin/com/github/pozo/investmentfunds/api}/grabber/processors/CsvProcessor.kt (88%) rename {grabber/src/main/kotlin/com/github/pozo/investmentfunds => src/main/kotlin/com/github/pozo/investmentfunds/api}/grabber/processors/ISINProcessor.kt (81%) rename {grabber/src/main/kotlin/com/github/pozo/investmentfunds => src/main/kotlin/com/github/pozo/investmentfunds/api}/grabber/processors/RedisProcessor.kt (81%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesAPI.kt (100%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesController.kt (94%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesService.kt (94%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/redis/RedisService.kt (100%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsAPI.kt (100%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsController.kt (96%) rename {api/src => src}/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsService.kt (94%) rename {domain/src/main/kotlin/com/github/pozo/investmentfunds => src/main/kotlin/com/github/pozo/investmentfunds/domain}/DataFlowConstants.kt (79%) rename domain/src/main/kotlin/com/github/pozo/investmentfunds/Fund.kt => src/main/kotlin/com/github/pozo/investmentfunds/domain/FundHeaders.kt (92%) rename {domain/src/main/kotlin/com/github/pozo/investmentfunds => src/main/kotlin/com/github/pozo/investmentfunds/domain}/ISIN.kt (71%) rename domain/src/main/kotlin/com/github/pozo/investmentfunds/Rate.kt => src/main/kotlin/com/github/pozo/investmentfunds/domain/RateHeaders.kt (93%) rename {domain/src/main/kotlin/com/github/pozo/investmentfunds => src/main/kotlin/com/github/pozo/investmentfunds/domain}/RedisHashKey.kt (93%) create mode 100644 src/main/resources/application.properties rename {api/src => src}/main/resources/static/favicon.ico (100%) rename {api/src => src}/test/kotlin/com/github/pozo/investmentfunds/api/funds/FundsControllerTest.kt (100%) rename {api/src => src}/test/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsControllerTest.kt (100%) diff --git a/.github/workflows/deploy-grabber.yaml b/.github/workflows/deploy-grabber.yaml deleted file mode 100644 index 413f1aa..0000000 --- a/.github/workflows/deploy-grabber.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: Deploy investment funds grabber to ECS - -on: - pull_request: - branches: - - main - push: - paths: - - grabber/** - branches: - - main - -jobs: - deploy: - runs-on: ubuntu-latest - - # These permissions are needed to interact with GitHub's OIDC Token endpoint. - permissions: - id-token: write - contents: read - - steps: - - name: Checkout repo - uses: actions/checkout@v3 - - - name: Configure AWS credentials for build and push - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_GITHUB_DOCKER_BUILD_AND_PUSH_ROLE }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Log in to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - - name: Build and push grabber Docker image - id: build-and-push-grabber-docker-image - uses: docker/build-push-action@v5 - with: - file: Dockerfile-grabber - push: true - tags: | - ${{ secrets.AWS_ECR_REPOSITORY }}/investmentfunds/grabber:latest - ${{ secrets.AWS_ECR_REPOSITORY }}/investmentfunds/grabber:${{ github.sha }} - - - name: Configure AWS credentials for deploy - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.AWS_GITHUB_ECS_DEPLOY_ROLE }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Download task definition - id: download-task-definition - run: | - aws ecs describe-task-definition --task-definition ${{ vars.AWS_ECS_TASK_DEFINITION_FAMILY }} --query taskDefinition > task-definition.json - - - name: Fill in the new image ID in the Amazon ECS task definition - id: ecs-render-task-definition - uses: aws-actions/amazon-ecs-render-task-definition@v1 - with: - task-definition: task-definition.json - container-name: ${{ vars.AWS_ECS_GRABBER_CONTAINER_NAME }} - image: ${{ secrets.AWS_ECR_REPOSITORY }}/investmentfunds/grabber:${{ github.sha }} - - - name: Deploy to Amazon ECS service - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 - with: - task-definition: ${{ steps.ecs-render-task-definition.outputs.task-definition }} - service: ${{ vars.AWS_ECS_GRABBER_SERVICE_NAME }} - cluster: ${{ vars.AWS_ECS_CLUSTER_NAME }} diff --git a/.github/workflows/deploy-api.yaml b/.github/workflows/deploy.yaml similarity index 97% rename from .github/workflows/deploy-api.yaml rename to .github/workflows/deploy.yaml index 68152d8..1cc2246 100644 --- a/.github/workflows/deploy-api.yaml +++ b/.github/workflows/deploy.yaml @@ -5,8 +5,6 @@ on: branches: - main push: - paths: - - api/** branches: - main @@ -37,7 +35,6 @@ jobs: id: build-and-push-api-docker-image uses: docker/build-push-action@v5 with: - file: Dockerfile-api push: true tags: | ${{ secrets.AWS_ECR_REPOSITORY }}/investmentfunds/api:latest diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fe594a6..9b2c052 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -11,12 +11,5 @@ jobs: runs-on: ubuntu-latest steps: - - name: Build grabber Docker image - uses: docker/build-push-action@v5 - with: - file: Dockerfile-api - - name: Build api Docker image uses: docker/build-push-action@v5 - with: - file: Dockerfile-grabber diff --git a/.gitignore b/.gitignore index 93bd748..d3603b4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ isin-list /build/ +notes .idea/ *.iml *.mwb.bak @@ -7,4 +8,5 @@ isin-list target out .java-version -*/.terragrunt-cache \ No newline at end of file +*/.terragrunt-cache +*/.clasp.json \ No newline at end of file diff --git a/Dockerfile-api b/Dockerfile similarity index 72% rename from Dockerfile-api rename to Dockerfile index 541e4e6..3b6a721 100644 --- a/Dockerfile-api +++ b/Dockerfile @@ -2,9 +2,7 @@ FROM maven:3.8.5-amazoncorretto-17 AS build WORKDIR /app -COPY api api -COPY domain domain -COPY grabber grabber +COPY src src COPY pom.xml . RUN mvn clean package @@ -12,7 +10,7 @@ FROM amazoncorretto:17-alpine-jdk WORKDIR /app -COPY --from=build /app/api/target/api*.jar app.jar +COPY --from=build /app/target/api*.jar app.jar EXPOSE 8080 ENV JAVA_OPTS="" diff --git a/Dockerfile-grabber b/Dockerfile-grabber deleted file mode 100644 index b49d9f4..0000000 --- a/Dockerfile-grabber +++ /dev/null @@ -1,18 +0,0 @@ -FROM maven:3.8.5-amazoncorretto-17 AS build - -WORKDIR /app - -COPY api api -COPY domain domain -COPY grabber grabber -COPY pom.xml . - -RUN mvn clean package -FROM amazoncorretto:17-alpine-jdk - -WORKDIR /app - -COPY --from=build /app/grabber/target/grabber*executable-jar.jar app.jar - -ENV JAVA_OPTS="" -ENTRYPOINT [ "sh", "-c", "java $JAVA_OPTS -Djava.security.egd=file:/dev/./urandom -jar app.jar" ] diff --git a/HELP.md b/HELP.md deleted file mode 100644 index 05dc945..0000000 --- a/HELP.md +++ /dev/null @@ -1,21 +0,0 @@ -# Getting Started - -### Reference Documentation -For further reference, please consider the following sections: - -* [Official Apache Maven documentation](https://maven.apache.org/guides/index.html) -* [Spring Boot Maven Plugin Reference Guide](https://docs.spring.io/spring-boot/docs/3.2.3/maven-plugin/reference/html/) -* [Create an OCI image](https://docs.spring.io/spring-boot/docs/3.2.3/maven-plugin/reference/html/#build-image) -* [Spring Web](https://docs.spring.io/spring-boot/docs/3.2.3/reference/htmlsingle/index.html#web) -* [Spring Security](https://docs.spring.io/spring-boot/docs/3.2.3/reference/htmlsingle/index.html#web.security) - -### Guides -The following guides illustrate how to use some features concretely: - -* [Building a RESTful Web Service](https://spring.io/guides/gs/rest-service/) -* [Serving Web Content with Spring MVC](https://spring.io/guides/gs/serving-web-content/) -* [Building REST services with Spring](https://spring.io/guides/tutorials/rest/) -* [Securing a Web Application](https://spring.io/guides/gs/securing-web/) -* [Spring Boot and OAuth2](https://spring.io/guides/tutorials/spring-boot-oauth2/) -* [Authenticating a User with LDAP](https://spring.io/guides/gs/authenticating-ldap/) - diff --git a/api/README.md b/README.md similarity index 100% rename from api/README.md rename to README.md diff --git a/api/pom.xml b/api/pom.xml deleted file mode 100644 index f4cca3c..0000000 --- a/api/pom.xml +++ /dev/null @@ -1,131 +0,0 @@ - - - 4.0.0 - - com.github.pozo - investment-funds - 1.0.0-SNAPSHOT - - - api - - - UTF-8 - official - 1.8 - - - - - mavenCentral - https://repo1.maven.org/maven2/ - - - - - src/main/kotlin - src/test/kotlin - - - org.springframework.boot - spring-boot-maven-plugin - - - org.jetbrains.kotlin - kotlin-maven-plugin - - - compile - compile - - compile - - - - test-compile - test-compile - - test-compile - - - - - - maven-surefire-plugin - 2.22.2 - - - maven-failsafe-plugin - 2.22.2 - - - - - - - com.github.pozo - domain - 1.0.0-SNAPSHOT - - - org.springdoc - springdoc-openapi-starter-webmvc-api - 2.5.0 - - - redis.clients - jedis - 5.1.2 - - - org.springframework.boot - spring-boot-starter-data-redis - - - org.springframework.boot - spring-boot-starter-web - - - org.springframework.boot - spring-boot-starter-validation - - - com.fasterxml.jackson.module - jackson-module-kotlin - - - org.jetbrains.kotlin - kotlin-reflect - - - org.jetbrains.kotlin - kotlin-stdlib - - - org.springframework.boot - spring-boot-starter-test - test - - - org.jetbrains.kotlin - kotlin-test-junit5 - 1.9.22 - test - - - org.junit.jupiter - junit-jupiter-engine - 5.10.0 - test - - - com.ninja-squad - springmockk - 4.0.2 - test - - - - \ No newline at end of file diff --git a/api/src/main/resources/application.properties b/api/src/main/resources/application.properties deleted file mode 100644 index 8bc5bc8..0000000 --- a/api/src/main/resources/application.properties +++ /dev/null @@ -1,2 +0,0 @@ -server.error.whitelabel.enabled=false -springdoc.api-docs.path=/ \ No newline at end of file diff --git a/domain/pom.xml b/domain/pom.xml deleted file mode 100644 index c29094d..0000000 --- a/domain/pom.xml +++ /dev/null @@ -1,91 +0,0 @@ - - - 4.0.0 - - com.github.pozo - investment-funds - 1.0.0-SNAPSHOT - - - domain - - - UTF-8 - official - 1.8 - - - - - mavenCentral - https://repo1.maven.org/maven2/ - - - - - src/main/kotlin - src/test/kotlin - - - org.jetbrains.kotlin - kotlin-maven-plugin - 1.9.22 - - - compile - compile - - compile - - - - test-compile - test-compile - - test-compile - - - - - - maven-surefire-plugin - 2.22.2 - - - maven-failsafe-plugin - 2.22.2 - - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - MainKt - - - - - - - - org.jetbrains.kotlin - kotlin-test-junit5 - 1.9.22 - test - - - org.junit.jupiter - junit-jupiter-engine - 5.10.0 - test - - - org.jetbrains.kotlin - kotlin-stdlib - 1.9.22 - - - - \ No newline at end of file diff --git a/grabber/pom.xml b/grabber/pom.xml deleted file mode 100644 index 9d10cb5..0000000 --- a/grabber/pom.xml +++ /dev/null @@ -1,186 +0,0 @@ - - - 4.0.0 - - com.github.pozo - investment-funds - 1.0.0-SNAPSHOT - - - grabber - - - UTF-8 - UTF-8 - - official - 1.8 - 2.23.1 - - - - - mavenCentral - https://repo1.maven.org/maven2/ - - - - - src/main/kotlin - src/test/kotlin - - - org.jetbrains.kotlin - kotlin-maven-plugin - - - compile - compile - - compile - - - - test-compile - test-compile - - test-compile - - - - - - maven-surefire-plugin - 2.22.2 - - - maven-failsafe-plugin - 2.22.2 - - - org.apache.camel - camel-maven-plugin - 4.5.0 - - true - com.github.pozo.investmentfunds.grabber.InvestmentFundsApplication - - - - org.apache.maven.plugins - maven-shade-plugin - - - shade-my-jar - package - - shade - - - true - executable-jar - - - com.github.pozo.investmentfunds.grabber.InvestmentFundsApplication - - - META-INF/services/org/apache/camel/TypeConverterLoader - - - - - - - - - - - - - org.apache.camel - camel-bom - 4.5.0 - pom - import - - - - - - - com.github.pozo - domain - 1.0.0-SNAPSHOT - - - - org.apache.camel - camel-core-engine - - - org.apache.camel - camel-main - - - org.apache.camel - camel-bean - - - org.apache.camel - camel-core - - - org.apache.camel - camel-csv - - - org.apache.camel.springboot - camel-spring-redis-starter - 4.4.1 - - - org.apache.camel - camel-http - - - org.apache.camel - camel-timer - - - org.apache.camel - camel-log - - - - org.slf4j - slf4j-log4j12 - 2.0.12 - - - redis.clients - jedis - 5.1.2 - - - - org.jetbrains.kotlin - kotlin-test-junit5 - 1.9.22 - test - - - org.junit.jupiter - junit-jupiter-engine - 5.10.0 - test - - - org.jetbrains.kotlin - kotlin-stdlib - 1.9.22 - - - - \ No newline at end of file diff --git a/grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/InvestmentFundsApplication.java b/grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/InvestmentFundsApplication.java deleted file mode 100644 index bcecb95..0000000 --- a/grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/InvestmentFundsApplication.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.github.pozo.investmentfunds.grabber; - -import org.apache.camel.main.Main; - -/** - * Main class that boot the Camel application - */ -public final class InvestmentFundsApplication { - - private InvestmentFundsApplication() { } - - public static void main(String[] args) throws Exception { - // use Camels Main class - Main main = new Main(InvestmentFundsApplication.class); - // now keep the application running until the JVM is terminated (ctrl + c or sigterm) - main.run(args); - } - -} diff --git a/grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/Test.kt b/grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/Test.kt deleted file mode 100644 index 0eb136f..0000000 --- a/grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/Test.kt +++ /dev/null @@ -1,14 +0,0 @@ -//package com.github.pozo.investmentfunds.grabber -// -//import java.util.regex.Matcher -//import java.util.regex.Pattern -// -//fun main() { -// val regexPattern = "[A-Z]{2}[A-Z0-9]{9}" // Matches two letters followed by 9 alphanumeric characters -// val pattern = Pattern.compile(regexPattern) -// val matcher: Matcher = pattern.matcher("href=\"/alapoldal?isin=HU0000716402\" target=\"_blank\">Accorde Abacus Alap - + 4.0.0 org.springframework.boot spring-boot-starter-parent 3.2.4 - - com.github.pozo - investment-funds - 1.0.0-SNAPSHOT - pom - investment-funds - investment-funds api - - - grabber - api - domain - + + api 17 + UTF-8 + 1.9.22 + official + 1.8 + + 4.6.0 - + + + mavenCentral + https://repo1.maven.org/maven2/ + + + + + src/main/kotlin + src/test/kotlin + + + org.springframework.boot + spring-boot-maven-plugin + + + org.jetbrains.kotlin + kotlin-maven-plugin + + + compile + compile + + compile + + + + test-compile + test-compile + + test-compile + + + + + + maven-surefire-plugin + 2.22.2 + + + maven-failsafe-plugin + 2.22.2 + + + + + + + + org.apache.camel + camel-bom + ${camel.version} + pom + import + + + + + + + org.springdoc + springdoc-openapi-starter-webmvc-api + 2.5.0 + + + redis.clients + jedis + 5.1.2 + + + org.springframework.boot + spring-boot-starter-data-redis + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-validation + + + org.apache.camel.springboot + camel-spring-boot-starter + ${camel.version} + + + org.apache.camel + camel-http + + + org.apache.camel + camel-csv + + + com.fasterxml.jackson.module + jackson-module-kotlin + + + org.jetbrains.kotlin + kotlin-reflect + + + org.jetbrains.kotlin + kotlin-stdlib + + + org.springframework.boot + spring-boot-starter-test + test + + + org.jetbrains.kotlin + kotlin-test-junit5 + 1.9.22 + test + + + org.junit.jupiter + junit-jupiter-engine + 5.10.0 + test + + + com.ninja-squad + springmockk + 4.0.2 + test + + + + \ No newline at end of file diff --git a/sheets-extension/Code.gs b/sheets-extension/Code.gs index ea9f3e1..47af8fa 100644 --- a/sheets-extension/Code.gs +++ b/sheets-extension/Code.gs @@ -1,6 +1,6 @@ const BASE_API_URL = 'https://arrabona-software.solutions'; -function ISIN(isin, attribute = 'rate', startDate = null, endDate = null) { +function FUND_RATE_LOOKUP(isin, attribute = 'rate', startDate = null, endDate = null) { var apiUrl = `${BASE_API_URL}/sheets/rates/${isin}`; var dataToPost = {}; @@ -35,4 +35,8 @@ function ISIN(isin, attribute = 'rate', startDate = null, endDate = null) { Logger.log(reducedData); return reducedData; +} + +function FUND_LOOKUP(attribute = 'name', value = null) { + } \ No newline at end of file diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/DateValidator.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/DateValidator.kt similarity index 100% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/DateValidator.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/DateValidator.kt diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/InvestmentFundsApplication.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/InvestmentFundsApplication.kt similarity index 100% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/InvestmentFundsApplication.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/InvestmentFundsApplication.kt diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/ValidDate.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/ValidDate.kt similarity index 100% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/ValidDate.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/ValidDate.kt diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/error/GlobalExceptionHandler.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/error/GlobalExceptionHandler.kt similarity index 100% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/error/GlobalExceptionHandler.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/error/GlobalExceptionHandler.kt diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsAPI.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsAPI.kt similarity index 100% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsAPI.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsAPI.kt diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsController.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsController.kt similarity index 100% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsController.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsController.kt diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsService.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsService.kt similarity index 100% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsService.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/funds/FundsService.kt diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberAPI.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberAPI.kt similarity index 100% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberAPI.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberAPI.kt diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberController.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberController.kt similarity index 100% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberController.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberController.kt diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberService.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberService.kt similarity index 57% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberService.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberService.kt index d3f91ce..708fe8e 100644 --- a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberService.kt +++ b/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/GrabberService.kt @@ -1,14 +1,19 @@ package com.github.pozo.investmentfunds.api.grabber -import com.github.pozo.investmentfunds.DataFlowConstants import com.github.pozo.investmentfunds.api.redis.RedisService +import com.github.pozo.investmentfunds.domain.DataFlowConstants +import org.apache.camel.ProducerTemplate import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import java.text.SimpleDateFormat import java.util.* + @Service -class GrabberService : GrabberAPI { +class GrabberService constructor( + @Autowired val producerTemplate: ProducerTemplate +) : GrabberAPI { private val logger = LoggerFactory.getLogger(GrabberService::class.java) @@ -17,16 +22,16 @@ class GrabberService : GrabberAPI { override fun trigger() { val latestEndDate = RedisService.jedis.get(DataFlowConstants.GRAB_DATA_LATEST_DATE_KEY.field) - logger.info("Initiating CSV retrieval mechanism via Redis. The last trigger date was $latestEndDate.") + logger.info("Initiating CSV retrieval mechanism. The last trigger date was $latestEndDate.") if (latestEndDate == null || latestEndDate.isEmpty()) { - RedisService.jedis.publish( - DataFlowConstants.GRAB_DATA_COMMAND_CHANNEL_NAME.field, - "2024.05.01${DataFlowConstants.GRAB_DATA_COMMAND_SEPARATOR.field}${format.format(Date())}" + producerTemplate.sendBody( + "direct:grab-data", + "2024.06.01${DataFlowConstants.GRAB_DATA_COMMAND_SEPARATOR.field}${format.format(Date())}" ) } else { - RedisService.jedis.publish( - DataFlowConstants.GRAB_DATA_COMMAND_CHANNEL_NAME.field, + producerTemplate.sendBody( + "direct:grab-data", "$latestEndDate,${format.format(Date())}" ) } diff --git a/grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/InvestmentFundsRoutes.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/InvestmentFundsRoutes.kt similarity index 64% rename from grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/InvestmentFundsRoutes.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/InvestmentFundsRoutes.kt index 73d47d7..37dccae 100644 --- a/grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/InvestmentFundsRoutes.kt +++ b/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/InvestmentFundsRoutes.kt @@ -1,22 +1,23 @@ -package com.github.pozo.investmentfunds.grabber +package com.github.pozo.investmentfunds.api.grabber -import com.github.pozo.investmentfunds.DataFlowConstants -import com.github.pozo.investmentfunds.grabber.processors.CsvProcessor -import com.github.pozo.investmentfunds.grabber.processors.CsvProcessor.ISIN_HEADER_NAME -import com.github.pozo.investmentfunds.grabber.processors.ISINProcessor -import com.github.pozo.investmentfunds.grabber.processors.ISINProcessor.END_DATE_HEADER_NAME -import com.github.pozo.investmentfunds.grabber.processors.ISINProcessor.ISIN_LIST_HEADER_NAME -import com.github.pozo.investmentfunds.grabber.processors.ISINProcessor.START_DATE_HEADER_NAME -import com.github.pozo.investmentfunds.grabber.processors.RedisProcessor +import com.github.pozo.investmentfunds.api.grabber.processors.CsvProcessor +import com.github.pozo.investmentfunds.api.grabber.processors.CsvProcessor.ISIN_HEADER_NAME +import com.github.pozo.investmentfunds.api.grabber.processors.ISINProcessor +import com.github.pozo.investmentfunds.api.grabber.processors.ISINProcessor.END_DATE_HEADER_NAME +import com.github.pozo.investmentfunds.api.grabber.processors.ISINProcessor.ISIN_LIST_HEADER_NAME +import com.github.pozo.investmentfunds.api.grabber.processors.ISINProcessor.START_DATE_HEADER_NAME +import com.github.pozo.investmentfunds.api.grabber.processors.RedisProcessor import org.apache.camel.LoggingLevel import org.apache.camel.builder.RouteBuilder +import org.apache.camel.model.dataformat.CsvDataFormat +import org.springframework.beans.factory.annotation.Value +import org.springframework.stereotype.Component -import org.apache.camel.dataformat.csv.CsvDataFormat -import org.springframework.data.redis.serializer.StringRedisSerializer -import java.util.* - -class InvestmentFundsRoutes : RouteBuilder() { +@Component +class InvestmentFundsRoutes constructor( + @Value("\${processors.isin-group-size}") private val isinGroupSize: Int +) : RouteBuilder() { companion object { const val FUND_DATA_ROUTE_NAME = "direct:fund-data" @@ -24,16 +25,11 @@ class InvestmentFundsRoutes : RouteBuilder() { } override fun configure() { - val isinGroupSize: Int = resolveProperty("processors.isin-group-size") { it.toInt() }.orElseThrow() - - val redisHost: String = resolveProperty("redis.host").orElseThrow() - val redisPort: Int = resolveProperty("redis.port") { it.toInt() }.orElseThrow() - - context.registry.bind("stringRedisSerializer", StringRedisSerializer()) - from("spring-redis://$redisHost:$redisPort?command=SUBSCRIBE&channels=${DataFlowConstants.GRAB_DATA_COMMAND_CHANNEL_NAME.field}&serializer=#stringRedisSerializer") + from("direct:grab-data") .onCompletion().onCompleteOnly().to("direct:end-csv-processing").end() - .filter(ISINProcessor.isValidIntervalHeaderValues()) + .filter(ISINProcessor.isValidIntervalValues()) .process(ISINProcessor.setISINIntervalHeaderValues()) + .log("Received message with body: \${body} and headers: \${headers}") .to("https://www.bamosz.hu/egyes-alapok-kivalasztasa") .convertBodyTo(String::class.java, "UTF-8") .process(ISINProcessor.extractISINList()) @@ -57,9 +53,9 @@ class InvestmentFundsRoutes : RouteBuilder() { .convertBodyTo(String::class.java, "ISO-8859-2") .convertBodyTo(String::class.java, "UTF-8") .unmarshal(CsvDataFormat().apply { - delimiter = ',' - quote = '"' - escape = '\\' + delimiter = "," + quote = "\"" + escape = "\\" }) .log(LoggingLevel.INFO, "Vertically split CSV files according to ISIN numbers.") @@ -88,16 +84,8 @@ class InvestmentFundsRoutes : RouteBuilder() { .end() from("direct:end-csv-processing") - .filter(ISINProcessor.isValidIntervalHeaderValues()) + .filter { exchange -> exchange.message.headers[START_DATE_HEADER_NAME] != null && exchange.message.headers[END_DATE_HEADER_NAME] != null } .log("All CSV processed for (\${header.$START_DATE_HEADER_NAME}-\${header.$END_DATE_HEADER_NAME})") .process(RedisProcessor.saveMetaData()) } - - private fun resolveProperty(propertyName: String): Optional = - camelContext.propertiesComponent.resolveProperty(propertyName) - - private fun resolveProperty(propertyName: String, operation: (String) -> T): Optional = - resolveProperty(propertyName) - .map { operation(it) } - } diff --git a/grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/processors/CsvProcessor.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/processors/CsvProcessor.kt similarity index 88% rename from grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/processors/CsvProcessor.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/processors/CsvProcessor.kt index 7e8db92..d071803 100644 --- a/grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/processors/CsvProcessor.kt +++ b/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/processors/CsvProcessor.kt @@ -1,14 +1,13 @@ -package com.github.pozo.investmentfunds.grabber.processors +package com.github.pozo.investmentfunds.api.grabber.processors -import com.github.pozo.investmentfunds.FundHeaders -import com.github.pozo.investmentfunds.grabber.InvestmentFundsRoutes.Companion.FUND_DATA_ROUTE_NAME -import com.github.pozo.investmentfunds.grabber.InvestmentFundsRoutes.Companion.RATE_DATA_ROUTE_NAME -import com.github.pozo.investmentfunds.grabber.processors.CsvProcessor.DataRow +import com.github.pozo.investmentfunds.api.grabber.InvestmentFundsRoutes.Companion.FUND_DATA_ROUTE_NAME +import com.github.pozo.investmentfunds.api.grabber.InvestmentFundsRoutes.Companion.RATE_DATA_ROUTE_NAME +import com.github.pozo.investmentfunds.domain.FundHeaders import org.apache.camel.Exchange import java.util.stream.Collectors import java.util.stream.IntStream -typealias VerticalPiece = MutableList +typealias VerticalPiece = MutableList typealias CSVLine = List object CsvProcessor { diff --git a/grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/processors/ISINProcessor.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/processors/ISINProcessor.kt similarity index 81% rename from grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/processors/ISINProcessor.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/processors/ISINProcessor.kt index 52bc5fb..7f0c5d4 100644 --- a/grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/processors/ISINProcessor.kt +++ b/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/processors/ISINProcessor.kt @@ -1,7 +1,8 @@ -package com.github.pozo.investmentfunds.grabber.processors +package com.github.pozo.investmentfunds.api.grabber.processors -import com.github.pozo.investmentfunds.DataFlowConstants +import com.github.pozo.investmentfunds.domain.DataFlowConstants import org.apache.camel.Exchange +import org.slf4j.LoggerFactory import java.text.SimpleDateFormat import java.util.* import java.util.regex.Matcher @@ -10,6 +11,8 @@ import java.util.regex.Pattern object ISINProcessor { + private val logger = LoggerFactory.getLogger(ISINProcessor::class.java) + private const val ISIN_PATTERN = "HU[0-9]{10}" const val ISIN_LIST_HEADER_NAME = "isin-list" @@ -29,6 +32,7 @@ object ISINProcessor { while (matcher.find()) { isinSet.add(matcher.group()) } + logger.info("The extracted ISIN set is '$isinSet'") exchange.message.body = isinSet.joinToString(separator = "\n") } @@ -37,7 +41,7 @@ object ISINProcessor { exchange.message.setHeader(ISIN_LIST_HEADER_NAME, message.replace('\n', ',')) } - fun isValidIntervalHeaderValues(): (exchange: Exchange) -> Boolean { + fun isValidIntervalValues(): (exchange: Exchange) -> Boolean { return { exchange -> val message: String = exchange.message.body as String @@ -45,12 +49,14 @@ object ISINProcessor { val (start, end) = message.trim().split(DataFlowConstants.GRAB_DATA_COMMAND_SEPARATOR.field) val startDate = format.parse(start) val endDate = format.parse(end) + logger.info("The extracted start date is '$startDate' and end date is '$endDate'") !startDate.before(format.parse(DataFlowConstants.START_YEAR_DATE.field)) && !startDate.after(endDate) && !endDate.after(Date()) && !isSameDay(startDate, endDate) } catch (e: Exception) { + logger.error("The given interval values are not correct '$message'", e) false } } @@ -66,10 +72,12 @@ object ISINProcessor { && startDateCalendar[Calendar.MONTH] == endDateCalendar[Calendar.MONTH] && startDateCalendar[Calendar.DAY_OF_MONTH] == endDateCalendar[Calendar.DAY_OF_MONTH] } + fun setISINIntervalHeaderValues(): (exchange: Exchange) -> Unit = { exchange -> val message: String = exchange.message.body as String val (startDate, endDate) = message.trim().split(DataFlowConstants.GRAB_DATA_COMMAND_SEPARATOR.field) + logger.info("The extracted start date is '$startDate' and end date is '$endDate'") exchange.message.setHeader(START_DATE_HEADER_NAME, startDate) exchange.message.setHeader(END_DATE_HEADER_NAME, endDate) } diff --git a/grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/processors/RedisProcessor.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/processors/RedisProcessor.kt similarity index 81% rename from grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/processors/RedisProcessor.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/processors/RedisProcessor.kt index e88c655..2e43e84 100644 --- a/grabber/src/main/kotlin/com/github/pozo/investmentfunds/grabber/processors/RedisProcessor.kt +++ b/src/main/kotlin/com/github/pozo/investmentfunds/api/grabber/processors/RedisProcessor.kt @@ -1,11 +1,11 @@ -package com.github.pozo.investmentfunds.grabber.processors - -import com.github.pozo.investmentfunds.FundHeaders -import com.github.pozo.investmentfunds.RateHeaders -import com.github.pozo.investmentfunds.DataFlowConstants -import com.github.pozo.investmentfunds.RedisHashKey -import com.github.pozo.investmentfunds.grabber.processors.CsvProcessor.ISIN_HEADER_NAME -import com.github.pozo.investmentfunds.grabber.processors.ISINProcessor.END_DATE_HEADER_NAME +package com.github.pozo.investmentfunds.api.grabber.processors + +import com.github.pozo.investmentfunds.api.grabber.processors.CsvProcessor.ISIN_HEADER_NAME +import com.github.pozo.investmentfunds.api.grabber.processors.ISINProcessor.END_DATE_HEADER_NAME +import com.github.pozo.investmentfunds.domain.DataFlowConstants +import com.github.pozo.investmentfunds.domain.FundHeaders +import com.github.pozo.investmentfunds.domain.RateHeaders +import com.github.pozo.investmentfunds.domain.RedisHashKey import org.apache.camel.Exchange import redis.clients.jedis.JedisPooled diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesAPI.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesAPI.kt similarity index 100% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesAPI.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesAPI.kt diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesController.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesController.kt similarity index 94% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesController.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesController.kt index 94b02e3..63c1500 100644 --- a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesController.kt +++ b/src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesController.kt @@ -1,7 +1,7 @@ package com.github.pozo.investmentfunds.api.rates -import com.github.pozo.investmentfunds.ISIN.ISIN_REGEX_PATTERN import com.github.pozo.investmentfunds.api.ValidDate +import com.github.pozo.investmentfunds.domain.ISIN.ISIN_REGEX_PATTERN import jakarta.validation.constraints.Pattern import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesService.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesService.kt similarity index 94% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesService.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesService.kt index c92df2c..2d17ad5 100644 --- a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesService.kt +++ b/src/main/kotlin/com/github/pozo/investmentfunds/api/rates/RatesService.kt @@ -1,7 +1,7 @@ package com.github.pozo.investmentfunds.api.rates -import com.github.pozo.investmentfunds.RedisHashKey import com.github.pozo.investmentfunds.api.redis.RedisService +import com.github.pozo.investmentfunds.domain.RedisHashKey import org.springframework.stereotype.Service @Service diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/redis/RedisService.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/redis/RedisService.kt similarity index 100% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/redis/RedisService.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/redis/RedisService.kt diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsAPI.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsAPI.kt similarity index 100% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsAPI.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsAPI.kt diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsController.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsController.kt similarity index 96% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsController.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsController.kt index 67d3f65..139961a 100644 --- a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsController.kt +++ b/src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsController.kt @@ -1,7 +1,7 @@ package com.github.pozo.investmentfunds.api.sheets -import com.github.pozo.investmentfunds.ISIN.ISIN_REGEX_PATTERN import com.github.pozo.investmentfunds.api.ValidDate +import com.github.pozo.investmentfunds.domain.ISIN.ISIN_REGEX_PATTERN import jakarta.validation.Valid import jakarta.validation.constraints.NotBlank import jakarta.validation.constraints.Pattern diff --git a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsService.kt b/src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsService.kt similarity index 94% rename from api/src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsService.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsService.kt index e93e378..d9ea102 100644 --- a/api/src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsService.kt +++ b/src/main/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsService.kt @@ -1,12 +1,11 @@ package com.github.pozo.investmentfunds.api.sheets -import com.github.pozo.investmentfunds.DataFlowConstants -import com.github.pozo.investmentfunds.RateHeaders -import com.github.pozo.investmentfunds.RedisHashKey import com.github.pozo.investmentfunds.api.redis.RedisService +import com.github.pozo.investmentfunds.domain.DataFlowConstants +import com.github.pozo.investmentfunds.domain.RateHeaders +import com.github.pozo.investmentfunds.domain.RedisHashKey import org.springframework.stereotype.Service import java.text.SimpleDateFormat -import java.util.* @Service class SheetsService() : SheetsAPI { diff --git a/domain/src/main/kotlin/com/github/pozo/investmentfunds/DataFlowConstants.kt b/src/main/kotlin/com/github/pozo/investmentfunds/domain/DataFlowConstants.kt similarity index 79% rename from domain/src/main/kotlin/com/github/pozo/investmentfunds/DataFlowConstants.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/domain/DataFlowConstants.kt index 319b2b5..ca99cd5 100644 --- a/domain/src/main/kotlin/com/github/pozo/investmentfunds/DataFlowConstants.kt +++ b/src/main/kotlin/com/github/pozo/investmentfunds/domain/DataFlowConstants.kt @@ -1,10 +1,9 @@ -package com.github.pozo.investmentfunds +package com.github.pozo.investmentfunds.domain enum class DataFlowConstants(val field: String) { START_YEAR("1992"),// there is no data before this date START_YEAR_DATE("1992.01.01"), - GRAB_DATA_COMMAND_CHANNEL_NAME("grab-data"), GRAB_DATA_COMMAND_DATE_FORMAT("yyyy.MM.dd"), GRAB_DATA_COMMAND_SEPARATOR(","), GRAB_DATA_LATEST_DATE_KEY("meta#last-successful-grabbing-ending-date"), diff --git a/domain/src/main/kotlin/com/github/pozo/investmentfunds/Fund.kt b/src/main/kotlin/com/github/pozo/investmentfunds/domain/FundHeaders.kt similarity index 92% rename from domain/src/main/kotlin/com/github/pozo/investmentfunds/Fund.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/domain/FundHeaders.kt index f83bb14..d0c9f68 100644 --- a/domain/src/main/kotlin/com/github/pozo/investmentfunds/Fund.kt +++ b/src/main/kotlin/com/github/pozo/investmentfunds/domain/FundHeaders.kt @@ -1,4 +1,4 @@ -package com.github.pozo.investmentfunds +package com.github.pozo.investmentfunds.domain enum class FundHeaders(val field: String) { ISIN("ISIN kód"), diff --git a/domain/src/main/kotlin/com/github/pozo/investmentfunds/ISIN.kt b/src/main/kotlin/com/github/pozo/investmentfunds/domain/ISIN.kt similarity index 71% rename from domain/src/main/kotlin/com/github/pozo/investmentfunds/ISIN.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/domain/ISIN.kt index 35dbaa5..e538203 100644 --- a/domain/src/main/kotlin/com/github/pozo/investmentfunds/ISIN.kt +++ b/src/main/kotlin/com/github/pozo/investmentfunds/domain/ISIN.kt @@ -1,4 +1,4 @@ -package com.github.pozo.investmentfunds +package com.github.pozo.investmentfunds.domain object ISIN { diff --git a/domain/src/main/kotlin/com/github/pozo/investmentfunds/Rate.kt b/src/main/kotlin/com/github/pozo/investmentfunds/domain/RateHeaders.kt similarity index 93% rename from domain/src/main/kotlin/com/github/pozo/investmentfunds/Rate.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/domain/RateHeaders.kt index bd1351a..c00c4b2 100644 --- a/domain/src/main/kotlin/com/github/pozo/investmentfunds/Rate.kt +++ b/src/main/kotlin/com/github/pozo/investmentfunds/domain/RateHeaders.kt @@ -1,4 +1,4 @@ -package com.github.pozo.investmentfunds +package com.github.pozo.investmentfunds.domain enum class RateHeaders(val field: String) { DATE("Dátum"), diff --git a/domain/src/main/kotlin/com/github/pozo/investmentfunds/RedisHashKey.kt b/src/main/kotlin/com/github/pozo/investmentfunds/domain/RedisHashKey.kt similarity index 93% rename from domain/src/main/kotlin/com/github/pozo/investmentfunds/RedisHashKey.kt rename to src/main/kotlin/com/github/pozo/investmentfunds/domain/RedisHashKey.kt index bf19f28..8b9cbf1 100644 --- a/domain/src/main/kotlin/com/github/pozo/investmentfunds/RedisHashKey.kt +++ b/src/main/kotlin/com/github/pozo/investmentfunds/domain/RedisHashKey.kt @@ -1,4 +1,4 @@ -package com.github.pozo.investmentfunds +package com.github.pozo.investmentfunds.domain import java.time.LocalDate import java.time.format.DateTimeFormatter diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties new file mode 100644 index 0000000..80636ff --- /dev/null +++ b/src/main/resources/application.properties @@ -0,0 +1,4 @@ +server.error.whitelabel.enabled=false +springdoc.api-docs.path=/ + +processors.isin-group-size=10 \ No newline at end of file diff --git a/api/src/main/resources/static/favicon.ico b/src/main/resources/static/favicon.ico similarity index 100% rename from api/src/main/resources/static/favicon.ico rename to src/main/resources/static/favicon.ico diff --git a/api/src/test/kotlin/com/github/pozo/investmentfunds/api/funds/FundsControllerTest.kt b/src/test/kotlin/com/github/pozo/investmentfunds/api/funds/FundsControllerTest.kt similarity index 100% rename from api/src/test/kotlin/com/github/pozo/investmentfunds/api/funds/FundsControllerTest.kt rename to src/test/kotlin/com/github/pozo/investmentfunds/api/funds/FundsControllerTest.kt diff --git a/api/src/test/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsControllerTest.kt b/src/test/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsControllerTest.kt similarity index 100% rename from api/src/test/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsControllerTest.kt rename to src/test/kotlin/com/github/pozo/investmentfunds/api/sheets/SheetsControllerTest.kt diff --git a/terraform/ecr.tf b/terraform/ecr.tf index 677adba..91c39fc 100644 --- a/terraform/ecr.tf +++ b/terraform/ecr.tf @@ -3,19 +3,6 @@ resource "aws_ecr_repository" "investmentfunds-api" { name = "investmentfunds/api" force_delete = true - encryption_configuration { - encryption_type = "AES256" - } - image_scanning_configuration { - scan_on_push = true - } -} - -resource "aws_ecr_repository" "investmentfunds-grabber" { - image_tag_mutability = "MUTABLE" - name = "investmentfunds/grabber" - force_delete = true - encryption_configuration { encryption_type = "AES256" } diff --git a/terraform/ecs.tf b/terraform/ecs.tf index f3fcc6c..a898573 100644 --- a/terraform/ecs.tf +++ b/terraform/ecs.tf @@ -81,20 +81,6 @@ resource "aws_ecs_task_definition" "investment-funds" { ], links = ["investmentfunds-redis"] }, - { - name = var.ecs_grabber_container_name - image = "${var.ecr_repository}/investmentfunds/grabber" - cpu = 1 - memory = 1536 - essential = true - environment = [ - { - name = "JAVA_OPTS" - value = "-Xmx1536m -Xms1536m" - } - ], - links = ["investmentfunds-redis"] - }, { name = "investmentfunds-redis" image = "redis" diff --git a/terraform/github-roles.tf b/terraform/github-roles.tf index b570baa..564a134 100644 --- a/terraform/github-roles.tf +++ b/terraform/github-roles.tf @@ -46,8 +46,7 @@ resource "aws_iam_policy" "github_build_actions_policy" { "ecr:UploadLayerPart", ] Resource = [ - aws_ecr_repository.investmentfunds-api.arn, - aws_ecr_repository.investmentfunds-grabber.arn + aws_ecr_repository.investmentfunds-api.arn ] }, { diff --git a/terraform/github.tf b/terraform/github.tf index 397cf16..fe3188c 100644 --- a/terraform/github.tf +++ b/terraform/github.tf @@ -34,18 +34,6 @@ resource "github_actions_variable" "aws_ecs_task_definition_family" { value = aws_ecs_task_definition.investment-funds.family } -resource "github_actions_variable" "aws_ecs_grabber_service_name" { - repository = var.github_repository - variable_name = "AWS_ECS_GRABBER_SERVICE_NAME" - value = aws_ecs_service.investment-funds.name -} - -resource "github_actions_variable" "aws_ecs_grabber_container_name" { - repository = var.github_repository - variable_name = "AWS_ECS_GRABBER_CONTAINER_NAME" - value = var.ecs_grabber_container_name -} - resource "github_actions_variable" "aws_ecs_api_service_name" { repository = var.github_repository variable_name = "AWS_ECS_API_SERVICE_NAME" diff --git a/terraform/variables.tf b/terraform/variables.tf index 4226746..d26cddb 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -25,9 +25,6 @@ variable "ecs_service_name" { variable "ecs_api_container_name" { type = string } -variable "ecs_grabber_container_name" { - type = string -} variable "github_token" { type = string }