From 106f39601ae4511b6812f20799b5fef05c7b9985 Mon Sep 17 00:00:00 2001 From: Maxence Maireaux Date: Wed, 25 Jan 2023 13:41:59 +0100 Subject: [PATCH] feat: Switch to monorepo (#21) * feat: first commit * feat: import wallet, fctl sdk and some stuff * fix: make hooks enabled only if changes in the code * feat: refine scripts * chore: disable pre commit hooks * chore: update payments * feat: make all hooks pass this has required modifications on almost each project * git subrepo push services/webhooks subrepo: subdir: "services/webhooks" merged: "ca5d19c5" upstream: origin: "webhooks" branch: "main" commit: "ca5d19c5" git-subrepo: version: "0.4.5" origin: "ssh://git@github.com/ingydotnet/git-subrepo" commit: "dbb99be" * chore: push backport * chore: backports * chore: backports * chore: backports * chore: backports * chore: backports * chore: remove test code * chore: backports * fix: branch * git subrepo push --branch=chore/backports --force services/fctl subrepo: subdir: "services/fctl" merged: "0d98fdaa" upstream: origin: "fctl" branch: "chore/backports" commit: "0d98fdaa" git-subrepo: version: "0.4.5" origin: "ssh://git@github.com/ingydotnet/git-subrepo" commit: "dbb99be" * feat(fctl): fix argument * git subrepo push --branch=chore/backports --force services/fctl subrepo: subdir: "services/fctl" merged: "5a7a41f9" upstream: origin: "fctl" branch: "chore/backports" commit: "5a7a41f9" git-subrepo: version: "0.4.5" origin: "ssh://git@github.com/ingydotnet/git-subrepo" commit: "dbb99be" * feat: remove ledger * git subrepo clone ledger services/ledger subrepo: subdir: "services/ledger" merged: "ad57138a" upstream: origin: "ledger" branch: "main" commit: "ad57138a" git-subrepo: version: "0.4.5" origin: "ssh://git@github.com/ingydotnet/git-subrepo" commit: "dbb99be" * feat(auth): fix dependencies * chore: backports * feat: remove ledger * feat: add ledger * feat: remove auth * feat: fetch auth main * feat: remove search * feat: add search * feat: remove submodules * feat: reomve payments * feat: add payments from main * feat: add 'yarn install' before generate sdk * feat: Update CI * feat: remove fctl * feat: add fctl * chore: go mod tidy * git subrepo push services/fctl subrepo: subdir: "services/fctl" merged: "5d4c919" upstream: origin: "fctl" branch: "main" commit: "5d4c919" git-subrepo: version: "0.4.5" origin: "ssh://git@github.com/ingydotnet/git-subrepo" commit: "dbb99be" * feat: add go-project-template * feat: update go-project-template under new go libs * git subrepo push libs/go-project-template subrepo: subdir: "libs/go-project-template" merged: "40e49f0" upstream: origin: "go-project-template" branch: "main" commit: "40e49f0" git-subrepo: version: "0.4.5" origin: "ssh://git@github.com/ingydotnet/git-subrepo" commit: "dbb99be" * fix: sdk hook * feat: Update CI * feat: Update CI * feat: Update CI * feat: Update CI * feat: Update * feat: CI Update * feat: Clear CI * feat: CI clear * fee * feat: CI * feat: Update CI * feat: Update CI * feat: Desactivate release mode * feat: Update CI * feat: Update CI * ci: upgrade * feat: update go-libs * feat(orchetration): init project * git subrepo push libs/go-project-template subrepo: subdir: "libs/go-project-template" merged: "cbead19" upstream: origin: "go-project-template" branch: "main" commit: "cbead19" git-subrepo: version: "0.4.5" origin: "ssh://git@github.com/ingydotnet/git-subrepo" commit: "dbb99be" * fix: commitlint * feat: little clean & fix pre-commit * fix: for pre-commit * ci: Improvement * ci: Improvement * ci: Improvement * ci: Improvement * ci: Improvement * chore: add machine * ci: Improvement * ci: Improvement * ci: Improvement * ci: Improvement * feat: Add all SDK * ci: Improvement * feat: Update taskfile & CI * feat: Update taskfile & CI * feat: List foliders in find * feat: List foliders in find * feat: List foliders in find * feat: List foliders in find * feat: List foliders in find * feat: List foliders in find * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * chore: Empty-Commit * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: Add GoBuild * feat: big bang * feat: big bang * feat: big bang * ci: Update * ci: Add release * ci: Update .goreleaser.yml * feat(orchestration): first working version * feat(operator): simplify * feat(operator): factorize ingress, services, auth clients... * feat(operator): remove resources * fix: auth configuration causing pods reboot * feat: Update gitpod configuration * feat(gitpod): Add Garden * feat(gitpod): Add rsync * feat(gitpod): Add packages * feat(gitpod): Add packages * feat(gitpod): Optimize configuration * feat(gitpod): restore default config * feat(gitpod): Add port 80 & 443 on qemu * feat(gitpod): Ignore .garden fodler * feat(gitpod): Upgrade qemu * feat: add 'stack' label on created objects * feat: add taskfile operator to root taskfile * feat(operator): add lint to taskfile * feat(operator): add build.Dockerfile * fix(operator): fix build.Dockerfile * fix(operator): restore service names * fix(operator): ingresses * fix(operator): fix middlewares * fix(operator): duplicate middlewares entries * fix(operator): control ingress path prefix * feat(operator): disable ingress on service if not specified in configuration * fix(fctl): membership uri * fix(orchestration): application/json content-type * feat: wip * fix: orchestration run ids * fix: catch panic on workflow executions Co-authored-by: Ragot Geoffrey Co-authored-by: Antoine Gelloz --- .github/ISSUE_TEMPLATE/bug_report.md | 27 + .github/ISSUE_TEMPLATE/feature_request.md | 22 + .github/dependabot.yml | 7 + .github/workflows/codeql.yml | 33 + .github/workflows/main.yml | 92 ++ .gitignore | 9 + .gitrepo | 12 + .golangci.yml | 112 +++ .goreleaser.yml | 106 +++ .pre-commit-config.yaml | 22 + Dockerfile | 35 + Makefile | 11 + README.md | 32 + Taskfile.yml | 35 + cmd/migrate.go | 73 ++ cmd/root.go | 92 ++ cmd/server.go | 194 ++++ cmd/version.go | 21 + codecov.yml | 4 + commitlint.config.js | 5 + docker-compose.yml | 40 + docs/connectors.md | 157 ++++ docs/development.md | 24 + docs/samples-payin.json | 7 + docs/tuto-connector.md | 580 ++++++++++++ go.mod | 122 +++ go.sum | 803 ++++++++++++++++ internal/app/api/accounts.go | 109 +++ internal/app/api/connector.go | 247 +++++ internal/app/api/connectorconfigs.go | 41 + internal/app/api/connectormodule.go | 83 ++ internal/app/api/health.go | 27 + internal/app/api/module.go | 170 ++++ internal/app/api/payments.go | 247 +++++ internal/app/api/readconnectors.go | 52 ++ internal/app/api/recovery.go | 20 + internal/app/api/router.go | 84 ++ internal/app/api/stripe.go | 111 +++ .../app/connectors/bankingcircle/client.go | 246 +++++ .../app/connectors/bankingcircle/config.go | 49 + .../app/connectors/bankingcircle/errors.go | 20 + .../app/connectors/bankingcircle/loader.go | 33 + .../bankingcircle/task_fetch_payments.go | 80 ++ .../connectors/bankingcircle/task_resolve.go | 49 + .../app/connectors/configtemplate/template.go | 35 + .../app/connectors/configtemplate/types.go | 9 + .../connectors/currencycloud/client/auth.go | 50 + .../connectors/currencycloud/client/client.go | 71 ++ .../currencycloud/client/transactions.go | 60 ++ .../app/connectors/currencycloud/config.go | 86 ++ .../app/connectors/currencycloud/connector.go | 46 + .../app/connectors/currencycloud/errors.go | 20 + .../app/connectors/currencycloud/loader.go | 32 + .../currencycloud/task_fetch_transactions.go | 122 +++ .../connectors/currencycloud/task_resolve.go | 44 + internal/app/connectors/dummypay/config.go | 64 ++ .../app/connectors/dummypay/config_test.go | 58 ++ internal/app/connectors/dummypay/connector.go | 81 ++ .../app/connectors/dummypay/connector_test.go | 79 ++ internal/app/connectors/dummypay/errors.go | 20 + internal/app/connectors/dummypay/fs.go | 12 + internal/app/connectors/dummypay/fs_test.go | 7 + internal/app/connectors/dummypay/loader.go | 54 ++ .../app/connectors/dummypay/loader_test.go | 30 + internal/app/connectors/dummypay/payment.go | 19 + .../app/connectors/dummypay/remove_files.go | 33 + .../connectors/dummypay/task_descriptor.go | 34 + .../connectors/dummypay/task_generate_file.go | 166 ++++ .../app/connectors/dummypay/task_ingest.go | 74 ++ .../connectors/dummypay/task_read_files.go | 78 ++ internal/app/connectors/dummypay/task_test.go | 41 + internal/app/connectors/duration.go | 41 + .../app/connectors/modulr/client/accounts.go | 44 + .../app/connectors/modulr/client/client.go | 62 ++ .../connectors/modulr/client/transactions.go | 41 + internal/app/connectors/modulr/config.go | 39 + internal/app/connectors/modulr/connector.go | 54 ++ internal/app/connectors/modulr/errors.go | 14 + internal/app/connectors/modulr/hmac/hmac.go | 59 ++ .../app/connectors/modulr/hmac/hmac_test.go | 71 ++ .../modulr/hmac/signature_generator.go | 32 + .../connectors/modulr/hmac/signature_test.go | 68 ++ internal/app/connectors/modulr/loader.go | 32 + .../connectors/modulr/task_fetch_accounts.go | 46 + .../modulr/task_fetch_transactions.go | 76 ++ .../app/connectors/modulr/task_resolve.go | 47 + internal/app/connectors/stripe/client.go | 124 +++ internal/app/connectors/stripe/client_test.go | 281 ++++++ internal/app/connectors/stripe/config.go | 47 + internal/app/connectors/stripe/connector.go | 59 ++ internal/app/connectors/stripe/descriptor.go | 7 + internal/app/connectors/stripe/ingester.go | 20 + internal/app/connectors/stripe/loader.go | 43 + internal/app/connectors/stripe/runner.go | 85 ++ internal/app/connectors/stripe/runner_test.go | 51 + internal/app/connectors/stripe/state.go | 11 + .../stripe/task_connected_account.go | 63 ++ internal/app/connectors/stripe/task_main.go | 77 ++ internal/app/connectors/stripe/timeline.go | 197 ++++ .../app/connectors/stripe/timeline_test.go | 67 ++ .../app/connectors/stripe/timeline_trigger.go | 121 +++ .../stripe/timeline_trigger_test.go | 105 +++ internal/app/connectors/stripe/translate.go | 325 +++++++ internal/app/connectors/stripe/utils_test.go | 199 ++++ internal/app/connectors/wise/client.go | 175 ++++ internal/app/connectors/wise/config.go | 31 + internal/app/connectors/wise/connector.go | 54 ++ internal/app/connectors/wise/errors.go | 11 + internal/app/connectors/wise/loader.go | 32 + .../connectors/wise/task_fetch_profiles.go | 44 + .../connectors/wise/task_fetch_transfers.go | 134 +++ internal/app/connectors/wise/task_resolve.go | 39 + internal/app/ingestion/accounts.go | 64 ++ internal/app/ingestion/ingester.go | 45 + internal/app/ingestion/payments.go | 86 ++ internal/app/integration/connector.go | 96 ++ internal/app/integration/loader.go | 97 ++ internal/app/integration/manager.go | 275 ++++++ internal/app/integration/manager_test.go | 207 +++++ internal/app/integration/store.go | 20 + internal/app/integration/storememory.go | 99 ++ internal/app/integration/taskscheduler.go | 15 + internal/app/messages/accounts.go | 37 + internal/app/messages/connectors.go | 25 + internal/app/messages/event.go | 25 + internal/app/messages/payments.go | 45 + .../app/migrations/001_initiate_schemas.go | 39 + internal/app/migrations/002_connectors.go | 43 + internal/app/migrations/003_tasks.go | 55 ++ internal/app/migrations/004_accounts.go | 43 + internal/app/migrations/005_payments.go | 106 +++ internal/app/models/account.go | 28 + internal/app/models/adjustment.go | 26 + internal/app/models/connector.go | 74 ++ internal/app/models/metadata.go | 26 + internal/app/models/payment.go | 96 ++ internal/app/models/task.go | 86 ++ internal/app/storage/accounts.go | 72 ++ internal/app/storage/connectors.go | 155 +++ internal/app/storage/error.go | 22 + internal/app/storage/module.go | 58 ++ internal/app/storage/paginate.go | 129 +++ internal/app/storage/paginate_test.go | 241 +++++ internal/app/storage/payments.go | 167 ++++ internal/app/storage/ping.go | 5 + internal/app/storage/repository.go | 19 + internal/app/storage/sort.go | 33 + internal/app/storage/task.go | 202 ++++ internal/app/task/context.go | 30 + internal/app/task/resolver.go | 13 + internal/app/task/scheduler.go | 367 ++++++++ internal/app/task/scheduler_test.go | 217 +++++ internal/app/task/state.go | 40 + internal/app/task/store.go | 21 + internal/app/task/storememory.go | 209 +++++ internal/app/task/task.go | 3 + main.go | 8 + openapi.yaml | 879 ++++++++++++++++++ 158 files changed, 13546 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/main.yml create mode 100644 .gitignore create mode 100644 .gitrepo create mode 100644 .golangci.yml create mode 100644 .goreleaser.yml create mode 100644 .pre-commit-config.yaml create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 README.md create mode 100644 Taskfile.yml create mode 100644 cmd/migrate.go create mode 100644 cmd/root.go create mode 100644 cmd/server.go create mode 100644 cmd/version.go create mode 100644 codecov.yml create mode 100644 commitlint.config.js create mode 100644 docker-compose.yml create mode 100644 docs/connectors.md create mode 100644 docs/development.md create mode 100644 docs/samples-payin.json create mode 100644 docs/tuto-connector.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/app/api/accounts.go create mode 100644 internal/app/api/connector.go create mode 100644 internal/app/api/connectorconfigs.go create mode 100644 internal/app/api/connectormodule.go create mode 100644 internal/app/api/health.go create mode 100644 internal/app/api/module.go create mode 100644 internal/app/api/payments.go create mode 100644 internal/app/api/readconnectors.go create mode 100644 internal/app/api/recovery.go create mode 100644 internal/app/api/router.go create mode 100644 internal/app/api/stripe.go create mode 100644 internal/app/connectors/bankingcircle/client.go create mode 100644 internal/app/connectors/bankingcircle/config.go create mode 100644 internal/app/connectors/bankingcircle/errors.go create mode 100644 internal/app/connectors/bankingcircle/loader.go create mode 100644 internal/app/connectors/bankingcircle/task_fetch_payments.go create mode 100644 internal/app/connectors/bankingcircle/task_resolve.go create mode 100644 internal/app/connectors/configtemplate/template.go create mode 100644 internal/app/connectors/configtemplate/types.go create mode 100644 internal/app/connectors/currencycloud/client/auth.go create mode 100644 internal/app/connectors/currencycloud/client/client.go create mode 100644 internal/app/connectors/currencycloud/client/transactions.go create mode 100644 internal/app/connectors/currencycloud/config.go create mode 100644 internal/app/connectors/currencycloud/connector.go create mode 100644 internal/app/connectors/currencycloud/errors.go create mode 100644 internal/app/connectors/currencycloud/loader.go create mode 100644 internal/app/connectors/currencycloud/task_fetch_transactions.go create mode 100644 internal/app/connectors/currencycloud/task_resolve.go create mode 100644 internal/app/connectors/dummypay/config.go create mode 100644 internal/app/connectors/dummypay/config_test.go create mode 100644 internal/app/connectors/dummypay/connector.go create mode 100644 internal/app/connectors/dummypay/connector_test.go create mode 100644 internal/app/connectors/dummypay/errors.go create mode 100644 internal/app/connectors/dummypay/fs.go create mode 100644 internal/app/connectors/dummypay/fs_test.go create mode 100644 internal/app/connectors/dummypay/loader.go create mode 100644 internal/app/connectors/dummypay/loader_test.go create mode 100644 internal/app/connectors/dummypay/payment.go create mode 100644 internal/app/connectors/dummypay/remove_files.go create mode 100644 internal/app/connectors/dummypay/task_descriptor.go create mode 100644 internal/app/connectors/dummypay/task_generate_file.go create mode 100644 internal/app/connectors/dummypay/task_ingest.go create mode 100644 internal/app/connectors/dummypay/task_read_files.go create mode 100644 internal/app/connectors/dummypay/task_test.go create mode 100644 internal/app/connectors/duration.go create mode 100644 internal/app/connectors/modulr/client/accounts.go create mode 100644 internal/app/connectors/modulr/client/client.go create mode 100644 internal/app/connectors/modulr/client/transactions.go create mode 100644 internal/app/connectors/modulr/config.go create mode 100644 internal/app/connectors/modulr/connector.go create mode 100644 internal/app/connectors/modulr/errors.go create mode 100644 internal/app/connectors/modulr/hmac/hmac.go create mode 100644 internal/app/connectors/modulr/hmac/hmac_test.go create mode 100644 internal/app/connectors/modulr/hmac/signature_generator.go create mode 100644 internal/app/connectors/modulr/hmac/signature_test.go create mode 100644 internal/app/connectors/modulr/loader.go create mode 100644 internal/app/connectors/modulr/task_fetch_accounts.go create mode 100644 internal/app/connectors/modulr/task_fetch_transactions.go create mode 100644 internal/app/connectors/modulr/task_resolve.go create mode 100644 internal/app/connectors/stripe/client.go create mode 100644 internal/app/connectors/stripe/client_test.go create mode 100644 internal/app/connectors/stripe/config.go create mode 100644 internal/app/connectors/stripe/connector.go create mode 100644 internal/app/connectors/stripe/descriptor.go create mode 100644 internal/app/connectors/stripe/ingester.go create mode 100644 internal/app/connectors/stripe/loader.go create mode 100644 internal/app/connectors/stripe/runner.go create mode 100644 internal/app/connectors/stripe/runner_test.go create mode 100644 internal/app/connectors/stripe/state.go create mode 100644 internal/app/connectors/stripe/task_connected_account.go create mode 100644 internal/app/connectors/stripe/task_main.go create mode 100644 internal/app/connectors/stripe/timeline.go create mode 100644 internal/app/connectors/stripe/timeline_test.go create mode 100644 internal/app/connectors/stripe/timeline_trigger.go create mode 100644 internal/app/connectors/stripe/timeline_trigger_test.go create mode 100644 internal/app/connectors/stripe/translate.go create mode 100644 internal/app/connectors/stripe/utils_test.go create mode 100644 internal/app/connectors/wise/client.go create mode 100644 internal/app/connectors/wise/config.go create mode 100644 internal/app/connectors/wise/connector.go create mode 100644 internal/app/connectors/wise/errors.go create mode 100644 internal/app/connectors/wise/loader.go create mode 100644 internal/app/connectors/wise/task_fetch_profiles.go create mode 100644 internal/app/connectors/wise/task_fetch_transfers.go create mode 100644 internal/app/connectors/wise/task_resolve.go create mode 100644 internal/app/ingestion/accounts.go create mode 100644 internal/app/ingestion/ingester.go create mode 100644 internal/app/ingestion/payments.go create mode 100644 internal/app/integration/connector.go create mode 100644 internal/app/integration/loader.go create mode 100644 internal/app/integration/manager.go create mode 100644 internal/app/integration/manager_test.go create mode 100644 internal/app/integration/store.go create mode 100644 internal/app/integration/storememory.go create mode 100644 internal/app/integration/taskscheduler.go create mode 100644 internal/app/messages/accounts.go create mode 100644 internal/app/messages/connectors.go create mode 100644 internal/app/messages/event.go create mode 100644 internal/app/messages/payments.go create mode 100644 internal/app/migrations/001_initiate_schemas.go create mode 100644 internal/app/migrations/002_connectors.go create mode 100644 internal/app/migrations/003_tasks.go create mode 100644 internal/app/migrations/004_accounts.go create mode 100644 internal/app/migrations/005_payments.go create mode 100644 internal/app/models/account.go create mode 100644 internal/app/models/adjustment.go create mode 100644 internal/app/models/connector.go create mode 100644 internal/app/models/metadata.go create mode 100644 internal/app/models/payment.go create mode 100644 internal/app/models/task.go create mode 100644 internal/app/storage/accounts.go create mode 100644 internal/app/storage/connectors.go create mode 100644 internal/app/storage/error.go create mode 100644 internal/app/storage/module.go create mode 100644 internal/app/storage/paginate.go create mode 100644 internal/app/storage/paginate_test.go create mode 100644 internal/app/storage/payments.go create mode 100644 internal/app/storage/ping.go create mode 100644 internal/app/storage/repository.go create mode 100644 internal/app/storage/sort.go create mode 100644 internal/app/storage/task.go create mode 100644 internal/app/task/context.go create mode 100644 internal/app/task/resolver.go create mode 100644 internal/app/task/scheduler.go create mode 100644 internal/app/task/scheduler_test.go create mode 100644 internal/app/task/state.go create mode 100644 internal/app/task/store.go create mode 100644 internal/app/task/storememory.go create mode 100644 internal/app/task/task.go create mode 100644 main.go create mode 100644 openapi.yaml diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..f7757cdf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,27 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Logs** +If applicable, add logs to help explain your problem. + +**Environment (please complete the following information):** + - OS: [e.g. ubuntu 20.04] + - Numary Version [e.g. 1.0.0-beta.4] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..b36319ba --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,22 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement, rfc +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Summary** + +**Solution proposal** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..99602049 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000..773d9fba --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,33 @@ +name: "CodeQL" + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + schedule: + - cron: '35 21 * * 5' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + strategy: + fail-fast: false + matrix: + language: [ 'go' ] + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000..3416ff0a --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,92 @@ +on: + push: + branches: + - 'main' + - 'features/**' + - 'feature/**' + - 'feat/**' + - 'fix/**' + - 'hotfix/**' + pull_request: + types: [ assigned, opened, synchronize, reopened ] + release: + types: [ prereleased, released ] + +name: Main +jobs: + pr-style: + if: github.event_name == 'pull_request' + uses: numary/gh-workflows/.github/workflows/pr-style.yml@main + + lint: + uses: numary/gh-workflows/.github/workflows/golang-lint.yml@main + + test: + uses: numary/gh-workflows/.github/workflows/golang-test.yml@main + + goreleaser-build: + if: github.event_name != 'release' + uses: numary/gh-workflows/.github/workflows/goreleaser-build.yml@main + needs: + - lint + - test + + goreleaser-release: + if: github.event_name == 'release' + uses: numary/gh-workflows/.github/workflows/goreleaser-release.yml@main + secrets: + FURY_TOKEN: ${{ secrets.FURY_TOKEN }} + NUMARY_GITHUB_TOKEN: ${{ secrets.NUMARY_GITHUB_TOKEN }} + needs: + - lint + - test + + docker-build-push: + runs-on: ubuntu-latest + needs: + - lint + - test + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + cache: true + - run: go mod vendor + - uses: docker/setup-qemu-action@v2 + - uses: docker/setup-buildx-action@v2 + - uses: docker/login-action@v2 + with: + registry: ghcr.io + username: "NumaryBot" + password: ${{ secrets.NUMARY_GITHUB_TOKEN }} + - if: github.event.action == 'released' + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ghcr.io/formancehq/payments:latest,ghcr.io/formancehq/payments:${{ github.event.release.tag_name }} + build-args: | + APP_SHA=${{ github.sha }} + VERSION=${{ github.event.release.tag_name }} + - if: github.event.action == 'prereleased' + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ghcr.io/formancehq/payments:${{ github.event.release.tag_name }} + build-args: | + APP_SHA=${{ github.sha }} + VERSION=${{ github.event.release.tag_name }} + - if: github.event.action != 'released' || github.event.action != 'prereleased' + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ghcr.io/formancehq/payments:${{ github.sha }} + build-args: | + APP_SHA=${{ github.sha }} + VERSION=develop diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..63a9a7f5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.DS_Store +.idea +vendor +.cloud/ressources/.terraform +.cloud/ressources/.terraform.lock.hcl +/.cloud/helm/charts/ +coverage.out +dist/ +.env diff --git a/.gitrepo b/.gitrepo new file mode 100644 index 00000000..09dd1d76 --- /dev/null +++ b/.gitrepo @@ -0,0 +1,12 @@ +; DO NOT EDIT (unless you know what you are doing) +; +; This subdirectory is a git "subrepo", and this file is maintained by the +; git-subrepo command. See https://github.com/ingydotnet/git-subrepo#readme +; +[subrepo] + remote = payments + branch = main + commit = ae1fa5ec4508bc3a7603bd15953228b6a6152761 + parent = e11fe64274a7326f57cf409cd542ce7a08b05233 + method = merge + cmdver = 0.4.5 diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..e1845e02 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,112 @@ +linters-settings: + dupl: + threshold: 30 + funlen: + lines: 150 + statements: 30 + goconst: + min-len: 2 + min-occurrences: 2 + gocritic: + enabled-tags: + - diagnostic + - experimental + - opinionated + - performance + - style + cyclop: + max-complexity: 30 + gocyclo: + min-complexity: 30 + goimports: + local-prefixes: github.com/formancehq/payments + govet: + check-shadowing: true + lll: + line-length: 120 + nakedret: + max-func-lines: 0 + tagliatelle: + case: + rules: + json: goCamel + yaml: goCamel + bson: goCamel + gomnd: + ignored-numbers: + - '2' + - '5' + - '10' + - '100' + - '64' + - '256' + - '512' + varnamelen: + ignore-names: + - id + - i + - db + - m + - r + - c + - fn + - w + - h + - to + - fs + +linters: + enable-all: true + disable: + - deadcode # deprecated + - maligned #deprecated + - golint # deprecated + - ifshort # deprecated + - structcheck # deprecated + - varcheck # deprecated + - interfacer # deprecated + - scopelint #deprecated + - nosnakecase # deprecated + - rowserrcheck # disabled due to generics + - sqlclosecheck # disabled due to generics + - structcheck # disabled due to generics + - wastedassign # disabled due to generics + - gci # conflicts with gofumpt + - goimports # conflicts with gofumpt + - testpackage # Disabled by design + - wrapcheck # Disabled by design + - exhaustivestruct # Disabled by design + - exhaustruct # Disabled by design + - exhaustive # Disabled by design + - dupl # Disabled by design + - godox # Disabled by design + - lll # Disabled by design + - funlen # Disabled by design + - misspell # Disabled by design + - ireturn # Disabled by design + - wsl # Disabled by design + - gocritic # TODO: FIX. Seems to have issues with generics + - gocognit # TODO: FIX + - goerr113 # TODO: FIX + - noctx # TODO: FIX + - contextcheck # TODO: FIX + - containedctx # TODO: FIX + +issues: + exclude-rules: + - path: _test\.go + linters: + - goerr113 + - varnamelen + + - path: internal/app/migrations/ + linters: + - gochecknoinits + - varnamelen + + - linters: + - nolintlint + text: "should be written without leading space" + +run: + timeout: 5m diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 00000000..56fe0d0e --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,106 @@ +project_name: payments +env: + - GO111MODULE=on + - GOPROXY=https://proxy.golang.org +before: + hooks: + - go mod download + +builds: + - binary: payments + id: payments + ldflags: + - -X github.com/formancehq/payments/cmd.BuildDate={{ .Date }} + - -X github.com/formancehq/payments/cmd.Version={{ .Version }} + - -X github.com/formancehq/payments/cmd.Commit={{ .ShortCommit }} + - -extldflags "-static" + env: + - CGO_ENABLED=0 + goos: + - windows + - linux + - darwin + goarch: + - amd64 + - arm64 + flags: + - -tags=json1 + +archives: + - id: "payments" + builds: + - payments + format: tar.gz + format_overrides: + - goos: windows + format: zip + name_template: "{{.ProjectName}}_{{.Os}}-{{.Arch}}" + + +checksum: + name_template: 'checksums.txt' + +snapshot: + name_template: "{{ .Tag }}" + +changelog: + sort: asc + use: github + filters: + exclude: + - '^docs:' + - '^test:' + - '^spec:' + - Merge pull request + - Merge remote-tracking branch + - Merge branch + - go mod tidy + groups: + - title: 'New Features' + regexp: "^.*feat[(\\w)]*:+.*$" + order: 0 + - title: 'Bug fixes' + regexp: "^.*fix[(\\w)]*:+.*$" + order: 10 + - title: Other work + order: 999 + +release: + prerelease: auto + footer: | + **Full Changelog**: https://github.com/formancehq/payments/compare/{{ .PreviousTag }}...{{ .Tag }} + ## What to do next? + - Read the [documentation](https://docs.formance.com/oss/payments/get-started/installation) + - Join our [Discord server](https://discord.gg/xyHvcbzk4w) + +brews: + - tap: + owner: numary + name: homebrew-tap + name: payments + folder: Formula + homepage: https://formance.com + skip_upload: 'false' + test: | + system "#{bin}/payments version" + install: | + bin.install "payments" + +nfpms: + - id: packages + package_name: payments + file_name_template: "{{ .ProjectName }}_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + builds: + - payments + homepage: https://formance.com + maintainer: Maxence Maireaux + formats: + - deb + - rpm + +publishers: + - name: fury.io + ids: + - packages + dir: "{{ dir .ArtifactPath }}" + cmd: curl -F package=@{{ .ArtifactName }} https://{{ .Env.FURY_TOKEN }}@push.fury.io/numary/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..7975b4e9 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +exclude: client +fail_fast: true +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.3.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + exclude: .cloud + - id: check-added-large-files +- repo: https://github.com/formancehq/pre-commit-hooks + rev: dd079f7c30ad72446d615f55a000d4f875e79633 + hooks: + - id: gogenerate + files: openapi.yaml + - id: gomodtidy + - id: goimports + - id: gofmt + - id: golangci-lint + - id: gotests + - id: commitlint diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..4cdf518c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM golang:1.19.3-bullseye AS builder + +RUN apt-get update && \ + apt-get install -y gcc-aarch64-linux-gnu gcc-x86-64-linux-gnu && \ + ln -s /usr/bin/aarch64-linux-gnu-gcc /usr/bin/arm64-linux-gnu-gcc && \ + ln -s /usr/bin/x86_64-linux-gnu-gcc /usr/bin/amd64-linux-gnu-gcc + +ARG TARGETARCH +ARG APP_SHA +ARG VERSION + +WORKDIR /go/src/github.com/formancehq/payments + +# get deps first so it's cached +COPY . . + +RUN go mod vendor + +RUN CGO_ENABLED=0 GOOS=linux GOARCH=$TARGETARCH \ + CC=$TARGETARCH-linux-gnu-gcc \ + go build -o bin/payments \ + -ldflags="-X github.com/formancehq/payments/cmd.Version=${VERSION} \ + -X github.com/formancehq/payments/cmd.BuildDate=$(date +%s) \ + -X github.com/formancehq/payments/cmd.Commit=${APP_SHA}" ./ + +FROM scratch + +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /go/src/github.com/formancehq/payments/bin/payments /usr/local/bin/payments + +EXPOSE 8080 + +ENTRYPOINT ["payments"] + +CMD ["server"] diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..f6057fd7 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +docker-build: + DOCKER_BUILDKIT=1 docker build -t payments:local --build-arg BUILDPLATFORM=amd64 . + +lint: + golangci-lint run + +lint-fix: + golangci-lint run --fix + +run-tests: + go test -race -count=1 ./... diff --git a/README.md b/README.md new file mode 100644 index 00000000..bff14a25 --- /dev/null +++ b/README.md @@ -0,0 +1,32 @@ +# Formance Payments [![test](https://github.com/formancehq/payments/actions/workflows/main.yml/badge.svg)](https://github.com/formancehq/payments/actions/workflows/main.yml) [![goreportcard](https://goreportcard.com/badge/github.com/formancehq/payments)](https://goreportcard.com/report/github.com/formancehq/payments) [![discord](https://img.shields.io/discord/846686859869814784?label=chat%20@%20discord)](https://discord.gg/xyHvcbzk4w) + +# Getting started + +Payments works as a standalone binary, the latest of which can be downloaded from the [releases page](https://github.com/formancehq/payments/releases). You can move the binary to any executable path, such as to `/usr/local/bin`. Installations using brew, apt, yum or docker are also [available](https://docs.formance.com/oss/payments/get-started/installation). + +```SHELL +payments +``` + +# What is it? + +Basically, a framework. + +A framework to ingest payin and payout coming from different payment providers (PSP). + +The framework contains connectors. Each connector is basically a translator for a PSP. +Translator, because the main role of a connector is to translate specific PSP payin/payout formats to a generalized format used at Formance. + +Because it is a framework, it is extensible. Please follow the guide below if you want to add your connector. + +# Contribute + +Please follow [this guide](./docs/development.md) if you want to contribute. + +# Roadmap & Community + +We keep an open roadmap of the upcoming releases and features [here](https://numary.notion.site/OSS-Roadmap-4535fa5716fb4f618027201afcc6f204). + +If you need help, want to show us what you built or just hang out and chat about paymentss you are more than welcome on our [Discord](https://discord.gg/xyHvcbzk4w) - looking forward to see you there! + +![Frame 1 (2)](https://user-images.githubusercontent.com/1770991/134163361-d86c5728-6075-4510-8de7-06df1f6ed740.png) diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 00000000..dd44154b --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,35 @@ +version: '3' + +vars: + BINARY_NAME: payments + PKG: ./... + COVERAGE_FILE: coverage.out + FAILFAST: -failfast + TIMEOUT: 10m + +tasks: + default: + cmds: + - task: lint + - task: tests + + lint: + cmds: + - golangci-lint run -v --fix {{.PKG}} + + tests: + cmds: + - go test -v {{.FAILFAST}} -coverpkg {{.PKG}} -coverprofile {{.COVERAGE_FILE}} -covermode atomic -timeout {{.TIMEOUT}} {{.PKG}} + + build: + cmds: + - go build -o {{.BINARY_NAME}} + + install: + cmds: + - go install -o {{.BINARY_NAME}} + + clean: + cmds: + - go clean + - rm -f {{.BINARY_NAME}} {{.COVERAGE_FILE}} diff --git a/cmd/migrate.go b/cmd/migrate.go new file mode 100644 index 00000000..0d12d6b6 --- /dev/null +++ b/cmd/migrate.go @@ -0,0 +1,73 @@ +package cmd + +import ( + "fmt" + "log" + + "github.com/spf13/viper" + + // allow blank import to initiate migrations. + _ "github.com/formancehq/payments/internal/app/migrations" + _ "github.com/lib/pq" + + "github.com/pressly/goose/v3" + "github.com/spf13/cobra" +) + +func newMigrate() *cobra.Command { + return &cobra.Command{ + Use: "migrate", + Short: "Run migrations", + RunE: runMigrate, + } +} + +// Usage: `go run cmd/main.go migrate --postgres-uri {uri} {command}` +/* +Commands: + up Migrate the DB to the most recent version available + up-by-one Migrate the DB up by 1 + up-to VERSION Migrate the DB to a specific VERSION + down Roll back the version by 1 + down-to VERSION Roll back to a specific VERSION + redo Re-run the latest migration + reset Roll back all migrations + status Dump the migration status for the current DB + version Print the current version of the database + create NAME [sql|go] Creates new migration file with the current timestamp + fix Apply sequential ordering to migrations +*/ + +func runMigrate(cmd *cobra.Command, args []string) error { + postgresURI := viper.GetString(postgresURIFlag) + if postgresURI == "" { + postgresURI = cmd.Flag(postgresURIFlag).Value.String() + } + + if postgresURI == "" { + return fmt.Errorf("postgres uri is not set") + } + + database, err := goose.OpenDBWithDriver("postgres", postgresURI) + if err != nil { + return fmt.Errorf("failed to open database: %w", err) + } + + defer func() { + if err = database.Close(); err != nil { + log.Fatalf("failed to close DB: %v\n", err) + } + }() + + if len(args) == 0 { + return fmt.Errorf("missing migration direction") + } + + command := args[0] + + if err = goose.Run(command, database, ".", args[1:]...); err != nil { + log.Printf("migrate %v: %v", command, err) + } + + return nil +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 00000000..4f3facb8 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,92 @@ +//nolint:gochecknoglobals,golint,revive // allow for cobra & logrus init +package cmd + +import ( + "fmt" + "os" + "strings" + + _ "github.com/bombsimon/logrusr/v3" + "github.com/formancehq/go-libs/otlp/otlptraces" + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +const ( + debugFlag = "debug" +) + +var ( + Version = "develop" + BuildDate = "-" + Commit = "-" +) + +func rootCommand() *cobra.Command { + viper.SetDefault("version", Version) + + root := &cobra.Command{ + Use: "payments", + Short: "payments", + DisableAutoGenTag: true, + } + + version := newVersion() + root.AddCommand(version) + + server := newServer() + root.AddCommand(newServer()) + + migrate := newMigrate() + root.AddCommand(migrate) + + root.PersistentFlags().Bool(debugFlag, false, "Debug mode") + + migrate.Flags().String(postgresURIFlag, "postgres://localhost/payments", "PostgreSQL DB address") + + server.Flags().BoolP("toggle", "t", false, "Help message for toggle") + server.Flags().String(postgresURIFlag, "postgres://localhost/payments", "PostgreSQL DB address") + server.Flags().String(envFlag, "local", "Environment") + server.Flags().Bool(publisherKafkaEnabledFlag, false, "Publish write events to kafka") + server.Flags().StringSlice(publisherKafkaBrokerFlag, []string{}, "Kafka address is kafka enabled") + server.Flags().StringSlice(publisherTopicMappingFlag, + []string{}, "Define mapping between internal event types and topics") + server.Flags().Bool(publisherHTTPEnabledFlag, false, "Sent write event to http endpoint") + server.Flags().Bool(publisherKafkaSASLEnabled, false, "Enable SASL authentication on kafka publisher") + server.Flags().String(publisherKafkaSASLUsername, "", "SASL username") + server.Flags().String(publisherKafkaSASLPassword, "", "SASL password") + server.Flags().String(publisherKafkaSASLMechanism, "", "SASL authentication mechanism") + server.Flags().Int(publisherKafkaSASLScramSHASize, 512, "SASL SCRAM SHA size") + server.Flags().Bool(publisherKafkaTLSEnabled, false, "Enable TLS to connect on kafka") + server.Flags().Bool(authBasicEnabledFlag, false, "Enable basic auth") + server.Flags().StringSlice(authBasicCredentialsFlag, []string{}, + "HTTP basic auth credentials (:)") + server.Flags().Bool(authBearerEnabledFlag, false, "Enable bearer auth") + server.Flags().String(authBearerIntrospectURLFlag, "", "OAuth2 introspect URL") + server.Flags().StringSlice(authBearerAudienceFlag, []string{}, "Allowed audiences") + server.Flags().Bool(authBearerAudiencesWildcardFlag, false, "Don't check audience") + server.Flags().Bool(authBearerUseScopesFlag, + false, "Use scopes as defined by rfc https://datatracker.ietf.org/doc/html/rfc8693") + + otlptraces.InitOTLPTracesFlags(server.Flags()) + + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_", "-", "_")) + viper.AutomaticEnv() + + err := viper.BindPFlags(root.Flags()) + if err != nil { + panic(err) + } + + return root +} + +func Execute() { + if err := rootCommand().Execute(); err != nil { + if _, err = fmt.Fprintln(os.Stderr, err); err != nil { + panic(err) + } + + os.Exit(1) + } +} diff --git a/cmd/server.go b/cmd/server.go new file mode 100644 index 00000000..b872227f --- /dev/null +++ b/cmd/server.go @@ -0,0 +1,194 @@ +package cmd + +import ( + "strings" + + "github.com/formancehq/go-libs/otlp/otlptraces" + + "github.com/bombsimon/logrusr/v3" + sharedapi "github.com/formancehq/go-libs/api" + "github.com/formancehq/payments/internal/app/api" + "github.com/formancehq/payments/internal/app/storage" + "github.com/pkg/errors" + "go.opentelemetry.io/otel" + + "github.com/Shopify/sarama" + "github.com/ThreeDotsLabs/watermill/message" + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/go-libs/logging/logginglogrus" + "github.com/formancehq/go-libs/publish" + "github.com/formancehq/go-libs/publish/publishhttp" + "github.com/formancehq/go-libs/publish/publishkafka" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/uptrace/opentelemetry-go-extra/otellogrus" + "github.com/xdg-go/scram" + "go.uber.org/fx" +) + +//nolint:gosec // false positive +const ( + postgresURIFlag = "postgres-uri" + otelTracesFlag = "otel-traces" + envFlag = "env" + publisherKafkaEnabledFlag = "publisher-kafka-enabled" + publisherKafkaBrokerFlag = "publisher-kafka-broker" + publisherKafkaSASLEnabled = "publisher-kafka-sasl-enabled" + publisherKafkaSASLUsername = "publisher-kafka-sasl-username" + publisherKafkaSASLPassword = "publisher-kafka-sasl-password" + publisherKafkaSASLMechanism = "publisher-kafka-sasl-mechanism" + publisherKafkaSASLScramSHASize = "publisher-kafka-sasl-scram-sha-size" + publisherKafkaTLSEnabled = "publisher-kafka-tls-enabled" + publisherTopicMappingFlag = "publisher-topic-mapping" + publisherHTTPEnabledFlag = "publisher-http-enabled" + authBasicEnabledFlag = "auth-basic-enabled" + authBasicCredentialsFlag = "auth-basic-credentials" + authBearerEnabledFlag = "auth-bearer-enabled" + authBearerIntrospectURLFlag = "auth-bearer-introspect-url" + authBearerAudienceFlag = "auth-bearer-audience" + authBearerAudiencesWildcardFlag = "auth-bearer-audiences-wildcard" + authBearerUseScopesFlag = "auth-bearer-use-scopes" + + serviceName = "Payments" +) + +func newServer() *cobra.Command { + return &cobra.Command{ + Use: "server", + Short: "Launch server", + SilenceUsage: true, + RunE: runServer, + } +} + +func runServer(cmd *cobra.Command, args []string) error { + setLogger() + + databaseOptions, err := prepareDatabaseOptions() + if err != nil { + return err + } + + options := make([]fx.Option, 0) + + if !viper.GetBool(debugFlag) { + options = append(options, fx.NopLogger) + } + + options = append(options, databaseOptions) + options = append(options, otlptraces.CLITracesModule(viper.GetViper())) + + options = append(options, + fx.Provide(fx.Annotate(func(p message.Publisher) *publish.TopicMapperPublisher { + return publish.NewTopicMapperPublisher(p, topicsMapping()) + }, fx.As(new(publish.Publisher))))) + + options = append(options, api.HTTPModule(sharedapi.ServiceInfo{ + Version: Version, + })) + options = append(options, publish.Module()) + + switch { + case viper.GetBool(publisherHTTPEnabledFlag): + options = append(options, publishhttp.Module()) + case viper.GetBool(publisherKafkaEnabledFlag): + options = append(options, + publishkafka.Module(serviceName, viper.GetStringSlice(publisherKafkaBrokerFlag)...), + publishkafka.ProvideSaramaOption( + publishkafka.WithConsumerReturnErrors(), + publishkafka.WithProducerReturnSuccess(), + ), + ) + + if viper.GetBool(publisherKafkaTLSEnabled) { + options = append(options, publishkafka.ProvideSaramaOption(publishkafka.WithTLS())) + } + + if viper.GetBool(publisherKafkaSASLEnabled) { + options = append(options, publishkafka.ProvideSaramaOption( + publishkafka.WithSASLEnabled(), + publishkafka.WithSASLCredentials( + viper.GetString(publisherKafkaSASLUsername), + viper.GetString(publisherKafkaSASLPassword), + ), + publishkafka.WithSASLMechanism(sarama.SASLMechanism(viper.GetString(publisherKafkaSASLMechanism))), + publishkafka.WithSASLScramClient(setSCRAMClient), + )) + } + } + + err = fx.New(options...).Start(cmd.Context()) + if err != nil { + return err + } + + <-cmd.Context().Done() + + return nil +} + +func setLogger() { + log := logrus.New() + + if viper.GetBool(debugFlag) { + log.SetLevel(logrus.DebugLevel) + } + + if viper.GetBool(otelTracesFlag) { + log.AddHook(otellogrus.NewHook(otellogrus.WithLevels( + logrus.PanicLevel, + logrus.FatalLevel, + logrus.ErrorLevel, + logrus.WarnLevel, + ))) + log.SetFormatter(&logrus.JSONFormatter{}) + } + + logging.SetFactory(logging.StaticLoggerFactory(logginglogrus.New(log))) + + // Add a dedicated logger for opentelemetry in case of error + otel.SetLogger(logrusr.New(logrus.New().WithField("component", "otlp"))) +} + +func prepareDatabaseOptions() (fx.Option, error) { + postgresURI := viper.GetString(postgresURIFlag) + if postgresURI == "" { + return nil, errors.New("missing postgres uri") + } + + return storage.Module(postgresURI), nil +} + +func topicsMapping() map[string]string { + topics := viper.GetStringSlice(publisherTopicMappingFlag) + mapping := make(map[string]string) + + for _, topic := range topics { + parts := strings.SplitN(topic, ":", 2) + if len(parts) != 2 { + panic("invalid topic flag") + } + + mapping[parts[0]] = parts[1] + } + + return mapping +} + +func setSCRAMClient() sarama.SCRAMClient { + var fn scram.HashGeneratorFcn + + switch viper.GetInt(publisherKafkaSASLScramSHASize) { + case 512: + fn = publishkafka.SHA512 + case 256: + fn = publishkafka.SHA256 + default: + panic("sha size not handled") + } + + return &publishkafka.XDGSCRAMClient{ + HashGeneratorFcn: fn, + } +} diff --git a/cmd/version.go b/cmd/version.go new file mode 100644 index 00000000..9cad9086 --- /dev/null +++ b/cmd/version.go @@ -0,0 +1,21 @@ +package cmd + +import ( + "log" + + "github.com/spf13/cobra" +) + +func newVersion() *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Get version", + Run: printVersion, + } +} + +func printVersion(cmd *cobra.Command, args []string) { + log.Printf("Version: %s \n", Version) + log.Printf("Date: %s \n", BuildDate) + log.Printf("Commit: %s \n", Commit) +} diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..dad420a1 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,4 @@ +ignore: + - "docs" + - ".github" + - ".devcontainer" diff --git a/commitlint.config.js b/commitlint.config.js new file mode 100644 index 00000000..3580f602 --- /dev/null +++ b/commitlint.config.js @@ -0,0 +1,5 @@ +module.exports = { + extends: [ + '@commitlint/config-conventional' + ] +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..d4cd17c5 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,40 @@ +version: '3.8' +volumes: + postgres: + +services: + postgres: + image: "postgres:14-alpine" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U payments"] + interval: 10s + timeout: 5s + retries: 5 + ports: + - "5432:5432" + environment: + POSTGRES_USER: "payments" + POSTGRES_PASSWORD: "payments" + POSTGRES_DB: "payments" + PGDATA: /data/postgres + volumes: + - postgres:/data/postgres + + payments: + image: golang:1.19.3-alpine3.16 + command: go run ./ migrate up && go run ./ server + healthcheck: + test: [ "CMD", "curl", "-f", "http://127.0.0.1:8080/_healthcheck" ] + interval: 10s + timeout: 5s + retries: 5 + depends_on: + - postgres + ports: + - "8080:8080" + volumes: + - .:/app + working_dir: /app + environment: + DEBUG: ${DEBUG:-"true"} + POSTGRES_URI: postgres://payments:payments@postgres/payments?sslmode=disable diff --git a/docs/connectors.md b/docs/connectors.md new file mode 100644 index 00000000..3d7c31ee --- /dev/null +++ b/docs/connectors.md @@ -0,0 +1,157 @@ +# Connectors + +## Currently supported connectors configs and typical default values + +- [BankingCircle](#bankingcircle) +- [CurrencyCloud](#currencycloud) +- [Modulr](#modulr) +- [Stripe](#stripe) +- [Wise](#wise) +- [DummyPay](#dummypay) + +### BankingCircle + +Docs: [https://docs.bankingcircleconnect.com/](https://docs.bankingcircleconnect.com/) + +```golang + Username string `json:"username" yaml:"username" bson:"username"` + Password string `json:"password" yaml:"password" bson:"password"` + Endpoint string `json:"endpoint" yaml:"endpoint" bson:"endpoint"` + AuthorizationEndpoint string `json:"authorizationEndpoint" yaml:"authorizationEndpoint" bson:"authorizationEndpoint"` +``` + +#### Sandbox defaults +```json +{ + "username": "username", + "password": "password", + "endpoint": "https://sandbox.bankingcircle.com", + "authorizationEndpoint": "https://authorizationsandbox.bankingcircleconnect.com" +} +``` + +#### Production defaults +```json +{ + "username": "username", + "password": "password", + "endpoint": " https://www.bankingcircleconnect.com/", + "authorizationEndpoint": "https://authorization.bankingcircleconnect.com" +} +``` + +### CurrencyCloud + +Docs: [https://www.currencycloud.com/developers/](https://www.currencycloud.com/developers/) + +```golang + LoginID string `json:"loginID" bson:"loginID"` + APIKey string `json:"apiKey" bson:"apiKey"` + Endpoint string `json:"endpoint" bson:"endpoint"` + PollingPeriod Duration `json:"pollingPeriod" bson:"pollingPeriod"` +``` + +#### Demo defaults +```json +{ + "loginID": "loginID", + "apiKey": "apiKey", + "endpoint": "https://devapi.currencycloud.com", + "pollingPeriod": "1m" +} +``` + +#### Production defaults +```json +{ + "loginID": "loginID", + "apiKey": "apiKey", + "endpoint": "https://api.currencycloud.com", + "pollingPeriod": "1m" +} +``` + +### Modulr + +Docs: [https://www.modulrfinance.com/modulr-api](https://www.modulrfinance.com/modulr-api) + +```golang + APIKey string `json:"apiKey" bson:"apiKey"` + APISecret string `json:"apiSecret" bson:"apiSecret"` + Endpoint string `json:"endpoint" bson:"endpoint"` +``` + +#### Sandbox defaults +```json +{ + "apiKey": "apiKey", + "apiSecret": "apiSecret", + "endpoint": "https://api-sandbox.modulrfinance.com" +} +``` + +#### Production defaults +```json +{ + "apiKey": "apiKey", + "apiSecret": "apiSecret", + "endpoint": "https://api.modulrfinance.com" +} +``` + +### Stripe + +Docs: [https://stripe.com/docs/development](https://stripe.com/docs/development) + +Sandbox/Production environment selection is controlled by api key and api secret types. + +```golang + PollingPeriod connectors.Duration `json:"pollingPeriod" yaml:"pollingPeriod" bson:"pollingPeriod"` + APIKey string `json:"apiKey" yaml:"apiKey" bson:"apiKey"` + PageSize uint64 `json:"pageSize" yaml:"pageSize" bson:"pageSize"` +``` + +#### Defaults +```json +{ + "pollingPeriod": "1m", + "apiKey": "apiKey", + "pageSize": 100 +} +``` + +### Wise + +Docs: [https://api-docs.wise.com/](https://api-docs.wise.com/) + +Sandbox/Production environment selection is controlled by api key and api secret types. + +```golang + APIKey string `json:"apiKey +```` + +#### Defaults +```json +{ + "apiKey": "apiKey" +} +``` + +### DummyPay + +This connector is used only for testing purposes. It does not connect to any real payment provider. + +```golang + Directory string `json:"directory" yaml:"directory" bson:"directory"` + FilePollingPeriod connectors.Duration `json:"filePollingPeriod" yaml:"filePollingPeriod" bson:"filePollingPeriod"` + FileGenerationPeriod connectors.Duration `json:"fileGenerationPeriod" yaml:"fileGenerationPeriod" bson:"fileGenerationPeriod"` +``` + +#### Defaults +```json +{ + "directory": "/tmp/payments", + "filePollingPeriod": "30s", + "fileGenerationPeriod": "10s" +} +``` diff --git a/docs/development.md b/docs/development.md new file mode 100644 index 00000000..881e93c5 --- /dev/null +++ b/docs/development.md @@ -0,0 +1,24 @@ +# Development + +## Run + +A docker-compose contains all the stuff required to launch the service. + +Currently, the service use MongoDB as database, and it takes few seconds to start and is not ready when the payments service try to connect to him. +You can start MongoDB before and wait before start payments service using two terminal : +``` +docker compose up mongodb # Run on first terminal +``` +and +``` +docker compose up payments # Run on second terminal +``` + +Tests can be started regularly using standard go tooling, just use : +``` +go test ./... +``` + +## Develop a connector + +Want to develop a connector? [Follow this link](./tuto-connector.md) diff --git a/docs/samples-payin.json b/docs/samples-payin.json new file mode 100644 index 00000000..4259ffba --- /dev/null +++ b/docs/samples-payin.json @@ -0,0 +1,7 @@ +{ + "type": "payin", + "reference": "001", + "status": "succeeded", + "asset": "USD", + "initialAmount": 100 +} diff --git a/docs/tuto-connector.md b/docs/tuto-connector.md new file mode 100644 index 00000000..8e6c25a3 --- /dev/null +++ b/docs/tuto-connector.md @@ -0,0 +1,580 @@ +# Tutorial connector + +_referenced in `/pkg/bridge/connectors/dummypay`_ + +We are going to create a fake connector which read a directory. +In this directory, a fake bank service will create files. +Each files contain a payin or a payout as a json object. + +First, to create a connector, we need a loader. + +## The loader object + +```go +type Loader[ConnectorConfig payments.ConnectorConfigObject] interface { + // Name has to return the name of the connector. It must be constant and unique + Name() string + // Load is in charge of loading the connector + // It takes a logger and a ConnectorConfig object. + // At this point, the config must have been validated + Load(logger sharedlogging.Logger, config ConnectorConfig) Connector + // ApplyDefaults is used to fill default values of the provided configuration object. + ApplyDefaults(t ConnectorConfig) ConnectorConfig + // AllowTasks define how many task the connector can run + // If too many tasks are scheduled by the connector, + // those will be set to pending state and restarted later when some other tasks will be terminated + AllowTasks() int +} +``` + +A connector has a name. + +This name is provided by the loader by the method Name(). +Also, each connector define a config object using generics which has to implement interface payments.ConnectorConfigObject. +This interface only have a method Validate() error which is used by the code to validate an external config is valid before load the connector with it. +Since, some properties of the config may have some optional, the loader is also in charge of configuring default values on it. +This is done using the method ```ApplyDefaults(Config)```. + +The framework provide the capabilities to run tasks. +So each connector can start any number of tasks. +Those tasks will be scheduled by the framework. For example, if the service is restarted, the tasks will be restarted at reboot. +The number of tasks a connector can schedule is defined by the method AllowTasks(). + +To implement Loader interface, you can create your own struct implementing required methods, or you can use some utilities provided by the framework. +Let`s create a basic loader. + +```go +type ( + Config struct {} + TaskDescriptor struct {} +) + +func (cfg Config) Validate() error { + return nil +} + +var Loader = integration.NewLoaderBuilder[Config]("example").Build() +``` + +Here, we built our loader. +The name of the connector is "example". +For now, the ```Config``` and ```TaskDescriptor``` are just empty structs, we will change it later. +Also, we didn't define any logic on our connector. + +It's time to plug our connector on the core. +Edit the file cmd/root.go and go at the end of the method HTTPModule(), you should find a code like this : +```go + ... + cdi.ConnectorModule[stripe.Config, stripe.TaskDescriptor]( + viper.GetBool(authBearerUseScopesFlag), + stripe.NewLoader(), + ), + ... +``` + +You can add your connector bellow that : +```go + ... + cdi.ConnectorModule[example.Config, example.TaskDescriptor]( + viper.GetBool(authBearerUseScopesFlag), + example.Loader, + ), + ... +``` + +Now you, can run the service and you should see something like this : +```bash +payments-payments-1 | time="2022-07-01T09:12:21Z" level=info msg="Restoring state" component=connector-manager provider=example +payments-payments-1 | time="2022-07-01T09:12:21Z" level=info msg="Not installed, skip" component=connector-manager provider=example +``` + +This indicates your connector is properly integrated. +You can install it like this : +```bash +curl http://localhost:8080/connectors/example -X POST +``` + +The service will display something like this : +```bash +payments-payments-1 | time="2022-07-01T10:04:53Z" level=info msg="Install connector example" component=connector-manager config="{}" provider=example +payments-payments-1 | time="2022-07-01T10:04:53Z" level=info msg="Connector installed" component=connector-manager provider=example +``` + +Your connector was installed! +It makes nothing but it is installed. + +Let's uninstall it before continue : +```bash +curl http://localhost:8080/connectors/example -X DELETE +``` + +You should see something like this : +```bash +payments-payments-1 | time="2022-07-01T10:06:16Z" level=info msg="Uninstalling connector" component=connector-manager provider=example +payments-payments-1 | time="2022-07-01T10:06:16Z" level=info msg="Stopping scheduler..." component=scheduler provider=example +payments-payments-1 | time="2022-07-01T10:06:16Z" level=info msg="Connector uninstalled" component=connector-manager provider=example +``` + +It's to time to add a bit of logic to our connector. + +As you may have noticed, the ```Loader``` has method named ```Load``` : +```go +... +Load(logger sharedlogging.Logger, config ConnectorConfig) Connector[TaskDescriptor] +... +``` + +The Load function take a logger provided by the framework and a config, probably provided by the api endpoint. +It has to return a Connector object. Here the interface : +```go +// Connector provide entry point to a payment provider +type Connector interface { + // Install is used to start the connector. The implementation if in charge of scheduling all required resources. + Install(ctx task.ConnectorContext) error + // Uninstall is used to uninstall the connector. It has to close all related resources opened by the connector. + Uninstall(ctx context.Context) error + // Resolve is used to recover state of a failed or restarted task + Resolve(descriptor TaskDescriptor) task.Task +} +``` + +When you made ```curl http://localhost:8080/connectors/example -X POST```, the framework called the ```Install()``` method. +When you made ```curl http://localhost:8080/connectors/example -X DELETE```, the framework called the ```Uninstall(ctx context.Context) error``` method. + +It's time to add some logic. We have to modify our loader but before let's add some property to our config : +```go +type ( + Config struct { + Directory string + } + ... +) + +func (cfg Config) Validate() error { + if cfg.Directory == "" { + return errors.New("missing directory to watch") + } + return nil +} +``` + +Here we defined only one property to our connector, "Directory", which indicates the directory when json files will be pushed. +Now, modify our loader : +```go +var Loader = integration.NewLoaderBuilder[Config]("example"). + WithLoad(func(logger sharedlogging.Logger, config Config) integration.Connector { + return integration.NewConnectorBuilder(). + WithInstall(func(ctx task.ConnectorContext) error { + return errors.New("not implemented") + }). + Build() + }). + Build() +``` + +Here we create a connector using a builtin builder, but you can implement the interface if you want. +We define a ```Install``` method which only returns an errors when installed. +You can retry to install your connector and see the error on the http response. + +The ```Install``` method take a ```task.ConnectorContext``` parameter : +```go +type ConnectorContext interface { + Context() context.Context + Scheduler() Scheduler[TaskDescriptor] +} +``` + +Basically this context provides two things : +* a ```context.Context``` : If the connector make long-running processing, it should listen on this context to abort if necessary. +* a ```Scheduler```: A scheduler to run tasks + +But, what is a task ? + +A task is like a process that the framework will handle for you. It is basically a simple function. +When installed, a connector has the opportunity to schedule some tasks and let the system handle them for him. +A task has a descriptor. +The descriptor must be immutable and represents a specific task in the system. It can be anything. +A task also have a state. The state can change and the framework provides necessary apis to do that. We will come on that later. +As the descriptor, the state is freely defined by the connector. + +In our case, the main task seems evident as we have to list the target repository. +Secondary tasks will be defined to read each files present in the directory. +We can define our task descriptor to a string. The value will be the file name in case of secondary tasks and a hardcoded value of "directory" for the main task. + +Before add the logic, let's modify our previously introduced task descriptor : +```go +type ( + ... + TaskDescriptor string + ... +) + +``` + +Add some logic on our connector : +```go + ... + WithInstall(func(ctx task.ConnectorContext) error { + return ctx.Scheduler().Schedule("directory", true) + }). + ... +``` + +Here we instruct the framework to create the task with the descriptor "directory". +Cool! The framework can handle the task, restart it, log/save errors etc... +But it doesn't know about the logic. + +To do that, we have to use the last method of the connector : ```Resolve(descriptor TaskDescriptor) task.Task``` +This method is in charge of providing a ```task.Task``` instance given a descriptor. + +So, when calling ```ctx.Scheduler().Schedule("directory")```, the framework will call the ```Resolve``` method with "directory" as parameter. + +Let's implement the resolve method : +```go + ... + WithInstall(func(ctx task.ConnectorContext) error { + return ctx.Scheduler().Schedule("directory") + }). + WithResolve(func(descriptor models.TaskDescriptor) task.Task { + if descriptor == "directory" { + return func() { + // TODO + } + } + // Secondary tasks + return func() { + // TODO + } + }). + ... +``` + +Now, we have to implement the logic for each task. + +Let's start with the main task which read the directory : +```go + ... + WithResolve(func(descriptor models.TaskDescriptor) task.Task { + if descriptor == "directory" { + return func(ctx context.Context, logget sharedlogging.Logger, scheduler task.Scheduler) + for { + select { + case <-ctx.Done(): + return nil + case <-time.After(10 * time.Second): // Could be configurable using Config object + logger.Infof("Opening directory '%s'...", config.Directory) + dir, err := os.ReadDir(config.Directory) + if err != nil { + logger.Errorf("Error opening directory '%s': %s", config.Directory, err) + continue + } + + logger.Infof("Found %d files", len(dir)) + for _, file := range dir { + err = scheduler.Schedule(TaskDescriptor(file.Name())) + if err != nil { + logger.Errorf("Error scheduling task '%s': %s", file.Name(), err) + continue + } + } + } + } + } + } + return func() error { + return errors.New("not implemented") + } + }). + ... +``` + +Let's test our implementation. + +Start the server as usual and issue a curl request to install the connector : +```bash +curl http://localhost:8080/connectors/example -X POST -d '{"directory": "/tmp/payments"}' +``` + +Here we instruct the connector to watch the directory /tmp/payments. Check the app logs, you should see something like this : +```bash +payments-payments-1 | time="2022-07-01T12:29:05Z" level=info msg="Install connector example" component=connector-manager config="{/tmp/payments}" provider=example +payments-payments-1 | time="2022-07-01T12:29:05Z" level=info msg="Starting task..." component=scheduler provider=example task-id="ImRpcmVjdG9yeSI=" +payments-payments-1 | time="2022-07-01T12:29:05Z" level=info msg="Connector installed" component=connector-manager provider=example +payments-payments-1 | time="2022-07-01T13:26:51Z" level=info msg="Opening directory '/tmp/payments'..." component=scheduler provider=example task-id="ImRpcmVjdG9yeSI=" +payments-payments-1 | time="2022-07-01T13:26:51Z" level=error msg="Error opening directory '/tmp/payments': open /tmp/payments: no such file or directory" component=scheduler provider=example task-id="ImRpcmVjdG9yeSI=" +``` + +As expected, the task trigger an error because of non-existent /tmp/payments directory. + +You can see the tasks on api too : +```bash +curl http://localhost:8080/connectors/example/tasks | jq + +[ + { + "provider": "example", + "descriptor": "directory", + "createdAt": "2022-07-01T13:26:41.749Z", + "status": "active", + "error": "", + "state": {}, + "id": "ImRpcmVjdG9yeSI=" + } +] +``` + +As you can see, a task has an id. This id is simply the descriptor of the task encoded in canonical json and encoded as base 64. + +Let's create the missing directory: +```bash +docker compose exec payments mkdir /tmp/payments +``` + +After a few seconds, you should see thoses logs on app : +```bash +payments-payments-1 | time="2022-07-01T13:29:21Z" level=info msg="Opening directory '/tmp/payments'..." component=scheduler provider=example task-id="ImRpcmVjdG9yeSI=" +payments-payments-1 | time="2022-07-01T13:29:21Z" level=info msg="Found 0 files" component=scheduler provider=example task-id="ImRpcmVjdG9yeSI=" +``` + +Ok, create a payin file : +```bash +docker compose cp docs/samples-payin.json payments:/tmp/payments/001.json +``` + +You should see those lines on logs : +```bash +payments-payments-1 | time="2022-07-01T13:33:51Z" level=info msg="Opening directory '/tmp/payments'..." component=scheduler provider=example task-id="ImRpcmVjdG9yeSI=" +payments-payments-1 | time="2022-07-01T13:33:51Z" level=info msg="Found 1 files" component=scheduler provider=example task-id="ImRpcmVjdG9yeSI=" +payments-payments-1 | time="2022-07-01T13:33:52Z" level=info msg="Starting task..." component=scheduler provider=example task-id="IjAwMS5qc29uIg==" +payments-payments-1 | time="2022-07-01T13:33:52Z" level=error msg="Task terminated with error: not implemented" component=scheduler provider=example task-id="IjAwMS5qc29uIg==" +``` + +The log show our connector detect the file and trigger a new task for the file. +The task terminate with an error as the ```Resolve``` function does not handle the descriptor. We will do this later. + +Again, you can view the tasks on the api : +```bash +[ + { + "provider": "example", + "descriptor": "directory", + "createdAt": "2022-07-01T13:26:41.749Z", + "status": "active", + "error": "", + "state": "XXX", + "id": "ImRpcmVjdG9yeSI=" + }, + { + "provider": "example", + "descriptor": "001.json", + "createdAt": "2022-07-01T13:33:31.935Z", + "status": "failed", + "error": "not implemented", + "state": "XXX", + "id": "IjAwMS5qc29uIg==" + } +] +``` + +As you can see, as the first task is still active, the second is flagged as failed with an error message. + +It's time to implement the second task : +```go + ... + file, err := os.Open(filepath.Join(config.Directory, string(descriptor))) + if err != nil { + return err + } + + type JsonPayment struct { + payments.Data + Reference string `json:"reference"` + Type string `json:"type"` + } + + jsonPayment := &JsonPayment{} + err = json.NewDecoder(file).Decode(jsonPayment) + if err != nil { + return err + } + + return ingester.Ingest(ctx, ingestion.Batch{ + { + Referenced: payments.Referenced{ + Reference: jsonPayment.Reference, + Type: jsonPayment.Type, + }, + Payment: &jsonPayment.Data, + Forward: true, + }, + }, struct{}{}) + ... +``` + +Now restart the service, uninstall the connector, and reinstall it. + +Here the logs : +```bash +payments-payments-1 | time="2022-07-01T14:25:20Z" level=info msg="Install connector example" component=connector-manager config="{/tmp/payments}" provider=example +payments-payments-1 | time="2022-07-01T14:25:20Z" level=info msg="Starting task..." component=scheduler provider=example task-id="ImRpcmVjdG9yeSI=" +payments-payments-1 | time="2022-07-01T14:25:20Z" level=info msg="Connector installed" component=connector-manager provider=example +payments-payments-1 | time="2022-07-01T14:25:30Z" level=info msg="Opening directory '/tmp/payments'..." component=scheduler provider=example task-id="ImRpcmVjdG9yeSI=" +payments-payments-1 | time="2022-07-01T14:25:30Z" level=info msg="Found 1 files" component=scheduler provider=example task-id="ImRpcmVjdG9yeSI=" +payments-payments-1 | time="2022-07-01T14:25:30Z" level=info msg="Starting task..." component=scheduler provider=example task-id="IjAwMS5qc29uIg==" +payments-payments-1 | time="2022-07-01T14:25:30Z" level=info msg="Task terminated with success" component=scheduler provider=example task-id="IjAwMS5qc29uIg==" +``` + +As you can see, this time the second task has been started and was terminated with success. + +It should have created a payment on database. Let's check : +```bash +curl http://localhost:8080/payments | jq + +{ + "data": [ + { + "id": "eyJwcm92aWRlciI6ImV4YW1wbGUiLCJyZWZlcmVuY2UiOiIwMDEiLCJ0eXBlIjoicGF5aW4ifQ==", + "reference": "001", + "type": "payin", + "provider": "example", + "status": "succeeded", + "initialAmount": 100, + "scheme": "", + "asset": "USD", + "createdAt": "0001-01-01T00:00:00Z", + "raw": null, + "adjustments": [ + { + "status": "succeeded", + "amount": 100, + "date": "0001-01-01T00:00:00Z", + "raw": null, + "absolute": false + } + ] + } + ] +} +``` + +The last important part is the ```Ingester```. + +In the code of the second task, you should have seen this part : +```go +return ingester.Ingest(ctx.Context(), ingestion.Batch{ + { + Referenced: payments.Referenced{ + Reference: jsonPayment.Reference, + Type: jsonPayment.Type, + }, + Payment: &jsonPayment.Data, + Forward: true, + }, +}, struct{}{}) +``` +The ingester is in charge of accepting payments from a task and an eventual state to be persisted. + +In our case, we don't alter the state, but we could if we want (we passed an empty struct). + +If the connector is restarted, the task will be restarted with the previously state. + +The complete code : +```go +package example + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "time" + + "github.com/formancehq/go-libs/sharedlogging" + payments "github.com/formancehq/payments/pkg" + "github.com/formancehq/payments/pkg/bridge/ingestion" + "github.com/formancehq/payments/pkg/bridge/integration" + "github.com/formancehq/payments/pkg/bridge/task" +) + +type ( + Config struct { + Directory string + } + TaskDescriptor string +) + +func (cfg Config) Validate() error { + if cfg.Directory == "" { + return errors.New("missing directory to watch") + } + return nil +} + +var Loader = integration.NewLoaderBuilder[Config]("example"). + WithLoad(func(logger sharedlogging.Logger, config Config) integration.Connector { + return integration.NewConnectorBuilder(). + WithInstall(func(ctx task.ConnectorContext) error { + return ctx.Scheduler().Schedule("directory", false) + }). + WithResolve(func(descriptor models.TaskDescriptor) task.Task { + if descriptor == "directory" { + return func(ctx context.Context, logger sharedlogging.Logger, scheduler task.Scheduler) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(10 * time.Second): // Could be configurable using Config object + logger.Infof("Opening directory '%s'...", config.Directory) + dir, err := os.ReadDir(config.Directory) + if err != nil { + logger.Errorf("Error opening directory '%s': %s", config.Directory, err) + continue + } + + logger.Infof("Found %d files", len(dir)) + for _, file := range dir { + err = scheduler.Schedule(TaskDescriptor(file.Name()), false) + if err != nil { + logger.Errorf("Error scheduling task '%s': %s", file.Name(), err) + continue + } + } + } + } + } + } + return func(ctx context.Context, ingester ingestion.Ingester, resolver task.StateResolver) error { + file, err := os.Open(filepath.Join(config.Directory, string(descriptor))) + if err != nil { + return err + } + + type JsonPayment struct { + payments.Data + Reference string `json:"reference"` + Type string `json:"type"` + } + + jsonPayment := &JsonPayment{} + err = json.NewDecoder(file).Decode(jsonPayment) + if err != nil { + return err + } + + return ingester.Ingest(ctx, ingestion.Batch{ + { + Referenced: payments.Referenced{ + Reference: jsonPayment.Reference, + Type: jsonPayment.Type, + }, + Payment: &jsonPayment.Data, + Forward: true, + }, + }, struct{}{}) + } + }). + Build() + }). + Build() +``` diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..a2b85546 --- /dev/null +++ b/go.mod @@ -0,0 +1,122 @@ +module github.com/formancehq/payments + +go 1.19 + +require ( + github.com/Shopify/sarama v1.37.2 + github.com/ThreeDotsLabs/watermill v1.1.1 + github.com/bombsimon/logrusr/v3 v3.1.0 + github.com/davecgh/go-spew v1.1.1 + github.com/formancehq/go-libs v1.4.1 + github.com/google/uuid v1.3.0 + github.com/gorilla/mux v1.8.0 + github.com/jackc/pgx/v5 v5.2.0 + github.com/lib/pq v1.10.7 + github.com/pkg/errors v0.9.1 + github.com/pressly/goose/v3 v3.7.0 + github.com/rs/cors v1.8.3 + github.com/sirupsen/logrus v1.9.0 + github.com/spf13/afero v1.9.3 + github.com/spf13/cobra v1.6.1 + github.com/spf13/viper v1.14.0 + github.com/stretchr/testify v1.8.1 + github.com/stripe/stripe-go/v72 v72.122.0 + github.com/uptrace/bun v1.1.9 + github.com/uptrace/bun/dialect/pgdialect v1.1.9 + github.com/uptrace/bun/extra/bundebug v1.1.9 + github.com/uptrace/bun/extra/bunotel v1.1.9 + github.com/uptrace/opentelemetry-go-extra/otellogrus v0.1.17 + github.com/xdg-go/scram v1.1.2 + go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.37.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0 + go.opentelemetry.io/otel v1.11.2 + go.opentelemetry.io/otel/trace v1.11.2 + go.uber.org/dig v1.16.0 + go.uber.org/fx v1.19.0 + golang.org/x/sync v0.1.0 +) + +require ( + github.com/ThreeDotsLabs/watermill-http v1.1.4 // indirect + github.com/ThreeDotsLabs/watermill-kafka/v2 v2.2.2 // indirect + github.com/ajg/form v1.5.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/eapache/go-resiliency v1.3.0 // indirect + github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect + github.com/eapache/queue v1.1.0 // indirect + github.com/fatih/color v1.13.0 // indirect + github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-chi/chi v4.1.2+incompatible // indirect + github.com/go-chi/render v1.0.2 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang/glog v1.0.0 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jcmturner/aescts/v2 v2.0.0 // indirect + github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect + github.com/jcmturner/gofork v1.7.6 // indirect + github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect + github.com/jcmturner/rpc/v2 v2.0.3 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/klauspost/compress v1.15.14 // indirect + github.com/lithammer/shortuuid/v3 v3.0.7 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/pierrec/lz4/v4 v4.1.17 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect + github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.4.1 // indirect + github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect + github.com/uptrace/opentelemetry-go-extra/otelsql v0.1.17 // indirect + github.com/uptrace/opentelemetry-go-extra/otelutil v0.1.17 // indirect + github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect + github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect + github.com/xdg-go/pbkdf2 v1.0.0 // indirect + github.com/xdg-go/stringprep v1.0.4 // indirect + go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.37.0 // indirect + go.opentelemetry.io/contrib/propagators/b3 v1.12.0 // indirect + go.opentelemetry.io/otel/exporters/jaeger v1.11.2 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.2 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.11.2 // indirect + go.opentelemetry.io/otel/metric v0.34.0 // indirect + go.opentelemetry.io/otel/sdk v1.11.2 // indirect + go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.uber.org/atomic v1.10.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + go.uber.org/zap v1.24.0 // indirect + golang.org/x/crypto v0.5.0 // indirect + golang.org/x/net v0.5.0 // indirect + golang.org/x/sys v0.4.0 // indirect + golang.org/x/text v0.6.0 // indirect + google.golang.org/genproto v0.0.0-20230109162033-3c3c17ce83e6 // indirect + google.golang.org/grpc v1.51.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..b567c6ba --- /dev/null +++ b/go.sum @@ -0,0 +1,803 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Shopify/sarama v1.37.2 h1:LoBbU0yJPte0cE5TZCGdlzZRmMgMtZU/XgnUKZg9Cv4= +github.com/Shopify/sarama v1.37.2/go.mod h1:Nxye/E+YPru//Bpaorfhc3JsSGYwCaDDj+R4bK52U5o= +github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= +github.com/ThreeDotsLabs/watermill v1.1.0/go.mod h1:Qd1xNFxolCAHCzcMrm6RnjW0manbvN+DJVWc1MWRFlI= +github.com/ThreeDotsLabs/watermill v1.1.1 h1:+9NXqWQvplzxBru2CIInvVOZeKUnM+Nysg42fInl5sY= +github.com/ThreeDotsLabs/watermill v1.1.1/go.mod h1:Qd1xNFxolCAHCzcMrm6RnjW0manbvN+DJVWc1MWRFlI= +github.com/ThreeDotsLabs/watermill-http v1.1.4 h1:wRM54z/BPnIWjGbXMrOnwOlrCAESzoSNxTAHiLysFA4= +github.com/ThreeDotsLabs/watermill-http v1.1.4/go.mod h1:mkQ9CC0pxTZerNwr281rBoOy355vYt/lePkmYSX/BRg= +github.com/ThreeDotsLabs/watermill-kafka/v2 v2.2.2 h1:COB5neqVL8jGwoz1Y9dawQ7Xhxid1XXX8+1CI/PebVU= +github.com/ThreeDotsLabs/watermill-kafka/v2 v2.2.2/go.mod h1:U001oyrHo+df3Q7hIXgKqxY2OW6woz64+GNuIxZokbM= +github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bombsimon/logrusr/v3 v3.1.0 h1:zORbLM943D+hDMGgyjMhSAz/iDz86ZV72qaak/CA0zQ= +github.com/bombsimon/logrusr/v3 v3.1.0/go.mod h1:PksPPgSFEL2I52pla2glgCyyd2OqOHAnFF5E+g8Ixco= +github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= +github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= +github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= +github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= +github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/formancehq/go-libs v1.4.1 h1:rUKfUyZFq9aid+JUIqKN8Mk80Lx06Bx7AreHj+9vyTo= +github.com/formancehq/go-libs v1.4.1/go.mod h1:IK1zDIGRPi/o8sSApIc1W0VC1Y0DGdADzxvpbqlD8fk= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= +github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns= +github.com/go-chi/render v1.0.2 h1:4ER/udB0+fMWB2Jlf15RV3F4A2FDuYi/9f+lFttR/Lg= +github.com/go-chi/render v1.0.2/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.0 h1:1JYBfzqrWPcCclBwxFCPAou9n+q86mfnu7NAeHfte7A= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.0/go.mod h1:YDZoGHuwE+ov0c8smSH49WLF3F2LaWnYYuDVd+EWrc0= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.2.0 h1:NdPpngX0Y6z6XDFKqmFQaE+bCtkqzvQIOt1wvBlAqs8= +github.com/jackc/pgx/v5 v5.2.0/go.mod h1:Ptn7zmohNsWEsdxRawMzk3gaKma2obW+NWTnKa0S4nk= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= +github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.3 h1:iTonLeSJOn7MVUtyMT+arAn5AKAPrkilzhGw8wE/Tq8= +github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= +github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc= +github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lithammer/shortuuid/v3 v3.0.4/go.mod h1:RviRjexKqIzx/7r1peoAITm6m7gnif/h+0zmolKJjzw= +github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= +github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= +github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pressly/goose/v3 v3.7.0 h1:jblaZul15uCIEKHRu5KUdA+5wDA7E60JC0TOthdrtf8= +github.com/pressly/goose/v3 v3.7.0/go.mod h1:N5gqPdIzdxf3BiPWdmoPreIwHStkxsvKWE5xjUvfYNk= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= +github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= +github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stripe/stripe-go/v72 v72.122.0 h1:eRXWqnEwGny6dneQ5BsxGzUCED5n180u8n665JHlut8= +github.com/stripe/stripe-go/v72 v72.122.0/go.mod h1:QwqJQtduHubZht9mek5sds9CtQcKFdsykV9ZepRWwo0= +github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= +github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo= +github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs= +github.com/uptrace/bun v1.1.9 h1:6zs+YJcgw8oj67c+YmI8edQokDFeyR4BE/ykNWjGYYs= +github.com/uptrace/bun v1.1.9/go.mod h1:fpYRCGyruLCyP7dNjMfqulYn4VBP/fH0enc0j0yW/Cs= +github.com/uptrace/bun/dialect/pgdialect v1.1.9 h1:V23SU89WfjqtePLFPRXVXCwmSyYb0XKeg8Z6BMXgyHg= +github.com/uptrace/bun/dialect/pgdialect v1.1.9/go.mod h1:+ux7PjC4NYsNMdGE9b2ERxCi2jJai8Z8zniXFExq0Ns= +github.com/uptrace/bun/extra/bundebug v1.1.9 h1:rs7H4MxYrcPk9QlaJRY7XlEQxRHCVhki2X/UC8+VJvo= +github.com/uptrace/bun/extra/bundebug v1.1.9/go.mod h1:ohdrG1K6GGU3x6gTIQ2izqZ1RWP0T8StdSq93lBDi68= +github.com/uptrace/bun/extra/bunotel v1.1.9 h1:LVZ09KKJGfleDQ4rtVSFcpG4iNUvwvuJyQLA21tyNRA= +github.com/uptrace/bun/extra/bunotel v1.1.9/go.mod h1:Vzax3Xl5s3+ZDnz5NHHTz9Qa9UMabvGVdn0YksALhHo= +github.com/uptrace/opentelemetry-go-extra/otellogrus v0.1.17 h1:FeCTrRenM5ucXWMpq3u4Wh2nWov9Co68aM2gINGlJRU= +github.com/uptrace/opentelemetry-go-extra/otellogrus v0.1.17/go.mod h1:CXKQH9iiW89FahjDENpC7ES9iUQTIyTE2V2aQlLQme8= +github.com/uptrace/opentelemetry-go-extra/otelsql v0.1.17 h1:LJgQBGDf/u2RxdAiQvb47lZ0PuQYZutJgjmxLPaFKLU= +github.com/uptrace/opentelemetry-go-extra/otelsql v0.1.17/go.mod h1:Ohf26FVZL1ckvgqrwLoZfb8SRwlCe6otyKlpolfdflI= +github.com/uptrace/opentelemetry-go-extra/otelutil v0.1.17 h1:fcKgoKi1dGCFr1zTP0mKzZDGcMliY2hBmBjpGVf/ee4= +github.com/uptrace/opentelemetry-go-extra/otelutil v0.1.17/go.mod h1:wl/W+O/95rYcMa67D9qQ+8/IJEztbyYSUkdT7L6t+p4= +github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= +github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= +github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= +github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.37.0 h1:fv05KgzYZm0lDIqIfXAuktVpd7WgKuRO05kJjsDNFC0= +go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.37.0/go.mod h1:SOPIANa+vHjORB2YtPx5IttHBQ6vtvfbVk8kCgIJ70g= +go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.37.0 h1:MlbQ16t8LOeui5xk9tCXawxP6kPSio/Jjl3EvCTFy+M= +go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux v0.37.0/go.mod h1:L2aUfzscu1vQEIoYXNTkCrw1ICYXWcZ+f9DtK17xYwA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0 h1:yt2NKzK7Vyo6h0+X8BA4FpreZQTlVEIarnsBP/H5mzs= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0/go.mod h1:+ARmXlUlc51J7sZeCBkBJNdHGySrdOzgzxp6VWRWM1U= +go.opentelemetry.io/contrib/propagators/b3 v1.12.0 h1:OtfTF8bneN8qTeo/j92kcvc0iDDm4bm/c3RzaUJfiu0= +go.opentelemetry.io/contrib/propagators/b3 v1.12.0/go.mod h1:0JDB4elfPUWGsCH/qhaMkDzP1l8nB0ANVx8zXuAYEwg= +go.opentelemetry.io/otel v1.11.2 h1:YBZcQlsVekzFsFbjygXMOXSs6pialIZxcjfO/mBDmR0= +go.opentelemetry.io/otel v1.11.2/go.mod h1:7p4EUV+AqgdlNV9gL97IgUZiVR3yrFXYo53f9BM3tRI= +go.opentelemetry.io/otel/exporters/jaeger v1.11.2 h1:ES8/j2+aB+3/BUw51ioxa50V9btN1eew/2J7N7n1tsE= +go.opentelemetry.io/otel/exporters/jaeger v1.11.2/go.mod h1:nwcF/DK4Hk0auZ/a5vw20uMsaJSXbzeeimhN5f9d0Lc= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 h1:htgM8vZIF8oPSCxa341e3IZ4yr/sKxgu8KZYllByiVY= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2/go.mod h1:rqbht/LlhVBgn5+k3M5QK96K5Xb0DvXpMJ5SFQpY6uw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2 h1:fqR1kli93643au1RKo0Uma3d2aPQKT+WBKfTSBaKbOc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2/go.mod h1:5Qn6qvgkMsLDX+sYK64rHb1FPhpn0UtxF+ouX1uhyJE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.2 h1:ERwKPn9Aer7Gxsc0+ZlutlH1bEEAUXAUhqm3Y45ABbk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.2/go.mod h1:jWZUM2MWhWCJ9J9xVbRx7tzK1mXKpAlze4CeulycwVY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2 h1:Us8tbCmuN16zAnK5TC69AtODLycKbwnskQzaB6DfFhc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2/go.mod h1:GZWSQQky8AgdJj50r1KJm8oiQiIPaAX7uZCFQX9GzC8= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.11.2 h1:BhEVgvuE1NWLLuMLvC6sif791F45KFHi5GhOs1KunZU= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.11.2/go.mod h1:bx//lU66dPzNT+Y0hHA12ciKoMOH9iixEwCqC1OeQWQ= +go.opentelemetry.io/otel/metric v0.34.0 h1:MCPoQxcg/26EuuJwpYN1mZTeCYAUGx8ABxfW07YkjP8= +go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2bR7bw5JqUm/AP8= +go.opentelemetry.io/otel/sdk v1.11.2 h1:GF4JoaEx7iihdMFu30sOyRx52HDHOkl9xQ8SMqNXUiU= +go.opentelemetry.io/otel/sdk v1.11.2/go.mod h1:wZ1WxImwpq+lVRo4vsmSOxdd+xwoUJ6rqyLc3SyX9aU= +go.opentelemetry.io/otel/trace v1.11.2 h1:Xf7hWSF2Glv0DE3MH7fBHvtpSBsjcBUe5MYAmZM/+y0= +go.opentelemetry.io/otel/trace v1.11.2/go.mod h1:4N+yC7QEz7TTsG9BSRLNAa63eg5E06ObSbKPmxQ/pKA= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= +go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.16.0 h1:O48QoUEj4ePocypAIE5jz+SrxVdG/izHM1CZ/Yjrwww= +go.uber.org/dig v1.16.0/go.mod h1:557JTAUZT5bUK0SvCwikmLPPtdQhfvLYtO5tJgQSbnk= +go.uber.org/fx v1.19.0 h1:QetyAKH/ya3Avfg+s84DljRV+svcSAo8k+2y+B+ZaRQ= +go.uber.org/fx v1.19.0/go.mod h1:bGK+AEy7XUwTBkqCsK/vDyFF0JJOA6X5KWpNC0e6qTA= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= +golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20230109162033-3c3c17ce83e6 h1:uUn6GsgKK2eCI0bWeRMgRCcqDaQXYDuB+5tXA5Xeg/8= +google.golang.org/genproto v0.0.0-20230109162033-3c3c17ce83e6/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= +modernc.org/cc/v3 v3.36.1 h1:CICrjwr/1M4+6OQ4HJZ/AHxjcwe67r5vPUF518MkO8A= +modernc.org/ccgo/v3 v3.16.8 h1:G0QNlTqI5uVgczBWfGKs7B++EPwCfXPWGD2MdeKloDs= +modernc.org/libc v1.16.19 h1:S8flPn5ZeXx6iw/8yNa986hwTQDrY8RXU7tObZuAozo= +modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8= +modernc.org/memory v1.1.1 h1:bDOL0DIDLQv7bWhP3gMvIrnoFw+Eo6F7a2QK9HPDiFU= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/sqlite v1.18.1 h1:ko32eKt3jf7eqIkCgPAeHMBXw3riNSLhl2f3loEF7o8= +modernc.org/strutil v1.1.2 h1:iFBDH6j1Z0bN/Q9udJnnFoFpENA4252qe/7/5woE5MI= +modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/app/api/accounts.go b/internal/app/api/accounts.go new file mode 100644 index 00000000..b35fd41b --- /dev/null +++ b/internal/app/api/accounts.go @@ -0,0 +1,109 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "time" + + "github.com/formancehq/payments/internal/app/models" + "github.com/formancehq/payments/internal/app/storage" + + "github.com/formancehq/go-libs/api" + "github.com/pkg/errors" +) + +type listAccountsRepository interface { + ListAccounts(ctx context.Context, pagination storage.Paginator) ([]*models.Account, storage.PaginationDetails, error) +} + +type accountResponse struct { + ID string `json:"id"` + Reference string `json:"reference"` + CreatedAt time.Time `json:"createdAt"` + Provider string `json:"provider"` + Type models.AccountType `json:"type"` +} + +func listAccountsHandler(repo listAccountsRepository) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var sorter storage.Sorter + + if sortParams := r.URL.Query()["sort"]; sortParams != nil { + for _, s := range sortParams { + parts := strings.SplitN(s, ":", 2) + + var order storage.SortOrder + + if len(parts) > 1 { + //nolint:goconst // allow duplicate string + switch parts[1] { + case "asc", "ASC": + order = storage.SortOrderAsc + case "dsc", "desc", "DSC", "DESC": + order = storage.SortOrderDesc + default: + handleValidationError(w, r, errors.New("sort order not well specified, got "+parts[1])) + + return + } + } + + column := parts[0] + + sorter.Add(column, order) + } + } + + pageSize, err := pageSizeQueryParam(r) + if err != nil { + handleValidationError(w, r, err) + + return + } + + pagination, err := storage.Paginate(pageSize, r.URL.Query().Get("cursor"), sorter) + if err != nil { + handleValidationError(w, r, err) + + return + } + + ret, paginationDetails, err := repo.ListAccounts(r.Context(), pagination) + if err != nil { + handleServerError(w, r, err) + + return + } + + data := make([]*accountResponse, len(ret)) + + for i := range ret { + data[i] = &accountResponse{ + ID: ret[i].ID.String(), + Reference: ret[i].Reference, + CreatedAt: ret[i].CreatedAt, + Provider: ret[i].Provider, + Type: ret[i].Type, + } + } + + err = json.NewEncoder(w).Encode(api.BaseResponse[*accountResponse]{ + Cursor: &api.Cursor[*accountResponse]{ + PageSize: paginationDetails.PageSize, + HasMore: paginationDetails.HasMore, + Previous: paginationDetails.PreviousPage, + Next: paginationDetails.NextPage, + Data: data, + }, + }) + if err != nil { + handleServerError(w, r, err) + + return + } + } +} diff --git a/internal/app/api/connector.go b/internal/app/api/connector.go new file mode 100644 index 00000000..7cbf60e2 --- /dev/null +++ b/internal/app/api/connector.go @@ -0,0 +1,247 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "time" + + "github.com/formancehq/payments/internal/app/storage" + + "github.com/google/uuid" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/gorilla/mux" + + "github.com/formancehq/go-libs/api" + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/integration" +) + +func handleErrorBadRequest(w http.ResponseWriter, r *http.Request, err error) { + w.WriteHeader(http.StatusBadRequest) + + logging.GetLogger(r.Context()).Error(err) + // TODO: Opentracing + err = json.NewEncoder(w).Encode(api.ErrorResponse{ + ErrorCode: http.StatusText(http.StatusBadRequest), + ErrorMessage: err.Error(), + }) + if err != nil { + panic(err) + } +} + +func handleError(w http.ResponseWriter, r *http.Request, err error) { + w.WriteHeader(http.StatusInternalServerError) + + logging.GetLogger(r.Context()).Error(err) + // TODO: Opentracing + err = json.NewEncoder(w).Encode(api.ErrorResponse{ + ErrorCode: "INTERNAL", + ErrorMessage: err.Error(), + }) + if err != nil { + panic(err) + } +} + +func readConfig[Config models.ConnectorConfigObject](connectorManager *integration.ConnectorManager[Config], +) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + config, err := connectorManager.ReadConfig(r.Context()) + if err != nil { + handleError(w, r, err) + + return + } + + err = json.NewEncoder(w).Encode(api.BaseResponse[Config]{ + Data: config, + }) + if err != nil { + panic(err) + } + } +} + +type listTasksResponseElement struct { + ID string `json:"id"` + ConnectorID string `json:"connectorID"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + Descriptor json.RawMessage `json:"descriptor"` + Status models.TaskStatus `json:"status"` + State json.RawMessage `json:"state"` + Error string `json:"error"` +} + +func listTasks[Config models.ConnectorConfigObject](connectorManager *integration.ConnectorManager[Config], +) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + pageSize, err := pageSizeQueryParam(r) + if err != nil { + handleValidationError(w, r, err) + + return + } + + pagination, err := storage.Paginate(pageSize, r.URL.Query().Get("cursor"), nil) + if err != nil { + handleValidationError(w, r, err) + + return + } + + tasks, paginationDetails, err := connectorManager.ListTasksStates(r.Context(), pagination) + if err != nil { + handleError(w, r, err) + + return + } + + data := make([]listTasksResponseElement, len(tasks)) + for i, task := range tasks { + data[i] = listTasksResponseElement{ + ID: task.ID.String(), + ConnectorID: task.ConnectorID.String(), + CreatedAt: task.CreatedAt.Format(time.RFC3339), + UpdatedAt: task.UpdatedAt.Format(time.RFC3339), + Descriptor: task.Descriptor, + Status: task.Status, + State: task.State, + Error: task.Error, + } + } + + err = json.NewEncoder(w).Encode(api.BaseResponse[listTasksResponseElement]{ + Cursor: &api.Cursor[listTasksResponseElement]{ + PageSize: paginationDetails.PageSize, + HasMore: paginationDetails.HasMore, + Previous: paginationDetails.PreviousPage, + Next: paginationDetails.NextPage, + Data: data, + }, + }) + if err != nil { + panic(err) + } + } +} + +func readTask[Config models.ConnectorConfigObject](connectorManager *integration.ConnectorManager[Config], +) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + taskID, err := uuid.Parse(mux.Vars(r)["taskID"]) + if err != nil { + handleErrorBadRequest(w, r, err) + + return + } + + task, err := connectorManager.ReadTaskState(r.Context(), taskID) + if err != nil { + handleError(w, r, err) + + return + } + + data := listTasksResponseElement{ + ID: task.ID.String(), + ConnectorID: task.ConnectorID.String(), + CreatedAt: task.CreatedAt.Format(time.RFC3339), + UpdatedAt: task.UpdatedAt.Format(time.RFC3339), + Descriptor: task.Descriptor, + Status: task.Status, + State: task.State, + Error: task.Error, + } + + err = json.NewEncoder(w).Encode(api.BaseResponse[listTasksResponseElement]{ + Data: &data, + }) + if err != nil { + panic(err) + } + } +} + +func uninstall[Config models.ConnectorConfigObject](connectorManager *integration.ConnectorManager[Config], +) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + err := connectorManager.Uninstall(r.Context()) + if err != nil { + handleError(w, r, err) + + return + } + + w.WriteHeader(http.StatusNoContent) + } +} + +func install[Config models.ConnectorConfigObject](connectorManager *integration.ConnectorManager[Config], +) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + installed, err := connectorManager.IsInstalled(context.Background()) + if err != nil { + handleError(w, r, err) + + return + } + + if installed { + handleError(w, r, integration.ErrAlreadyInstalled) + + return + } + + var config Config + if r.ContentLength > 0 { + err = json.NewDecoder(r.Body).Decode(&config) + if err != nil { + handleError(w, r, err) + + return + } + } + + err = connectorManager.Install(r.Context(), config) + if err != nil { + handleError(w, r, err) + + return + } + + w.WriteHeader(http.StatusNoContent) + } +} + +func reset[Config models.ConnectorConfigObject](connectorManager *integration.ConnectorManager[Config], +) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + installed, err := connectorManager.IsInstalled(context.Background()) + if err != nil { + handleError(w, r, err) + + return + } + + if !installed { + handleError(w, r, errors.New("connector not installed")) + + return + } + + err = connectorManager.Reset(r.Context()) + if err != nil { + handleError(w, r, err) + + return + } + + w.WriteHeader(http.StatusNoContent) + } +} diff --git a/internal/app/api/connectorconfigs.go b/internal/app/api/connectorconfigs.go new file mode 100644 index 00000000..683ffe3a --- /dev/null +++ b/internal/app/api/connectorconfigs.go @@ -0,0 +1,41 @@ +package api + +import ( + "encoding/json" + "net/http" + + "github.com/formancehq/go-libs/api" + + "github.com/formancehq/payments/internal/app/connectors/bankingcircle" + "github.com/formancehq/payments/internal/app/connectors/configtemplate" + "github.com/formancehq/payments/internal/app/connectors/currencycloud" + "github.com/formancehq/payments/internal/app/connectors/dummypay" + "github.com/formancehq/payments/internal/app/connectors/modulr" + "github.com/formancehq/payments/internal/app/connectors/stripe" + "github.com/formancehq/payments/internal/app/connectors/wise" +) + +func connectorConfigsHandler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + // TODO: It's not ideal to re-identify available connectors + // Refactor it when refactoring the HTTP lib. + + configs := configtemplate.BuildConfigs( + bankingcircle.Config{}, + currencycloud.Config{}, + dummypay.Config{}, + modulr.Config{}, + stripe.Config{}, + wise.Config{}, + ) + + err := json.NewEncoder(w).Encode(api.BaseResponse[configtemplate.Configs]{ + Data: &configs, + }) + if err != nil { + handleServerError(w, r, err) + + return + } + } +} diff --git a/internal/app/api/connectormodule.go b/internal/app/api/connectormodule.go new file mode 100644 index 00000000..be8f0d40 --- /dev/null +++ b/internal/app/api/connectormodule.go @@ -0,0 +1,83 @@ +package api + +import ( + "context" + "net/http" + + "github.com/google/uuid" + + "github.com/pkg/errors" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/payments/internal/app/storage" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/go-libs/publish" + "github.com/formancehq/payments/internal/app/ingestion" + "github.com/formancehq/payments/internal/app/integration" + "github.com/formancehq/payments/internal/app/task" + "go.uber.org/dig" + "go.uber.org/fx" +) + +type connectorHandler struct { + Handler http.Handler + Provider models.ConnectorProvider +} + +func addConnector[ConnectorConfig models.ConnectorConfigObject](loader integration.Loader[ConnectorConfig], +) fx.Option { + return fx.Options( + fx.Provide(func(store *storage.Storage, + publisher publish.Publisher, + ) *integration.ConnectorManager[ConnectorConfig] { + logger := logging.GetLogger(context.Background()) + + schedulerFactory := integration.TaskSchedulerFactoryFn(func( + resolver task.Resolver, maxTasks int, + ) *task.DefaultTaskScheduler { + return task.NewDefaultScheduler(loader.Name(), logger, + store, func(ctx context.Context, + descriptor models.TaskDescriptor, + taskID uuid.UUID, + ) (*dig.Container, error) { + container := dig.New() + + if err := container.Provide(func() ingestion.Ingester { + return ingestion.NewDefaultIngester(loader.Name(), descriptor, store, + logger.WithFields(map[string]interface{}{ + "task-id": taskID.String(), + }), publisher) + }); err != nil { + return nil, err + } + + return container, nil + }, resolver, maxTasks) + }) + + return integration.NewConnectorManager[ConnectorConfig](logger, + store, loader, schedulerFactory, publisher) + }), + fx.Provide(fx.Annotate(func(cm *integration.ConnectorManager[ConnectorConfig], + ) connectorHandler { + return connectorHandler{ + Handler: connectorRouter(loader.Name(), cm), + Provider: loader.Name(), + } + }, fx.ResultTags(`group:"connectorHandlers"`))), + fx.Invoke(func(lc fx.Lifecycle, cm *integration.ConnectorManager[ConnectorConfig]) { + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + err := cm.Restore(ctx) + if err != nil && !errors.Is(err, integration.ErrNotInstalled) { + return err + } + + return nil + }, + }) + }), + ) +} diff --git a/internal/app/api/health.go b/internal/app/api/health.go new file mode 100644 index 00000000..898f21b3 --- /dev/null +++ b/internal/app/api/health.go @@ -0,0 +1,27 @@ +package api + +import ( + "net/http" +) + +type healthRepository interface { + Ping() error +} + +func healthHandler(repo healthRepository) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if err := repo.Ping(); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + + return + } + + w.WriteHeader(http.StatusOK) + } +} + +func liveHandler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + } +} diff --git a/internal/app/api/module.go b/internal/app/api/module.go new file mode 100644 index 00000000..ebe3a8d6 --- /dev/null +++ b/internal/app/api/module.go @@ -0,0 +1,170 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "runtime/debug" + "strconv" + "strings" + "time" + + "github.com/formancehq/go-libs/api" + "github.com/formancehq/go-libs/logging" + + "github.com/formancehq/payments/internal/app/connectors/bankingcircle" + "github.com/formancehq/payments/internal/app/connectors/currencycloud" + + "github.com/formancehq/go-libs/auth" + "github.com/formancehq/go-libs/oauth2/oauth2introspect" + "github.com/formancehq/go-libs/otlp" + "github.com/formancehq/payments/internal/app/connectors/dummypay" + "github.com/formancehq/payments/internal/app/connectors/modulr" + "github.com/formancehq/payments/internal/app/connectors/stripe" + "github.com/formancehq/payments/internal/app/connectors/wise" + "github.com/gorilla/mux" + "github.com/rs/cors" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "go.uber.org/fx" +) + +//nolint:gosec // false positive +const ( + otelTracesFlag = "otel-traces" + authBasicEnabledFlag = "auth-basic-enabled" + authBasicCredentialsFlag = "auth-basic-credentials" + authBearerEnabledFlag = "auth-bearer-enabled" + authBearerIntrospectURLFlag = "auth-bearer-introspect-url" + authBearerAudienceFlag = "auth-bearer-audience" + authBearerAudiencesWildcardFlag = "auth-bearer-audiences-wildcard" + + serviceName = "Payments" +) + +func HTTPModule(serviceInfo api.ServiceInfo) fx.Option { + return fx.Options( + fx.Invoke(func(m *mux.Router, lc fx.Lifecycle) { + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + go func() { + //nolint:gomnd // allow timeout values + srv := &http.Server{ + Handler: m, + Addr: "0.0.0.0:8080", + WriteTimeout: 15 * time.Second, + ReadTimeout: 15 * time.Second, + } + + err := srv.ListenAndServe() + if err != nil { + panic(err) + } + }() + + return nil + }, + }) + }), + fx.Supply(serviceInfo), + fx.Provide(fx.Annotate(httpRouter, fx.ParamTags(``, ``, `group:"connectorHandlers"`))), + addConnector[dummypay.Config](dummypay.NewLoader()), + addConnector[modulr.Config](modulr.NewLoader()), + addConnector[stripe.Config](stripe.NewLoader()), + addConnector[wise.Config](wise.NewLoader()), + addConnector[currencycloud.Config](currencycloud.NewLoader()), + addConnector[bankingcircle.Config](bankingcircle.NewLoader()), + ) +} + +func httpRecoveryFunc(ctx context.Context, e interface{}) { + if viper.GetBool(otelTracesFlag) { + otlp.RecordAsError(ctx, e) + } else { + logrus.Errorln(e) + debug.PrintStack() + } +} + +func httpCorsHandler() func(http.Handler) http.Handler { + return cors.New(cors.Options{ + AllowedOrigins: []string{"*"}, + AllowedMethods: []string{http.MethodGet, http.MethodPost, http.MethodPut}, + AllowCredentials: true, + }).Handler +} + +func httpServeFunc(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + handler.ServeHTTP(w, r) + }) +} + +func sharedAuthMethods() []auth.Method { + methods := make([]auth.Method, 0) + + if viper.GetBool(authBasicEnabledFlag) { + credentials := auth.Credentials{} + + for _, kv := range viper.GetStringSlice(authBasicCredentialsFlag) { + parts := strings.SplitN(kv, ":", 2) + credentials[parts[0]] = auth.Credential{ + Password: parts[1], + } + } + + methods = append(methods, auth.NewHTTPBasicMethod(credentials)) + } + + if viper.GetBool(authBearerEnabledFlag) { + methods = append(methods, auth.NewHttpBearerMethod( + auth.NewIntrospectionValidator( + oauth2introspect.NewIntrospecter(viper.GetString(authBearerIntrospectURLFlag)), + viper.GetBool(authBearerAudiencesWildcardFlag), + auth.AudienceIn(viper.GetStringSlice(authBearerAudienceFlag)...), + ), + )) + } + + return methods +} + +func handleServerError(w http.ResponseWriter, r *http.Request, err error) { + w.WriteHeader(http.StatusInternalServerError) + logging.GetLogger(r.Context()).Error(err) + // TODO: Opentracing + err = json.NewEncoder(w).Encode(api.ErrorResponse{ + ErrorCode: "INTERNAL", + ErrorMessage: err.Error(), + }) + if err != nil { + panic(err) + } +} + +func handleValidationError(w http.ResponseWriter, r *http.Request, err error) { + w.WriteHeader(http.StatusBadRequest) + logging.GetLogger(r.Context()).Error(err) + // TODO: Opentracing + err = json.NewEncoder(w).Encode(api.ErrorResponse{ + ErrorCode: "VALIDATION", + ErrorMessage: err.Error(), + }) + if err != nil { + panic(err) + } +} + +func pageSizeQueryParam(r *http.Request) (int, error) { + if value := r.URL.Query().Get("pageSize"); value != "" { + ret, err := strconv.ParseInt(value, 10, 32) + if err != nil { + return 0, err + } + + return int(ret), nil + } + + return 0, nil +} diff --git a/internal/app/api/payments.go b/internal/app/api/payments.go new file mode 100644 index 00000000..abbf2e7e --- /dev/null +++ b/internal/app/api/payments.go @@ -0,0 +1,247 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/formancehq/payments/internal/app/models" + "github.com/formancehq/payments/internal/app/storage" + + "github.com/formancehq/go-libs/api" + "github.com/gorilla/mux" + "github.com/pkg/errors" +) + +type listPaymentsRepository interface { + ListPayments(ctx context.Context, pagination storage.Paginator) ([]*models.Payment, storage.PaginationDetails, error) +} + +type paymentResponse struct { + ID string `json:"id"` + Reference string `json:"reference"` + AccountID string `json:"accountID"` + Type string `json:"type"` + Provider models.ConnectorProvider `json:"provider"` + Status models.PaymentStatus `json:"status"` + InitialAmount int64 `json:"initialAmount"` + Scheme models.PaymentScheme `json:"scheme"` + Asset string `json:"asset"` + CreatedAt time.Time `json:"createdAt"` + Raw interface{} `json:"raw"` + Adjustments []paymentAdjustment `json:"adjustments"` + Metadata []paymentMetadata `json:"metadata"` +} + +type paymentMetadata struct { + Key string `json:"key"` + Value string `json:"value"` + Changelog []paymentMetadataChangelog `json:"changelog"` +} + +type paymentMetadataChangelog struct { + Timestamp string `json:"timestamp"` + Value string `json:"value"` +} + +type paymentAdjustment struct { + Status models.PaymentStatus `json:"status" bson:"status"` + Amount int64 `json:"amount" bson:"amount"` + Date time.Time `json:"date" bson:"date"` + Raw interface{} `json:"raw" bson:"raw"` + Absolute bool `json:"absolute" bson:"absolute"` +} + +func listPaymentsHandler(repo listPaymentsRepository) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + var sorter storage.Sorter + + if sortParams := r.URL.Query()["sort"]; sortParams != nil { + for _, s := range sortParams { + parts := strings.SplitN(s, ":", 2) + + var order storage.SortOrder + + if len(parts) > 1 { + switch parts[1] { + case "asc", "ASC": + order = storage.SortOrderAsc + case "dsc", "desc", "DSC", "DESC": + order = storage.SortOrderDesc + default: + handleValidationError(w, r, errors.New("sort order not well specified, got "+parts[1])) + + return + } + } + + column := parts[0] + + sorter.Add(column, order) + } + } + + pageSize, err := pageSizeQueryParam(r) + if err != nil { + handleValidationError(w, r, err) + + return + } + + pagination, err := storage.Paginate(pageSize, r.URL.Query().Get("cursor"), sorter) + if err != nil { + handleValidationError(w, r, err) + + return + } + + ret, paginationDetails, err := repo.ListPayments(r.Context(), pagination) + if err != nil { + handleServerError(w, r, err) + + return + } + + data := make([]*paymentResponse, len(ret)) + + for i := range ret { + data[i] = &paymentResponse{ + ID: ret[i].ID.String(), + Reference: ret[i].Reference, + Type: ret[i].Type.String(), + Provider: ret[i].Connector.Provider, + Status: ret[i].Status, + InitialAmount: ret[i].Amount, + Scheme: ret[i].Scheme, + Asset: ret[i].Asset.String(), + CreatedAt: ret[i].CreatedAt, + Raw: ret[i].RawData, + Adjustments: make([]paymentAdjustment, len(ret[i].Adjustments)), + } + + if ret[i].AccountID != uuid.Nil { + data[i].AccountID = ret[i].AccountID.String() + } + + for adjustmentIdx := range ret[i].Adjustments { + data[i].Adjustments[adjustmentIdx] = paymentAdjustment{ + Status: ret[i].Adjustments[adjustmentIdx].Status, + Amount: ret[i].Adjustments[adjustmentIdx].Amount, + Date: ret[i].Adjustments[adjustmentIdx].CreatedAt, + Raw: ret[i].Adjustments[adjustmentIdx].RawData, + Absolute: ret[i].Adjustments[adjustmentIdx].Absolute, + } + } + + for metadataIDx := range ret[i].Metadata { + data[i].Metadata = append(data[i].Metadata, + paymentMetadata{ + Key: ret[i].Metadata[metadataIDx].Key, + Value: ret[i].Metadata[metadataIDx].Value, + }) + + for changelogIdx := range ret[i].Metadata[metadataIDx].Changelog { + data[i].Metadata[metadataIDx].Changelog = append(data[i].Metadata[metadataIDx].Changelog, + paymentMetadataChangelog{ + Timestamp: ret[i].Metadata[metadataIDx].Changelog[changelogIdx].CreatedAt.Format(time.RFC3339), + Value: ret[i].Metadata[metadataIDx].Changelog[changelogIdx].Value, + }) + } + } + } + + err = json.NewEncoder(w).Encode(api.BaseResponse[*paymentResponse]{ + Cursor: &api.Cursor[*paymentResponse]{ + PageSize: paginationDetails.PageSize, + HasMore: paginationDetails.HasMore, + Previous: paginationDetails.PreviousPage, + Next: paginationDetails.NextPage, + Data: data, + }, + }) + if err != nil { + handleServerError(w, r, err) + + return + } + } +} + +type readPaymentRepository interface { + GetPayment(ctx context.Context, id string) (*models.Payment, error) +} + +func readPaymentHandler(repo readPaymentRepository) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + paymentID := mux.Vars(r)["paymentID"] + + payment, err := repo.GetPayment(r.Context(), paymentID) + if err != nil { + handleServerError(w, r, err) + + return + } + + data := paymentResponse{ + ID: payment.ID.String(), + Reference: payment.Reference, + Type: payment.Type.String(), + Provider: payment.Connector.Provider, + Status: payment.Status, + InitialAmount: payment.Amount, + Scheme: payment.Scheme, + Asset: payment.Asset.String(), + CreatedAt: payment.CreatedAt, + Raw: payment.RawData, + Adjustments: make([]paymentAdjustment, len(payment.Adjustments)), + Metadata: make([]paymentMetadata, len(payment.Metadata)), + } + + if payment.AccountID != uuid.Nil { + data.AccountID = payment.AccountID.String() + } + + for i := range payment.Adjustments { + data.Adjustments[i] = paymentAdjustment{ + Status: payment.Adjustments[i].Status, + Amount: payment.Adjustments[i].Amount, + Date: payment.Adjustments[i].CreatedAt, + Raw: payment.Adjustments[i].RawData, + Absolute: payment.Adjustments[i].Absolute, + } + } + + for metadataIDx := range payment.Metadata { + data.Metadata = append(data.Metadata, + paymentMetadata{ + Key: payment.Metadata[metadataIDx].Key, + Value: payment.Metadata[metadataIDx].Value, + }) + + for changelogIdx := range payment.Metadata[metadataIDx].Changelog { + data.Metadata[metadataIDx].Changelog = append(data.Metadata[metadataIDx].Changelog, + paymentMetadataChangelog{ + Timestamp: payment.Metadata[metadataIDx].Changelog[changelogIdx].CreatedAt.Format(time.RFC3339), + Value: payment.Metadata[metadataIDx].Changelog[changelogIdx].Value, + }) + } + } + + err = json.NewEncoder(w).Encode(api.BaseResponse[paymentResponse]{ + Data: &data, + }) + if err != nil { + handleServerError(w, r, err) + + return + } + } +} diff --git a/internal/app/api/readconnectors.go b/internal/app/api/readconnectors.go new file mode 100644 index 00000000..8457d1e6 --- /dev/null +++ b/internal/app/api/readconnectors.go @@ -0,0 +1,52 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/go-libs/api" +) + +type readConnectorsRepository interface { + ListConnectors(ctx context.Context) ([]*models.Connector, error) +} + +type readConnectorsResponseElement struct { + Provider models.ConnectorProvider `json:"provider" bson:"provider"` + Enabled bool `json:"enabled" bson:"enabled"` + + // TODO: remove disabled field when frontend switches to using enabled + Disabled bool `json:"disabled" bson:"disabled"` +} + +func readConnectorsHandler(repo readConnectorsRepository) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + res, err := repo.ListConnectors(r.Context()) + if err != nil { + handleError(w, r, err) + + return + } + + data := make([]readConnectorsResponseElement, len(res)) + + for i := range res { + data[i] = readConnectorsResponseElement{ + Provider: res[i].Provider, + Enabled: res[i].Enabled, + Disabled: !res[i].Enabled, + } + } + + err = json.NewEncoder(w).Encode( + api.BaseResponse[[]readConnectorsResponseElement]{ + Data: &data, + }) + if err != nil { + panic(err) + } + } +} diff --git a/internal/app/api/recovery.go b/internal/app/api/recovery.go new file mode 100644 index 00000000..fce3087e --- /dev/null +++ b/internal/app/api/recovery.go @@ -0,0 +1,20 @@ +package api + +import ( + "context" + "net/http" +) + +func recoveryHandler(reporter func(ctx context.Context, e interface{})) func(h http.Handler) http.Handler { + return func(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if e := recover(); e != nil { + w.WriteHeader(http.StatusInternalServerError) + reporter(r.Context(), e) + } + }() + h.ServeHTTP(w, r) + }) + } +} diff --git a/internal/app/api/router.go b/internal/app/api/router.go new file mode 100644 index 00000000..a1e68c9b --- /dev/null +++ b/internal/app/api/router.go @@ -0,0 +1,84 @@ +package api + +import ( + "net/http" + + "github.com/formancehq/go-libs/api" + "github.com/formancehq/go-libs/auth" + "github.com/formancehq/payments/internal/app/integration" + "github.com/formancehq/payments/internal/app/models" + "github.com/formancehq/payments/internal/app/storage" + "github.com/gorilla/mux" + "github.com/spf13/viper" + "go.opentelemetry.io/contrib/instrumentation/github.com/gorilla/mux/otelmux" +) + +func httpRouter(store *storage.Storage, serviceInfo api.ServiceInfo, connectorHandlers []connectorHandler) (*mux.Router, error) { + rootMux := mux.NewRouter() + + if viper.GetBool(otelTracesFlag) { + rootMux.Use(otelmux.Middleware(serviceName)) + } + + rootMux.Use(recoveryHandler(httpRecoveryFunc)) + rootMux.Use(httpCorsHandler()) + rootMux.Use(httpServeFunc) + + rootMux.Path("/_health").Handler(healthHandler(store)) + rootMux.Path("/_live").Handler(liveHandler()) + rootMux.Path("/_info").Handler(api.InfoHandler(serviceInfo)) + + authGroup := rootMux.Name("authenticated").Subrouter() + + if methods := sharedAuthMethods(); len(methods) > 0 { + authGroup.Use(auth.Middleware(methods...)) + } + + authGroup.Path("/payments").Methods(http.MethodGet).Handler(listPaymentsHandler(store)) + authGroup.Path("/payments/{paymentID}").Methods(http.MethodGet).Handler(readPaymentHandler(store)) + + authGroup.Path("/accounts").Methods(http.MethodGet).Handler(listAccountsHandler(store)) + + authGroup.HandleFunc("/connectors", readConnectorsHandler(store)) + + connectorGroup := authGroup.PathPrefix("/connectors").Subrouter() + + connectorGroup.Path("/configs").Handler(connectorConfigsHandler()) + + // TODO: It's not ideal to define it explicitly here + // Refactor it when refactoring the HTTP lib. + connectorGroup.Path("/stripe/transfers").Methods(http.MethodPost). + Handler(handleStripeTransfers(store)) + + for _, h := range connectorHandlers { + connectorGroup.PathPrefix("/" + h.Provider.String()).Handler( + http.StripPrefix("/connectors", h.Handler)) + + connectorGroup.PathPrefix("/" + h.Provider.StringLower()).Handler( + http.StripPrefix("/connectors", h.Handler)) + } + + return rootMux, nil +} + +func connectorRouter[Config models.ConnectorConfigObject]( + provider models.ConnectorProvider, + manager *integration.ConnectorManager[Config], +) *mux.Router { + r := mux.NewRouter() + + addRoute(r, provider, "", http.MethodPost, install(manager)) + addRoute(r, provider, "", http.MethodDelete, uninstall(manager)) + addRoute(r, provider, "/config", http.MethodGet, readConfig(manager)) + addRoute(r, provider, "/reset", http.MethodPost, reset(manager)) + addRoute(r, provider, "/tasks", http.MethodGet, listTasks(manager)) + addRoute(r, provider, "/tasks/{taskID}", http.MethodGet, readTask(manager)) + + return r +} + +func addRoute(r *mux.Router, provider models.ConnectorProvider, path, method string, handler http.Handler) { + r.Path("/" + provider.String() + path).Methods(method).Handler(handler) + + r.Path("/" + provider.StringLower() + path).Methods(method).Handler(handler) +} diff --git a/internal/app/api/stripe.go b/internal/app/api/stripe.go new file mode 100644 index 00000000..1f17272b --- /dev/null +++ b/internal/app/api/stripe.go @@ -0,0 +1,111 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + + "github.com/formancehq/go-libs/api" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/pkg/errors" + + stripeConnector "github.com/formancehq/payments/internal/app/connectors/stripe" + "github.com/stripe/stripe-go/v72" + "github.com/stripe/stripe-go/v72/transfer" +) + +type stripeTransferRequest struct { + Amount int64 `json:"amount"` + Asset string `json:"asset"` + Destination string `json:"destination"` + Metadata map[string]string `json:"metadata"` + + currency string +} + +func (req *stripeTransferRequest) validate() error { + if req.Amount <= 0 { + return errors.New("amount must be greater than 0") + } + + if req.Asset == "" { + return errors.New("asset is required") + } + + if req.Asset != "USD/2" && req.Asset != "EUR/2" { + return errors.New("asset must be USD/2 or EUR/2") + } + + req.currency = req.Asset[:3] + + if req.Destination == "" { + return errors.New("destination is required") + } + + return nil +} + +type stripeTransfersRepository interface { + GetConfig(ctx context.Context, connectorName models.ConnectorProvider, cfg any) error +} + +func handleStripeTransfers(repo stripeTransfersRepository) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + var cfg stripeConnector.Config + + if err := repo.GetConfig(r.Context(), stripeConnector.Name, &cfg); err != nil { + handleError(w, r, err) + + return + } + + stripe.Key = cfg.APIKey + + var transferRequest stripeTransferRequest + + err := json.NewDecoder(r.Body).Decode(&transferRequest) + if err != nil { + handleError(w, r, err) + + return + } + + err = transferRequest.validate() + if err != nil { + handleError(w, r, err) + + return + } + + params := &stripe.TransferParams{ + Params: stripe.Params{ + Context: r.Context(), + }, + Amount: stripe.Int64(transferRequest.Amount), + Currency: stripe.String(transferRequest.currency), + Destination: stripe.String(transferRequest.Destination), + } + + for k, v := range transferRequest.Metadata { + params.AddMetadata(k, v) + } + + transferResponse, err := transfer.New(params) + if err != nil { + handleServerError(w, r, err) + + return + } + + err = json.NewEncoder(w).Encode(api.BaseResponse[stripe.Transfer]{ + Data: transferResponse, + }) + if err != nil { + handleServerError(w, r, err) + + return + } + } +} diff --git a/internal/app/connectors/bankingcircle/client.go b/internal/app/connectors/bankingcircle/client.go new file mode 100644 index 00000000..8666b637 --- /dev/null +++ b/internal/app/connectors/bankingcircle/client.go @@ -0,0 +1,246 @@ +package bankingcircle + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/formancehq/go-libs/logging" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" +) + +type client struct { + httpClient *http.Client + + username string + password string + + endpoint string + authorizationEndpoint string + + logger logging.Logger + + accessToken string + accessTokenExpiresAt time.Time +} + +func newHTTPClient() *http.Client { + return &http.Client{ + Timeout: 10 * time.Second, + Transport: otelhttp.NewTransport(http.DefaultTransport), + } +} + +func newClient(username, password, endpoint, authorizationEndpoint string, logger logging.Logger) (*client, error) { + c := &client{ + httpClient: newHTTPClient(), + + username: username, + password: password, + endpoint: endpoint, + authorizationEndpoint: authorizationEndpoint, + + logger: logger, + } + + if err := c.login(context.TODO()); err != nil { + return nil, err + } + + return c, nil +} + +func (c *client) login(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + c.authorizationEndpoint+"/api/v1/authorizations/authorize", http.NoBody) + if err != nil { + return fmt.Errorf("failed to create login request: %w", err) + } + + req.SetBasicAuth(c.username, c.password) + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to login: %w", err) + } + + defer func() { + err = resp.Body.Close() + if err != nil { + c.logger.Error(err) + } + }() + + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read login response body: %w", err) + } + + //nolint:tagliatelle // allow for client-side structures + type response struct { + AccessToken string `json:"access_token"` + ExpiresIn int `json:"expires_in"` + } + + var res response + + if err = json.Unmarshal(responseBody, &res); err != nil { + return fmt.Errorf("failed to unmarshal login response: %w", err) + } + + c.accessToken = res.AccessToken + c.accessTokenExpiresAt = time.Now().Add(time.Duration(res.ExpiresIn) * time.Second) + + return nil +} + +func (c *client) ensureAccessTokenIsValid(ctx context.Context) error { + if c.accessTokenExpiresAt.After(time.Now()) { + return nil + } + + return c.login(ctx) +} + +//nolint:tagliatelle // allow for client-side structures +type payment struct { + PaymentID string `json:"paymentId"` + TransactionReference string `json:"transactionReference"` + ConcurrencyToken string `json:"concurrencyToken"` + Classification string `json:"classification"` + Status string `json:"status"` + Errors interface{} `json:"errors"` + LastChangedTimestamp time.Time `json:"lastChangedTimestamp"` + DebtorInformation struct { + PaymentBulkID interface{} `json:"paymentBulkId"` + AccountID string `json:"accountId"` + Account struct { + Account string `json:"account"` + FinancialInstitution string `json:"financialInstitution"` + Country string `json:"country"` + } `json:"account"` + VibanID interface{} `json:"vibanId"` + Viban struct { + Account string `json:"account"` + FinancialInstitution string `json:"financialInstitution"` + Country string `json:"country"` + } `json:"viban"` + InstructedDate interface{} `json:"instructedDate"` + DebitAmount struct { + Currency string `json:"currency"` + Amount float64 `json:"amount"` + } `json:"debitAmount"` + DebitValueDate time.Time `json:"debitValueDate"` + FxRate interface{} `json:"fxRate"` + Instruction interface{} `json:"instruction"` + } `json:"debtorInformation"` + Transfer struct { + DebtorAccount interface{} `json:"debtorAccount"` + DebtorName interface{} `json:"debtorName"` + DebtorAddress interface{} `json:"debtorAddress"` + Amount struct { + Currency string `json:"currency"` + Amount float64 `json:"amount"` + } `json:"amount"` + ValueDate interface{} `json:"valueDate"` + ChargeBearer interface{} `json:"chargeBearer"` + RemittanceInformation interface{} `json:"remittanceInformation"` + CreditorAccount interface{} `json:"creditorAccount"` + CreditorName interface{} `json:"creditorName"` + CreditorAddress interface{} `json:"creditorAddress"` + } `json:"transfer"` + CreditorInformation struct { + AccountID string `json:"accountId"` + Account struct { + Account string `json:"account"` + FinancialInstitution string `json:"financialInstitution"` + Country string `json:"country"` + } `json:"account"` + VibanID interface{} `json:"vibanId"` + Viban struct { + Account string `json:"account"` + FinancialInstitution string `json:"financialInstitution"` + Country string `json:"country"` + } `json:"viban"` + CreditAmount struct { + Currency string `json:"currency"` + Amount float64 `json:"amount"` + } `json:"creditAmount"` + CreditValueDate time.Time `json:"creditValueDate"` + FxRate interface{} `json:"fxRate"` + } `json:"creditorInformation"` +} + +func (c *client) getAllPayments(ctx context.Context) ([]*payment, error) { + var payments []*payment + + for page := 0; ; page++ { + pagedPayments, err := c.getPayments(ctx, page) + if err != nil { + return nil, err + } + + if len(pagedPayments) == 0 { + break + } + + payments = append(payments, pagedPayments...) + } + + return payments, nil +} + +func (c *client) getPayments(ctx context.Context, page int) ([]*payment, error) { + if err := c.ensureAccessTokenIsValid(ctx); err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.endpoint+"/api/v1/payments/singles", http.NoBody) + if err != nil { + return nil, fmt.Errorf("failed to create login request: %w", err) + } + + q := req.URL.Query() + q.Add("PageSize", "5000") + q.Add("PageNumber", fmt.Sprint(page)) + + req.URL.RawQuery = q.Encode() + + req.Header.Set("Authorization", "Bearer "+c.accessToken) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to login: %w", err) + } + + defer func() { + err = resp.Body.Close() + if err != nil { + c.logger.Error(err) + } + }() + + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read login response body: %w", err) + } + + type response struct { + Result []*payment `json:"result"` + PageInfo struct { + CurrentPage int `json:"currentPage"` + PageSize int `json:"pageSize"` + } `json:"pageInfo"` + } + + var res response + + if err = json.Unmarshal(responseBody, &res); err != nil { + return nil, fmt.Errorf("failed to unmarshal login response: %w", err) + } + + return res.Result, nil +} diff --git a/internal/app/connectors/bankingcircle/config.go b/internal/app/connectors/bankingcircle/config.go new file mode 100644 index 00000000..233e09c6 --- /dev/null +++ b/internal/app/connectors/bankingcircle/config.go @@ -0,0 +1,49 @@ +package bankingcircle + +import ( + "encoding/json" + + "github.com/formancehq/payments/internal/app/connectors/configtemplate" +) + +type Config struct { + Username string `json:"username" yaml:"username" bson:"username"` + Password string `json:"password" yaml:"password" bson:"password"` + Endpoint string `json:"endpoint" yaml:"endpoint" bson:"endpoint"` + AuthorizationEndpoint string `json:"authorizationEndpoint" yaml:"authorizationEndpoint" bson:"authorizationEndpoint"` +} + +func (c Config) Validate() error { + if c.Username == "" { + return ErrMissingUsername + } + + if c.Password == "" { + return ErrMissingPassword + } + + if c.Endpoint == "" { + return ErrMissingEndpoint + } + + if c.AuthorizationEndpoint == "" { + return ErrMissingAuthorizationEndpoint + } + + return nil +} + +func (c Config) Marshal() ([]byte, error) { + return json.Marshal(c) +} + +func (c Config) BuildTemplate() (string, configtemplate.Config) { + cfg := configtemplate.NewConfig() + + cfg.AddParameter("username", configtemplate.TypeString, true) + cfg.AddParameter("password", configtemplate.TypeString, true) + cfg.AddParameter("endpoint", configtemplate.TypeString, true) + cfg.AddParameter("authorizationEndpoint", configtemplate.TypeString, true) + + return Name.String(), cfg +} diff --git a/internal/app/connectors/bankingcircle/errors.go b/internal/app/connectors/bankingcircle/errors.go new file mode 100644 index 00000000..7a7c18ac --- /dev/null +++ b/internal/app/connectors/bankingcircle/errors.go @@ -0,0 +1,20 @@ +package bankingcircle + +import "github.com/pkg/errors" + +var ( + // ErrMissingTask is returned when the task is missing. + ErrMissingTask = errors.New("task is not implemented") + + // ErrMissingUsername is returned when the username is missing. + ErrMissingUsername = errors.New("missing username from config") + + // ErrMissingPassword is returned when the password is missing. + ErrMissingPassword = errors.New("missing password from config") + + // ErrMissingEndpoint is returned when the endpoint is missing. + ErrMissingEndpoint = errors.New("missing endpoint from config") + + // ErrMissingAuthorizationEndpoint is returned when the authorization endpoint is missing. + ErrMissingAuthorizationEndpoint = errors.New("missing authorization endpoint from config") +) diff --git a/internal/app/connectors/bankingcircle/loader.go b/internal/app/connectors/bankingcircle/loader.go new file mode 100644 index 00000000..9dac0639 --- /dev/null +++ b/internal/app/connectors/bankingcircle/loader.go @@ -0,0 +1,33 @@ +package bankingcircle + +import ( + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/integration" + "github.com/formancehq/payments/internal/app/models" + "github.com/formancehq/payments/internal/app/task" +) + +const Name = models.ConnectorProviderBankingCircle + +// NewLoader creates a new loader. +func NewLoader() integration.Loader[Config] { + loader := integration.NewLoaderBuilder[Config](Name). + WithLoad(func(logger logging.Logger, config Config) integration.Connector { + return integration.NewConnectorBuilder(). + WithInstall(func(ctx task.ConnectorContext) error { + taskDescriptor, err := models.EncodeTaskDescriptor(TaskDescriptor{ + Name: "Fetch payments from source", + Key: taskNameFetchPayments, + }) + if err != nil { + return err + } + + return ctx.Scheduler().Schedule(taskDescriptor, false) + }). + WithResolve(resolveTasks(logger, config)). + Build() + }).Build() + + return loader +} diff --git a/internal/app/connectors/bankingcircle/task_fetch_payments.go b/internal/app/connectors/bankingcircle/task_fetch_payments.go new file mode 100644 index 00000000..a3f9181f --- /dev/null +++ b/internal/app/connectors/bankingcircle/task_fetch_payments.go @@ -0,0 +1,80 @@ +package bankingcircle + +import ( + "context" + "encoding/json" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/payments/internal/app/ingestion" + "github.com/formancehq/payments/internal/app/task" + + "github.com/formancehq/go-libs/logging" +) + +func taskFetchPayments(logger logging.Logger, client *client) task.Task { + return func( + ctx context.Context, + scheduler task.Scheduler, + ingester ingestion.Ingester, + ) error { + paymentsList, err := client.getAllPayments(ctx) + if err != nil { + return err + } + + batch := ingestion.PaymentBatch{} + + for _, paymentEl := range paymentsList { + logger.Info(paymentEl) + + raw, err := json.Marshal(paymentEl) + if err != nil { + return err + } + + batchElement := ingestion.PaymentBatchElement{ + Payment: &models.Payment{ + Reference: paymentEl.TransactionReference, + Type: matchPaymentType(paymentEl.Classification), + Status: matchPaymentStatus(paymentEl.Status), + Scheme: models.PaymentSchemeOther, + Amount: int64(paymentEl.Transfer.Amount.Amount * 100), + Asset: models.PaymentAsset(paymentEl.Transfer.Amount.Currency + "/2"), + RawData: raw, + }, + } + + batch = append(batch, batchElement) + } + + return ingester.IngestPayments(ctx, batch, struct{}{}) + } +} + +func matchPaymentStatus(paymentStatus string) models.PaymentStatus { + switch paymentStatus { + case "Processed": + return models.PaymentStatusSucceeded + // On MissingFunding - the payment is still in progress. + // If there will be funds available within 10 days - the payment will be processed. + // Otherwise - it will be cancelled. + case "PendingProcessing", "MissingFunding": + return models.PaymentStatusPending + case "Rejected", "Cancelled", "Reversed", "Returned": + return models.PaymentStatusFailed + } + + return models.PaymentStatusOther +} + +func matchPaymentType(paymentType string) models.PaymentType { + switch paymentType { + case "Incoming": + return models.PaymentTypePayIn + case "Outgoing": + return models.PaymentTypePayOut + } + + return models.PaymentTypeOther +} diff --git a/internal/app/connectors/bankingcircle/task_resolve.go b/internal/app/connectors/bankingcircle/task_resolve.go new file mode 100644 index 00000000..19d49e1b --- /dev/null +++ b/internal/app/connectors/bankingcircle/task_resolve.go @@ -0,0 +1,49 @@ +package bankingcircle + +import ( + "fmt" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/payments/internal/app/task" + + "github.com/formancehq/go-libs/logging" +) + +const ( + taskNameFetchPayments = "fetch-payments" +) + +// TaskDescriptor is the definition of a task. +type TaskDescriptor struct { + Name string `json:"name" yaml:"name" bson:"name"` + Key string `json:"key" yaml:"key" bson:"key"` +} + +func resolveTasks(logger logging.Logger, config Config) func(taskDefinition models.TaskDescriptor) task.Task { + bankingCircleClient, err := newClient(config.Username, config.Password, config.Endpoint, config.AuthorizationEndpoint, logger) + if err != nil { + logger.Error(err) + + return nil + } + + return func(taskDefinition models.TaskDescriptor) task.Task { + taskDescriptor, err := models.DecodeTaskDescriptor[TaskDescriptor](taskDefinition) + if err != nil { + logger.Error(err) + + return nil + } + + switch taskDescriptor.Key { + case taskNameFetchPayments: + return taskFetchPayments(logger, bankingCircleClient) + } + + // This should never happen. + return func() error { + return fmt.Errorf("key '%s': %w", taskDescriptor.Key, ErrMissingTask) + } + } +} diff --git a/internal/app/connectors/configtemplate/template.go b/internal/app/connectors/configtemplate/template.go new file mode 100644 index 00000000..2158be9d --- /dev/null +++ b/internal/app/connectors/configtemplate/template.go @@ -0,0 +1,35 @@ +package configtemplate + +type Configs map[string]Config + +type Config map[string]Parameter + +type Parameter struct { + DataType Type `json:"dataType"` + Required bool `json:"required"` +} + +type TemplateBuilder interface { + BuildTemplate() (string, Config) +} + +func BuildConfigs(builders ...TemplateBuilder) Configs { + configs := make(map[string]Config) + for _, builder := range builders { + name, config := builder.BuildTemplate() + configs[name] = config + } + + return configs +} + +func NewConfig() Config { + return make(map[string]Parameter) +} + +func (c *Config) AddParameter(name string, dataType Type, required bool) { + (*c)[name] = Parameter{ + DataType: dataType, + Required: required, + } +} diff --git a/internal/app/connectors/configtemplate/types.go b/internal/app/connectors/configtemplate/types.go new file mode 100644 index 00000000..a44fed1b --- /dev/null +++ b/internal/app/connectors/configtemplate/types.go @@ -0,0 +1,9 @@ +package configtemplate + +type Type string + +const ( + TypeString Type = "string" + TypeDurationNs Type = "duration ns" + TypeDurationUnsignedInteger Type = "unsigned integer" +) diff --git a/internal/app/connectors/currencycloud/client/auth.go b/internal/app/connectors/currencycloud/client/auth.go new file mode 100644 index 00000000..a045687c --- /dev/null +++ b/internal/app/connectors/currencycloud/client/auth.go @@ -0,0 +1,50 @@ +package client + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" +) + +func (c *Client) authenticate(ctx context.Context) (string, error) { + form := make(url.Values) + + form.Add("login_id", c.loginID) + form.Add("api_key", c.apiKey) + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.buildEndpoint("v2/authenticate/api"), strings.NewReader(form.Encode())) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + req.Header.Add("Accept", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return "", fmt.Errorf("failed to do get request: %w", err) + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + //nolint:tagliatelle // allow for client code + type response struct { + AuthToken string `json:"auth_token"` + } + + var res response + + if err = json.NewDecoder(resp.Body).Decode(&res); err != nil { + return "", fmt.Errorf("failed to decode response body: %w", err) + } + + return res.AuthToken, nil +} diff --git a/internal/app/connectors/currencycloud/client/client.go b/internal/app/connectors/currencycloud/client/client.go new file mode 100644 index 00000000..d402d85e --- /dev/null +++ b/internal/app/connectors/currencycloud/client/client.go @@ -0,0 +1,71 @@ +package client + +import ( + "context" + "fmt" + "net/http" + + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" +) + +type apiTransport struct { + authToken string + underlying *otelhttp.Transport +} + +func (t *apiTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Add("X-Auth-Token", t.authToken) + + return t.underlying.RoundTrip(req) +} + +type Client struct { + httpClient *http.Client + endpoint string + loginID string + apiKey string +} + +func (c *Client) buildEndpoint(path string, args ...interface{}) string { + return fmt.Sprintf("%s/%s", c.endpoint, fmt.Sprintf(path, args...)) +} + +const devAPIEndpoint = "https://devapi.currencycloud.com" + +func newAuthenticatedHTTPClient(authToken string) *http.Client { + return &http.Client{ + Transport: &apiTransport{ + authToken: authToken, + underlying: otelhttp.NewTransport(http.DefaultTransport), + }, + } +} + +func newHTTPClient() *http.Client { + return &http.Client{ + Transport: otelhttp.NewTransport(http.DefaultTransport), + } +} + +// NewClient creates a new client for the CurrencyCloud API. +func NewClient(ctx context.Context, loginID, apiKey, endpoint string) (*Client, error) { + if endpoint == "" { + endpoint = devAPIEndpoint + } + + c := &Client{ + httpClient: newHTTPClient(), + endpoint: endpoint, + loginID: loginID, + apiKey: apiKey, + } + + authToken, err := c.authenticate(ctx) + if err != nil { + return nil, err + } + + c.httpClient = newAuthenticatedHTTPClient(authToken) + + return c, nil +} diff --git a/internal/app/connectors/currencycloud/client/transactions.go b/internal/app/connectors/currencycloud/client/transactions.go new file mode 100644 index 00000000..cbf67b6d --- /dev/null +++ b/internal/app/connectors/currencycloud/client/transactions.go @@ -0,0 +1,60 @@ +package client + +import ( + "context" + "encoding/json" + "fmt" + "net/http" +) + +//nolint:tagliatelle // allow different styled tags in client +type Transaction struct { + ID string `json:"id"` + Currency string `json:"currency"` + Type string `json:"type"` + Status string `json:"status"` + CreatedAt string `json:"created_at"` + Action string `json:"action"` + + Amount string `json:"amount"` +} + +func (c *Client) GetTransactions(ctx context.Context, page int) ([]Transaction, int, error) { + if page < 1 { + return nil, 0, fmt.Errorf("page must be greater than 0") + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, + c.buildEndpoint("v2/transactions/find?page=%d", page), http.NoBody) + if err != nil { + return nil, 0, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Add("Accept", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, 0, err + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, 0, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + //nolint:tagliatelle // allow for client code + type response struct { + Transactions []Transaction `json:"transactions"` + Pagination struct { + NextPage int `json:"next_page"` + } + } + + var res response + if err = json.NewDecoder(resp.Body).Decode(&res); err != nil { + return nil, 0, err + } + + return res.Transactions, res.Pagination.NextPage, nil +} diff --git a/internal/app/connectors/currencycloud/config.go b/internal/app/connectors/currencycloud/config.go new file mode 100644 index 00000000..ea9c3673 --- /dev/null +++ b/internal/app/connectors/currencycloud/config.go @@ -0,0 +1,86 @@ +package currencycloud + +import ( + "encoding/json" + "time" + + "github.com/formancehq/payments/internal/app/connectors/configtemplate" +) + +type Config struct { + LoginID string `json:"loginID" bson:"loginID"` + APIKey string `json:"apiKey" bson:"apiKey"` + Endpoint string `json:"endpoint" bson:"endpoint"` + PollingPeriod Duration `json:"pollingPeriod" bson:"pollingPeriod"` +} + +func (c Config) Validate() error { + if c.APIKey == "" { + return ErrMissingAPIKey + } + + if c.LoginID == "" { + return ErrMissingLoginID + } + + if c.PollingPeriod == 0 { + return ErrMissingPollingPeriod + } + + return nil +} + +func (c Config) Marshal() ([]byte, error) { + return json.Marshal(c) +} + +type Duration time.Duration + +func (d *Duration) String() string { + return time.Duration(*d).String() +} + +func (d *Duration) Duration() time.Duration { + return time.Duration(*d) +} + +func (d *Duration) MarshalJSON() ([]byte, error) { + return json.Marshal(time.Duration(*d).String()) +} + +func (d *Duration) UnmarshalJSON(b []byte) error { + var durationValue interface{} + + if err := json.Unmarshal(b, &durationValue); err != nil { + return err + } + + switch value := durationValue.(type) { + case float64: + *d = Duration(time.Duration(value)) + + return nil + case string: + tmp, err := time.ParseDuration(value) + if err != nil { + return err + } + + *d = Duration(tmp) + + return nil + default: + return ErrDurationInvalid + } +} + +func (c Config) BuildTemplate() (string, configtemplate.Config) { + cfg := configtemplate.NewConfig() + + cfg.AddParameter("loginID", configtemplate.TypeString, true) + cfg.AddParameter("apiKey", configtemplate.TypeString, true) + cfg.AddParameter("endpoint", configtemplate.TypeString, false) + cfg.AddParameter("pollingPeriod", configtemplate.TypeDurationNs, true) + + return Name.String(), cfg +} diff --git a/internal/app/connectors/currencycloud/connector.go b/internal/app/connectors/currencycloud/connector.go new file mode 100644 index 00000000..69105ba5 --- /dev/null +++ b/internal/app/connectors/currencycloud/connector.go @@ -0,0 +1,46 @@ +package currencycloud + +import ( + "context" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/integration" + "github.com/formancehq/payments/internal/app/task" +) + +const Name = models.ConnectorProviderCurrencyCloud + +type Connector struct { + logger logging.Logger + cfg Config +} + +func (c *Connector) Install(ctx task.ConnectorContext) error { + taskDescriptor, err := models.EncodeTaskDescriptor(TaskDescriptor{Name: taskNameFetchTransactions}) + if err != nil { + return err + } + + return ctx.Scheduler().Schedule(taskDescriptor, true) +} + +func (c *Connector) Uninstall(ctx context.Context) error { + return nil +} + +func (c *Connector) Resolve(descriptor models.TaskDescriptor) task.Task { + return resolveTasks(c.logger, c.cfg) +} + +var _ integration.Connector = &Connector{} + +func newConnector(logger logging.Logger, cfg Config) *Connector { + return &Connector{ + logger: logger.WithFields(map[string]any{ + "component": "connector", + }), + cfg: cfg, + } +} diff --git a/internal/app/connectors/currencycloud/errors.go b/internal/app/connectors/currencycloud/errors.go new file mode 100644 index 00000000..30f5712e --- /dev/null +++ b/internal/app/connectors/currencycloud/errors.go @@ -0,0 +1,20 @@ +package currencycloud + +import "github.com/pkg/errors" + +var ( + // ErrMissingTask is returned when the task is missing. + ErrMissingTask = errors.New("task is not implemented") + + // ErrMissingAPIKey is returned when the api key is missing from config. + ErrMissingAPIKey = errors.New("missing apiKey from config") + + // ErrMissingLoginID is returned when the login id is missing from config. + ErrMissingLoginID = errors.New("missing loginID from config") + + // ErrMissingPollingPeriod is returned when the polling period is missing from config. + ErrMissingPollingPeriod = errors.New("missing pollingPeriod from config") + + // ErrDurationInvalid is returned when the duration is invalid. + ErrDurationInvalid = errors.New("duration is invalid") +) diff --git a/internal/app/connectors/currencycloud/loader.go b/internal/app/connectors/currencycloud/loader.go new file mode 100644 index 00000000..aaf6d793 --- /dev/null +++ b/internal/app/connectors/currencycloud/loader.go @@ -0,0 +1,32 @@ +package currencycloud + +import ( + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/integration" + "github.com/formancehq/payments/internal/app/models" +) + +type Loader struct{} + +const allowedTasks = 50 + +func (l *Loader) AllowTasks() int { + return allowedTasks +} + +func (l *Loader) Name() models.ConnectorProvider { + return Name +} + +func (l *Loader) Load(logger logging.Logger, config Config) integration.Connector { + return newConnector(logger, config) +} + +func (l *Loader) ApplyDefaults(cfg Config) Config { + return cfg +} + +// NewLoader creates a new loader. +func NewLoader() *Loader { + return &Loader{} +} diff --git a/internal/app/connectors/currencycloud/task_fetch_transactions.go b/internal/app/connectors/currencycloud/task_fetch_transactions.go new file mode 100644 index 00000000..a188d5a0 --- /dev/null +++ b/internal/app/connectors/currencycloud/task_fetch_transactions.go @@ -0,0 +1,122 @@ +package currencycloud + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "time" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/payments/internal/app/connectors/currencycloud/client" + + "github.com/formancehq/payments/internal/app/ingestion" + "github.com/formancehq/payments/internal/app/task" + + "github.com/formancehq/go-libs/logging" +) + +func taskFetchTransactions(logger logging.Logger, client *client.Client, config Config) task.Task { + return func( + ctx context.Context, + ingester ingestion.Ingester, + ) error { + for { + select { + case <-ctx.Done(): + return nil + case <-time.After(config.PollingPeriod.Duration()): + if err := ingestTransactions(ctx, logger, client, ingester); err != nil { + return err + } + } + } + } +} + +func ingestTransactions(ctx context.Context, logger logging.Logger, + client *client.Client, ingester ingestion.Ingester, +) error { + page := 1 + + for { + if page < 0 { + break + } + + logger.Info("Fetching transactions") + + transactions, nextPage, err := client.GetTransactions(ctx, page) + if err != nil { + return err + } + + page = nextPage + + batch := ingestion.PaymentBatch{} + + for _, transaction := range transactions { + logger.Info(transaction) + + var amount float64 + + amount, err = strconv.ParseFloat(transaction.Amount, 64) + if err != nil { + return fmt.Errorf("failed to parse amount: %w", err) + } + + var rawData json.RawMessage + + rawData, err = json.Marshal(transaction) + if err != nil { + return fmt.Errorf("failed to marshal transaction: %w", err) + } + + batchElement := ingestion.PaymentBatchElement{ + Payment: &models.Payment{ + Reference: transaction.ID, + Type: matchTransactionType(transaction.Type), + Status: matchTransactionStatus(transaction.Status), + Scheme: models.PaymentSchemeOther, + Amount: int64(amount * 100), + Asset: models.PaymentAsset(fmt.Sprintf("%s/2", transaction.Currency)), + RawData: rawData, + }, + } + + batch = append(batch, batchElement) + } + + err = ingester.IngestPayments(ctx, batch, struct{}{}) + if err != nil { + return err + } + } + + return nil +} + +func matchTransactionType(transactionType string) models.PaymentType { + switch transactionType { + case "credit": + return models.PaymentTypePayOut + case "debit": + return models.PaymentTypePayIn + } + + return models.PaymentTypeOther +} + +func matchTransactionStatus(transactionStatus string) models.PaymentStatus { + switch transactionStatus { + case "completed": + return models.PaymentStatusSucceeded + case "pending": + return models.PaymentStatusPending + case "deleted": + return models.PaymentStatusFailed + } + + return models.PaymentStatusOther +} diff --git a/internal/app/connectors/currencycloud/task_resolve.go b/internal/app/connectors/currencycloud/task_resolve.go new file mode 100644 index 00000000..c1bd1e36 --- /dev/null +++ b/internal/app/connectors/currencycloud/task_resolve.go @@ -0,0 +1,44 @@ +package currencycloud + +import ( + "context" + "fmt" + + "github.com/formancehq/payments/internal/app/connectors/currencycloud/client" + + "github.com/formancehq/payments/internal/app/task" + + "github.com/formancehq/go-libs/logging" +) + +const ( + taskNameFetchTransactions = "fetch-transactions" +) + +// TaskDescriptor is the definition of a task. +type TaskDescriptor struct { + Name string `json:"name" yaml:"name" bson:"name"` +} + +func resolveTasks(logger logging.Logger, config Config) task.Task { + return func(ctx context.Context, taskDescriptor TaskDescriptor) task.Task { + currencyCloudClient, err := client.NewClient(ctx, config.LoginID, config.APIKey, config.Endpoint) + if err != nil { + return func(ctx context.Context, taskDefinition TaskDescriptor) task.Task { + return func() error { + return fmt.Errorf("failed to initiate client: %w", err) + } + } + } + + switch taskDescriptor.Name { + case taskNameFetchTransactions: + return taskFetchTransactions(logger, currencyCloudClient, config) + } + + // This should never happen. + return func() error { + return fmt.Errorf("key '%s': %w", taskDescriptor.Name, ErrMissingTask) + } + } +} diff --git a/internal/app/connectors/dummypay/config.go b/internal/app/connectors/dummypay/config.go new file mode 100644 index 00000000..f45ffe50 --- /dev/null +++ b/internal/app/connectors/dummypay/config.go @@ -0,0 +1,64 @@ +package dummypay + +import ( + "encoding/json" + "fmt" + + "github.com/formancehq/payments/internal/app/connectors/configtemplate" + + "github.com/formancehq/payments/internal/app/connectors" +) + +// Config is the configuration for the dummy payment connector. +type Config struct { + // Directory is the directory where the files are stored. + Directory string `json:"directory" yaml:"directory" bson:"directory"` + + // FilePollingPeriod is the period between file polling. + FilePollingPeriod connectors.Duration `json:"filePollingPeriod" yaml:"filePollingPeriod" bson:"filePollingPeriod"` + + // FileGenerationPeriod is the period between file generation + FileGenerationPeriod connectors.Duration `json:"fileGenerationPeriod" yaml:"fileGenerationPeriod" bson:"fileGenerationPeriod"` +} + +// String returns a string representation of the configuration. +func (c Config) String() string { + return fmt.Sprintf("directory: %s, filePollingPeriod: %s, fileGenerationPeriod: %s", + c.Directory, c.FilePollingPeriod.String(), c.FileGenerationPeriod.String()) +} + +func (c Config) Marshal() ([]byte, error) { + return json.Marshal(c) +} + +// Validate validates the configuration. +func (c Config) Validate() error { + // require directory path to be present + if c.Directory == "" { + return ErrMissingDirectory + } + + // check if file polling period is set properly + if c.FilePollingPeriod.Duration <= 0 { + return fmt.Errorf("filePollingPeriod must be greater than 0: %w", + ErrFilePollingPeriodInvalid) + } + + // check if file generation period is set properly + if c.FileGenerationPeriod.Duration <= 0 { + return fmt.Errorf("fileGenerationPeriod must be greater than 0: %w", + ErrFileGenerationPeriodInvalid) + } + + return nil +} + +func (c Config) BuildTemplate() (string, configtemplate.Config) { + cfg := configtemplate.NewConfig() + + cfg.AddParameter("directory", configtemplate.TypeString, true) + cfg.AddParameter("filePollingPeriod", configtemplate.TypeDurationNs, true) + cfg.AddParameter("fileGenerationPeriod", configtemplate.TypeDurationNs, false) + + return Name.String(), cfg +} diff --git a/internal/app/connectors/dummypay/config_test.go b/internal/app/connectors/dummypay/config_test.go new file mode 100644 index 00000000..3375c6da --- /dev/null +++ b/internal/app/connectors/dummypay/config_test.go @@ -0,0 +1,58 @@ +package dummypay + +import ( + "os" + "testing" + "time" + + "github.com/formancehq/payments/internal/app/connectors" + "github.com/stretchr/testify/assert" +) + +// TestConfigString tests the string representation of the config. +func TestConfigString(t *testing.T) { + t.Parallel() + + config := Config{ + Directory: "test", + FilePollingPeriod: connectors.Duration{Duration: time.Second}, + FileGenerationPeriod: connectors.Duration{Duration: time.Minute}, + } + + assert.Equal(t, "directory: test, filePollingPeriod: 1s, fileGenerationPeriod: 1m0s", config.String()) +} + +// TestConfigValidate tests the validation of the config. +func TestConfigValidate(t *testing.T) { + t.Parallel() + + var config Config + + // fail on missing directory + assert.EqualError(t, config.Validate(), ErrMissingDirectory.Error()) + + // fail on missing RW access to directory + config.Directory = "/non-existing" + assert.Error(t, config.Validate()) + + // set directory with RW access + userHomeDir, err := os.UserHomeDir() + if err != nil { + t.Error(err) + } + + config.Directory = userHomeDir + + // fail on invalid file polling period + config.FilePollingPeriod.Duration = -1 + assert.ErrorIs(t, config.Validate(), ErrFilePollingPeriodInvalid) + + // fail on invalid file generation period + config.FilePollingPeriod.Duration = 1 + config.FileGenerationPeriod.Duration = -1 + assert.ErrorIs(t, config.Validate(), ErrFileGenerationPeriodInvalid) + + // success + config.FileGenerationPeriod.Duration = 1 + assert.NoError(t, config.Validate()) +} diff --git a/internal/app/connectors/dummypay/connector.go b/internal/app/connectors/dummypay/connector.go new file mode 100644 index 00000000..ca394866 --- /dev/null +++ b/internal/app/connectors/dummypay/connector.go @@ -0,0 +1,81 @@ +package dummypay + +import ( + "context" + "fmt" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/payments/internal/app/task" + + "github.com/formancehq/go-libs/logging" +) + +// Name is the name of the connector. +const Name = models.ConnectorProviderDummyPay + +// Connector is the connector for the dummy payment connector. +type Connector struct { + logger logging.Logger + cfg Config + fs fs +} + +// Install executes post-installation steps to read and generate files. +// It is called after the connector is installed. +func (c *Connector) Install(ctx task.ConnectorContext) error { + readFilesDescriptor, err := models.EncodeTaskDescriptor(newTaskReadFiles()) + if err != nil { + return fmt.Errorf("failed to create read files task descriptor: %w", err) + } + + if err = ctx.Scheduler().Schedule(readFilesDescriptor, true); err != nil { + return fmt.Errorf("failed to schedule task to read files: %w", err) + } + + generateFilesDescriptor, err := models.EncodeTaskDescriptor(newTaskGenerateFiles()) + if err != nil { + return fmt.Errorf("failed to create generate files task descriptor: %w", err) + } + + if err = ctx.Scheduler().Schedule(generateFilesDescriptor, true); err != nil { + return fmt.Errorf("failed to schedule task to generate files: %w", err) + } + + return nil +} + +// Uninstall executes pre-uninstallation steps to remove the generated files. +// It is called before the connector is uninstalled. +func (c *Connector) Uninstall(ctx context.Context) error { + c.logger.Infof("Removing generated files from '%s'...", c.cfg.Directory) + + err := removeFiles(c.cfg, c.fs) + if err != nil { + return fmt.Errorf("failed to remove generated files: %w", err) + } + + return nil +} + +// Resolve resolves a task execution request based on the task descriptor. +func (c *Connector) Resolve(descriptor models.TaskDescriptor) task.Task { + taskDescriptor, err := models.DecodeTaskDescriptor[TaskDescriptor](descriptor) + if err != nil { + panic(err) + } + + c.logger.Infof("Executing '%s' task...", taskDescriptor.Key) + + return handleResolve(c.cfg, taskDescriptor, c.fs) +} + +func newConnector(logger logging.Logger, cfg Config, fs fs) *Connector { + return &Connector{ + logger: logger.WithFields(map[string]any{ + "component": "connector", + }), + cfg: cfg, + fs: fs, + } +} diff --git a/internal/app/connectors/dummypay/connector_test.go b/internal/app/connectors/dummypay/connector_test.go new file mode 100644 index 00000000..39576448 --- /dev/null +++ b/internal/app/connectors/dummypay/connector_test.go @@ -0,0 +1,79 @@ +package dummypay + +import ( + "context" + "reflect" + "testing" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/payments/internal/app/task" + + "github.com/formancehq/go-libs/logging" + "github.com/stretchr/testify/assert" +) + +// Create a minimal mock for connector installation. +type ( + mockConnectorContext struct { + ctx context.Context + } + mockScheduler struct{} +) + +func (mcc *mockConnectorContext) Context() context.Context { + return mcc.ctx +} + +func (mcc mockScheduler) Schedule(p models.TaskDescriptor, restart bool) error { + return nil +} + +func (mcc *mockConnectorContext) Scheduler() task.Scheduler { + return mockScheduler{} +} + +func TestConnector(t *testing.T) { + t.Parallel() + + config := Config{} + logger := logging.GetLogger(context.Background()) + + fileSystem := newTestFS() + + connector := newConnector(logger, config, fileSystem) + + err := connector.Install(new(mockConnectorContext)) + assert.NoErrorf(t, err, "Install() failed") + + testCases := []struct { + key taskKey + task task.Task + }{ + {taskKeyReadFiles, taskReadFiles(config, fileSystem)}, + {taskKeyGenerateFiles, taskGenerateFiles(config, fileSystem)}, + {taskKeyIngest, taskIngest(config, TaskDescriptor{}, fileSystem)}, + } + + for _, testCase := range testCases { + var taskDescriptor models.TaskDescriptor + + taskDescriptor, err = models.EncodeTaskDescriptor(TaskDescriptor{Key: testCase.key}) + assert.NoErrorf(t, err, "EncodeTaskDescriptor() failed") + + assert.EqualValues(t, + reflect.ValueOf(testCase.task).String(), + reflect.ValueOf(connector.Resolve(taskDescriptor)).String(), + ) + } + + taskDescriptor, err := models.EncodeTaskDescriptor(TaskDescriptor{Key: "test"}) + assert.NoErrorf(t, err, "EncodeTaskDescriptor() failed") + + assert.EqualValues(t, + reflect.ValueOf(func() error { return nil }).String(), + reflect.ValueOf(connector.Resolve(taskDescriptor)).String(), + ) + + assert.NoError(t, connector.Uninstall(context.Background())) +} diff --git a/internal/app/connectors/dummypay/errors.go b/internal/app/connectors/dummypay/errors.go new file mode 100644 index 00000000..40c127aa --- /dev/null +++ b/internal/app/connectors/dummypay/errors.go @@ -0,0 +1,20 @@ +package dummypay + +import "github.com/pkg/errors" + +var ( + // ErrMissingDirectory is returned when the directory is missing. + ErrMissingDirectory = errors.New("missing directory to watch") + + // ErrFilePollingPeriodInvalid is returned when the file polling period is invalid. + ErrFilePollingPeriodInvalid = errors.New("file polling period is invalid") + + // ErrFileGenerationPeriodInvalid is returned when the file generation period is invalid. + ErrFileGenerationPeriodInvalid = errors.New("file generation period is invalid") + + // ErrMissingTask is returned when the task is missing. + ErrMissingTask = errors.New("task is not implemented") + + // ErrDurationInvalid is returned when the duration is invalid. + ErrDurationInvalid = errors.New("duration is invalid") +) diff --git a/internal/app/connectors/dummypay/fs.go b/internal/app/connectors/dummypay/fs.go new file mode 100644 index 00000000..5224ccda --- /dev/null +++ b/internal/app/connectors/dummypay/fs.go @@ -0,0 +1,12 @@ +package dummypay + +import ( + "github.com/spf13/afero" +) + +type fs afero.Fs + +// newFS creates a new file system access point. +func newFS() fs { + return afero.NewOsFs() +} diff --git a/internal/app/connectors/dummypay/fs_test.go b/internal/app/connectors/dummypay/fs_test.go new file mode 100644 index 00000000..36d374a5 --- /dev/null +++ b/internal/app/connectors/dummypay/fs_test.go @@ -0,0 +1,7 @@ +package dummypay + +import "github.com/spf13/afero" + +func newTestFS() fs { + return afero.NewMemMapFs() +} diff --git a/internal/app/connectors/dummypay/loader.go b/internal/app/connectors/dummypay/loader.go new file mode 100644 index 00000000..319caa3b --- /dev/null +++ b/internal/app/connectors/dummypay/loader.go @@ -0,0 +1,54 @@ +package dummypay + +import ( + "time" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/connectors" + "github.com/formancehq/payments/internal/app/integration" +) + +type Loader struct{} + +// Name returns the name of the connector. +func (l *Loader) Name() models.ConnectorProvider { + return Name +} + +// AllowTasks returns the amount of tasks that are allowed to be scheduled. +func (l *Loader) AllowTasks() int { + return 10 +} + +const ( + // defaultFilePollingPeriod is the default period between file polling. + defaultFilePollingPeriod = 10 * time.Second + + // defaultFileGenerationPeriod is the default period between file generation. + defaultFileGenerationPeriod = 5 * time.Second +) + +// ApplyDefaults applies default values to the configuration. +func (l *Loader) ApplyDefaults(cfg Config) Config { + if cfg.FileGenerationPeriod.Duration == 0 { + cfg.FileGenerationPeriod = connectors.Duration{Duration: defaultFileGenerationPeriod} + } + + if cfg.FilePollingPeriod.Duration == 0 { + cfg.FilePollingPeriod = connectors.Duration{Duration: defaultFilePollingPeriod} + } + + return cfg +} + +// Load returns the connector. +func (l *Loader) Load(logger logging.Logger, config Config) integration.Connector { + return newConnector(logger, config, newFS()) +} + +// NewLoader creates a new loader. +func NewLoader() *Loader { + return &Loader{} +} diff --git a/internal/app/connectors/dummypay/loader_test.go b/internal/app/connectors/dummypay/loader_test.go new file mode 100644 index 00000000..3e0776e1 --- /dev/null +++ b/internal/app/connectors/dummypay/loader_test.go @@ -0,0 +1,30 @@ +package dummypay + +import ( + "context" + "testing" + "time" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/connectors" + "github.com/stretchr/testify/assert" +) + +// TestLoader tests the loader. +func TestLoader(t *testing.T) { + t.Parallel() + + config := Config{} + logger := logging.GetLogger(context.Background()) + + loader := NewLoader() + + assert.Equal(t, Name, loader.Name()) + assert.Equal(t, 10, loader.AllowTasks()) + assert.Equal(t, Config{ + FilePollingPeriod: connectors.Duration{Duration: 10 * time.Second}, + FileGenerationPeriod: connectors.Duration{Duration: 5 * time.Second}, + }, loader.ApplyDefaults(config)) + + assert.EqualValues(t, newConnector(logger, config, newFS()), loader.Load(logger, config)) +} diff --git a/internal/app/connectors/dummypay/payment.go b/internal/app/connectors/dummypay/payment.go new file mode 100644 index 00000000..96026e1a --- /dev/null +++ b/internal/app/connectors/dummypay/payment.go @@ -0,0 +1,19 @@ +package dummypay + +import ( + "encoding/json" + + "github.com/formancehq/payments/internal/app/models" +) + +// payment represents a payment structure used in the generated files. +type payment struct { + Reference string `json:"reference"` + Amount int64 `json:"amount"` + Type models.PaymentType `json:"type"` + Status models.PaymentStatus `json:"status"` + Scheme models.PaymentScheme `json:"scheme"` + Asset models.PaymentAsset `json:"asset"` + + RawData json.RawMessage `json:"rawData"` +} diff --git a/internal/app/connectors/dummypay/remove_files.go b/internal/app/connectors/dummypay/remove_files.go new file mode 100644 index 00000000..16e27188 --- /dev/null +++ b/internal/app/connectors/dummypay/remove_files.go @@ -0,0 +1,33 @@ +package dummypay + +import ( + "fmt" + "strings" + + "github.com/spf13/afero" +) + +// removeFiles removes all files from the given directory. +// Only removes files that has generatedFilePrefix in the name. +func removeFiles(config Config, fs fs) error { + dir, err := afero.ReadDir(fs, config.Directory) + if err != nil { + return fmt.Errorf("failed to open directory '%s': %w", config.Directory, err) + } + + // iterate over all files in the directory + for _, file := range dir { + // skip files that do not match the generatedFilePrefix + if !strings.HasPrefix(file.Name(), generatedFilePrefix) { + continue + } + + // remove the file + err = fs.Remove(fmt.Sprintf("%s/%s", config.Directory, file.Name())) + if err != nil { + return fmt.Errorf("failed to remove file '%s': %w", file.Name(), err) + } + } + + return nil +} diff --git a/internal/app/connectors/dummypay/task_descriptor.go b/internal/app/connectors/dummypay/task_descriptor.go new file mode 100644 index 00000000..eae5fd83 --- /dev/null +++ b/internal/app/connectors/dummypay/task_descriptor.go @@ -0,0 +1,34 @@ +package dummypay + +import ( + "fmt" + + "github.com/formancehq/payments/internal/app/task" +) + +// taskKey defines a unique key of the task. +type taskKey string + +// TaskDescriptor represents a task descriptor. +type TaskDescriptor struct { + Name string `json:"name" bson:"name" yaml:"name"` + Key taskKey `json:"key" bson:"key" yaml:"key"` + FileName string `json:"fileName" bson:"fileName" yaml:"fileName"` +} + +// handleResolve resolves a task execution request based on the task descriptor. +func handleResolve(config Config, descriptor TaskDescriptor, fs fs) task.Task { + switch descriptor.Key { + case taskKeyReadFiles: + return taskReadFiles(config, fs) + case taskKeyIngest: + return taskIngest(config, descriptor, fs) + case taskKeyGenerateFiles: + return taskGenerateFiles(config, fs) + } + + // This should never happen. + return func() error { + return fmt.Errorf("key '%s': %w", descriptor.Key, ErrMissingTask) + } +} diff --git a/internal/app/connectors/dummypay/task_generate_file.go b/internal/app/connectors/dummypay/task_generate_file.go new file mode 100644 index 00000000..e9740c15 --- /dev/null +++ b/internal/app/connectors/dummypay/task_generate_file.go @@ -0,0 +1,166 @@ +package dummypay + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "os" + "time" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/payments/internal/app/task" +) + +const ( + taskKeyGenerateFiles = "generate-files" + asset = "DUMMYCOIN" + generatedFilePrefix = "dummypay-generated-file" +) + +// newTaskGenerateFiles returns a new task descriptor for the task that generates files. +func newTaskGenerateFiles() TaskDescriptor { + return TaskDescriptor{ + Name: "Generate files into a directory", + Key: taskKeyGenerateFiles, + } +} + +// taskGenerateFiles generates payment files to a given directory. +func taskGenerateFiles(config Config, fs fs) task.Task { + return func(ctx context.Context) error { + for { + select { + case <-ctx.Done(): + return nil + case <-time.After(config.FileGenerationPeriod.Duration): + err := generateFile(config, fs) + if err != nil { + return err + } + } + } + } +} + +func generateFile(config Config, fs fs) error { + err := fs.Mkdir(config.Directory, 0o777) //nolint:gomnd + if err != nil && !os.IsExist(err) { + return fmt.Errorf( + "failed to create dummypay config directory '%s': %w", config.Directory, err) + } + + key := fmt.Sprintf("%s-%d", generatedFilePrefix, time.Now().UnixNano()) + fileKey := fmt.Sprintf("%s/%s.json", config.Directory, key) + + var paymentObj payment + + // Generate a random payment. + paymentObj.Reference = key + paymentObj.Type = generateRandomType() + paymentObj.Status = generateRandomStatus() + paymentObj.Amount = int64(generateRandomNumber()) + paymentObj.Asset = asset + paymentObj.Scheme = generateRandomScheme() + + file, err := fs.Create(fileKey) + if err != nil { + return fmt.Errorf("failed to create file: %w", err) + } + + // Encode the payment object as JSON to a new file. + err = json.NewEncoder(file).Encode(&paymentObj) + if err != nil { + // Close the file before returning. + if fileCloseErr := file.Close(); fileCloseErr != nil { + return fmt.Errorf("failed to close file: %w", fileCloseErr) + } + + return fmt.Errorf("failed to encode json into file: %w", err) + } + + // Close the file. + if err = file.Close(); err != nil { + return fmt.Errorf("failed to close file: %w", err) + } + + return nil +} + +// nMax is the maximum number that can be generated +// with the minimum being 0. +const nMax = 10000 + +// generateRandomNumber generates a random number between 0 and nMax. +func generateRandomNumber() int { + rand.Seed(time.Now().UnixNano()) + + //nolint:gosec // allow weak random number generator as it is not used for security + value := rand.Intn(nMax) + + return value +} + +// generateRandomType generates a random payment type. +func generateRandomType() models.PaymentType { + // 50% chance. + paymentType := models.PaymentTypePayIn + + // 50% chance. + if generateRandomNumber() > nMax/2 { + paymentType = models.PaymentTypePayOut + } + + return paymentType +} + +// generateRandomStatus generates a random payment status. +func generateRandomStatus() models.PaymentStatus { + // ~50% chance. + paymentStatus := models.PaymentStatusSucceeded + + num := generateRandomNumber() + + switch { + case num < nMax/4: // 25% chance + paymentStatus = models.PaymentStatusPending + case num < nMax/3: // ~9% chance + paymentStatus = models.PaymentStatusFailed + case num < nMax/2: // ~16% chance + paymentStatus = models.PaymentStatusCancelled + } + + return paymentStatus +} + +// generateRandomScheme generates a random payment scheme. +func generateRandomScheme() models.PaymentScheme { + num := generateRandomNumber() / 1000 //nolint:gomnd // allow for random number + + paymentScheme := models.PaymentSchemeCardMasterCard + + availableSchemes := []models.PaymentScheme{ + models.PaymentSchemeCardMasterCard, + models.PaymentSchemeCardVisa, + models.PaymentSchemeCardDiscover, + models.PaymentSchemeCardJCB, + models.PaymentSchemeCardUnionPay, + models.PaymentSchemeCardAmex, + models.PaymentSchemeCardDiners, + models.PaymentSchemeSepaDebit, + models.PaymentSchemeSepaCredit, + models.PaymentSchemeApplePay, + models.PaymentSchemeGooglePay, + models.PaymentSchemeA2A, + models.PaymentSchemeACHDebit, + models.PaymentSchemeACH, + models.PaymentSchemeRTP, + } + + if num < len(availableSchemes) { + paymentScheme = availableSchemes[num] + } + + return paymentScheme +} diff --git a/internal/app/connectors/dummypay/task_ingest.go b/internal/app/connectors/dummypay/task_ingest.go new file mode 100644 index 00000000..7bb313f3 --- /dev/null +++ b/internal/app/connectors/dummypay/task_ingest.go @@ -0,0 +1,74 @@ +package dummypay + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + + "github.com/formancehq/payments/internal/app/ingestion" + "github.com/formancehq/payments/internal/app/models" + "github.com/formancehq/payments/internal/app/task" +) + +const taskKeyIngest = "ingest" + +// newTaskIngest returns a new task descriptor for the ingest task. +func newTaskIngest(filePath string) TaskDescriptor { + return TaskDescriptor{ + Name: "Ingest payments from read files", + Key: taskKeyIngest, + FileName: filePath, + } +} + +// taskIngest ingests a payment file. +func taskIngest(config Config, descriptor TaskDescriptor, fs fs) task.Task { + return func(ctx context.Context, ingester ingestion.Ingester) error { + ingestionPayload, err := parseIngestionPayload(config, descriptor, fs) + if err != nil { + return err + } + + // Ingest the payment into the system. + err = ingester.IngestPayments(ctx, ingestionPayload, struct{}{}) + if err != nil { + return fmt.Errorf("failed to ingest file '%s': %w", descriptor.FileName, err) + } + + return nil + } +} + +func parseIngestionPayload(config Config, descriptor TaskDescriptor, fs fs) (ingestion.PaymentBatch, error) { + // Open the file. + file, err := fs.Open(filepath.Join(config.Directory, descriptor.FileName)) + if err != nil { + return nil, fmt.Errorf("failed to open file '%s': %w", descriptor.FileName, err) + } + + defer file.Close() + + var paymentElement payment + + // Decode the JSON file. + err = json.NewDecoder(file).Decode(&paymentElement) + if err != nil { + return nil, fmt.Errorf("failed to decode file '%s': %w", descriptor.FileName, err) + } + + ingestionPayload := ingestion.PaymentBatch{ingestion.PaymentBatchElement{ + Payment: &models.Payment{ + Reference: paymentElement.Reference, + Amount: paymentElement.Amount, + Type: paymentElement.Type, + Status: paymentElement.Status, + Scheme: paymentElement.Scheme, + Asset: paymentElement.Asset, + RawData: paymentElement.RawData, + }, + Update: true, + }} + + return ingestionPayload, nil +} diff --git a/internal/app/connectors/dummypay/task_read_files.go b/internal/app/connectors/dummypay/task_read_files.go new file mode 100644 index 00000000..a88c0aa9 --- /dev/null +++ b/internal/app/connectors/dummypay/task_read_files.go @@ -0,0 +1,78 @@ +package dummypay + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/task" + "github.com/spf13/afero" +) + +const taskKeyReadFiles = "read-files" + +// newTaskReadFiles creates a new task descriptor for the taskReadFiles task. +func newTaskReadFiles() TaskDescriptor { + return TaskDescriptor{ + Name: "Read Files from directory", + Key: taskKeyReadFiles, + } +} + +// taskReadFiles creates a task that reads files from a given directory. +// Only reads files with the generatedFilePrefix in their name. +func taskReadFiles(config Config, fs fs) task.Task { + return func(ctx context.Context, logger logging.Logger, + scheduler task.Scheduler, + ) error { + for { + select { + case <-ctx.Done(): + return nil + case <-time.After(config.FilePollingPeriod.Duration): + files, err := parseFilesToIngest(config, fs) + if err != nil { + return fmt.Errorf("error parsing files to ingest: %w", err) + } + + for _, file := range files { + descriptor, err := models.EncodeTaskDescriptor(newTaskIngest(file)) + if err != nil { + return err + } + + // schedule a task to ingest the file into the payments system. + err = scheduler.Schedule(descriptor, true) + if err != nil { + return fmt.Errorf("failed to schedule task to ingest file '%s': %w", file, err) + } + } + } + } + } +} + +func parseFilesToIngest(config Config, fs fs) ([]string, error) { + dir, err := afero.ReadDir(fs, config.Directory) + if err != nil { + return nil, fmt.Errorf("error reading directory '%s': %w", config.Directory, err) + } + + var files []string //nolint:prealloc // length is unknown + + // iterate over all files in the directory. + for _, file := range dir { + // skip files that do not match the generatedFilePrefix. + if !strings.HasPrefix(file.Name(), generatedFilePrefix) { + continue + } + + files = append(files, file.Name()) + } + + return files, nil +} diff --git a/internal/app/connectors/dummypay/task_test.go b/internal/app/connectors/dummypay/task_test.go new file mode 100644 index 00000000..1c5169b2 --- /dev/null +++ b/internal/app/connectors/dummypay/task_test.go @@ -0,0 +1,41 @@ +package dummypay + +import ( + "testing" + + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" +) + +func TestTasks(t *testing.T) { + t.Parallel() + + config := Config{Directory: "/tmp"} + fs := newTestFS() + + // test generating files + err := generateFile(config, fs) + assert.NoError(t, err) + + files, err := afero.ReadDir(fs, config.Directory) + assert.NoError(t, err) + assert.Len(t, files, 1) + + // test reading files + filesList, err := parseFilesToIngest(config, fs) + assert.NoError(t, err) + assert.Len(t, filesList, 1) + + // test ingesting files + payload, err := parseIngestionPayload(config, TaskDescriptor{Key: taskKeyIngest, FileName: files[0].Name()}, fs) + assert.NoError(t, err) + assert.Len(t, payload, 1) + + // test removing files + err = removeFiles(config, fs) + assert.NoError(t, err) + + files, err = afero.ReadDir(fs, config.Directory) + assert.NoError(t, err) + assert.Len(t, files, 0) +} diff --git a/internal/app/connectors/duration.go b/internal/app/connectors/duration.go new file mode 100644 index 00000000..8c4e27d5 --- /dev/null +++ b/internal/app/connectors/duration.go @@ -0,0 +1,41 @@ +package connectors + +import ( + "encoding/json" + "fmt" + "time" +) + +type Duration struct { + time.Duration `json:"duration"` +} + +func (d *Duration) UnmarshalJSON(b []byte) error { + var rawValue any + + if err := json.Unmarshal(b, &rawValue); err != nil { + return fmt.Errorf("custom Duration UnmarshalJSON: %w", err) + } + + switch value := rawValue.(type) { + case string: + var err error + d.Duration, err = time.ParseDuration(value) + if err != nil { + return fmt.Errorf("custom Duration UnmarshalJSON: time.ParseDuration: %w", err) + } + + return nil + case map[string]interface{}: + switch val := value["duration"].(type) { + case float64: + d.Duration = time.Duration(int64(val)) + + return nil + default: + return fmt.Errorf("custom Duration UnmarshalJSON from map: invalid type: value:%v, type:%T", val, val) + } + default: + return fmt.Errorf("custom Duration UnmarshalJSON: invalid type: value:%v, type:%T", value, value) + } +} diff --git a/internal/app/connectors/modulr/client/accounts.go b/internal/app/connectors/modulr/client/accounts.go new file mode 100644 index 00000000..194e250d --- /dev/null +++ b/internal/app/connectors/modulr/client/accounts.go @@ -0,0 +1,44 @@ +package client + +import ( + "encoding/json" + "fmt" + "net/http" +) + +//nolint:tagliatelle // allow for clients +type Account struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Balance string `json:"balance"` + Currency string `json:"currency"` + CustomerID string `json:"customerId"` + Identifiers []struct { + AccountNumber string `json:"accountNumber"` + SortCode string `json:"sortCode"` + Type string `json:"type"` + } `json:"identifiers"` + DirectDebit bool `json:"directDebit"` + CreatedDate string `json:"createdDate"` +} + +func (m *Client) GetAccounts() ([]Account, error) { + resp, err := m.httpClient.Get(m.buildEndpoint("accounts")) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + var res responseWrapper[[]Account] + if err = json.NewDecoder(resp.Body).Decode(&res); err != nil { + return nil, err + } + + return res.Content, nil +} diff --git a/internal/app/connectors/modulr/client/client.go b/internal/app/connectors/modulr/client/client.go new file mode 100644 index 00000000..06a050de --- /dev/null +++ b/internal/app/connectors/modulr/client/client.go @@ -0,0 +1,62 @@ +package client + +import ( + "fmt" + "net/http" + + "github.com/formancehq/payments/internal/app/connectors/modulr/hmac" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" +) + +type apiTransport struct { + apiKey string + headers map[string]string + underlying http.RoundTripper +} + +func (t *apiTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Add("Authorization", t.apiKey) + + return t.underlying.RoundTrip(req) +} + +type responseWrapper[t any] struct { + Content t `json:"content"` + Size int `json:"size"` + TotalSize int `json:"totalSize"` + Page int `json:"page"` + TotalPages int `json:"totalPages"` +} + +type Client struct { + httpClient *http.Client + endpoint string +} + +func (m *Client) buildEndpoint(path string, args ...interface{}) string { + return fmt.Sprintf("%s/%s", m.endpoint, fmt.Sprintf(path, args...)) +} + +const sandboxAPIEndpoint = "https://api-sandbox.modulrfinance.com/api-sandbox-token" + +func NewClient(apiKey, apiSecret, endpoint string) (*Client, error) { + if endpoint == "" { + endpoint = sandboxAPIEndpoint + } + + headers, err := hmac.GenerateHeaders(apiKey, apiSecret, "", false) + if err != nil { + return nil, fmt.Errorf("failed to generate headers: %w", err) + } + + return &Client{ + httpClient: &http.Client{ + Transport: &apiTransport{ + headers: headers, + apiKey: apiKey, + underlying: otelhttp.NewTransport(http.DefaultTransport), + }, + }, + endpoint: endpoint, + }, nil +} diff --git a/internal/app/connectors/modulr/client/transactions.go b/internal/app/connectors/modulr/client/transactions.go new file mode 100644 index 00000000..a9b90b3d --- /dev/null +++ b/internal/app/connectors/modulr/client/transactions.go @@ -0,0 +1,41 @@ +package client + +import ( + "encoding/json" + "fmt" + "net/http" +) + +//nolint:tagliatelle // allow different styled tags in client +type Transaction struct { + ID string `json:"id"` + Type string `json:"type"` + Amount float64 `json:"amount"` + Credit bool `json:"credit"` + SourceID string `json:"sourceId"` + Description string `json:"description"` + PostedDate string `json:"postedDate"` + TransactionDate string `json:"transactionDate"` + Account Account `json:"account"` + AdditionalInfo interface{} `json:"additionalInfo"` +} + +func (m *Client) GetTransactions(accountID string) ([]Transaction, error) { + resp, err := m.httpClient.Get(m.buildEndpoint("accounts/%s/transactions", accountID)) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + var res responseWrapper[[]Transaction] + if err = json.NewDecoder(resp.Body).Decode(&res); err != nil { + return nil, err + } + + return res.Content, nil +} diff --git a/internal/app/connectors/modulr/config.go b/internal/app/connectors/modulr/config.go new file mode 100644 index 00000000..da0ba040 --- /dev/null +++ b/internal/app/connectors/modulr/config.go @@ -0,0 +1,39 @@ +package modulr + +import ( + "encoding/json" + + "github.com/formancehq/payments/internal/app/connectors/configtemplate" +) + +type Config struct { + APIKey string `json:"apiKey" bson:"apiKey"` + APISecret string `json:"apiSecret" bson:"apiSecret"` + Endpoint string `json:"endpoint" bson:"endpoint"` +} + +func (c Config) Validate() error { + if c.APIKey == "" { + return ErrMissingAPIKey + } + + if c.APISecret == "" { + return ErrMissingAPISecret + } + + return nil +} + +func (c Config) Marshal() ([]byte, error) { + return json.Marshal(c) +} + +func (c Config) BuildTemplate() (string, configtemplate.Config) { + cfg := configtemplate.NewConfig() + + cfg.AddParameter("apiKey", configtemplate.TypeString, true) + cfg.AddParameter("apiSecret", configtemplate.TypeString, true) + cfg.AddParameter("endpoint", configtemplate.TypeString, false) + + return Name.String(), cfg +} diff --git a/internal/app/connectors/modulr/connector.go b/internal/app/connectors/modulr/connector.go new file mode 100644 index 00000000..f660d132 --- /dev/null +++ b/internal/app/connectors/modulr/connector.go @@ -0,0 +1,54 @@ +package modulr + +import ( + "context" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/integration" + "github.com/formancehq/payments/internal/app/task" +) + +const Name = models.ConnectorProviderModulr + +type Connector struct { + logger logging.Logger + cfg Config +} + +func (c *Connector) Install(ctx task.ConnectorContext) error { + taskDescriptor, err := models.EncodeTaskDescriptor(TaskDescriptor{ + Name: "Fetch accounts from client", + Key: taskNameFetchAccounts, + }) + if err != nil { + return err + } + + return ctx.Scheduler().Schedule(taskDescriptor, false) +} + +func (c *Connector) Uninstall(ctx context.Context) error { + return nil +} + +func (c *Connector) Resolve(descriptor models.TaskDescriptor) task.Task { + taskDescriptor, err := models.DecodeTaskDescriptor[TaskDescriptor](descriptor) + if err != nil { + return nil + } + + return resolveTasks(c.logger, c.cfg)(taskDescriptor) +} + +var _ integration.Connector = &Connector{} + +func newConnector(logger logging.Logger, cfg Config) *Connector { + return &Connector{ + logger: logger.WithFields(map[string]any{ + "component": "connector", + }), + cfg: cfg, + } +} diff --git a/internal/app/connectors/modulr/errors.go b/internal/app/connectors/modulr/errors.go new file mode 100644 index 00000000..9eda9006 --- /dev/null +++ b/internal/app/connectors/modulr/errors.go @@ -0,0 +1,14 @@ +package modulr + +import "github.com/pkg/errors" + +var ( + // ErrMissingTask is returned when the task is missing. + ErrMissingTask = errors.New("task is not implemented") + + // ErrMissingAPIKey is returned when the api key is missing from config. + ErrMissingAPIKey = errors.New("missing apiKey from config") + + // ErrMissingAPISecret is returned when the api secret is missing from config. + ErrMissingAPISecret = errors.New("missing apiSecret from config") +) diff --git a/internal/app/connectors/modulr/hmac/hmac.go b/internal/app/connectors/modulr/hmac/hmac.go new file mode 100644 index 00000000..f3bb8e31 --- /dev/null +++ b/internal/app/connectors/modulr/hmac/hmac.go @@ -0,0 +1,59 @@ +package hmac + +import ( + "time" + + "github.com/google/uuid" + "github.com/pkg/errors" +) + +const ( + authorizationHeader = "Authorization" + dateHeader = "Date" + emptyString = "" + nonceHeader = "x-mod-nonce" + retry = "x-mod-retry" + retryTrue = "true" + retryFalse = "false" +) + +var ErrInvalidCredentials = errors.New("invalid api credentials") + +func GenerateHeaders(apiKey, apiSecret, nonce string, hasRetry bool) (map[string]string, error) { + if apiKey == "" || apiSecret == "" { + return nil, ErrInvalidCredentials + } + + return constructHeadersMap(apiKey, apiSecret, nonce, hasRetry, time.Now()), nil +} + +func constructHeadersMap(apiKey, apiSecret, nonce string, hasRetry bool, + timestamp time.Time, +) map[string]string { + headers := make(map[string]string) + date := timestamp.Format(time.RFC1123) + nonce = generateNonceIfEmpty(nonce) + + headers[dateHeader] = date + headers[authorizationHeader] = buildSignature(apiKey, apiSecret, nonce, date) + headers[nonceHeader] = nonce + headers[retry] = parseRetryBool(hasRetry) + + return headers +} + +func generateNonceIfEmpty(nonce string) string { + if nonce == emptyString { + nonce = uuid.New().String() + } + + return nonce +} + +func parseRetryBool(hasRetry bool) string { + if hasRetry { + return retryTrue + } + + return retryFalse +} diff --git a/internal/app/connectors/modulr/hmac/hmac_test.go b/internal/app/connectors/modulr/hmac/hmac_test.go new file mode 100644 index 00000000..456ae503 --- /dev/null +++ b/internal/app/connectors/modulr/hmac/hmac_test.go @@ -0,0 +1,71 @@ +package hmac + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestGenerateReturnsAnHMACString(t *testing.T) { + t.Parallel() + + headers, _ := GenerateHeaders("api_key", "api_secret", "", false) + expectedSignature := "Signature keyId=\"api_key\",algorithm=\"hmac-sha1\",headers=\"date x-mod-nonce\",signature=\"" + assert.Equal(t, expectedSignature, headers["Authorization"][0:86], "generate should return the hmac headers") +} + +func TestGenerateReturnsADateHeader(t *testing.T) { + t.Parallel() + + timestamp := time.Date(2020, 1, 2, 15, 4, 5, 0, time.UTC) + + headers := constructHeadersMap("api_key", "api_secret", "", false, timestamp) + + expectedDate := "Thu, 02 Jan 2020 15:04:05 UTC" + + assert.Equal(t, expectedDate, headers["Date"]) +} + +func TestGenerateReturnsANonceHeaderWithExpectedValue(t *testing.T) { + t.Parallel() + + nonce := "thisIsTheNonce" + headers, _ := GenerateHeaders("api_key", "api_secret", nonce, false) + assert.Equal(t, nonce, headers["x-mod-nonce"]) +} + +func TestGenerateReturnsARetryHeaderWithTrueIfRetryIsExpected(t *testing.T) { + t.Parallel() + + headers, _ := GenerateHeaders("api_key", "api_secret", "", true) + assert.Equal(t, "true", headers["x-mod-retry"]) +} + +func TestGenerateReturnsARetryHeaderWithFalseIfRetryIsNotExpected(t *testing.T) { + t.Parallel() + + headers, _ := GenerateHeaders("api_key", "api_secret", "", false) + assert.Equal(t, "false", headers["x-mod-retry"]) +} + +func TestGenerateReturnsAGeneratedNonceHeaderIfNonceIsEmpty(t *testing.T) { + t.Parallel() + + headers, _ := GenerateHeaders("api_key", "api_secret", "", false) + assert.True(t, headers["x-mod-nonce"] != "", "x-mod-nonce header should have been populated") +} + +func TestGenerateThrowsErrorIfApiKeyIsNull(t *testing.T) { + t.Parallel() + + _, err := GenerateHeaders("", "api_secret", "", false) + assert.ErrorIs(t, err, ErrInvalidCredentials) +} + +func TestGenerateThrowsErrorIfApiSecretIsNull(t *testing.T) { + t.Parallel() + + _, err := GenerateHeaders("api_key", "", "", false) + assert.ErrorIs(t, err, ErrInvalidCredentials) +} diff --git a/internal/app/connectors/modulr/hmac/signature_generator.go b/internal/app/connectors/modulr/hmac/signature_generator.go new file mode 100644 index 00000000..70658950 --- /dev/null +++ b/internal/app/connectors/modulr/hmac/signature_generator.go @@ -0,0 +1,32 @@ +package hmac + +import ( + "crypto/hmac" + "crypto/sha1" //nolint:gosec // we need sha1 for the hmac + "encoding/base64" + "net/url" +) + +const ( + algorithm = "algorithm=\"hmac-sha1\"," + datePrefix = "date: " + headers = "headers=\"date x-mod-nonce\"," + prefix = "signature=\"" + suffix = "\"" + newline = "\n" + nonceKey = "x-mod-nonce: " + keyIDPrefix = "Signature keyId=\"" +) + +func buildSignature(apiKey, apiSecret, nonce, date string) string { + keyID := keyIDPrefix + apiKey + "\"," + + mac := hmac.New(sha1.New, []byte(apiSecret)) + mac.Write([]byte(datePrefix + date + newline + nonceKey + nonce)) + + encodedMac := mac.Sum(nil) + base64Encoded := base64.StdEncoding.EncodeToString(encodedMac) + encodedSignature := prefix + url.QueryEscape(base64Encoded) + suffix + + return keyID + algorithm + headers + encodedSignature +} diff --git a/internal/app/connectors/modulr/hmac/signature_test.go b/internal/app/connectors/modulr/hmac/signature_test.go new file mode 100644 index 00000000..1670098f --- /dev/null +++ b/internal/app/connectors/modulr/hmac/signature_test.go @@ -0,0 +1,68 @@ +package hmac + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestGenerateReturnsSignatureWithKeyId(t *testing.T) { + t.Parallel() + + signature := buildSignature("api_key", "api_secret", "", date()) + expectedPrefix := "Signature keyId=\"api_key\"," + hasKeyID := strings.HasPrefix(signature, expectedPrefix) + assert.True(t, hasKeyID, "HMAC signature must contain the keyId") +} + +func TestGenerateReturnsSignatureWithAlgorithm(t *testing.T) { + t.Parallel() + + signature := buildSignature("api_key", "api_secret", "", date()) + expectedAlgorithm := "algorithm=\"hmac-sha1\"," + actualValue := signature[26:48] + assert.Equal(t, expectedAlgorithm, actualValue, "HMAC signature must contain the algorithm used") +} + +func TestGenerateReturnsSignatureWithHeaders(t *testing.T) { + t.Parallel() + + signature := buildSignature("api_key", "api_secret", "", date()) + expectedHeaders := "headers=\"date x-mod-nonce\"," + actualValue := signature[48:75] + assert.Equal(t, expectedHeaders, actualValue, "HMAC signature must contain the headers") +} + +func TestGenerateReturnsSignatureWithSignatureValue(t *testing.T) { + t.Parallel() + + signature := buildSignature("api_key", "api_secret", "", date()) + expectedSignature := "signature=\"" + actualValue := signature[75:86] + assert.Equal(t, expectedSignature, actualValue, "HMAC signature must contain the signature") +} + +func TestGenerateReturnsHashedSignature(t *testing.T) { + t.Parallel() + + signature := buildSignature("api_key", "api_secret", "", date()) + actualValue := signature[86:117] + assert.True(t, actualValue != "", "Encoded HMAC signature should be present") +} + +func TestGenerateAcceptsANonce(t *testing.T) { + t.Parallel() + + signature := buildSignature("api_key", "api_secret", "nonce", date()) + actualValue := signature[86:116] + expected := "9V8gi5Mp9MsL%2FO7mV6qZlBM9%2FR" + assert.Equal(t, expected, actualValue, "HMAC signature must contain the signature") +} + +func date() string { + now, _ := time.Parse(time.RFC1123, "Mon, 02 Jan 2020 15:04:05 GMT") + + return now.String() +} diff --git a/internal/app/connectors/modulr/loader.go b/internal/app/connectors/modulr/loader.go new file mode 100644 index 00000000..8c7bf028 --- /dev/null +++ b/internal/app/connectors/modulr/loader.go @@ -0,0 +1,32 @@ +package modulr + +import ( + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/integration" + "github.com/formancehq/payments/internal/app/models" +) + +type Loader struct{} + +const allowedTasks = 50 + +func (l *Loader) AllowTasks() int { + return allowedTasks +} + +func (l *Loader) Name() models.ConnectorProvider { + return Name +} + +func (l *Loader) Load(logger logging.Logger, config Config) integration.Connector { + return newConnector(logger, config) +} + +func (l *Loader) ApplyDefaults(cfg Config) Config { + return cfg +} + +// NewLoader creates a new loader. +func NewLoader() *Loader { + return &Loader{} +} diff --git a/internal/app/connectors/modulr/task_fetch_accounts.go b/internal/app/connectors/modulr/task_fetch_accounts.go new file mode 100644 index 00000000..62af588b --- /dev/null +++ b/internal/app/connectors/modulr/task_fetch_accounts.go @@ -0,0 +1,46 @@ +package modulr + +import ( + "context" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/payments/internal/app/connectors/modulr/client" + "github.com/formancehq/payments/internal/app/task" + + "github.com/formancehq/go-libs/logging" +) + +func taskFetchAccounts(logger logging.Logger, client *client.Client) task.Task { + return func( + ctx context.Context, + scheduler task.Scheduler, + ) error { + logger.Info(taskNameFetchAccounts) + + accounts, err := client.GetAccounts() + if err != nil { + return err + } + + for _, account := range accounts { + logger.Infof("scheduling fetch-transactions: %s", account.ID) + + transactionsTask, err := models.EncodeTaskDescriptor(TaskDescriptor{ + Name: "Fetch transactions from client by account", + Key: taskNameFetchTransactions, + AccountID: account.ID, + }) + if err != nil { + return err + } + + err = scheduler.Schedule(transactionsTask, false) + if err != nil { + return err + } + } + + return nil + } +} diff --git a/internal/app/connectors/modulr/task_fetch_transactions.go b/internal/app/connectors/modulr/task_fetch_transactions.go new file mode 100644 index 00000000..d767ccf6 --- /dev/null +++ b/internal/app/connectors/modulr/task_fetch_transactions.go @@ -0,0 +1,76 @@ +package modulr + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/payments/internal/app/connectors/modulr/client" + "github.com/formancehq/payments/internal/app/ingestion" + "github.com/formancehq/payments/internal/app/task" + + "github.com/formancehq/go-libs/logging" +) + +func taskFetchTransactions(logger logging.Logger, client *client.Client, accountID string) task.Task { + return func( + ctx context.Context, + ingester ingestion.Ingester, + ) error { + logger.Info("Fetching transactions for account", accountID) + + transactions, err := client.GetTransactions(accountID) + if err != nil { + return err + } + + batch := ingestion.PaymentBatch{} + + for _, transaction := range transactions { + logger.Info(transaction) + + rawData, err := json.Marshal(transaction) + if err != nil { + return fmt.Errorf("failed to marshal transaction: %w", err) + } + + batchElement := ingestion.PaymentBatchElement{ + Payment: &models.Payment{ + Reference: transaction.ID, + Type: matchTransactionType(transaction.Type), + Status: models.PaymentStatusSucceeded, + Scheme: models.PaymentSchemeOther, + Amount: int64(transaction.Amount * 100), + Asset: models.PaymentAsset(fmt.Sprintf("%s/2", transaction.Account.Currency)), + RawData: rawData, + }, + } + + batch = append(batch, batchElement) + } + + return ingester.IngestPayments(ctx, batch, struct{}{}) + } +} + +func matchTransactionType(transactionType string) models.PaymentType { + if transactionType == "PI_REV" || + transactionType == "PO_REV" || + transactionType == "ADHOC" || + transactionType == "INT_INTERC" { + return models.PaymentTypeOther + } + + if strings.HasPrefix(transactionType, "PI_") { + return models.PaymentTypePayIn + } + + if strings.HasPrefix(transactionType, "PO_") { + return models.PaymentTypePayOut + } + + return models.PaymentTypeOther +} diff --git a/internal/app/connectors/modulr/task_resolve.go b/internal/app/connectors/modulr/task_resolve.go new file mode 100644 index 00000000..121cb99f --- /dev/null +++ b/internal/app/connectors/modulr/task_resolve.go @@ -0,0 +1,47 @@ +package modulr + +import ( + "fmt" + + "github.com/formancehq/payments/internal/app/connectors/modulr/client" + "github.com/formancehq/payments/internal/app/task" + + "github.com/formancehq/go-libs/logging" +) + +const ( + taskNameFetchTransactions = "fetch-transactions" + taskNameFetchAccounts = "fetch-accounts" +) + +// TaskDescriptor is the definition of a task. +type TaskDescriptor struct { + Name string `json:"name" yaml:"name" bson:"name"` + Key string `json:"key" yaml:"key" bson:"key"` + AccountID string `json:"accountID" yaml:"accountID" bson:"accountID"` +} + +func resolveTasks(logger logging.Logger, config Config) func(taskDefinition TaskDescriptor) task.Task { + modulrClient, err := client.NewClient(config.APIKey, config.APISecret, config.Endpoint) + if err != nil { + return func(taskDefinition TaskDescriptor) task.Task { + return func() error { + return fmt.Errorf("key '%s': %w", taskDefinition.Key, ErrMissingTask) + } + } + } + + return func(taskDefinition TaskDescriptor) task.Task { + switch taskDefinition.Key { + case taskNameFetchAccounts: + return taskFetchAccounts(logger, modulrClient) + case taskNameFetchTransactions: + return taskFetchTransactions(logger, modulrClient, taskDefinition.AccountID) + } + + // This should never happen. + return func() error { + return fmt.Errorf("key '%s': %w", taskDefinition.Key, ErrMissingTask) + } + } +} diff --git a/internal/app/connectors/stripe/client.go b/internal/app/connectors/stripe/client.go new file mode 100644 index 00000000..3b67826d --- /dev/null +++ b/internal/app/connectors/stripe/client.go @@ -0,0 +1,124 @@ +package stripe + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + + "github.com/pkg/errors" + "github.com/stripe/stripe-go/v72" +) + +type ClientOption interface { + apply(req *http.Request) +} +type ClientOptionFn func(req *http.Request) + +func (fn ClientOptionFn) apply(req *http.Request) { + fn(req) +} + +func QueryParam(key, value string) ClientOptionFn { + return func(req *http.Request) { + q := req.URL.Query() + q.Set(key, value) + req.URL.RawQuery = q.Encode() + } +} + +type Client interface { + BalanceTransactions(ctx context.Context, options ...ClientOption) ([]*stripe.BalanceTransaction, bool, error) + ForAccount(account string) Client +} + +type DefaultClient struct { + httpClient *http.Client + apiKey string + stripeAccount string +} + +func NewDefaultClient(apiKey string) *DefaultClient { + return &DefaultClient{ + httpClient: newHTTPClient(), + apiKey: apiKey, + } +} + +func (d *DefaultClient) ForAccount(account string) Client { + cp := *d + cp.stripeAccount = account + + return &cp +} + +func (d *DefaultClient) BalanceTransactions(ctx context.Context, + options ...ClientOption, +) ([]*stripe.BalanceTransaction, bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, balanceTransactionsEndpoint, nil) + if err != nil { + return nil, false, errors.Wrap(err, "creating http request") + } + + for _, opt := range options { + opt.apply(req) + } + + if d.stripeAccount != "" { + req.Header.Set("Stripe-Account", d.stripeAccount) + } + + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.SetBasicAuth(d.apiKey, "") // gfyrag: really weird authentication right? + + var httpResponse *http.Response + + httpResponse, err = d.httpClient.Do(req) + if err != nil { + return nil, false, errors.Wrap(err, "doing request") + } + defer httpResponse.Body.Close() + + if httpResponse.StatusCode != http.StatusOK { + return nil, false, fmt.Errorf("unexpected status code: %d", httpResponse.StatusCode) + } + + type listResponse struct { + ListResponse + Data []json.RawMessage `json:"data"` + } + + rsp := &listResponse{} + + err = json.NewDecoder(httpResponse.Body).Decode(rsp) + if err != nil { + return nil, false, errors.Wrap(err, "decoding response") + } + + asBalanceTransactions := make([]*stripe.BalanceTransaction, 0) + + if len(rsp.Data) > 0 { + for _, data := range rsp.Data { + asBalanceTransaction := &stripe.BalanceTransaction{} + + err = json.Unmarshal(data, &asBalanceTransaction) + if err != nil { + return nil, false, err + } + + asBalanceTransactions = append(asBalanceTransactions, asBalanceTransaction) + } + } + + return asBalanceTransactions, rsp.HasMore, nil +} + +func newHTTPClient() *http.Client { + return &http.Client{ + Transport: otelhttp.NewTransport(http.DefaultTransport), + } +} + +var _ Client = &DefaultClient{} diff --git a/internal/app/connectors/stripe/client_test.go b/internal/app/connectors/stripe/client_test.go new file mode 100644 index 00000000..f6d0fe6e --- /dev/null +++ b/internal/app/connectors/stripe/client_test.go @@ -0,0 +1,281 @@ +package stripe + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "reflect" + "sync" + "testing" + "time" + + "github.com/stripe/stripe-go/v72" +) + +type httpMockExpectation interface { + handle(t *testing.T, r *http.Request) (*http.Response, error) +} + +type httpMock struct { + t *testing.T + expectations []httpMockExpectation + mu sync.Mutex +} + +func (mock *httpMock) RoundTrip(request *http.Request) (*http.Response, error) { + mock.mu.Lock() + defer mock.mu.Unlock() + + if len(mock.expectations) == 0 { + return nil, fmt.Errorf("no more expectations") + } + + expectations := mock.expectations[0] + if len(mock.expectations) == 1 { + mock.expectations = make([]httpMockExpectation, 0) + } else { + mock.expectations = mock.expectations[1:] + } + + return expectations.handle(mock.t, request) +} + +var _ http.RoundTripper = &httpMock{} + +type HTTPExpect[REQUEST any, RESPONSE any] struct { + statusCode int + path string + method string + requestBody *REQUEST + responseBody *RESPONSE + queryParams map[string]any +} + +func (e *HTTPExpect[REQUEST, RESPONSE]) handle(t *testing.T, request *http.Request) (*http.Response, error) { + t.Helper() + + if e.path != request.URL.Path { + return nil, fmt.Errorf("expected url was '%s', got, '%s'", e.path, request.URL.Path) + } + + if e.method != request.Method { + return nil, fmt.Errorf("expected method was '%s', got, '%s'", e.method, request.Method) + } + + if e.requestBody != nil { + body := new(REQUEST) + + err := json.NewDecoder(request.Body).Decode(body) + if err != nil { + panic(err) + } + + if !reflect.DeepEqual(*e.responseBody, *body) { + return nil, fmt.Errorf("mismatch body") + } + } + + for key, value := range e.queryParams { + qpvalue := "" + + switch value.(type) { + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + qpvalue = fmt.Sprintf("%d", value) + default: + qpvalue = fmt.Sprintf("%s", value) + } + + if rvalue := request.URL.Query().Get(key); rvalue != qpvalue { + return nil, fmt.Errorf("expected query param '%s' with value '%s', got '%s'", key, qpvalue, rvalue) + } + } + + data := make([]byte, 0) + + if e.responseBody != nil { + var err error + + data, err = json.Marshal(e.responseBody) + if err != nil { + panic(err) + } + } + + return &http.Response{ + StatusCode: e.statusCode, + Body: io.NopCloser(bytes.NewReader(data)), + ContentLength: int64(len(data)), + Request: request, + }, nil +} + +func (e *HTTPExpect[REQUEST, RESPONSE]) Path(p string) *HTTPExpect[REQUEST, RESPONSE] { + e.path = p + + return e +} + +func (e *HTTPExpect[REQUEST, RESPONSE]) Method(p string) *HTTPExpect[REQUEST, RESPONSE] { + e.method = p + + return e +} + +func (e *HTTPExpect[REQUEST, RESPONSE]) Body(body *REQUEST) *HTTPExpect[REQUEST, RESPONSE] { + e.requestBody = body + + return e +} + +func (e *HTTPExpect[REQUEST, RESPONSE]) QueryParam(key string, value any) *HTTPExpect[REQUEST, RESPONSE] { + e.queryParams[key] = value + + return e +} + +func (e *HTTPExpect[REQUEST, RESPONSE]) RespondsWith(statusCode int, + body *RESPONSE, +) *HTTPExpect[REQUEST, RESPONSE] { + e.statusCode = statusCode + e.responseBody = body + + return e +} + +func Expect[REQUEST, RESPONSE any](mock *httpMock) *HTTPExpect[REQUEST, RESPONSE] { + expectations := &HTTPExpect[REQUEST, RESPONSE]{ + queryParams: map[string]any{}, + } + + mock.mu.Lock() + defer mock.mu.Unlock() + + mock.expectations = append(mock.expectations, expectations) + + return expectations +} + +type StripeBalanceTransactionListExpect struct { + *HTTPExpect[struct{}, MockedListResponse] +} + +func (e *StripeBalanceTransactionListExpect) Path(p string) *StripeBalanceTransactionListExpect { + e.HTTPExpect.Path(p) + + return e +} + +func (e *StripeBalanceTransactionListExpect) Method(p string) *StripeBalanceTransactionListExpect { + e.HTTPExpect.Method(p) + + return e +} + +func (e *StripeBalanceTransactionListExpect) QueryParam(key string, + value any, +) *StripeBalanceTransactionListExpect { + e.HTTPExpect.QueryParam(key, value) + + return e +} + +func (e *StripeBalanceTransactionListExpect) RespondsWith(statusCode int, hasMore bool, + body ...*stripe.BalanceTransaction, +) *StripeBalanceTransactionListExpect { + e.HTTPExpect.RespondsWith(statusCode, &MockedListResponse{ + HasMore: hasMore, + Data: body, + }) + + return e +} + +func (e *StripeBalanceTransactionListExpect) StartingAfter(v string) *StripeBalanceTransactionListExpect { + e.QueryParam("starting_after", v) + + return e +} + +func (e *StripeBalanceTransactionListExpect) CreatedLte(v time.Time) *StripeBalanceTransactionListExpect { + e.QueryParam("created[lte]", v.Unix()) + + return e +} + +func (e *StripeBalanceTransactionListExpect) Limit(v int) *StripeBalanceTransactionListExpect { + e.QueryParam("limit", v) + + return e +} + +func ExpectBalanceTransactionList(mock *httpMock) *StripeBalanceTransactionListExpect { + e := Expect[struct{}, MockedListResponse](mock) + e.Path("/v1/balance_transactions").Method(http.MethodGet) + + return &StripeBalanceTransactionListExpect{ + HTTPExpect: e, + } +} + +func DatePtr(t time.Time) *time.Time { + return &t +} + +type BalanceTransactionSource stripe.BalanceTransactionSource + +func (t *BalanceTransactionSource) MarshalJSON() ([]byte, error) { + type Aux BalanceTransactionSource + + return json.Marshal(struct { + Aux + Charge *stripe.Charge `json:"charge"` + Payout *stripe.Payout `json:"payout"` + Refund *stripe.Refund `json:"refund"` + Transfer *stripe.Transfer `json:"transfer"` + }{ + Aux: Aux(*t), + Charge: t.Charge, + Payout: t.Payout, + Refund: t.Refund, + Transfer: t.Transfer, + }) +} + +type BalanceTransaction stripe.BalanceTransaction + +func (t *BalanceTransaction) MarshalJSON() ([]byte, error) { + type Aux BalanceTransaction + + return json.Marshal(struct { + Aux + Source *BalanceTransactionSource `json:"source"` + }{ + Aux: Aux(*t), + Source: (*BalanceTransactionSource)(t.Source), + }) +} + +//nolint:tagliatelle // allow snake_case in client +type MockedListResponse struct { + HasMore bool `json:"has_more"` + Data []*stripe.BalanceTransaction `json:"data"` +} + +func (t *MockedListResponse) MarshalJSON() ([]byte, error) { + type Aux MockedListResponse + + txs := make([]*BalanceTransaction, 0) + for _, tx := range t.Data { + txs = append(txs, (*BalanceTransaction)(tx)) + } + + return json.Marshal(struct { + Aux + Data []*BalanceTransaction `json:"data"` + }{ + Aux: Aux(*t), + Data: txs, + }) +} diff --git a/internal/app/connectors/stripe/config.go b/internal/app/connectors/stripe/config.go new file mode 100644 index 00000000..2edb0634 --- /dev/null +++ b/internal/app/connectors/stripe/config.go @@ -0,0 +1,47 @@ +package stripe + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/formancehq/payments/internal/app/connectors/configtemplate" + + "github.com/formancehq/payments/internal/app/connectors" +) + +type Config struct { + PollingPeriod connectors.Duration `json:"pollingPeriod" yaml:"pollingPeriod" bson:"pollingPeriod"` + APIKey string `json:"apiKey" yaml:"apiKey" bson:"apiKey"` + TimelineConfig `bson:",inline"` +} + +func (c Config) String() string { + return fmt.Sprintf("pollingPeriod=%d, pageSize=%d, apiKey=%s", c.PollingPeriod, c.PageSize, c.APIKey) +} + +func (c Config) Validate() error { + if c.APIKey == "" { + return errors.New("missing api key") + } + + return nil +} + +func (c Config) Marshal() ([]byte, error) { + return json.Marshal(c) +} + +type TimelineConfig struct { + PageSize uint64 `json:"pageSize" yaml:"pageSize" bson:"pageSize"` +} + +func (c Config) BuildTemplate() (string, configtemplate.Config) { + cfg := configtemplate.NewConfig() + + cfg.AddParameter("apiKey", configtemplate.TypeString, true) + cfg.AddParameter("pollingPeriod", configtemplate.TypeDurationNs, false) + cfg.AddParameter("pageSize", configtemplate.TypeDurationUnsignedInteger, false) + + return Name.String(), cfg +} diff --git a/internal/app/connectors/stripe/connector.go b/internal/app/connectors/stripe/connector.go new file mode 100644 index 00000000..06596c0b --- /dev/null +++ b/internal/app/connectors/stripe/connector.go @@ -0,0 +1,59 @@ +package stripe + +import ( + "context" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/payments/internal/app/integration" + "github.com/formancehq/payments/internal/app/task" + + "github.com/formancehq/go-libs/logging" +) + +const Name = models.ConnectorProviderStripe + +type Connector struct { + logger logging.Logger + cfg Config +} + +func (c *Connector) Install(ctx task.ConnectorContext) error { + descriptor, err := models.EncodeTaskDescriptor(TaskDescriptor{ + Name: "Main task to periodically fetch transactions", + Main: true, + }) + if err != nil { + return err + } + + return ctx.Scheduler().Schedule(descriptor, false) +} + +func (c *Connector) Uninstall(ctx context.Context) error { + return nil +} + +func (c *Connector) Resolve(descriptor models.TaskDescriptor) task.Task { + taskDescriptor, err := models.DecodeTaskDescriptor[TaskDescriptor](descriptor) + if err != nil { + panic(err) + } + + if taskDescriptor.Main { + return MainTask(c.cfg) + } + + return ConnectedAccountTask(c.cfg, taskDescriptor.Account) +} + +var _ integration.Connector = &Connector{} + +func newConnector(logger logging.Logger, cfg Config) *Connector { + return &Connector{ + logger: logger.WithFields(map[string]any{ + "component": "connector", + }), + cfg: cfg, + } +} diff --git a/internal/app/connectors/stripe/descriptor.go b/internal/app/connectors/stripe/descriptor.go new file mode 100644 index 00000000..dd460de7 --- /dev/null +++ b/internal/app/connectors/stripe/descriptor.go @@ -0,0 +1,7 @@ +package stripe + +type TaskDescriptor struct { + Name string `json:"name" yaml:"name" bson:"name"` + Main bool `json:"main,omitempty" yaml:"main" bson:"main"` + Account string `json:"account,omitempty" yaml:"account" bson:"account"` +} diff --git a/internal/app/connectors/stripe/ingester.go b/internal/app/connectors/stripe/ingester.go new file mode 100644 index 00000000..3833ad1a --- /dev/null +++ b/internal/app/connectors/stripe/ingester.go @@ -0,0 +1,20 @@ +package stripe + +import ( + "context" + + "github.com/stripe/stripe-go/v72" +) + +type Ingester interface { + Ingest(ctx context.Context, batch []*stripe.BalanceTransaction, commitState TimelineState, tail bool) error +} + +type IngesterFn func(ctx context.Context, batch []*stripe.BalanceTransaction, + commitState TimelineState, tail bool) error + +func (fn IngesterFn) Ingest(ctx context.Context, batch []*stripe.BalanceTransaction, + commitState TimelineState, tail bool, +) error { + return fn(ctx, batch, commitState, tail) +} diff --git a/internal/app/connectors/stripe/loader.go b/internal/app/connectors/stripe/loader.go new file mode 100644 index 00000000..5bdeab4d --- /dev/null +++ b/internal/app/connectors/stripe/loader.go @@ -0,0 +1,43 @@ +package stripe + +import ( + "time" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/connectors" + "github.com/formancehq/payments/internal/app/integration" +) + +type Loader struct{} + +const allowedTasks = 50 + +func (l *Loader) AllowTasks() int { + return allowedTasks +} + +func (l *Loader) Name() models.ConnectorProvider { + return Name +} + +func (l *Loader) Load(logger logging.Logger, config Config) integration.Connector { + return newConnector(logger, config) +} + +func (l *Loader) ApplyDefaults(cfg Config) Config { + if cfg.PageSize == 0 { + cfg.PageSize = 10 + } + + if cfg.PollingPeriod.Duration == 0 { + cfg.PollingPeriod = connectors.Duration{Duration: 2 * time.Minute} + } + + return cfg +} + +func NewLoader() *Loader { + return &Loader{} +} diff --git a/internal/app/connectors/stripe/runner.go b/internal/app/connectors/stripe/runner.go new file mode 100644 index 00000000..0833f73c --- /dev/null +++ b/internal/app/connectors/stripe/runner.go @@ -0,0 +1,85 @@ +package stripe + +import ( + "context" + "time" + + "github.com/formancehq/go-libs/logging" +) + +func NewRunner( + logger logging.Logger, + trigger *TimelineTrigger, + pollingPeriod time.Duration, +) *Runner { + return &Runner{ + logger: logger.WithFields(map[string]interface{}{ + "component": "runner", + }), + trigger: trigger, + pollingPeriod: pollingPeriod, + stopChan: make(chan chan struct{}), + } +} + +type Runner struct { + stopChan chan chan struct{} + trigger *TimelineTrigger + logger logging.Logger + pollingPeriod time.Duration +} + +func (r *Runner) Stop(ctx context.Context) error { + ch := make(chan struct{}) + select { + case r.stopChan <- ch: + select { + case <-ch: + return nil + case <-ctx.Done(): + return ctx.Err() + } + case <-ctx.Done(): + return ctx.Err() + } +} + +func (r *Runner) Run(ctx context.Context) error { + r.logger.WithFields(map[string]interface{}{ + "polling-period": r.pollingPeriod, + }).Info("Starting runner") + defer r.trigger.Cancel(ctx) + + done := make(chan struct{}, 1) + fetch := func() { + defer func() { + done <- struct{}{} + }() + + if err := r.trigger.Fetch(ctx); err != nil { + r.logger.Errorf("Error fetching page: %s", err) + } + } + + go fetch() + + var timeChan <-chan time.Time + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case closeChannel := <-r.stopChan: + r.trigger.Cancel(ctx) + close(closeChannel) + + return nil + case <-done: + timeChan = time.After(r.pollingPeriod) + case <-timeChan: + timeChan = nil + + go fetch() + } + } +} diff --git a/internal/app/connectors/stripe/runner_test.go b/internal/app/connectors/stripe/runner_test.go new file mode 100644 index 00000000..13a556fd --- /dev/null +++ b/internal/app/connectors/stripe/runner_test.go @@ -0,0 +1,51 @@ +package stripe + +import ( + "context" + "testing" + "time" + + "github.com/stripe/stripe-go/v72" + + "github.com/formancehq/go-libs/logging" + "github.com/stretchr/testify/require" +) + +func TestStopTailing(t *testing.T) { + t.Parallel() + + NoOpIngester := IngesterFn(func(ctx context.Context, batch []*stripe.BalanceTransaction, + commitState TimelineState, tail bool, + ) error { + return nil + }) + + mock := NewClientMock(t, true) + timeline := NewTimeline(mock, TimelineConfig{ + PageSize: 2, + }, TimelineState{ + OldestID: "tx1", + MoreRecentID: "tx2", + }) + + logger := logging.GetLogger(context.Background()) + trigger := NewTimelineTrigger(logger, NoOpIngester, timeline) + r := NewRunner(logger, trigger, time.Second) + + go func() { + _ = r.Run(context.Background()) + }() + + defer func() { + _ = r.Stop(context.Background()) + }() + + require.False(t, timeline.state.NoMoreHistory) + + mock.Expect().RespondsWith(false) // Fetch head + mock.Expect().RespondsWith(false) // Fetch tail + + require.Eventually(t, func() bool { + return timeline.state.NoMoreHistory + }, time.Second, 10*time.Millisecond) +} diff --git a/internal/app/connectors/stripe/state.go b/internal/app/connectors/stripe/state.go new file mode 100644 index 00000000..9ae3edbc --- /dev/null +++ b/internal/app/connectors/stripe/state.go @@ -0,0 +1,11 @@ +package stripe + +import "time" + +type TimelineState struct { + OldestID string `bson:"oldestID,omitempty" json:"oldestID"` + OldestDate *time.Time `bson:"oldestDate,omitempty" json:"oldestDate"` + MoreRecentID string `bson:"moreRecentID,omitempty" json:"moreRecentID"` + MoreRecentDate *time.Time `bson:"moreRecentDate,omitempty" json:"moreRecentDate"` + NoMoreHistory bool `bson:"noMoreHistory" json:"noMoreHistory"` +} diff --git a/internal/app/connectors/stripe/task_connected_account.go b/internal/app/connectors/stripe/task_connected_account.go new file mode 100644 index 00000000..95cc1035 --- /dev/null +++ b/internal/app/connectors/stripe/task_connected_account.go @@ -0,0 +1,63 @@ +package stripe + +import ( + "context" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/ingestion" + "github.com/formancehq/payments/internal/app/task" + "github.com/stripe/stripe-go/v72" +) + +func ingestBatch(ctx context.Context, logger logging.Logger, ingester ingestion.Ingester, + bts []*stripe.BalanceTransaction, commitState TimelineState, tail bool, +) error { + batch := ingestion.PaymentBatch{} + + for i := range bts { + batchElement, handled := CreateBatchElement(bts[i], !tail) + + if !handled { + logger.Debugf("Balance transaction type not handled: %s", bts[i].Type) + + continue + } + + if batchElement.Adjustment == nil && batchElement.Payment == nil { + continue + } + + batch = append(batch, batchElement) + } + + logger.WithFields(map[string]interface{}{ + "state": commitState, + }).Debugf("updating state") + + err := ingester.IngestPayments(ctx, batch, commitState) + if err != nil { + return err + } + + return nil +} + +func ConnectedAccountTask(config Config, account string) func(ctx context.Context, logger logging.Logger, + ingester ingestion.Ingester, resolver task.StateResolver) error { + return func(ctx context.Context, logger logging.Logger, ingester ingestion.Ingester, + resolver task.StateResolver, + ) error { + logger.Infof("Create new trigger") + + trigger := NewTimelineTrigger( + logger, + IngesterFn(func(ctx context.Context, bts []*stripe.BalanceTransaction, commitState TimelineState, tail bool) error { + return ingestBatch(ctx, logger, ingester, bts, commitState, tail) + }), + NewTimeline(NewDefaultClient(config.APIKey). + ForAccount(account), config.TimelineConfig, task.MustResolveTo(ctx, resolver, TimelineState{})), + ) + + return trigger.Fetch(ctx) + } +} diff --git a/internal/app/connectors/stripe/task_main.go b/internal/app/connectors/stripe/task_main.go new file mode 100644 index 00000000..2d496fe3 --- /dev/null +++ b/internal/app/connectors/stripe/task_main.go @@ -0,0 +1,77 @@ +package stripe + +import ( + "context" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/ingestion" + "github.com/formancehq/payments/internal/app/task" + "github.com/pkg/errors" + "github.com/stripe/stripe-go/v72" +) + +func ingest( + ctx context.Context, + logger logging.Logger, + scheduler task.Scheduler, + ingester ingestion.Ingester, + bts []*stripe.BalanceTransaction, + commitState TimelineState, + tail bool, +) error { + err := ingestBatch(ctx, logger, ingester, bts, commitState, tail) + if err != nil { + return err + } + + connectedAccounts := make([]string, 0) + + for _, bt := range bts { + if bt.Type == stripe.BalanceTransactionTypeTransfer { + connectedAccounts = append(connectedAccounts, bt.Source.Transfer.Destination.ID) + } + } + + for _, connectedAccount := range connectedAccounts { + descriptor, err := models.EncodeTaskDescriptor(TaskDescriptor{ + Name: "Fetch balance transactions for a specific connected account", + Account: connectedAccount, + }) + if err != nil { + return errors.Wrap(err, "failed to transform task descriptor") + } + + err = scheduler.Schedule(descriptor, true) + if err != nil && !errors.Is(err, task.ErrAlreadyScheduled) { + return errors.Wrap(err, "scheduling connected account") + } + } + + return nil +} + +func MainTask(config Config) func(ctx context.Context, logger logging.Logger, resolver task.StateResolver, + scheduler task.Scheduler, ingester ingestion.Ingester) error { + return func(ctx context.Context, logger logging.Logger, resolver task.StateResolver, + scheduler task.Scheduler, ingester ingestion.Ingester, + ) error { + runner := NewRunner( + logger, + NewTimelineTrigger( + logger, + IngesterFn(func(ctx context.Context, batch []*stripe.BalanceTransaction, + commitState TimelineState, tail bool, + ) error { + return ingest(ctx, logger, scheduler, ingester, batch, commitState, tail) + }), + NewTimeline(NewDefaultClient(config.APIKey), + config.TimelineConfig, task.MustResolveTo(ctx, resolver, TimelineState{})), + ), + config.PollingPeriod.Duration, + ) + + return runner.Run(ctx) + } +} diff --git a/internal/app/connectors/stripe/timeline.go b/internal/app/connectors/stripe/timeline.go new file mode 100644 index 00000000..a65eef81 --- /dev/null +++ b/internal/app/connectors/stripe/timeline.go @@ -0,0 +1,197 @@ +package stripe + +import ( + "context" + "fmt" + "net/url" + "time" + + "github.com/stripe/stripe-go/v72" +) + +const ( + balanceTransactionsEndpoint = "https://api.stripe.com/v1/balance_transactions" +) + +//nolint:tagliatelle // allow different styled tags in client +type ListResponse struct { + HasMore bool `json:"has_more"` + Data []*stripe.BalanceTransaction `json:"data"` +} + +type TimelineOption interface { + apply(c *Timeline) +} +type TimelineOptionFn func(c *Timeline) + +func (fn TimelineOptionFn) apply(c *Timeline) { + fn(c) +} + +func WithStartingAt(v time.Time) TimelineOptionFn { + return func(c *Timeline) { + c.startingAt = v + } +} + +func NewTimeline(client Client, cfg TimelineConfig, state TimelineState, options ...TimelineOption) *Timeline { + defaultOptions := make([]TimelineOption, 0) + + c := &Timeline{ + config: cfg, + state: state, + client: client, + } + + options = append(defaultOptions, append([]TimelineOption{ + WithStartingAt(time.Now()), + }, options...)...) + + for _, opt := range options { + opt.apply(c) + } + + return c +} + +type Timeline struct { + state TimelineState + firstIDAfterStartingAt string + startingAt time.Time + config TimelineConfig + client Client +} + +func (tl *Timeline) doRequest(ctx context.Context, queryParams url.Values, + to *[]*stripe.BalanceTransaction, +) (bool, error) { + options := make([]ClientOption, 0) + options = append(options, QueryParam("limit", fmt.Sprintf("%d", tl.config.PageSize))) + options = append(options, QueryParam("expand[]", "data.source")) + + for k, v := range queryParams { + options = append(options, QueryParam(k, v[0])) + } + + txs, hasMore, err := tl.client.BalanceTransactions(ctx, options...) + if err != nil { + return false, err + } + + *to = txs + + return hasMore, nil +} + +func (tl *Timeline) init(ctx context.Context) error { + ret := make([]*stripe.BalanceTransaction, 0) + params := url.Values{} + params.Set("limit", "1") + params.Set("created[lt]", fmt.Sprintf("%d", tl.startingAt.Unix())) + + _, err := tl.doRequest(ctx, params, &ret) + if err != nil { + return err + } + + if len(ret) > 0 { + tl.firstIDAfterStartingAt = ret[0].ID + } + + return nil +} + +func (tl *Timeline) Tail(ctx context.Context, to *[]*stripe.BalanceTransaction) (bool, TimelineState, func(), error) { + queryParams := url.Values{} + + switch { + case tl.state.OldestID != "": + queryParams.Set("starting_after", tl.state.OldestID) + default: + queryParams.Set("created[lte]", fmt.Sprintf("%d", tl.startingAt.Unix())) + } + + hasMore, err := tl.doRequest(ctx, queryParams, to) + if err != nil { + return false, TimelineState{}, nil, err + } + + futureState := tl.state + + if len(*to) > 0 { + lastItem := (*to)[len(*to)-1] + futureState.OldestID = lastItem.ID + oldestDate := time.Unix(lastItem.Created, 0) + futureState.OldestDate = &oldestDate + + if futureState.MoreRecentID == "" { + firstItem := (*to)[0] + futureState.MoreRecentID = firstItem.ID + moreRecentDate := time.Unix(firstItem.Created, 0) + futureState.MoreRecentDate = &moreRecentDate + } + } + + futureState.NoMoreHistory = !hasMore + + return hasMore, futureState, func() { + tl.state = futureState + }, nil +} + +func (tl *Timeline) Head(ctx context.Context, to *[]*stripe.BalanceTransaction) (bool, TimelineState, func(), error) { + if tl.firstIDAfterStartingAt == "" && tl.state.MoreRecentID == "" { + err := tl.init(ctx) + if err != nil { + return false, TimelineState{}, nil, err + } + + if tl.firstIDAfterStartingAt == "" { + return false, TimelineState{ + NoMoreHistory: true, + }, func() {}, nil + } + } + + queryParams := url.Values{} + + switch { + case tl.state.MoreRecentID != "": + queryParams.Set("ending_before", tl.state.MoreRecentID) + case tl.firstIDAfterStartingAt != "": + queryParams.Set("ending_before", tl.firstIDAfterStartingAt) + } + + hasMore, err := tl.doRequest(ctx, queryParams, to) + if err != nil { + return false, TimelineState{}, nil, err + } + + futureState := tl.state + + if len(*to) > 0 { + firstItem := (*to)[0] + futureState.MoreRecentID = firstItem.ID + moreRecentDate := time.Unix(firstItem.Created, 0) + futureState.MoreRecentDate = &moreRecentDate + + if futureState.OldestID == "" { + lastItem := (*to)[len(*to)-1] + futureState.OldestID = lastItem.ID + oldestDate := time.Unix(lastItem.Created, 0) + futureState.OldestDate = &oldestDate + } + } + + for i, j := 0, len(*to)-1; i < j; i, j = i+1, j-1 { + (*to)[i], (*to)[j] = (*to)[j], (*to)[i] + } + + return hasMore, futureState, func() { + tl.state = futureState + }, nil +} + +func (tl *Timeline) State() TimelineState { + return tl.state +} diff --git a/internal/app/connectors/stripe/timeline_test.go b/internal/app/connectors/stripe/timeline_test.go new file mode 100644 index 00000000..f576717d --- /dev/null +++ b/internal/app/connectors/stripe/timeline_test.go @@ -0,0 +1,67 @@ +package stripe + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/stripe/stripe-go/v72" +) + +func TestTimeline(t *testing.T) { + t.Parallel() + + mock := NewClientMock(t, true) + ref := time.Now() + timeline := NewTimeline(mock, TimelineConfig{ + PageSize: 2, + }, TimelineState{}, WithStartingAt(ref)) + + tx1 := &stripe.BalanceTransaction{ + ID: "tx1", + Created: ref.Add(-time.Minute).Unix(), + } + + tx2 := &stripe.BalanceTransaction{ + ID: "tx2", + Created: ref.Add(-2 * time.Minute).Unix(), + } + + mock.Expect(). + Limit(2). + CreatedLte(ref). + RespondsWith(true, tx1, tx2) + + ret := make([]*stripe.BalanceTransaction, 0) + hasMore, state, commit, err := timeline.Tail(context.Background(), &ret) + require.NoError(t, err) + require.True(t, hasMore) + require.Equal(t, TimelineState{ + OldestID: "tx2", + OldestDate: DatePtr(time.Unix(tx2.Created, 0)), + MoreRecentID: "tx1", + MoreRecentDate: DatePtr(time.Unix(tx1.Created, 0)), + NoMoreHistory: false, + }, state) + + commit() + + tx3 := &stripe.BalanceTransaction{ + ID: "tx3", + Created: ref.Add(-3 * time.Minute).Unix(), + } + + mock.Expect().Limit(2).StartingAfter(tx2.ID).RespondsWith(false, tx3) + + hasMore, state, _, err = timeline.Tail(context.Background(), &ret) + require.NoError(t, err) + require.False(t, hasMore) + require.Equal(t, TimelineState{ + OldestID: "tx3", + OldestDate: DatePtr(time.Unix(tx3.Created, 0)), + MoreRecentID: "tx1", + MoreRecentDate: DatePtr(time.Unix(tx1.Created, 0)), + NoMoreHistory: true, + }, state) +} diff --git a/internal/app/connectors/stripe/timeline_trigger.go b/internal/app/connectors/stripe/timeline_trigger.go new file mode 100644 index 00000000..f930acc8 --- /dev/null +++ b/internal/app/connectors/stripe/timeline_trigger.go @@ -0,0 +1,121 @@ +package stripe + +import ( + "context" + + "github.com/formancehq/go-libs/logging" + "github.com/pkg/errors" + "github.com/stripe/stripe-go/v72" + "golang.org/x/sync/semaphore" +) + +func NewTimelineTrigger( + logger logging.Logger, + ingester Ingester, + timeline *Timeline, +) *TimelineTrigger { + return &TimelineTrigger{ + logger: logger.WithFields(map[string]interface{}{ + "component": "timeline-trigger", + }), + ingester: ingester, + timeline: timeline, + sem: semaphore.NewWeighted(1), + } +} + +type TimelineTrigger struct { + logger logging.Logger + ingester Ingester + timeline *Timeline + sem *semaphore.Weighted + cancel func() +} + +func (t *TimelineTrigger) Fetch(ctx context.Context) error { + if t.sem.TryAcquire(1) { + defer t.sem.Release(1) + + ctx, t.cancel = context.WithCancel(ctx) + if !t.timeline.State().NoMoreHistory { + if err := t.fetch(ctx, true); err != nil { + return err + } + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + if err := t.fetch(ctx, false); err != nil { + return err + } + } + } + + return nil +} + +func (t *TimelineTrigger) Cancel(ctx context.Context) { + if t.cancel != nil { + t.cancel() + + err := t.sem.Acquire(ctx, 1) + if err != nil { + panic(err) + } + + t.sem.Release(1) + } +} + +func (t *TimelineTrigger) fetch(ctx context.Context, tail bool) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + hasMore, err := t.triggerPage(ctx, tail) + if err != nil { + return errors.Wrap(err, "error triggering tail page") + } + + if !hasMore { + return nil + } + } + } +} + +func (t *TimelineTrigger) triggerPage(ctx context.Context, tail bool) (bool, error) { + logger := t.logger.WithFields(map[string]interface{}{ + "tail": tail, + }) + + logger.Debugf("Trigger page") + + ret := make([]*stripe.BalanceTransaction, 0) + method := t.timeline.Head + + if tail { + method = t.timeline.Tail + } + + hasMore, futureState, commitFn, err := method(ctx, &ret) + if err != nil { + return false, errors.Wrap(err, "fetching timeline") + } + + logger.Debug("Ingest batch") + + if len(ret) > 0 { + err = t.ingester.Ingest(ctx, ret, futureState, tail) + if err != nil { + return false, errors.Wrap(err, "ingesting batch") + } + } + + commitFn() + + return hasMore, nil +} diff --git a/internal/app/connectors/stripe/timeline_trigger_test.go b/internal/app/connectors/stripe/timeline_trigger_test.go new file mode 100644 index 00000000..9292272f --- /dev/null +++ b/internal/app/connectors/stripe/timeline_trigger_test.go @@ -0,0 +1,105 @@ +package stripe + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/formancehq/go-libs/logging" + "github.com/stretchr/testify/require" + "github.com/stripe/stripe-go/v72" +) + +func TestTimelineTrigger(t *testing.T) { + t.Parallel() + + const txCount = 12 + + mock := NewClientMock(t, true) + ref := time.Now().Add(-time.Minute * time.Duration(txCount) / 2) + timeline := NewTimeline(mock, TimelineConfig{ + PageSize: 2, + }, TimelineState{}, WithStartingAt(ref)) + + ingestedTx := make([]*stripe.BalanceTransaction, 0) + trigger := NewTimelineTrigger( + logging.GetLogger(context.Background()), + IngesterFn(func(ctx context.Context, batch []*stripe.BalanceTransaction, commitState TimelineState, tail bool) error { + ingestedTx = append(ingestedTx, batch...) + + return nil + }), + timeline, + ) + + allTxs := make([]*stripe.BalanceTransaction, txCount) + for i := 0; i < txCount/2; i++ { + allTxs[txCount/2+i] = &stripe.BalanceTransaction{ + ID: fmt.Sprintf("%d", txCount/2+i), + Created: ref.Add(-time.Duration(i) * time.Minute).Unix(), + } + allTxs[txCount/2-i-1] = &stripe.BalanceTransaction{ + ID: fmt.Sprintf("%d", txCount/2-i-1), + Created: ref.Add(time.Duration(i) * time.Minute).Unix(), + } + } + + for i := 0; i < txCount/2; i += 2 { + mock.Expect().Limit(2).RespondsWith(i < txCount/2-2, allTxs[txCount/2+i], allTxs[txCount/2+i+1]) + } + + for i := 0; i < txCount/2; i += 2 { + mock.Expect().Limit(2).RespondsWith(i < txCount/2-2, allTxs[txCount/2-i-2], allTxs[txCount/2-i-1]) + } + + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second)) + defer cancel() + + require.NoError(t, trigger.Fetch(ctx)) + require.Len(t, ingestedTx, txCount) +} + +func TestCancelTimelineTrigger(t *testing.T) { + t.Parallel() + + const txCount = 12 + + mock := NewClientMock(t, false) + ref := time.Now().Add(-time.Minute * time.Duration(txCount) / 2) + timeline := NewTimeline(mock, TimelineConfig{ + PageSize: 1, + }, TimelineState{}, WithStartingAt(ref)) + + waiting := make(chan struct{}) + trigger := NewTimelineTrigger( + logging.GetLogger(context.Background()), + IngesterFn(func(ctx context.Context, batch []*stripe.BalanceTransaction, commitState TimelineState, tail bool) error { + close(waiting) // Instruct the test the trigger is in fetching state + <-ctx.Done() + + return nil + }), + timeline, + ) + + allTxs := make([]*stripe.BalanceTransaction, txCount) + for i := 0; i < txCount; i++ { + allTxs[i] = &stripe.BalanceTransaction{ + ID: fmt.Sprintf("%d", i), + } + mock.Expect().Limit(1).RespondsWith(i < txCount-1, allTxs[i]) + } + + go func() { + // TODO: Handle error + _ = trigger.Fetch(context.Background()) + }() + select { + case <-time.After(time.Second): + t.Fatalf("timeout") + case <-waiting: + trigger.Cancel(context.Background()) + require.NotEmpty(t, mock.expectations) + } +} diff --git a/internal/app/connectors/stripe/translate.go b/internal/app/connectors/stripe/translate.go new file mode 100644 index 00000000..4249100c --- /dev/null +++ b/internal/app/connectors/stripe/translate.go @@ -0,0 +1,325 @@ +package stripe + +import ( + "encoding/json" + "fmt" + "log" + "runtime/debug" + "strings" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/formancehq/payments/internal/app/ingestion" + "github.com/formancehq/payments/internal/app/models" + "github.com/stripe/stripe-go/v72" +) + +type currency struct { + decimals int +} + +func currencies() map[string]currency { + return map[string]currency{ + "ARS": {2}, // Argentine Peso + "AMD": {2}, // Armenian Dram + "AWG": {2}, // Aruban Guilder + "AUD": {2}, // Australian Dollar + "BSD": {2}, // Bahamian Dollar + "BHD": {3}, // Bahraini Dinar + "BDT": {2}, // Bangladesh, Taka + "BZD": {2}, // Belize Dollar + "BMD": {2}, // Bermudian Dollar + "BOB": {2}, // Bolivia, Boliviano + "BAM": {2}, // Bosnia and Herzegovina, Convertible Marks + "BWP": {2}, // Botswana, Pula + "BRL": {2}, // Brazilian Real + "BND": {2}, // Brunei Dollar + "CAD": {2}, // Canadian Dollar + "KYD": {2}, // Cayman Islands Dollar + "CLP": {0}, // Chilean Peso + "CNY": {2}, // China Yuan Renminbi + "COP": {2}, // Colombian Peso + "CRC": {2}, // Costa Rican Colon + "HRK": {2}, // Croatian Kuna + "CUC": {2}, // Cuban Convertible Peso + "CUP": {2}, // Cuban Peso + "CYP": {2}, // Cyprus Pound + "CZK": {2}, // Czech Koruna + "DKK": {2}, // Danish Krone + "DOP": {2}, // Dominican Peso + "XCD": {2}, // East Caribbean Dollar + "EGP": {2}, // Egyptian Pound + "SVC": {2}, // El Salvador Colon + "ATS": {2}, // Euro + "BEF": {2}, // Euro + "DEM": {2}, // Euro + "EEK": {2}, // Euro + "ESP": {2}, // Euro + "EUR": {2}, // Euro + "FIM": {2}, // Euro + "FRF": {2}, // Euro + "GRD": {2}, // Euro + "IEP": {2}, // Euro + "ITL": {2}, // Euro + "LUF": {2}, // Euro + "NLG": {2}, // Euro + "PTE": {2}, // Euro + "GHC": {2}, // Ghana, Cedi + "GIP": {2}, // Gibraltar Pound + "GTQ": {2}, // Guatemala, Quetzal + "HNL": {2}, // Honduras, Lempira + "HKD": {2}, // Hong Kong Dollar + "HUF": {0}, // Hungary, Forint + "ISK": {0}, // Iceland Krona + "INR": {2}, // Indian Rupee + "IDR": {2}, // Indonesia, Rupiah + "IRR": {2}, // Iranian Rial + "JMD": {2}, // Jamaican Dollar + "JPY": {0}, // Japan, Yen + "JOD": {3}, // Jordanian Dinar + "KES": {2}, // Kenyan Shilling + "KWD": {3}, // Kuwaiti Dinar + "LVL": {2}, // Latvian Lats + "LBP": {0}, // Lebanese Pound + "LTL": {2}, // Lithuanian Litas + "MKD": {2}, // Macedonia, Denar + "MYR": {2}, // Malaysian Ringgit + "MTL": {2}, // Maltese Lira + "MUR": {0}, // Mauritius Rupee + "MXN": {2}, // Mexican Peso + "MZM": {2}, // Mozambique Metical + "NPR": {2}, // Nepalese Rupee + "ANG": {2}, // Netherlands Antillian Guilder + "ILS": {2}, // New Israeli Shekel + "TRY": {2}, // New Turkish Lira + "NZD": {2}, // New Zealand Dollar + "NOK": {2}, // Norwegian Krone + "PKR": {2}, // Pakistan Rupee + "PEN": {2}, // Peru, Nuevo Sol + "UYU": {2}, // Peso Uruguayo + "PHP": {2}, // Philippine Peso + "PLN": {2}, // Poland, Zloty + "GBP": {2}, // Pound Sterling + "OMR": {3}, // Rial Omani + "RON": {2}, // Romania, New Leu + "ROL": {2}, // Romania, Old Leu + "RUB": {2}, // Russian Ruble + "SAR": {2}, // Saudi Riyal + "SGD": {2}, // Singapore Dollar + "SKK": {2}, // Slovak Koruna + "SIT": {2}, // Slovenia, Tolar + "ZAR": {2}, // South Africa, Rand + "KRW": {0}, // South Korea, Won + "SZL": {2}, // Swaziland, Lilangeni + "SEK": {2}, // Swedish Krona + "CHF": {2}, // Swiss Franc + "TZS": {2}, // Tanzanian Shilling + "THB": {2}, // Thailand, Baht + "TOP": {2}, // Tonga, Paanga + "AED": {2}, // UAE Dirham + "UAH": {2}, // Ukraine, Hryvnia + "USD": {2}, // US Dollar + "VUV": {0}, // Vanuatu, Vatu + "VEF": {2}, // Venezuela Bolivares Fuertes + "VEB": {2}, // Venezuela, Bolivar + "VND": {0}, // Viet Nam, Dong + "ZWD": {2}, // Zimbabwe Dollar + } +} + +func CreateBatchElement(balanceTransaction *stripe.BalanceTransaction, forward bool) (ingestion.PaymentBatchElement, bool) { + var payment models.Payment // reference payments.Referenced + // paymentData *payments.Data + // adjustment *payments.Adjustment + + defer func() { + // DEBUG + if e := recover(); e != nil { + log.Println("Error translating transaction") + debug.PrintStack() + spew.Dump(balanceTransaction) + panic(e) + } + }() + + if balanceTransaction.Source == nil { + return ingestion.PaymentBatchElement{}, false + } + + if balanceTransaction.Source.Payout == nil && balanceTransaction.Source.Charge == nil { + return ingestion.PaymentBatchElement{}, false + } + + formatAsset := func(cur stripe.Currency) models.PaymentAsset { + asset := strings.ToUpper(string(cur)) + + def, ok := currencies()[asset] + if !ok { + return models.PaymentAsset(asset) + } + + if def.decimals == 0 { + return models.PaymentAsset(asset) + } + + return models.PaymentAsset(fmt.Sprintf("%s/%d", asset, def.decimals)) + } + + rawData, err := json.Marshal(balanceTransaction) + if err != nil { + return ingestion.PaymentBatchElement{}, false + } + + switch balanceTransaction.Type { + case stripe.BalanceTransactionTypeCharge: + payment = models.Payment{ + Reference: balanceTransaction.Source.Charge.ID, + Type: models.PaymentTypePayIn, + Status: models.PaymentStatusSucceeded, + Amount: balanceTransaction.Source.Charge.Amount, + Asset: formatAsset(balanceTransaction.Source.Charge.Currency), + RawData: rawData, + Scheme: models.PaymentScheme(balanceTransaction.Source.Charge.PaymentMethodDetails.Card.Brand), + CreatedAt: time.Unix(balanceTransaction.Created, 0), + } + case stripe.BalanceTransactionTypePayout: + payment = models.Payment{ + Reference: balanceTransaction.Source.Payout.ID, + Type: models.PaymentTypePayOut, + Status: convertPayoutStatus(balanceTransaction.Source.Payout.Status), + Amount: balanceTransaction.Source.Payout.Amount, + RawData: rawData, + Asset: formatAsset(balanceTransaction.Source.Payout.Currency), + Scheme: func() models.PaymentScheme { + switch balanceTransaction.Source.Payout.Type { + case stripe.PayoutTypeBank: + return models.PaymentSchemeSepaCredit + case stripe.PayoutTypeCard: + return models.PaymentScheme(balanceTransaction.Source.Payout.Card.Brand) + } + + return models.PaymentSchemeUnknown + }(), + CreatedAt: time.Unix(balanceTransaction.Created, 0), + } + case stripe.BalanceTransactionTypeTransfer: + payment = models.Payment{ + Reference: balanceTransaction.Source.Transfer.ID, + Type: models.PaymentTypePayOut, + Status: models.PaymentStatusSucceeded, + Amount: balanceTransaction.Source.Transfer.Amount, + RawData: rawData, + Asset: formatAsset(balanceTransaction.Source.Transfer.Currency), + Scheme: models.PaymentSchemeOther, + CreatedAt: time.Unix(balanceTransaction.Created, 0), + } + case stripe.BalanceTransactionTypeRefund: + payment = models.Payment{ + Reference: balanceTransaction.Source.Refund.Charge.ID, + Type: models.PaymentTypePayOut, + Adjustments: []*models.Adjustment{ + { + Reference: balanceTransaction.Source.Refund.Charge.ID, + Status: models.PaymentStatusSucceeded, + Amount: balanceTransaction.Amount, + CreatedAt: time.Unix(balanceTransaction.Created, 0), + RawData: rawData, + }, + }, + } + case stripe.BalanceTransactionTypePayment: + payment = models.Payment{ + Reference: balanceTransaction.Source.Charge.ID, + Type: models.PaymentTypePayIn, + Status: models.PaymentStatusSucceeded, + Amount: balanceTransaction.Source.Charge.Amount, + RawData: rawData, + Asset: formatAsset(balanceTransaction.Source.Charge.Currency), + Scheme: models.PaymentSchemeOther, + CreatedAt: time.Unix(balanceTransaction.Created, 0), + } + case stripe.BalanceTransactionTypePayoutCancel: + payment = models.Payment{ + Reference: balanceTransaction.Source.Payout.ID, + Type: models.PaymentTypePayOut, + Status: models.PaymentStatusFailed, + Adjustments: []*models.Adjustment{ + { + Reference: balanceTransaction.Source.Payout.ID, + Status: convertPayoutStatus(balanceTransaction.Source.Payout.Status), + CreatedAt: time.Unix(balanceTransaction.Created, 0), + RawData: rawData, + Absolute: true, + }, + }, + } + case stripe.BalanceTransactionTypePayoutFailure: + payment = models.Payment{ + Reference: balanceTransaction.Source.Payout.ID, + Type: models.PaymentTypePayIn, + Status: models.PaymentStatusFailed, + Adjustments: []*models.Adjustment{ + { + Reference: balanceTransaction.Source.Payout.ID, + Status: convertPayoutStatus(balanceTransaction.Source.Payout.Status), + CreatedAt: time.Unix(balanceTransaction.Created, 0), + RawData: rawData, + Absolute: true, + }, + }, + } + case stripe.BalanceTransactionTypePaymentRefund: + payment = models.Payment{ + Reference: balanceTransaction.Source.Refund.Charge.ID, + Type: models.PaymentTypePayOut, + Status: models.PaymentStatusSucceeded, + Adjustments: []*models.Adjustment{ + { + Reference: balanceTransaction.Source.Refund.Charge.ID, + Status: models.PaymentStatusSucceeded, + Amount: balanceTransaction.Amount, + CreatedAt: time.Unix(balanceTransaction.Created, 0), + RawData: rawData, + }, + }, + } + case stripe.BalanceTransactionTypeAdjustment: + payment = models.Payment{ + Reference: balanceTransaction.Source.Dispute.Charge.ID, + Type: models.PaymentTypePayOut, + Adjustments: []*models.Adjustment{ + { + Reference: balanceTransaction.Source.Dispute.Charge.ID, + Status: models.PaymentStatusCancelled, + Amount: balanceTransaction.Amount, + CreatedAt: time.Unix(balanceTransaction.Created, 0), + RawData: rawData, + }, + }, + } + case stripe.BalanceTransactionTypeStripeFee: + return ingestion.PaymentBatchElement{}, false + default: + return ingestion.PaymentBatchElement{}, false + } + + return ingestion.PaymentBatchElement{ + Payment: &payment, + Update: forward, + }, true +} + +func convertPayoutStatus(status stripe.PayoutStatus) models.PaymentStatus { + switch status { + case stripe.PayoutStatusCanceled: + return models.PaymentStatusCancelled + case stripe.PayoutStatusFailed: + return models.PaymentStatusFailed + case stripe.PayoutStatusInTransit, stripe.PayoutStatusPending: + return models.PaymentStatusPending + case stripe.PayoutStatusPaid: + return models.PaymentStatusSucceeded + } + + return models.PaymentStatusOther +} diff --git a/internal/app/connectors/stripe/utils_test.go b/internal/app/connectors/stripe/utils_test.go new file mode 100644 index 00000000..f36efb79 --- /dev/null +++ b/internal/app/connectors/stripe/utils_test.go @@ -0,0 +1,199 @@ +package stripe + +import ( + "context" + "flag" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "sync" + "testing" + "time" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/go-libs/logging/logginglogrus" + "github.com/sirupsen/logrus" + "github.com/stripe/stripe-go/v72" +) + +func TestMain(m *testing.M) { + flag.Parse() + + if testing.Verbose() { + l := logrus.New() + l.Level = logrus.DebugLevel + logging.SetFactory(logging.StaticLoggerFactory(logginglogrus.New(l))) + } + + os.Exit(m.Run()) +} + +type ClientMockExpectation struct { + query url.Values + hasMore bool + items []*stripe.BalanceTransaction +} + +func (e *ClientMockExpectation) QueryParam(key string, value any) *ClientMockExpectation { + var qpvalue string + switch value.(type) { + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: + qpvalue = fmt.Sprintf("%d", value) + default: + qpvalue = fmt.Sprintf("%s", value) + } + e.query.Set(key, qpvalue) + + return e +} + +func (e *ClientMockExpectation) StartingAfter(v string) *ClientMockExpectation { + e.QueryParam("starting_after", v) + + return e +} + +func (e *ClientMockExpectation) CreatedLte(v time.Time) *ClientMockExpectation { + e.QueryParam("created[lte]", v.Unix()) + + return e +} + +func (e *ClientMockExpectation) Limit(v int) *ClientMockExpectation { + e.QueryParam("limit", v) + + return e +} + +func (e *ClientMockExpectation) RespondsWith(hasMore bool, + txs ...*stripe.BalanceTransaction, +) *ClientMockExpectation { + e.hasMore = hasMore + e.items = txs + + return e +} + +func (e *ClientMockExpectation) handle(options ...ClientOption) ([]*stripe.BalanceTransaction, bool, error) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + + for _, option := range options { + option.apply(req) + } + + for key := range e.query { + if req.URL.Query().Get(key) != e.query.Get(key) { + return nil, false, fmt.Errorf("mismatch query params, expected query param '%s' "+ + "with value '%s', got '%s'", key, e.query.Get(key), req.URL.Query().Get(key)) + } + } + + return e.items, e.hasMore, nil +} + +type ClientMock struct { + expectations *FIFO[*ClientMockExpectation] +} + +func (m *ClientMock) ForAccount(account string) Client { + return m +} + +func (m *ClientMock) BalanceTransactions(ctx context.Context, + options ...ClientOption, +) ([]*stripe.BalanceTransaction, bool, error) { + e, ok := m.expectations.Pop() + if !ok { + return nil, false, fmt.Errorf("no more expectation") + } + + return e.handle(options...) +} + +func (m *ClientMock) Expect() *ClientMockExpectation { + e := &ClientMockExpectation{ + query: url.Values{}, + } + m.expectations.Push(e) + + return e +} + +func NewClientMock(t *testing.T, expectationsShouldBeConsumed bool) *ClientMock { + t.Helper() + + m := &ClientMock{ + expectations: &FIFO[*ClientMockExpectation]{}, + } + + if expectationsShouldBeConsumed { + t.Cleanup(func() { + if !m.expectations.Empty() && !t.Failed() { + t.Errorf("all expectations not consumed") + } + }) + } + + return m +} + +var _ Client = &ClientMock{} + +type FIFO[ITEM any] struct { + mu sync.Mutex + items []ITEM +} + +func (s *FIFO[ITEM]) Pop() (ITEM, bool) { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.items) == 0 { + var i ITEM + + return i, false + } + + ret := s.items[0] + + if len(s.items) == 1 { + s.items = make([]ITEM, 0) + + return ret, true + } + + s.items = s.items[1:] + + return ret, true +} + +func (s *FIFO[ITEM]) Peek() (ITEM, bool) { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.items) == 0 { + var i ITEM + + return i, false + } + + return s.items[0], true +} + +func (s *FIFO[ITEM]) Push(i ITEM) *FIFO[ITEM] { + s.mu.Lock() + defer s.mu.Unlock() + + s.items = append(s.items, i) + + return s +} + +func (s *FIFO[ITEM]) Empty() bool { + s.mu.Lock() + defer s.mu.Unlock() + + return len(s.items) == 0 +} diff --git a/internal/app/connectors/wise/client.go b/internal/app/connectors/wise/client.go new file mode 100644 index 00000000..754e3708 --- /dev/null +++ b/internal/app/connectors/wise/client.go @@ -0,0 +1,175 @@ +package wise + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" +) + +const apiEndpoint = "https://api.wise.com" + +type apiTransport struct { + APIKey string + underlying http.RoundTripper +} + +func (t *apiTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", t.APIKey)) + + return t.underlying.RoundTrip(req) +} + +type client struct { + httpClient *http.Client +} + +type profile struct { + ID uint64 `json:"id"` + Type string `json:"type"` +} + +type transfer struct { + ID uint64 `json:"id"` + Reference string `json:"reference"` + Status string `json:"status"` + SourceAccount uint64 `json:"sourceAccount"` + SourceCurrency string `json:"sourceCurrency"` + SourceValue float64 `json:"sourceValue"` + TargetAccount uint64 `json:"targetAccount"` + TargetCurrency string `json:"targetCurrency"` + TargetValue float64 `json:"targetValue"` + Business uint64 `json:"business"` + Created string `json:"created"` + //nolint:tagliatelle // allow for clients + CustomerTransactionID string `json:"customerTransactionId"` + Details struct { + Reference string `json:"reference"` + } `json:"details"` + Rate float64 `json:"rate"` + User uint64 `json:"user"` + + createdAt time.Time +} + +func (t *transfer) UnmarshalJSON(data []byte) error { + type Alias transfer + + aux := &struct { + Created string `json:"created"` + *Alias + }{ + Alias: (*Alias)(t), + } + + if err := json.Unmarshal(data, &aux); err != nil { + return err + } + + var err error + + t.createdAt, err = time.Parse("2006-01-02 15:04:05", aux.Created) + if err != nil { + return fmt.Errorf("failed to parse created time: %w", err) + } + + return nil +} + +func (w *client) endpoint(path string) string { + return fmt.Sprintf("%s/%s", apiEndpoint, path) +} + +func (w *client) getProfiles() ([]profile, error) { + var profiles []profile + + res, err := w.httpClient.Get(w.endpoint("v1/profiles")) + if err != nil { + return profiles, err + } + + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + err = json.Unmarshal(body, &profiles) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal profiles: %w", err) + } + + return profiles, nil +} + +func (w *client) getTransfers(ctx context.Context, profile *profile) ([]transfer, error) { + var transfers []transfer + + limit := 10 + offset := 0 + + for { + req, err := http.NewRequestWithContext(ctx, + http.MethodGet, w.endpoint("v1/transfers"), http.NoBody) + if err != nil { + return transfers, err + } + + q := req.URL.Query() + q.Add("limit", fmt.Sprintf("%d", limit)) + q.Add("profile", fmt.Sprintf("%d", profile.ID)) + q.Add("offset", fmt.Sprintf("%d", offset)) + req.URL.RawQuery = q.Encode() + + res, err := w.httpClient.Do(req) + if err != nil { + return transfers, err + } + + body, err := io.ReadAll(res.Body) + if err != nil { + res.Body.Close() + + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + if err = res.Body.Close(); err != nil { + return nil, fmt.Errorf("failed to close response body: %w", err) + } + + var transferList []transfer + + err = json.Unmarshal(body, &transferList) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal transfers: %w", err) + } + + transfers = append(transfers, transferList...) + + if len(transferList) < limit { + break + } + + offset += limit + } + + return transfers, nil +} + +func newClient(apiKey string) *client { + httpClient := &http.Client{ + Transport: &apiTransport{ + APIKey: apiKey, + underlying: otelhttp.NewTransport(http.DefaultTransport), + }, + } + + return &client{ + httpClient: httpClient, + } +} diff --git a/internal/app/connectors/wise/config.go b/internal/app/connectors/wise/config.go new file mode 100644 index 00000000..b1c01d51 --- /dev/null +++ b/internal/app/connectors/wise/config.go @@ -0,0 +1,31 @@ +package wise + +import ( + "encoding/json" + + "github.com/formancehq/payments/internal/app/connectors/configtemplate" +) + +type Config struct { + APIKey string `json:"apiKey" yaml:"apiKey" bson:"apiKey"` +} + +func (c Config) Validate() error { + if c.APIKey == "" { + return ErrMissingAPIKey + } + + return nil +} + +func (c Config) Marshal() ([]byte, error) { + return json.Marshal(c) +} + +func (c Config) BuildTemplate() (string, configtemplate.Config) { + cfg := configtemplate.NewConfig() + + cfg.AddParameter("apiKey", configtemplate.TypeString, true) + + return Name.String(), cfg +} diff --git a/internal/app/connectors/wise/connector.go b/internal/app/connectors/wise/connector.go new file mode 100644 index 00000000..ab71883b --- /dev/null +++ b/internal/app/connectors/wise/connector.go @@ -0,0 +1,54 @@ +package wise + +import ( + "context" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/integration" + "github.com/formancehq/payments/internal/app/task" +) + +const Name = models.ConnectorProviderWise + +type Connector struct { + logger logging.Logger + cfg Config +} + +func (c *Connector) Install(ctx task.ConnectorContext) error { + descriptor, err := models.EncodeTaskDescriptor(TaskDescriptor{ + Name: "Fetch profiles from client", + Key: taskNameFetchProfiles, + }) + if err != nil { + return err + } + + return ctx.Scheduler().Schedule(descriptor, true) +} + +func (c *Connector) Uninstall(ctx context.Context) error { + return nil +} + +func (c *Connector) Resolve(descriptor models.TaskDescriptor) task.Task { + taskDescriptor, err := models.DecodeTaskDescriptor[TaskDescriptor](descriptor) + if err != nil { + panic(err) + } + + return resolveTasks(c.logger, c.cfg)(taskDescriptor) +} + +var _ integration.Connector = &Connector{} + +func newConnector(logger logging.Logger, cfg Config) *Connector { + return &Connector{ + logger: logger.WithFields(map[string]any{ + "component": "connector", + }), + cfg: cfg, + } +} diff --git a/internal/app/connectors/wise/errors.go b/internal/app/connectors/wise/errors.go new file mode 100644 index 00000000..a4007d12 --- /dev/null +++ b/internal/app/connectors/wise/errors.go @@ -0,0 +1,11 @@ +package wise + +import "github.com/pkg/errors" + +var ( + // ErrMissingTask is returned when the task is missing. + ErrMissingTask = errors.New("task is not implemented") + + // ErrMissingAPIKey is returned when the api key is missing from config. + ErrMissingAPIKey = errors.New("missing apiKey from config") +) diff --git a/internal/app/connectors/wise/loader.go b/internal/app/connectors/wise/loader.go new file mode 100644 index 00000000..ee45ab29 --- /dev/null +++ b/internal/app/connectors/wise/loader.go @@ -0,0 +1,32 @@ +package wise + +import ( + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/integration" + "github.com/formancehq/payments/internal/app/models" +) + +type Loader struct{} + +const allowedTasks = 50 + +func (l *Loader) AllowTasks() int { + return allowedTasks +} + +func (l *Loader) Name() models.ConnectorProvider { + return Name +} + +func (l *Loader) Load(logger logging.Logger, config Config) integration.Connector { + return newConnector(logger, config) +} + +func (l *Loader) ApplyDefaults(cfg Config) Config { + return cfg +} + +// NewLoader creates a new loader. +func NewLoader() *Loader { + return &Loader{} +} diff --git a/internal/app/connectors/wise/task_fetch_profiles.go b/internal/app/connectors/wise/task_fetch_profiles.go new file mode 100644 index 00000000..01506762 --- /dev/null +++ b/internal/app/connectors/wise/task_fetch_profiles.go @@ -0,0 +1,44 @@ +package wise + +import ( + "context" + "fmt" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/payments/internal/app/task" + + "github.com/formancehq/go-libs/logging" +) + +func taskFetchProfiles(logger logging.Logger, client *client) task.Task { + return func( + ctx context.Context, + scheduler task.Scheduler, + ) error { + profiles, err := client.getProfiles() + if err != nil { + return err + } + + for _, profile := range profiles { + logger.Infof(fmt.Sprintf("scheduling fetch-transfers: %d", profile.ID)) + + descriptor, err := models.EncodeTaskDescriptor(TaskDescriptor{ + Name: "Fetch transfers from client by profile", + Key: taskNameFetchTransfers, + ProfileID: profile.ID, + }) + if err != nil { + return err + } + + err = scheduler.Schedule(descriptor, true) + if err != nil { + return err + } + } + + return nil + } +} diff --git a/internal/app/connectors/wise/task_fetch_transfers.go b/internal/app/connectors/wise/task_fetch_transfers.go new file mode 100644 index 00000000..58c2e433 --- /dev/null +++ b/internal/app/connectors/wise/task_fetch_transfers.go @@ -0,0 +1,134 @@ +package wise + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/ingestion" + "github.com/formancehq/payments/internal/app/task" +) + +func taskFetchTransfers(logger logging.Logger, client *client, profileID uint64) task.Task { + return func( + ctx context.Context, + scheduler task.Scheduler, + ingester ingestion.Ingester, + ) error { + transfers, err := client.getTransfers(ctx, &profile{ + ID: profileID, + }) + if err != nil { + return err + } + + if len(transfers) == 0 { + logger.Info("No transfers found") + + return nil + } + + var ( + accountBatch ingestion.AccountBatch + paymentBatch ingestion.PaymentBatch + ) + + for _, transfer := range transfers { + logger.Info(transfer) + + var rawData json.RawMessage + + rawData, err = json.Marshal(transfer) + if err != nil { + return fmt.Errorf("failed to marshal transfer: %w", err) + } + + batchElement := ingestion.PaymentBatchElement{ + Payment: &models.Payment{ + CreatedAt: transfer.createdAt, + Reference: fmt.Sprintf("%d", transfer.ID), + Type: models.PaymentTypeTransfer, + Status: matchTransferStatus(transfer.Status), + Scheme: models.PaymentSchemeOther, + Amount: int64(transfer.TargetValue * 100), + Asset: models.PaymentAsset(fmt.Sprintf("%s/2", transfer.TargetCurrency)), + RawData: rawData, + }, + } + + if transfer.SourceAccount != 0 { + ref := fmt.Sprintf("%d", transfer.SourceAccount) + + accountBatch = append(accountBatch, + ingestion.AccountBatchElement{ + Reference: ref, + Type: models.AccountTypeSource, + }, + ) + + batchElement.Payment.Account = &models.Account{Reference: ref} + } + + if transfer.TargetAccount != 0 { + ref := fmt.Sprintf("%d", transfer.TargetAccount) + + accountBatch = append(accountBatch, + ingestion.AccountBatchElement{ + Reference: ref, + Provider: models.ConnectorProviderWise.String(), + Type: models.AccountTypeTarget, + }, + ) + + batchElement.Payment.Account = &models.Account{Reference: ref} + } + + paymentBatch = append(paymentBatch, batchElement) + } + + if len(accountBatch) > 0 { + err = ingester.IngestAccounts(ctx, accountBatch) + if err != nil { + return err + } + } + + err = ingester.IngestPayments(ctx, paymentBatch, struct{}{}) + if err != nil { + return err + } + + // TODO: Implement proper looper & abstract the logic + + time.Sleep(time.Minute) + + descriptor, err := models.EncodeTaskDescriptor(TaskDescriptor{ + Name: "Fetch profiles from client", + Key: taskNameFetchProfiles, + }) + if err != nil { + return err + } + + return scheduler.Schedule(descriptor, true) + } +} + +func matchTransferStatus(status string) models.PaymentStatus { + switch status { + case "incoming_payment_waiting", "processing": + return models.PaymentStatusPending + case "funds_converted", "outgoing_payment_sent": + return models.PaymentStatusSucceeded + case "bounced_back", "funds_refunded": + return models.PaymentStatusFailed + case "cancelled": + return models.PaymentStatusCancelled + } + + return models.PaymentStatusOther +} diff --git a/internal/app/connectors/wise/task_resolve.go b/internal/app/connectors/wise/task_resolve.go new file mode 100644 index 00000000..4d7b7e34 --- /dev/null +++ b/internal/app/connectors/wise/task_resolve.go @@ -0,0 +1,39 @@ +package wise + +import ( + "fmt" + + "github.com/formancehq/payments/internal/app/task" + + "github.com/formancehq/go-libs/logging" +) + +const ( + taskNameFetchTransfers = "fetch-transfers" + taskNameFetchProfiles = "fetch-profiles" +) + +// TaskDescriptor is the definition of a task. +type TaskDescriptor struct { + Name string `json:"name" yaml:"name" bson:"name"` + Key string `json:"key" yaml:"key" bson:"key"` + ProfileID uint64 `json:"profileID" yaml:"profileID" bson:"profileID"` +} + +func resolveTasks(logger logging.Logger, config Config) func(taskDefinition TaskDescriptor) task.Task { + client := newClient(config.APIKey) + + return func(taskDefinition TaskDescriptor) task.Task { + switch taskDefinition.Key { + case taskNameFetchProfiles: + return taskFetchProfiles(logger, client) + case taskNameFetchTransfers: + return taskFetchTransfers(logger, client, taskDefinition.ProfileID) + } + + // This should never happen. + return func() error { + return fmt.Errorf("key '%s': %w", taskDefinition.Key, ErrMissingTask) + } + } +} diff --git a/internal/app/ingestion/accounts.go b/internal/app/ingestion/accounts.go new file mode 100644 index 00000000..d2a0e93a --- /dev/null +++ b/internal/app/ingestion/accounts.go @@ -0,0 +1,64 @@ +package ingestion + +import ( + "context" + "fmt" + "time" + + "github.com/formancehq/payments/internal/app/messages" + + "github.com/formancehq/payments/internal/app/models" +) + +type AccountBatchElement struct { + Reference string + Provider string + Type models.AccountType +} + +type AccountBatch []AccountBatchElement + +type AccountIngesterFn func(ctx context.Context, batch AccountBatch, commitState any) error + +func (fn AccountIngesterFn) IngestAccounts(ctx context.Context, batch AccountBatch, commitState any) error { + return fn(ctx, batch, commitState) +} + +func (i *DefaultIngester) IngestAccounts(ctx context.Context, batch AccountBatch) error { + startingAt := time.Now() + + i.logger.WithFields(map[string]interface{}{ + "size": len(batch), + "startingAt": startingAt, + }).Debugf("Ingest accounts batch") + + accounts := make([]models.Account, len(batch)) + + for batchIdx := range batch { + accounts[batchIdx] = models.Account{ + Reference: batch[batchIdx].Reference, + Provider: batch[batchIdx].Provider, + Type: batch[batchIdx].Type, + } + } + + if err := i.repo.UpsertAccounts(ctx, i.provider, accounts); err != nil { + return fmt.Errorf("error upserting accounts: %w", err) + } + + err := i.publisher.Publish(ctx, messages.TopicPayments, + messages.NewEventSavedAccounts(accounts)) + if err != nil { + i.logger.Errorf("Publishing message: %w", err) + } + + endedAt := time.Now() + + i.logger.WithFields(map[string]interface{}{ + "size": len(batch), + "endedAt": endedAt, + "latency": endedAt.Sub(startingAt).String(), + }).Debugf("Accounts batch ingested") + + return nil +} diff --git a/internal/app/ingestion/ingester.go b/internal/app/ingestion/ingester.go new file mode 100644 index 00000000..1bf9ca0f --- /dev/null +++ b/internal/app/ingestion/ingester.go @@ -0,0 +1,45 @@ +package ingestion + +import ( + "context" + "encoding/json" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/go-libs/publish" + "github.com/formancehq/payments/internal/app/models" +) + +type Ingester interface { + IngestPayments(ctx context.Context, batch PaymentBatch, commitState any) error + IngestAccounts(ctx context.Context, batch AccountBatch) error +} + +type DefaultIngester struct { + repo Repository + logger logging.Logger + provider models.ConnectorProvider + descriptor models.TaskDescriptor + publisher publish.Publisher +} + +type Repository interface { + UpsertAccounts(ctx context.Context, provider models.ConnectorProvider, accounts []models.Account) error + UpsertPayments(ctx context.Context, provider models.ConnectorProvider, payments []*models.Payment) error + UpdateTaskState(ctx context.Context, provider models.ConnectorProvider, descriptor models.TaskDescriptor, state json.RawMessage) error +} + +func NewDefaultIngester( + provider models.ConnectorProvider, + descriptor models.TaskDescriptor, + repo Repository, + logger logging.Logger, + publisher publish.Publisher, +) *DefaultIngester { + return &DefaultIngester{ + provider: provider, + descriptor: descriptor, + repo: repo, + logger: logger, + publisher: publisher, + } +} diff --git a/internal/app/ingestion/payments.go b/internal/app/ingestion/payments.go new file mode 100644 index 00000000..17c51173 --- /dev/null +++ b/internal/app/ingestion/payments.go @@ -0,0 +1,86 @@ +package ingestion + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/formancehq/payments/internal/app/messages" + + "github.com/formancehq/payments/internal/app/models" +) + +type PaymentBatchElement struct { + Payment *models.Payment + Adjustment *models.Adjustment + Metadata *models.Metadata + Update bool +} + +type PaymentBatch []PaymentBatchElement + +type IngesterFn func(ctx context.Context, batch PaymentBatch, commitState any) error + +func (fn IngesterFn) IngestPayments(ctx context.Context, batch PaymentBatch, commitState any) error { + return fn(ctx, batch, commitState) +} + +func (i *DefaultIngester) IngestPayments(ctx context.Context, batch PaymentBatch, commitState any) error { + startingAt := time.Now() + + i.logger.WithFields(map[string]interface{}{ + "size": len(batch), + "startingAt": startingAt, + }).Debugf("Ingest batch") + + var allPayments []*models.Payment //nolint:prealloc // length is unknown + + for batchIdx := range batch { + payment := batch[batchIdx].Payment + + if payment == nil { + continue + } + + allPayments = append(allPayments, payment) + } + + if err := i.repo.UpsertPayments(ctx, i.provider, allPayments); err != nil { + return fmt.Errorf("error upserting payments: %w", err) + } + + taskDescriptor, err := json.Marshal(i.descriptor) + if err != nil { + return fmt.Errorf("error marshaling task descriptor: %w", err) + } + + taskState, err := json.Marshal(commitState) + if err != nil { + return fmt.Errorf("error marshaling task state: %w", err) + } + + if err = i.repo.UpdateTaskState(ctx, i.provider, taskDescriptor, taskState); err != nil { + return fmt.Errorf("error updating task state: %w", err) + } + + for paymentIdx := range allPayments { + err = i.publisher.Publish(ctx, messages.TopicPayments, + messages.NewEventSavedPayments(allPayments[paymentIdx], i.provider)) + if err != nil { + i.logger.Errorf("Publishing message: %w", err) + + continue + } + } + + endedAt := time.Now() + + i.logger.WithFields(map[string]interface{}{ + "size": len(batch), + "endedAt": endedAt, + "latency": endedAt.Sub(startingAt).String(), + }).Debugf("Batch ingested") + + return nil +} diff --git a/internal/app/integration/connector.go b/internal/app/integration/connector.go new file mode 100644 index 00000000..687099d1 --- /dev/null +++ b/internal/app/integration/connector.go @@ -0,0 +1,96 @@ +package integration + +import ( + "context" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/payments/internal/app/task" +) + +// Connector provide entry point to a payment provider. +type Connector interface { + // Install is used to start the connector. The implementation if in charge of scheduling all required resources. + Install(ctx task.ConnectorContext) error + // Uninstall is used to uninstall the connector. It has to close all related resources opened by the connector. + Uninstall(ctx context.Context) error + // Resolve is used to recover state of a failed or restarted task + Resolve(descriptor models.TaskDescriptor) task.Task +} + +type ConnectorBuilder struct { + name string + uninstall func(ctx context.Context) error + resolve func(descriptor models.TaskDescriptor) task.Task + install func(ctx task.ConnectorContext) error +} + +func (b *ConnectorBuilder) WithUninstall( + uninstallFunction func(ctx context.Context) error, +) *ConnectorBuilder { + b.uninstall = uninstallFunction + + return b +} + +func (b *ConnectorBuilder) WithResolve(resolveFunction func(name models.TaskDescriptor) task.Task) *ConnectorBuilder { + b.resolve = resolveFunction + + return b +} + +func (b *ConnectorBuilder) WithInstall(installFunction func(ctx task.ConnectorContext) error) *ConnectorBuilder { + b.install = installFunction + + return b +} + +func (b *ConnectorBuilder) Build() Connector { + return &BuiltConnector{ + name: b.name, + uninstall: b.uninstall, + resolve: b.resolve, + install: b.install, + } +} + +func NewConnectorBuilder() *ConnectorBuilder { + return &ConnectorBuilder{} +} + +type BuiltConnector struct { + name string + uninstall func(ctx context.Context) error + resolve func(name models.TaskDescriptor) task.Task + install func(ctx task.ConnectorContext) error +} + +func (b *BuiltConnector) Name() string { + return b.name +} + +func (b *BuiltConnector) Install(ctx task.ConnectorContext) error { + if b.install != nil { + return b.install(ctx) + } + + return nil +} + +func (b *BuiltConnector) Uninstall(ctx context.Context) error { + if b.uninstall != nil { + return b.uninstall(ctx) + } + + return nil +} + +func (b *BuiltConnector) Resolve(name models.TaskDescriptor) task.Task { + if b.resolve != nil { + return b.resolve(name) + } + + return nil +} + +var _ Connector = &BuiltConnector{} diff --git a/internal/app/integration/loader.go b/internal/app/integration/loader.go new file mode 100644 index 00000000..519aa6c6 --- /dev/null +++ b/internal/app/integration/loader.go @@ -0,0 +1,97 @@ +package integration + +import ( + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/models" +) + +type Loader[ConnectorConfig models.ConnectorConfigObject] interface { + Name() models.ConnectorProvider + Load(logger logging.Logger, config ConnectorConfig) Connector + + // ApplyDefaults is used to fill default values of the provided configuration object + ApplyDefaults(t ConnectorConfig) ConnectorConfig + + // AllowTasks define how many task the connector can run + // If too many tasks are scheduled by the connector, + // those will be set to pending state and restarted later when some other tasks will be terminated + AllowTasks() int +} + +type LoaderBuilder[ConnectorConfig models.ConnectorConfigObject] struct { + loadFunction func(logger logging.Logger, config ConnectorConfig) Connector + applyDefaults func(t ConnectorConfig) ConnectorConfig + name models.ConnectorProvider + allowedTasks int +} + +func (b *LoaderBuilder[ConnectorConfig]) WithLoad(loadFunction func(logger logging.Logger, + config ConnectorConfig) Connector, +) *LoaderBuilder[ConnectorConfig] { + b.loadFunction = loadFunction + + return b +} + +func (b *LoaderBuilder[ConnectorConfig]) WithApplyDefaults( + applyDefaults func(t ConnectorConfig) ConnectorConfig, +) *LoaderBuilder[ConnectorConfig] { + b.applyDefaults = applyDefaults + + return b +} + +func (b *LoaderBuilder[ConnectorConfig]) WithAllowedTasks(v int) *LoaderBuilder[ConnectorConfig] { + b.allowedTasks = v + + return b +} + +func (b *LoaderBuilder[ConnectorConfig]) Build() *BuiltLoader[ConnectorConfig] { + return &BuiltLoader[ConnectorConfig]{ + loadFunction: b.loadFunction, + applyDefaults: b.applyDefaults, + name: b.name, + allowedTasks: b.allowedTasks, + } +} + +func NewLoaderBuilder[ConnectorConfig models.ConnectorConfigObject](name models.ConnectorProvider, +) *LoaderBuilder[ConnectorConfig] { + return &LoaderBuilder[ConnectorConfig]{ + name: name, + } +} + +type BuiltLoader[ConnectorConfig models.ConnectorConfigObject] struct { + loadFunction func(logger logging.Logger, config ConnectorConfig) Connector + applyDefaults func(t ConnectorConfig) ConnectorConfig + name models.ConnectorProvider + allowedTasks int +} + +func (b *BuiltLoader[ConnectorConfig]) AllowTasks() int { + return b.allowedTasks +} + +func (b *BuiltLoader[ConnectorConfig]) Name() models.ConnectorProvider { + return b.name +} + +func (b *BuiltLoader[ConnectorConfig]) Load(logger logging.Logger, config ConnectorConfig) Connector { + if b.loadFunction != nil { + return b.loadFunction(logger, config) + } + + return nil +} + +func (b *BuiltLoader[ConnectorConfig]) ApplyDefaults(t ConnectorConfig) ConnectorConfig { + if b.applyDefaults != nil { + return b.applyDefaults(t) + } + + return t +} + +var _ Loader[models.EmptyConnectorConfig] = &BuiltLoader[models.EmptyConnectorConfig]{} diff --git a/internal/app/integration/manager.go b/internal/app/integration/manager.go new file mode 100644 index 00000000..b3c106fb --- /dev/null +++ b/internal/app/integration/manager.go @@ -0,0 +1,275 @@ +package integration + +import ( + "context" + + "github.com/formancehq/payments/internal/app/messages" + + "github.com/formancehq/go-libs/publish" + + "github.com/formancehq/payments/internal/app/storage" + + "github.com/google/uuid" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/payments/internal/app/task" + "github.com/pkg/errors" +) + +var ( + ErrNotFound = errors.New("not found") + ErrAlreadyInstalled = errors.New("already installed") + ErrNotInstalled = errors.New("not installed") + ErrNotEnabled = errors.New("not enabled") + ErrAlreadyRunning = errors.New("already running") +) + +type ConnectorManager[Config models.ConnectorConfigObject] struct { + logger logging.Logger + loader Loader[Config] + connector Connector + store Repository + schedulerFactory TaskSchedulerFactory + scheduler *task.DefaultTaskScheduler + publisher publish.Publisher +} + +func (l *ConnectorManager[ConnectorConfig]) Enable(ctx context.Context) error { + l.logger.Info("Enabling connector") + + err := l.store.Enable(ctx, l.loader.Name()) + if err != nil { + return err + } + + return nil +} + +func (l *ConnectorManager[ConnectorConfig]) ReadConfig(ctx context.Context, +) (*ConnectorConfig, error) { + var config ConnectorConfig + + connector, err := l.store.GetConnector(ctx, l.loader.Name()) + if err != nil { + return &config, err + } + + err = connector.ParseConfig(&config) + if err != nil { + return &config, err + } + + config = l.loader.ApplyDefaults(config) + + return &config, nil +} + +func (l *ConnectorManager[ConnectorConfig]) load(config ConnectorConfig) { + l.connector = l.loader.Load(l.logger, config) + l.scheduler = l.schedulerFactory.Make(l.connector, l.loader.AllowTasks()) +} + +func (l *ConnectorManager[ConnectorConfig]) Install(ctx context.Context, config ConnectorConfig) error { + l.logger.WithFields(map[string]interface{}{ + "config": config, + }).Infof("Install connector %s", l.loader.Name()) + + isInstalled, err := l.store.IsInstalled(ctx, l.loader.Name()) + if err != nil { + l.logger.Errorf("Error checking if connector is installed: %s", err) + + return err + } + + if isInstalled { + l.logger.Errorf("Connector already installed") + + return ErrAlreadyInstalled + } + + config = l.loader.ApplyDefaults(config) + + if err = config.Validate(); err != nil { + return err + } + + l.load(config) + + cfg, err := config.Marshal() + if err != nil { + return err + } + + err = l.store.Install(ctx, l.loader.Name(), cfg) + if err != nil { + return err + } + + err = l.connector.Install(task.NewConnectorContext(context.Background(), l.scheduler)) + if err != nil { + l.logger.Errorf("Error starting connector: %s", err) + + return err + } + + l.logger.Infof("Connector installed") + + return nil +} + +func (l *ConnectorManager[ConnectorConfig]) Uninstall(ctx context.Context) error { + l.logger.Infof("Uninstalling connector") + + isInstalled, err := l.IsInstalled(ctx) + if err != nil { + l.logger.Errorf("Error checking if connector is installed: %s", err) + + return err + } + + if !isInstalled { + l.logger.Errorf("Connector not installed") + + return ErrNotInstalled + } + + err = l.scheduler.Shutdown(ctx) + if err != nil { + return err + } + + err = l.connector.Uninstall(ctx) + if err != nil { + return err + } + + err = l.store.Uninstall(ctx, l.loader.Name()) + if err != nil { + return err + } + + l.logger.Info("Connector uninstalled") + + return nil +} + +func (l *ConnectorManager[ConnectorConfig]) Restore(ctx context.Context) error { + l.logger.Info("Restoring state") + + installed, err := l.IsInstalled(ctx) + if err != nil { + return err + } + + if !installed { + l.logger.Info("Not installed, skip") + + return ErrNotInstalled + } + + enabled, err := l.IsEnabled(ctx) + if err != nil { + return err + } + + if !enabled { + l.logger.Info("Not enabled, skip") + + return ErrNotEnabled + } + + if l.connector != nil { + return ErrAlreadyRunning + } + + config, err := l.ReadConfig(ctx) + if err != nil { + return err + } + + l.load(*config) + + err = l.scheduler.Restore(ctx) + if err != nil { + l.logger.Errorf("Unable to restore scheduler: %s", err) + + return err + } + + l.logger.Info("State restored") + + return nil +} + +func (l *ConnectorManager[ConnectorConfig]) Disable(ctx context.Context) error { + l.logger.Info("Disabling connector") + + return l.store.Disable(ctx, l.loader.Name()) +} + +func (l *ConnectorManager[ConnectorConfig]) IsEnabled(ctx context.Context) (bool, error) { + return l.store.IsEnabled(ctx, l.loader.Name()) +} + +func (l *ConnectorManager[ConnectorConfig]) FindAll(ctx context.Context) ([]models.Connector, error) { + return l.store.FindAll(ctx) +} + +func (l *ConnectorManager[ConnectorConfig]) IsInstalled(ctx context.Context) (bool, error) { + return l.store.IsInstalled(ctx, l.loader.Name()) +} + +func (l *ConnectorManager[ConnectorConfig]) ListTasksStates(ctx context.Context, pagination storage.Paginator, +) ([]models.Task, storage.PaginationDetails, error) { + return l.scheduler.ListTasks(ctx, pagination) +} + +func (l *ConnectorManager[Config]) ReadTaskState(ctx context.Context, taskID uuid.UUID) (*models.Task, error) { + return l.scheduler.ReadTask(ctx, taskID) +} + +func (l *ConnectorManager[ConnectorConfig]) Reset(ctx context.Context) error { + config, err := l.ReadConfig(ctx) + if err != nil { + return err + } + + err = l.Uninstall(ctx) + if err != nil { + return err + } + + err = l.Install(ctx, *config) + if err != nil { + return err + } + + err = l.publisher.Publish(ctx, messages.TopicPayments, + messages.NewEventResetConnector(l.loader.Name())) + if err != nil { + l.logger.Errorf("Publishing message: %w", err) + } + + return nil +} + +func NewConnectorManager[ConnectorConfig models.ConnectorConfigObject]( + logger logging.Logger, + store Repository, + loader Loader[ConnectorConfig], + schedulerFactory TaskSchedulerFactory, + publisher publish.Publisher, +) *ConnectorManager[ConnectorConfig] { + return &ConnectorManager[ConnectorConfig]{ + logger: logger.WithFields(map[string]interface{}{ + "component": "connector-manager", + "provider": loader.Name(), + }), + store: store, + loader: loader, + schedulerFactory: schedulerFactory, + publisher: publisher, + } +} diff --git a/internal/app/integration/manager_test.go b/internal/app/integration/manager_test.go new file mode 100644 index 00000000..08620215 --- /dev/null +++ b/internal/app/integration/manager_test.go @@ -0,0 +1,207 @@ +package integration + +import ( + "context" + "testing" + + "github.com/google/uuid" + + "go.uber.org/dig" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/payments/internal/app/task" + + "github.com/formancehq/go-libs/logging" + "github.com/formancehq/go-libs/logging/logginglogrus" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func ChanClosed[T any](ch chan T) bool { + select { + case <-ch: + return true + default: + return false + } +} + +type testContext[ConnectorConfig models.ConnectorConfigObject] struct { + manager *ConnectorManager[ConnectorConfig] + taskStore task.Repository + connectorStore Repository + loader Loader[ConnectorConfig] + provider models.ConnectorProvider +} + +func withManager[ConnectorConfig models.ConnectorConfigObject](builder *ConnectorBuilder, + callback func(ctx *testContext[ConnectorConfig]), +) { + l := logrus.New() + if testing.Verbose() { + l.SetLevel(logrus.DebugLevel) + } + + DefaultContainerFactory := task.ContainerCreateFunc(func(ctx context.Context, descriptor models.TaskDescriptor, taskID uuid.UUID) (*dig.Container, error) { + return dig.New(), nil + }) + + logger := logginglogrus.New(l) + taskStore := task.NewInMemoryStore() + managerStore := NewInMemoryStore() + provider := models.ConnectorProvider(uuid.New().String()) + schedulerFactory := TaskSchedulerFactoryFn(func(resolver task.Resolver, + maxTasks int, + ) *task.DefaultTaskScheduler { + return task.NewDefaultScheduler(provider, logger, taskStore, + DefaultContainerFactory, resolver, maxTasks) + }) + + loader := NewLoaderBuilder[ConnectorConfig](provider). + WithLoad(func(logger logging.Logger, config ConnectorConfig) Connector { + return builder.Build() + }). + WithAllowedTasks(1). + Build() + manager := NewConnectorManager[ConnectorConfig](logger, managerStore, loader, + schedulerFactory, nil) + + defer func() { + _ = manager.Uninstall(context.Background()) + }() + + callback(&testContext[ConnectorConfig]{ + manager: manager, + taskStore: taskStore, + connectorStore: managerStore, + loader: loader, + provider: provider, + }) +} + +func TestInstallConnector(t *testing.T) { + t.Parallel() + + installed := make(chan struct{}) + builder := NewConnectorBuilder(). + WithInstall(func(ctx task.ConnectorContext) error { + close(installed) + + return nil + }) + withManager(builder, func(tc *testContext[models.EmptyConnectorConfig]) { + err := tc.manager.Install(context.Background(), models.EmptyConnectorConfig{}) + require.NoError(t, err) + require.True(t, ChanClosed(installed)) + + err = tc.manager.Install(context.Background(), models.EmptyConnectorConfig{}) + require.Equal(t, ErrAlreadyInstalled, err) + }) +} + +func TestUninstallConnector(t *testing.T) { + t.Parallel() + + uninstalled := make(chan struct{}) + taskTerminated := make(chan struct{}) + taskStarted := make(chan struct{}) + builder := NewConnectorBuilder(). + WithResolve(func(name models.TaskDescriptor) task.Task { + return func(ctx context.Context, stopChan task.StopChan) { + close(taskStarted) + defer close(taskTerminated) + select { + case flag := <-stopChan: + flag <- struct{}{} + case <-ctx.Done(): + } + } + }). + WithInstall(func(ctx task.ConnectorContext) error { + return ctx.Scheduler().Schedule([]byte(uuid.New().String()), false) + }). + WithUninstall(func(ctx context.Context) error { + close(uninstalled) + + return nil + }) + withManager(builder, func(tc *testContext[models.EmptyConnectorConfig]) { + err := tc.manager.Install(context.Background(), models.EmptyConnectorConfig{}) + require.NoError(t, err) + <-taskStarted + require.NoError(t, tc.manager.Uninstall(context.Background())) + require.True(t, ChanClosed(uninstalled)) + // TODO: We need to give a chance to the connector to properly stop execution + require.True(t, ChanClosed(taskTerminated)) + + isInstalled, err := tc.manager.IsInstalled(context.Background()) + require.NoError(t, err) + require.False(t, isInstalled) + }) +} + +func TestDisableConnector(t *testing.T) { + t.Parallel() + + uninstalled := make(chan struct{}) + builder := NewConnectorBuilder(). + WithUninstall(func(ctx context.Context) error { + close(uninstalled) + + return nil + }) + withManager[models.EmptyConnectorConfig](builder, func(tc *testContext[models.EmptyConnectorConfig]) { + err := tc.manager.Install(context.Background(), models.EmptyConnectorConfig{}) + require.NoError(t, err) + + enabled, err := tc.manager.IsEnabled(context.Background()) + require.NoError(t, err) + require.True(t, enabled) + + require.NoError(t, tc.manager.Disable(context.Background())) + enabled, err = tc.manager.IsEnabled(context.Background()) + require.NoError(t, err) + require.False(t, enabled) + }) +} + +func TestEnableConnector(t *testing.T) { + t.Parallel() + + builder := NewConnectorBuilder() + withManager[models.EmptyConnectorConfig](builder, func(tc *testContext[models.EmptyConnectorConfig]) { + err := tc.connectorStore.Enable(context.Background(), tc.loader.Name()) + require.NoError(t, err) + + err = tc.manager.Install(context.Background(), models.EmptyConnectorConfig{}) + require.NoError(t, err) + }) +} + +func TestRestoreEnabledConnector(t *testing.T) { + t.Parallel() + + builder := NewConnectorBuilder() + withManager(builder, func(tc *testContext[models.EmptyConnectorConfig]) { + cfg, err := models.EmptyConnectorConfig{}.Marshal() + require.NoError(t, err) + + err = tc.connectorStore.Install(context.Background(), tc.loader.Name(), cfg) + require.NoError(t, err) + + err = tc.manager.Restore(context.Background()) + require.NoError(t, err) + require.NotNil(t, tc.manager.connector) + }) +} + +func TestRestoreNotInstalledConnector(t *testing.T) { + t.Parallel() + + builder := NewConnectorBuilder() + withManager(builder, func(tc *testContext[models.EmptyConnectorConfig]) { + err := tc.manager.Restore(context.Background()) + require.Equal(t, ErrNotInstalled, err) + }) +} diff --git a/internal/app/integration/store.go b/internal/app/integration/store.go new file mode 100644 index 00000000..8fe0ea8d --- /dev/null +++ b/internal/app/integration/store.go @@ -0,0 +1,20 @@ +package integration + +import ( + "context" + "encoding/json" + + "github.com/formancehq/payments/internal/app/models" +) + +type Repository interface { + FindAll(ctx context.Context) ([]models.Connector, error) + IsInstalled(ctx context.Context, name models.ConnectorProvider) (bool, error) + Install(ctx context.Context, name models.ConnectorProvider, config json.RawMessage) error + Uninstall(ctx context.Context, name models.ConnectorProvider) error + UpdateConfig(ctx context.Context, name models.ConnectorProvider, config json.RawMessage) error + Enable(ctx context.Context, name models.ConnectorProvider) error + Disable(ctx context.Context, name models.ConnectorProvider) error + IsEnabled(ctx context.Context, name models.ConnectorProvider) (bool, error) + GetConnector(ctx context.Context, name models.ConnectorProvider) (*models.Connector, error) +} diff --git a/internal/app/integration/storememory.go b/internal/app/integration/storememory.go new file mode 100644 index 00000000..3208b602 --- /dev/null +++ b/internal/app/integration/storememory.go @@ -0,0 +1,99 @@ +package integration + +import ( + "context" + "encoding/json" + + "github.com/formancehq/payments/internal/app/models" +) + +type InMemoryConnectorStore struct { + installed map[models.ConnectorProvider]bool + disabled map[models.ConnectorProvider]bool + configs map[models.ConnectorProvider]json.RawMessage +} + +func (i *InMemoryConnectorStore) Uninstall(ctx context.Context, name models.ConnectorProvider) error { + delete(i.installed, name) + delete(i.configs, name) + delete(i.disabled, name) + + return nil +} + +func (i *InMemoryConnectorStore) FindAll(_ context.Context) ([]models.Connector, error) { + return []models.Connector{}, nil +} + +func (i *InMemoryConnectorStore) IsInstalled(ctx context.Context, name models.ConnectorProvider) (bool, error) { + return i.installed[name], nil +} + +func (i *InMemoryConnectorStore) Install(ctx context.Context, name models.ConnectorProvider, config json.RawMessage) error { + i.installed[name] = true + i.configs[name] = config + i.disabled[name] = false + + return nil +} + +func (i *InMemoryConnectorStore) UpdateConfig(ctx context.Context, name models.ConnectorProvider, config json.RawMessage) error { + i.configs[name] = config + + return nil +} + +func (i *InMemoryConnectorStore) Enable(ctx context.Context, name models.ConnectorProvider) error { + i.disabled[name] = false + + return nil +} + +func (i *InMemoryConnectorStore) Disable(ctx context.Context, name models.ConnectorProvider) error { + i.disabled[name] = true + + return nil +} + +func (i *InMemoryConnectorStore) IsEnabled(ctx context.Context, name models.ConnectorProvider) (bool, error) { + disabled, ok := i.disabled[name] + if !ok { + return false, nil + } + + return !disabled, nil +} + +func (i *InMemoryConnectorStore) GetConnector(ctx context.Context, name models.ConnectorProvider) (*models.Connector, error) { + cfg, ok := i.configs[name] + if !ok { + return nil, ErrNotFound + } + + return &models.Connector{ + Config: cfg, + }, nil +} + +func (i *InMemoryConnectorStore) ReadConfig(ctx context.Context, name models.ConnectorProvider, to interface{}) error { + connector, err := i.GetConnector(ctx, name) + if err != nil { + return err + } + + if err = connector.ParseConfig(to); err != nil { + return err + } + + return nil +} + +var _ Repository = &InMemoryConnectorStore{} + +func NewInMemoryStore() *InMemoryConnectorStore { + return &InMemoryConnectorStore{ + installed: make(map[models.ConnectorProvider]bool), + disabled: make(map[models.ConnectorProvider]bool), + configs: make(map[models.ConnectorProvider]json.RawMessage), + } +} diff --git a/internal/app/integration/taskscheduler.go b/internal/app/integration/taskscheduler.go new file mode 100644 index 00000000..4e6a80c2 --- /dev/null +++ b/internal/app/integration/taskscheduler.go @@ -0,0 +1,15 @@ +package integration + +import ( + "github.com/formancehq/payments/internal/app/task" +) + +type TaskSchedulerFactory interface { + Make(resolver task.Resolver, maxTasks int) *task.DefaultTaskScheduler +} + +type TaskSchedulerFactoryFn func(resolver task.Resolver, maxProcesses int) *task.DefaultTaskScheduler + +func (fn TaskSchedulerFactoryFn) Make(resolver task.Resolver, maxTasks int) *task.DefaultTaskScheduler { + return fn(resolver, maxTasks) +} diff --git a/internal/app/messages/accounts.go b/internal/app/messages/accounts.go new file mode 100644 index 00000000..d92113a2 --- /dev/null +++ b/internal/app/messages/accounts.go @@ -0,0 +1,37 @@ +package messages + +import ( + "time" + + "github.com/formancehq/payments/internal/app/models" +) + +type accountMessagePayload struct { + ID string `json:"id"` + CreatedAt time.Time `json:"createdAt"` + Reference string `json:"reference"` + Provider string `json:"provider"` + Type models.AccountType `json:"type"` +} + +func NewEventSavedAccounts(accounts []models.Account) EventMessage { + payload := make([]accountMessagePayload, len(accounts)) + + for accountIdx, account := range accounts { + payload[accountIdx] = accountMessagePayload{ + ID: account.ID.String(), + CreatedAt: account.CreatedAt, + Reference: account.Reference, + Provider: account.Provider, + Type: account.Type, + } + } + + return EventMessage{ + Date: time.Now().UTC(), + App: EventApp, + Version: EventVersion, + Type: EventTypeSavedAccounts, + Payload: payload, + } +} diff --git a/internal/app/messages/connectors.go b/internal/app/messages/connectors.go new file mode 100644 index 00000000..93032162 --- /dev/null +++ b/internal/app/messages/connectors.go @@ -0,0 +1,25 @@ +package messages + +import ( + "time" + + "github.com/formancehq/payments/internal/app/models" +) + +type connectorMessagePayload struct { + CreatedAt time.Time `json:"createdAt"` + Connector models.ConnectorProvider `json:"connector"` +} + +func NewEventResetConnector(connector models.ConnectorProvider) EventMessage { + return EventMessage{ + Date: time.Now().UTC(), + App: EventApp, + Version: EventVersion, + Type: EventTypeConnectorReset, + Payload: connectorMessagePayload{ + CreatedAt: time.Now().UTC(), + Connector: connector, + }, + } +} diff --git a/internal/app/messages/event.go b/internal/app/messages/event.go new file mode 100644 index 00000000..67d76b8f --- /dev/null +++ b/internal/app/messages/event.go @@ -0,0 +1,25 @@ +package messages + +import ( + "time" +) + +const ( + TopicPayments = "payments" + TopicConnectors = "connectors" + + EventVersion = "v1" + EventApp = "payments" + + EventTypeSavedPayments = "SAVED_PAYMENT" + EventTypeSavedAccounts = "SAVED_ACCOUNT" + EventTypeConnectorReset = "CONNECTOR_RESET" +) + +type EventMessage struct { + Date time.Time `json:"date"` + App string `json:"app"` + Version string `json:"version"` + Type string `json:"type"` + Payload any `json:"payload"` +} diff --git a/internal/app/messages/payments.go b/internal/app/messages/payments.go new file mode 100644 index 00000000..0121eafc --- /dev/null +++ b/internal/app/messages/payments.go @@ -0,0 +1,45 @@ +package messages + +import ( + "time" + + "github.com/formancehq/payments/internal/app/models" +) + +type paymentMessagePayload struct { + ID string `json:"id"` + Reference string `json:"reference"` + CreatedAt time.Time `json:"createdAt"` + Provider string `json:"provider"` + Type models.PaymentType `json:"type"` + Status models.PaymentStatus `json:"status"` + Scheme models.PaymentScheme `json:"scheme"` + Asset models.PaymentAsset `json:"asset"` + + // TODO: Remove 'initialAmount' once frontend has switched to 'amount + InitialAmount int64 `json:"initialAmount"` + Amount int64 `json:"amount"` +} + +func NewEventSavedPayments(payment *models.Payment, provider models.ConnectorProvider) EventMessage { + payload := paymentMessagePayload{ + ID: payment.ID.String(), + Reference: payment.Reference, + Type: payment.Type, + Status: payment.Status, + InitialAmount: payment.Amount, + Scheme: payment.Scheme, + Asset: payment.Asset, + CreatedAt: payment.CreatedAt, + Amount: payment.Amount, + Provider: provider.String(), + } + + return EventMessage{ + Date: time.Now().UTC(), + App: EventApp, + Version: EventVersion, + Type: EventTypeSavedPayments, + Payload: payload, + } +} diff --git a/internal/app/migrations/001_initiate_schemas.go b/internal/app/migrations/001_initiate_schemas.go new file mode 100644 index 00000000..62c23f48 --- /dev/null +++ b/internal/app/migrations/001_initiate_schemas.go @@ -0,0 +1,39 @@ +package migrations + +import ( + "database/sql" + + "github.com/pressly/goose/v3" +) + +func init() { + up := func(tx *sql.Tx) error { + _, err := tx.Exec(` + CREATE SCHEMA IF NOT EXISTS connectors; + CREATE SCHEMA IF NOT EXISTS tasks; + CREATE SCHEMA IF NOT EXISTS accounts; + CREATE SCHEMA IF NOT EXISTS payments; + `) + if err != nil { + return err + } + + return nil + } + + down := func(tx *sql.Tx) error { + _, err := tx.Exec(` + DROP SCHEMA IF EXISTS connectors; + DROP SCHEMA IF EXISTS tasks; + DROP SCHEMA IF EXISTS accounts; + DROP SCHEMA IF EXISTS payments; + `) + if err != nil { + return err + } + + return nil + } + + goose.AddMigration(up, down) +} diff --git a/internal/app/migrations/002_connectors.go b/internal/app/migrations/002_connectors.go new file mode 100644 index 00000000..4f8fc1ab --- /dev/null +++ b/internal/app/migrations/002_connectors.go @@ -0,0 +1,43 @@ +package migrations + +import ( + "database/sql" + + "github.com/pressly/goose/v3" +) + +func init() { + up := func(tx *sql.Tx) error { + _, err := tx.Exec(` + CREATE TYPE connector_provider AS ENUM ('BANKING-CIRCLE', 'CURRENCY-CLOUD', 'DUMMY-PAY', 'MODULR', 'STRIPE', 'WISE');; + + CREATE TABLE connectors.connector ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + created_at timestamp with time zone NOT NULL DEFAULT NOW() CHECK (created_at<=NOW()), + provider connector_provider NOT NULL UNIQUE, + enabled boolean NOT NULL DEFAULT false, + config json NULL, + CONSTRAINT connector_pk PRIMARY KEY (id) + ); + `) + if err != nil { + return err + } + + return nil + } + + down := func(tx *sql.Tx) error { + _, err := tx.Exec(` + DROP TABLE connectors.connector; + DROP TYPE connector_provider; + `) + if err != nil { + return err + } + + return nil + } + + goose.AddMigration(up, down) +} diff --git a/internal/app/migrations/003_tasks.go b/internal/app/migrations/003_tasks.go new file mode 100644 index 00000000..6a5ffc66 --- /dev/null +++ b/internal/app/migrations/003_tasks.go @@ -0,0 +1,55 @@ +package migrations + +import ( + "database/sql" + + "github.com/pressly/goose/v3" +) + +func init() { + up := func(tx *sql.Tx) error { + _, err := tx.Exec(` + CREATE TYPE task_status AS ENUM ('STOPPED', 'PENDING', 'ACTIVE', 'TERMINATED', 'FAILED');; + + CREATE TABLE tasks.task ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + connector_id uuid NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT NOW() CHECK (created_at<=NOW()), + updated_at timestamp with time zone NOT NULL DEFAULT NOW() CHECK (created_at<=updated_at), + name text NOT NULL, + descriptor json NULL, + status task_status NOT NULL, + error text NULL, + state json NULL, + CONSTRAINT task_pk PRIMARY KEY (id) + ); + + ALTER TABLE tasks.task ADD CONSTRAINT task_connector + FOREIGN KEY (connector_id) + REFERENCES connectors.connector (id) + ON DELETE CASCADE + NOT DEFERRABLE + INITIALLY IMMEDIATE + ; + `) + if err != nil { + return err + } + + return nil + } + + down := func(tx *sql.Tx) error { + _, err := tx.Exec(` + DROP TABLE tasks.task; + DROP TYPE task_status; + `) + if err != nil { + return err + } + + return nil + } + + goose.AddMigration(up, down) +} diff --git a/internal/app/migrations/004_accounts.go b/internal/app/migrations/004_accounts.go new file mode 100644 index 00000000..fe5930b7 --- /dev/null +++ b/internal/app/migrations/004_accounts.go @@ -0,0 +1,43 @@ +package migrations + +import ( + "database/sql" + + "github.com/pressly/goose/v3" +) + +func init() { + up := func(tx *sql.Tx) error { + _, err := tx.Exec(` + CREATE TYPE account_type AS ENUM('SOURCE', 'TARGET', 'UNKNOWN');; + + CREATE TABLE accounts.account ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + created_at timestamp with time zone NOT NULL DEFAULT NOW() CHECK (created_at<=NOW()), + reference text NOT NULL UNIQUE, + provider text NOT NULL, + type account_type NOT NULL, + CONSTRAINT account_pk PRIMARY KEY (id) + ); + `) + if err != nil { + return err + } + + return nil + } + + down := func(tx *sql.Tx) error { + _, err := tx.Exec(` + DROP TABLE accounts.account; + DROP TYPE account_type; + `) + if err != nil { + return err + } + + return nil + } + + goose.AddMigration(up, down) +} diff --git a/internal/app/migrations/005_payments.go b/internal/app/migrations/005_payments.go new file mode 100644 index 00000000..fa00a541 --- /dev/null +++ b/internal/app/migrations/005_payments.go @@ -0,0 +1,106 @@ +package migrations + +import ( + "database/sql" + + "github.com/pressly/goose/v3" +) + +func init() { + up := func(tx *sql.Tx) error { + _, err := tx.Exec(` + CREATE TYPE payment_type AS ENUM ('PAY-IN', 'PAYOUT', 'TRANSFER', 'OTHER'); + CREATE TYPE payment_status AS ENUM ('SUCCEEDED', 'CANCELLED', 'FAILED', 'PENDING', 'OTHER');; + + CREATE TABLE payments.adjustment ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + payment_id uuid NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT NOW() CHECK (created_at<=NOW()), + amount bigint NOT NULL DEFAULT 0, + reference text NOT NULL UNIQUE, + status payment_status NOT NULL, + absolute boolean NOT NULL DEFAULT FALSE, + raw_data json NULL, + CONSTRAINT adjustment_pk PRIMARY KEY (id) + ); + + CREATE TABLE payments.metadata ( + payment_id uuid NOT NULL, + created_at timestamp with time zone NOT NULL DEFAULT NOW() CHECK (created_at<=NOW()), + key text NOT NULL, + value text NOT NULL, + changelog jsonb NOT NULL, + CONSTRAINT metadata_pk PRIMARY KEY (payment_id,key) + ); + + CREATE TABLE payments.payment ( + id uuid NOT NULL DEFAULT gen_random_uuid(), + connector_id uuid NOT NULL, + account_id uuid DEFAULT NULL, + created_at timestamp with time zone NOT NULL DEFAULT NOW() CHECK (created_at<=NOW()), + reference text NOT NULL UNIQUE, + type payment_type NOT NULL, + status payment_status NOT NULL, + amount bigint NOT NULL DEFAULT 0, + raw_data json NULL, + scheme text NOT NULL, + asset text NOT NULL, + CONSTRAINT payment_pk PRIMARY KEY (id) + ); + + ALTER TABLE payments.adjustment ADD CONSTRAINT adjustment_payment + FOREIGN KEY (payment_id) + REFERENCES payments.payment (id) + ON DELETE CASCADE + NOT DEFERRABLE + INITIALLY IMMEDIATE + ; + + ALTER TABLE payments.metadata ADD CONSTRAINT metadata_payment + FOREIGN KEY (payment_id) + REFERENCES payments.payment (id) + ON DELETE CASCADE + NOT DEFERRABLE + INITIALLY IMMEDIATE + ; + + ALTER TABLE payments.payment ADD CONSTRAINT payment_account + FOREIGN KEY (account_id) + REFERENCES accounts.account (id) + ON DELETE CASCADE + NOT DEFERRABLE + INITIALLY IMMEDIATE + ; + + ALTER TABLE payments.payment ADD CONSTRAINT payment_connector + FOREIGN KEY (connector_id) + REFERENCES connectors.connector (id) + ON DELETE CASCADE + NOT DEFERRABLE + INITIALLY IMMEDIATE + ; + `) + if err != nil { + return err + } + + return nil + } + + down := func(tx *sql.Tx) error { + _, err := tx.Exec(` + DROP TABLE payments.adjustment; + DROP TABLE payments.metadata; + DROP TABLE payments.payment; + DROP TYPE payment_type; + DROP TYPE payment_status; + `) + if err != nil { + return err + } + + return nil + } + + goose.AddMigration(up, down) +} diff --git a/internal/app/models/account.go b/internal/app/models/account.go new file mode 100644 index 00000000..ab3aab58 --- /dev/null +++ b/internal/app/models/account.go @@ -0,0 +1,28 @@ +package models + +import ( + "time" + + "github.com/google/uuid" + "github.com/uptrace/bun" +) + +type Account struct { + bun.BaseModel `bun:"accounts.account"` + + ID uuid.UUID `bun:",pk,nullzero"` + CreatedAt time.Time `bun:",nullzero"` + Reference string + Provider string + Type AccountType + + Payments []*Payment `bun:"rel:has-many,join:id=account_id"` +} + +type AccountType string + +const ( + AccountTypeSource AccountType = "SOURCE" + AccountTypeTarget AccountType = "TARGET" + AccountTypeUnknown AccountType = "UNKNOWN" +) diff --git a/internal/app/models/adjustment.go b/internal/app/models/adjustment.go new file mode 100644 index 00000000..51e6e1b9 --- /dev/null +++ b/internal/app/models/adjustment.go @@ -0,0 +1,26 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/uptrace/bun" + + "github.com/google/uuid" +) + +type Adjustment struct { + bun.BaseModel `bun:"payments.adjustment"` + + ID uuid.UUID `bun:",pk,nullzero"` + PaymentID uuid.UUID `bun:",pk,nullzero"` + CreatedAt time.Time `bun:",nullzero"` + Reference string + Amount int64 + Status PaymentStatus + Absolute bool + + RawData json.RawMessage + + Payment *Payment `bun:"rel:has-one,join:payment_id=id"` +} diff --git a/internal/app/models/connector.go b/internal/app/models/connector.go new file mode 100644 index 00000000..892dd61e --- /dev/null +++ b/internal/app/models/connector.go @@ -0,0 +1,74 @@ +package models + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/uptrace/bun" + + "github.com/google/uuid" +) + +type Connector struct { + bun.BaseModel `bun:"connectors.connector"` + + ID uuid.UUID `bun:",pk,nullzero"` + CreatedAt time.Time `bun:",nullzero"` + Provider ConnectorProvider + Enabled bool + + // TODO: Enable DB-level encryption + Config json.RawMessage + + Tasks []*Task `bun:"rel:has-many,join:id=connector_id"` + Payments []*Payment `bun:"rel:has-many,join:id=connector_id"` +} + +type ConnectorProvider string + +const ( + ConnectorProviderBankingCircle ConnectorProvider = "BANKING-CIRCLE" + ConnectorProviderCurrencyCloud ConnectorProvider = "CURRENCY-CLOUD" + ConnectorProviderDummyPay ConnectorProvider = "DUMMY-PAY" + ConnectorProviderModulr ConnectorProvider = "MODULR" + ConnectorProviderStripe ConnectorProvider = "STRIPE" + ConnectorProviderWise ConnectorProvider = "WISE" +) + +func (p ConnectorProvider) String() string { + return string(p) +} + +func (p ConnectorProvider) StringLower() string { + return strings.ToLower(string(p)) +} + +func (c Connector) ParseConfig(to interface{}) error { + if c.Config == nil { + return nil + } + + err := json.Unmarshal(c.Config, to) + if err != nil { + return fmt.Errorf("failed to parse config (%s): %w", string(c.Config), err) + } + + return nil +} + +type ConnectorConfigObject interface { + Validate() error + Marshal() ([]byte, error) +} + +type EmptyConnectorConfig struct{} + +func (cfg EmptyConnectorConfig) Validate() error { + return nil +} + +func (cfg EmptyConnectorConfig) Marshal() ([]byte, error) { + return nil, nil +} diff --git a/internal/app/models/metadata.go b/internal/app/models/metadata.go new file mode 100644 index 00000000..fdfbb41b --- /dev/null +++ b/internal/app/models/metadata.go @@ -0,0 +1,26 @@ +package models + +import ( + "time" + + "github.com/uptrace/bun" + + "github.com/google/uuid" +) + +type Metadata struct { + bun.BaseModel `bun:"payments.metadata"` + + PaymentID uuid.UUID `bun:",pk,nullzero"` + CreatedAt time.Time `bun:",nullzero"` + Key string `bun:",pk,nullzero"` + Value string + + Changelog []MetadataChangelog `bun:",pk,nullzero"` + Payment *Payment `bun:"rel:has-one,join:payment_id=id"` +} + +type MetadataChangelog struct { + CreatedAt time.Time + Value string +} diff --git a/internal/app/models/payment.go b/internal/app/models/payment.go new file mode 100644 index 00000000..262d4a2a --- /dev/null +++ b/internal/app/models/payment.go @@ -0,0 +1,96 @@ +package models + +import ( + "encoding/json" + "time" + + "github.com/uptrace/bun" + + "github.com/google/uuid" +) + +type Payment struct { + bun.BaseModel `bun:"payments.payment"` + + ID uuid.UUID `bun:",pk,nullzero"` + ConnectorID uuid.UUID `bun:",nullzero"` + CreatedAt time.Time `bun:",nullzero"` + Reference string + Amount int64 + Type PaymentType + Status PaymentStatus + Scheme PaymentScheme + Asset PaymentAsset + + RawData json.RawMessage + + AccountID uuid.UUID `bun:",nullzero"` + + Account *Account `bun:"rel:has-one,join:account_id=id"` + Adjustments []*Adjustment `bun:"rel:has-many,join:id=payment_id"` + Metadata []*Metadata `bun:"rel:has-many,join:id=payment_id"` + Connector *Connector `bun:"rel:has-one,join:connector_id=id"` +} + +type ( + PaymentType string + PaymentStatus string + PaymentScheme string + PaymentAsset string +) + +const ( + PaymentTypePayIn PaymentType = "PAY-IN" + PaymentTypePayOut PaymentType = "PAYOUT" + PaymentTypeTransfer PaymentType = "TRANSFER" + PaymentTypeOther PaymentType = "OTHER" +) + +const ( + PaymentStatusPending PaymentStatus = "PENDING" + PaymentStatusSucceeded PaymentStatus = "SUCCEEDED" + PaymentStatusCancelled PaymentStatus = "CANCELLED" + PaymentStatusFailed PaymentStatus = "FAILED" + PaymentStatusOther PaymentStatus = "OTHER" +) + +const ( + PaymentSchemeUnknown PaymentScheme = "unknown" + PaymentSchemeOther PaymentScheme = "other" + + PaymentSchemeCardVisa PaymentScheme = "visa" + PaymentSchemeCardMasterCard PaymentScheme = "mastercard" + PaymentSchemeCardAmex PaymentScheme = "amex" + PaymentSchemeCardDiners PaymentScheme = "diners" + PaymentSchemeCardDiscover PaymentScheme = "discover" + PaymentSchemeCardJCB PaymentScheme = "jcb" + PaymentSchemeCardUnionPay PaymentScheme = "unionpay" + + PaymentSchemeSepaDebit PaymentScheme = "sepa debit" + PaymentSchemeSepaCredit PaymentScheme = "sepa credit" + PaymentSchemeSepa PaymentScheme = "sepa" + + PaymentSchemeApplePay PaymentScheme = "apple pay" + PaymentSchemeGooglePay PaymentScheme = "google pay" + + PaymentSchemeA2A PaymentScheme = "a2a" + PaymentSchemeACHDebit PaymentScheme = "ach debit" + PaymentSchemeACH PaymentScheme = "ach" + PaymentSchemeRTP PaymentScheme = "rtp" +) + +func (t PaymentType) String() string { + return string(t) +} + +func (t PaymentStatus) String() string { + return string(t) +} + +func (t PaymentScheme) String() string { + return string(t) +} + +func (t PaymentAsset) String() string { + return string(t) +} diff --git a/internal/app/models/task.go b/internal/app/models/task.go new file mode 100644 index 00000000..242f03c8 --- /dev/null +++ b/internal/app/models/task.go @@ -0,0 +1,86 @@ +package models + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "time" + + "github.com/uptrace/bun" + + "github.com/google/uuid" +) + +type Task struct { + bun.BaseModel `bun:"tasks.task"` + + ID uuid.UUID `bun:",pk,nullzero"` + ConnectorID uuid.UUID + CreatedAt time.Time `bun:",nullzero"` + UpdatedAt time.Time `bun:",nullzero"` + Name string + Descriptor json.RawMessage + Status TaskStatus + Error string + State json.RawMessage + + Connector *Connector `bun:"rel:belongs-to,join:connector_id=id"` +} + +func (t Task) GetDescriptor() TaskDescriptor { + return TaskDescriptor(t.Descriptor) +} + +type TaskDescriptor json.RawMessage + +func (td TaskDescriptor) ToMessage() json.RawMessage { + return json.RawMessage(td) +} + +func (td TaskDescriptor) EncodeToString() (string, error) { + data, err := json.Marshal(td) + if err != nil { + return "", fmt.Errorf("failed to encode task descriptor: %w", err) + } + + return base64.StdEncoding.EncodeToString(data), nil +} + +func EncodeTaskDescriptor(descriptor any) (TaskDescriptor, error) { + res, err := json.Marshal(descriptor) + if err != nil { + return nil, fmt.Errorf("failed to encode task descriptor: %w", err) + } + + return res, nil +} + +func DecodeTaskDescriptor[descriptor any](data TaskDescriptor) (descriptor, error) { + var res descriptor + + err := json.Unmarshal(data, &res) + if err != nil { + return res, fmt.Errorf("failed to decode task descriptor: %w", err) + } + + return res, nil +} + +type TaskStatus string + +const ( + TaskStatusStopped TaskStatus = "STOPPED" + TaskStatusPending TaskStatus = "PENDING" + TaskStatusActive TaskStatus = "ACTIVE" + TaskStatusTerminated TaskStatus = "TERMINATED" + TaskStatusFailed TaskStatus = "FAILED" +) + +func (t Task) ParseDescriptor(to interface{}) error { + err := json.Unmarshal(t.Descriptor, to) + if err != nil { + return fmt.Errorf("failed to parse descriptor: %w", err) + } + + return nil +} diff --git a/internal/app/storage/accounts.go b/internal/app/storage/accounts.go new file mode 100644 index 00000000..2ea80a2c --- /dev/null +++ b/internal/app/storage/accounts.go @@ -0,0 +1,72 @@ +package storage + +import ( + "context" + "fmt" + "time" + + "github.com/formancehq/payments/internal/app/models" +) + +func (s *Storage) UpsertAccounts(ctx context.Context, provider models.ConnectorProvider, accounts []models.Account) error { + if len(accounts) == 0 { + return nil + } + + accountsMap := make(map[string]models.Account) + for _, account := range accounts { + accountsMap[account.Reference] = account + } + + accounts = make([]models.Account, 0, len(accountsMap)) + for _, account := range accountsMap { + accounts = append(accounts, account) + } + + _, err := s.db.NewInsert(). + Model(&accounts). + On("CONFLICT (reference) DO UPDATE"). + Set("provider = EXCLUDED.provider"). + Set("type = EXCLUDED.type"). + Exec(ctx) + if err != nil { + return e("failed to create accounts", err) + } + + return nil +} + +func (s *Storage) ListAccounts(ctx context.Context, pagination Paginator) ([]*models.Account, PaginationDetails, error) { + var accounts []*models.Account + + query := s.db.NewSelect(). + Model(&accounts) + + query = pagination.apply(query, "account.created_at") + + err := query.Scan(ctx) + if err != nil { + return nil, PaginationDetails{}, fmt.Errorf("failed to list payments: %w", err) + } + + var ( + hasMore = len(accounts) > pagination.pageSize + firstReference, lastReference string + ) + + if hasMore { + accounts = accounts[:pagination.pageSize] + } + + if len(accounts) > 0 { + firstReference = accounts[0].CreatedAt.Format(time.RFC3339Nano) + lastReference = accounts[len(accounts)-1].CreatedAt.Format(time.RFC3339Nano) + } + + paginationDetails, err := pagination.paginationDetails(hasMore, firstReference, lastReference) + if err != nil { + return nil, PaginationDetails{}, fmt.Errorf("failed to get pagination details: %w", err) + } + + return accounts, paginationDetails, nil +} diff --git a/internal/app/storage/connectors.go b/internal/app/storage/connectors.go new file mode 100644 index 00000000..8ba0c154 --- /dev/null +++ b/internal/app/storage/connectors.go @@ -0,0 +1,155 @@ +package storage + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/formancehq/payments/internal/app/models" +) + +func (s *Storage) ListConnectors(ctx context.Context) ([]*models.Connector, error) { + var res []*models.Connector + err := s.db.NewSelect().Model(&res).Scan(ctx) + if err != nil { + return nil, e("list connectors", err) + } + + return res, nil +} + +func (s *Storage) GetConfig(ctx context.Context, connectorProvider models.ConnectorProvider, destination any) error { + var connector models.Connector + + err := s.db.NewSelect().Model(&connector). + Column("config"). + Where("provider = ?", connectorProvider). + Scan(ctx) + if err != nil { + return fmt.Errorf("failed to get config for connector %s: %w", connectorProvider, err) + } + + err = json.Unmarshal(connector.Config, destination) + if err != nil { + return fmt.Errorf("failed to unmarshal config for connector %s: %w", connectorProvider, err) + } + + return nil +} + +func (s *Storage) FindAll(ctx context.Context) ([]models.Connector, error) { + var connectors []models.Connector + + err := s.db.NewSelect().Model(&connectors).Scan(ctx) + if err != nil { + return nil, e("find all connectors", err) + } + + return connectors, err +} + +func (s *Storage) IsInstalled(ctx context.Context, provider models.ConnectorProvider) (bool, error) { + exists, err := s.db.NewSelect(). + Model(&models.Connector{}). + Where("provider = ?", provider). + Exists(ctx) + if err != nil { + return false, e("find connector", err) + } + + return exists, nil +} + +func (s *Storage) Install(ctx context.Context, provider models.ConnectorProvider, config json.RawMessage) error { + connector := models.Connector{ + Provider: provider, + Enabled: true, + Config: config, + } + + _, err := s.db.NewInsert().Model(&connector).Exec(ctx) + if err != nil { + return e("install connector", err) + } + + return nil +} + +func (s *Storage) Uninstall(ctx context.Context, provider models.ConnectorProvider) error { + _, err := s.db.NewDelete(). + Model(&models.Connector{}). + Where("provider = ?", provider). + Exec(ctx) + if err != nil { + return e("uninstall connector", err) + } + + return nil +} + +func (s *Storage) UpdateConfig(ctx context.Context, provider models.ConnectorProvider, config json.RawMessage) error { + _, err := s.db.NewUpdate(). + Model(&models.Connector{}). + Set("config = ?", config). + Where("provider = ?", provider). + Exec(ctx) + if err != nil { + return e("update connector config", err) + } + + return nil +} + +func (s *Storage) Enable(ctx context.Context, provider models.ConnectorProvider) error { + _, err := s.db.NewUpdate(). + Model(&models.Connector{}). + Set("enabled = TRUE"). + Where("provider = ?", provider). + Exec(ctx) + if err != nil { + return e("enable connector", err) + } + + return nil +} + +func (s *Storage) Disable(ctx context.Context, provider models.ConnectorProvider) error { + _, err := s.db.NewUpdate(). + Model(&models.Connector{}). + Set("enabled = TRUE"). + Where("provider = ?", provider). + Exec(ctx) + if err != nil { + return e("enable connector", err) + } + + return nil +} + +func (s *Storage) IsEnabled(ctx context.Context, provider models.ConnectorProvider) (bool, error) { + var connector models.Connector + + err := s.db.NewSelect(). + Model(&connector). + Where("provider = ?", provider). + Scan(ctx) + if err != nil { + return false, e("find connector", err) + } + + return connector.Enabled, nil +} + +func (s *Storage) GetConnector(ctx context.Context, provider models.ConnectorProvider) (*models.Connector, error) { + var connector models.Connector + + err := s.db.NewSelect(). + Model(&connector). + Where("provider = ?", provider). + Scan(ctx) + if err != nil { + return nil, e("find connector", err) + } + + return &connector, nil +} diff --git a/internal/app/storage/error.go b/internal/app/storage/error.go new file mode 100644 index 00000000..20de1f25 --- /dev/null +++ b/internal/app/storage/error.go @@ -0,0 +1,22 @@ +package storage + +import ( + "database/sql" + "fmt" + + "github.com/pkg/errors" +) + +var ErrNotFound = errors.New("not found") + +func e(msg string, err error) error { + if err == nil { + return nil + } + + if errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("%s: %w", msg, ErrNotFound) + } + + return fmt.Errorf("%s: %w", msg, err) +} diff --git a/internal/app/storage/module.go b/internal/app/storage/module.go new file mode 100644 index 00000000..eb82307a --- /dev/null +++ b/internal/app/storage/module.go @@ -0,0 +1,58 @@ +package storage + +import ( + "context" + "database/sql" + "fmt" + + "github.com/uptrace/bun/extra/bunotel" + + "github.com/uptrace/bun/dialect/pgdialect" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" + + "github.com/uptrace/bun" + + "github.com/formancehq/go-libs/logging" + "go.uber.org/fx" +) + +const dbName = "paymentsDB" + +func Module(uri string) fx.Option { + return fx.Options( + fx.Provide(func() (*pgx.ConnConfig, error) { + config, err := pgx.ParseConfig(uri) + if err != nil { + return nil, fmt.Errorf("parsing config: %w", err) + } + + return config, nil + }), + + fx.Provide(func(config *pgx.ConnConfig) *sql.DB { + return stdlib.OpenDB(*config) + }), + + fx.Provide(func(client *sql.DB) *Storage { + db := bun.NewDB(client, pgdialect.New()) + + db.AddQueryHook(bunotel.NewQueryHook(bunotel.WithDBName(dbName))) + + return newStorage(db) + }), + + fx.Invoke(func(lc fx.Lifecycle, repo *Storage) { + lc.Append(fx.Hook{ + OnStart: func(ctx context.Context) error { + logging.Debug("Ping database...") + + // TODO: Check migrations state and panic if migrations are not applied + + return nil + }, + }) + }), + ) +} diff --git a/internal/app/storage/paginate.go b/internal/app/storage/paginate.go new file mode 100644 index 00000000..8cf1ff56 --- /dev/null +++ b/internal/app/storage/paginate.go @@ -0,0 +1,129 @@ +package storage + +import ( + "encoding/base64" + "encoding/json" + "fmt" + + "github.com/uptrace/bun" +) + +const ( + defaultPageSize = 15 + maxPageSize = 100 +) + +type PaginationDetails struct { + PageSize int + HasMore bool + PreviousPage string + NextPage string +} + +type baseCursor struct { + Reference string `json:"reference"` + Sorter Sorter `json:"sorter"` + Next bool `json:"next"` +} + +func (c baseCursor) Encode() (string, error) { + bytes, err := json.Marshal(c) + if err != nil { + return "", fmt.Errorf("error marshaling baseCursor: %w", err) + } + + return base64.StdEncoding.EncodeToString(bytes), nil +} + +type Paginator struct { + pageSize int + token string + + cursor baseCursor + sorter Sorter +} + +func Paginate(pageSize int, token string, sorter Sorter) (Paginator, error) { + if pageSize == 0 { + pageSize = defaultPageSize + } + + if pageSize > maxPageSize { + pageSize = maxPageSize + } + + var cursor baseCursor + + if token != "" { + tokenBytes, err := base64.StdEncoding.DecodeString(token) + if err != nil { + return Paginator{}, fmt.Errorf("error decoding token: %w", err) + } + + err = json.Unmarshal(tokenBytes, &cursor) + if err != nil { + return Paginator{}, fmt.Errorf("error unmarshaling baseCursor: %w", err) + } + } + + return Paginator{pageSize, token, cursor, sorter}, nil +} + +func (p Paginator) apply(query *bun.SelectQuery, column string) *bun.SelectQuery { + query = query.Limit(p.pageSize + 1).Order(column + " DESC") + + if p.cursor.Reference == "" { + if p.sorter != nil { + query = p.sorter.apply(query) + } + + return query + } + + if p.cursor.Sorter != nil { + query = p.cursor.Sorter.apply(query) + } + + if p.cursor.Next { + return query.Where(fmt.Sprintf("%s < ?", column), p.cursor.Reference) + } + + return query.Where(fmt.Sprintf("%s > ?", column), p.cursor.Reference) +} + +func (p Paginator) paginationDetails(hasMore bool, firstReference, lastReference string) (PaginationDetails, error) { + var ( + previousPage string + nextPage string + err error + ) + + if p.cursor.Reference != "" { + previousPage, err = baseCursor{ + Reference: firstReference, + Sorter: p.sorter, + Next: false, + }.Encode() + if err != nil { + return PaginationDetails{}, fmt.Errorf("error encoding previous page cursor: %w", err) + } + } + + if hasMore { + nextPage, err = baseCursor{ + Reference: lastReference, + Sorter: p.sorter, + Next: true, + }.Encode() + if err != nil { + return PaginationDetails{}, fmt.Errorf("error encoding next page cursor: %w", err) + } + } + + return PaginationDetails{ + PageSize: p.pageSize, + HasMore: hasMore, + PreviousPage: previousPage, + NextPage: nextPage, + }, nil +} diff --git a/internal/app/storage/paginate_test.go b/internal/app/storage/paginate_test.go new file mode 100644 index 00000000..6418cf82 --- /dev/null +++ b/internal/app/storage/paginate_test.go @@ -0,0 +1,241 @@ +package storage + +import ( + "reflect" + "testing" + + "github.com/uptrace/bun" +) + +func TestPaginate(t *testing.T) { + t.Parallel() + + type args struct { + pageSize int + token string + sorter Sorter + } + + token, err := baseCursor{ + Reference: "", + Sorter: nil, + Next: false, + }.Encode() + if err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + args args + want Paginator + wantErr bool + }{ + { + name: "valid", + args: args{pageSize: 10, token: "", sorter: nil}, + want: Paginator{pageSize: 10, token: "", cursor: baseCursor{}, sorter: nil}, + wantErr: false, + }, + { + name: "invalid page size", + args: args{pageSize: 0, token: "", sorter: nil}, + want: Paginator{pageSize: defaultPageSize, token: "", cursor: baseCursor{}, sorter: nil}, + wantErr: false, + }, + { + name: "exceeding max page size", + args: args{pageSize: maxPageSize + 1, token: "", sorter: nil}, + want: Paginator{pageSize: maxPageSize, token: "", cursor: baseCursor{}, sorter: nil}, + wantErr: false, + }, + { + name: "token decode", + args: args{pageSize: 10, token: token, sorter: nil}, + want: Paginator{pageSize: 10, token: token, cursor: baseCursor{}, sorter: nil}, + wantErr: false, + }, + { + name: "invalid token", + args: args{pageSize: 10, token: "abc", sorter: nil}, + want: Paginator{pageSize: 0, token: "", cursor: baseCursor{}, sorter: nil}, + wantErr: true, + }, + } + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := Paginate(tt.args.pageSize, tt.args.token, tt.args.sorter) + if (err != nil) != tt.wantErr { + t.Errorf("Paginate() error = %v, wantErr %v", err, tt.wantErr) + + return + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Paginate() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPaginatorApply(t *testing.T) { + t.Parallel() + + type fields struct { + pageSize int + token string + cursor baseCursor + sorter Sorter + } + type args struct { + query *bun.SelectQuery + column string + } + tests := []struct { + name string + fields fields + args args + want *bun.SelectQuery + }{ + { + name: "no cursor", + fields: fields{pageSize: 10, token: "", cursor: baseCursor{}, sorter: nil}, + args: args{query: &bun.SelectQuery{}, column: "id"}, + want: nil, + }, + { + name: "with cursor", + fields: fields{pageSize: 10, token: "", cursor: baseCursor{Reference: "id", Sorter: Sorter{{Column: "id", Order: SortOrderDesc}}}, sorter: nil}, + args: args{query: &bun.SelectQuery{}, column: "id"}, + want: nil, + }, + { + name: "with cursor next", + fields: fields{pageSize: 10, token: "", cursor: baseCursor{Reference: "id", Next: true, Sorter: Sorter{{Column: "id", Order: SortOrderDesc}}}, sorter: nil}, + args: args{query: &bun.SelectQuery{}, column: "id"}, + want: nil, + }, + { + name: "with cursor no ref", + fields: fields{pageSize: 10, token: "", cursor: baseCursor{}, sorter: Sorter{{Column: "id", Order: SortOrderDesc}}}, + args: args{query: &bun.SelectQuery{}, column: "id"}, + want: nil, + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + p := Paginator{ + pageSize: tt.fields.pageSize, + token: tt.fields.token, + cursor: tt.fields.cursor, + sorter: tt.fields.sorter, + } + + if got := p.apply(tt.args.query, tt.args.column); tt.want != nil && !reflect.DeepEqual(got, tt.want) { + t.Errorf("apply() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPaginatorPaginationDetails(t *testing.T) { + t.Parallel() + + type fields struct { + pageSize int + token string + cursor baseCursor + sorter Sorter + } + + type args struct { + hasMore bool + firstReference string + lastReference string + } + + cursor := baseCursor{ + Reference: "abc", + Sorter: nil, + Next: false, + } + + token, err := cursor.Encode() + if err != nil { + t.Fatal(err) + } + + tokenNext, err := baseCursor{ + Reference: "", + Sorter: nil, + Next: true, + }.Encode() + if err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + fields fields + args args + want PaginationDetails + wantErr bool + }{ + { + name: "no cursor", + fields: fields{pageSize: 10, token: "", cursor: baseCursor{}, sorter: nil}, + args: args{hasMore: false, firstReference: "", lastReference: ""}, + want: PaginationDetails{PageSize: 10, HasMore: false}, + wantErr: false, + }, + { + name: "with cursor", + fields: fields{pageSize: 10, token: "", cursor: cursor, sorter: nil}, + args: args{hasMore: false, firstReference: "abc", lastReference: ""}, + want: PaginationDetails{PageSize: 10, HasMore: false, PreviousPage: token}, + wantErr: false, + }, + { + name: "has more", + fields: fields{pageSize: 10, token: "", cursor: baseCursor{}, sorter: nil}, + args: args{hasMore: true, firstReference: "", lastReference: ""}, + want: PaginationDetails{PageSize: 10, HasMore: true, NextPage: tokenNext}, + wantErr: false, + }, + } + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + p := Paginator{ + pageSize: tt.fields.pageSize, + token: tt.fields.token, + cursor: tt.fields.cursor, + sorter: tt.fields.sorter, + } + + got, err := p.paginationDetails(tt.args.hasMore, tt.args.firstReference, tt.args.lastReference) + if (err != nil) != tt.wantErr { + t.Errorf("paginationDetails() error = %v, wantErr %v", err, tt.wantErr) + + return + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("paginationDetails() got = %+v, want %+v", got, tt.want) + } + }) + } +} diff --git a/internal/app/storage/payments.go b/internal/app/storage/payments.go new file mode 100644 index 00000000..320eb488 --- /dev/null +++ b/internal/app/storage/payments.go @@ -0,0 +1,167 @@ +package storage + +import ( + "context" + "fmt" + "time" + + "github.com/uptrace/bun" + + "github.com/formancehq/payments/internal/app/models" +) + +func (s *Storage) ListPayments(ctx context.Context, pagination Paginator) ([]*models.Payment, PaginationDetails, error) { + var payments []*models.Payment + + query := s.db.NewSelect(). + Model(&payments). + Relation("Account"). + Relation("Connector"). + Relation("Metadata"). + Relation("Adjustments") + + query = pagination.apply(query, "payment.created_at") + + err := query.Scan(ctx) + if err != nil { + return nil, PaginationDetails{}, e("failed to list payments", err) + } + + var ( + hasMore = len(payments) > pagination.pageSize + firstReference, lastReference string + ) + + if hasMore { + payments = payments[:pagination.pageSize] + } + + if len(payments) > 0 { + firstReference = payments[0].CreatedAt.Format(time.RFC3339Nano) + lastReference = payments[len(payments)-1].CreatedAt.Format(time.RFC3339Nano) + } + + paginationDetails, err := pagination.paginationDetails(hasMore, firstReference, lastReference) + if err != nil { + return nil, PaginationDetails{}, fmt.Errorf("failed to get pagination details: %w", err) + } + + return payments, paginationDetails, nil +} + +func (s *Storage) GetPayment(ctx context.Context, id string) (*models.Payment, error) { + var payment models.Payment + + err := s.db.NewSelect(). + Model(&payment). + Relation("Connector"). + Relation("Metadata"). + Relation("Adjustments"). + Where("payment.id = ?", id). + Scan(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get payment %s: %w", id, err) + } + + return &payment, nil +} + +func (s *Storage) UpsertPayments(ctx context.Context, provider models.ConnectorProvider, payments []*models.Payment) error { + if len(payments) == 0 { + return nil + } + + connector, err := s.GetConnector(ctx, provider) + if err != nil { + return fmt.Errorf("failed to get connector: %w", err) + } + + var accountReferences []string + + for i := range payments { + payments[i].ConnectorID = connector.ID + + if payments[i].Account != nil && payments[i].Account.Reference != "" { + accountReferences = append(accountReferences, payments[i].Account.Reference) + } + } + + if len(accountReferences) > 0 { + var accounts []models.Account + + err = s.db.NewSelect().Model(&accounts). + Where("reference IN (?)", bun.In(accountReferences)). + Scan(ctx) + if err != nil { + return e("failed to get accounts", err) + } + + for i := range payments { + if payments[i].Account != nil && payments[i].Account.Reference != "" { + for j := range accounts { + if accounts[j].Reference == payments[i].Account.Reference { + payments[i].AccountID = accounts[j].ID + } + } + } + } + } + + _, err = s.db.NewInsert(). + Model(&payments). + On("CONFLICT (reference) DO NOTHING"). + Exec(ctx) + if err != nil { + return e("failed to create payments", err) + } + + var adjustments []*models.Adjustment + var metadata []*models.Metadata + + for i := range payments { + for _, adjustment := range payments[i].Adjustments { + if adjustment.Reference == "" { + continue + } + + adjustment.PaymentID = payments[i].ID + + adjustments = append(adjustments, adjustment) + } + + for _, data := range payments[i].Metadata { + data.PaymentID = payments[i].ID + data.Changelog = append(data.Changelog, + models.MetadataChangelog{ + CreatedAt: time.Now(), + Value: data.Value, + }) + + metadata = append(metadata, data) + } + } + + if len(adjustments) > 0 { + _, err = s.db.NewInsert(). + Model(&adjustments). + On("CONFLICT (reference) DO NOTHING"). + Exec(ctx) + if err != nil { + return e("failed to create adjustments", err) + } + } + + if len(metadata) > 0 { + _, err = s.db.NewInsert(). + Model(&metadata). + On("CONFLICT (payment_id, key) DO UPDATE"). + Set("value = EXCLUDED.value"). + Set("changelog = changelog || EXCLUDED.changelog"). + Exec(ctx) + if err != nil { + return e("failed to create metadata", err) + } + } + + return nil +} diff --git a/internal/app/storage/ping.go b/internal/app/storage/ping.go new file mode 100644 index 00000000..2832abb0 --- /dev/null +++ b/internal/app/storage/ping.go @@ -0,0 +1,5 @@ +package storage + +func (s *Storage) Ping() error { + return s.db.Ping() +} diff --git a/internal/app/storage/repository.go b/internal/app/storage/repository.go new file mode 100644 index 00000000..1804800e --- /dev/null +++ b/internal/app/storage/repository.go @@ -0,0 +1,19 @@ +package storage + +import ( + "github.com/uptrace/bun" + "github.com/uptrace/bun/extra/bundebug" +) + +type Storage struct { + db *bun.DB +} + +func newStorage(db *bun.DB) *Storage { + return &Storage{db: db} +} + +// nolint:unused // used for SQL debugging purposes +func (s *Storage) debug() { + s.db.AddQueryHook(bundebug.NewQueryHook(bundebug.WithVerbose(true))) +} diff --git a/internal/app/storage/sort.go b/internal/app/storage/sort.go new file mode 100644 index 00000000..1988d227 --- /dev/null +++ b/internal/app/storage/sort.go @@ -0,0 +1,33 @@ +package storage + +import ( + "fmt" + + "github.com/uptrace/bun" +) + +type SortOrder string + +const ( + SortOrderAsc SortOrder = "asc" + SortOrderDesc SortOrder = "desc" +) + +type sortExpression struct { + Column string `json:"column"` + Order SortOrder `json:"order"` +} + +type Sorter []sortExpression + +func (s Sorter) Add(column string, order SortOrder) Sorter { + return append(s, sortExpression{column, order}) +} + +func (s Sorter) apply(query *bun.SelectQuery) *bun.SelectQuery { + for _, expr := range s { + query = query.Order(fmt.Sprintf("%s %s", expr.Column, expr.Order)) + } + + return query +} diff --git a/internal/app/storage/task.go b/internal/app/storage/task.go new file mode 100644 index 00000000..e1176925 --- /dev/null +++ b/internal/app/storage/task.go @@ -0,0 +1,202 @@ +package storage + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/google/uuid" + + "github.com/pkg/errors" + + "github.com/formancehq/payments/internal/app/models" +) + +func (s *Storage) UpdateTaskStatus(ctx context.Context, provider models.ConnectorProvider, descriptor models.TaskDescriptor, status models.TaskStatus, taskError string) error { + connector, err := s.GetConnector(ctx, provider) + if err != nil { + return e("failed to get connector", err) + } + + _, err = s.db.NewUpdate().Model(&models.Task{}). + Set("status = ?", status). + Set("error = ?", taskError). + Where("descriptor::TEXT = ?::TEXT", descriptor.ToMessage()). + Where("connector_id = ?", connector.ID). + Exec(ctx) + if err != nil { + return e("failed to update task", err) + } + + return nil +} + +func (s *Storage) UpdateTaskState(ctx context.Context, provider models.ConnectorProvider, descriptor models.TaskDescriptor, state json.RawMessage) error { + connector, err := s.GetConnector(ctx, provider) + if err != nil { + return e("failed to get connector", err) + } + + _, err = s.db.NewUpdate().Model(&models.Task{}). + Set("state = ?", state). + Where("descriptor::TEXT = ?::TEXT", descriptor.ToMessage()). + Where("connector_id = ?", connector.ID). + Exec(ctx) + if err != nil { + return e("failed to update task", err) + } + + return nil +} + +func (s *Storage) FindAndUpsertTask(ctx context.Context, provider models.ConnectorProvider, descriptor models.TaskDescriptor, status models.TaskStatus, taskErr string) (*models.Task, error) { + _, err := s.GetTaskByDescriptor(ctx, provider, descriptor) + if err != nil && !errors.Is(err, ErrNotFound) { + return nil, e("failed to get task", err) + } + + if err == nil { + err = s.UpdateTaskStatus(ctx, provider, descriptor, status, taskErr) + if err != nil { + return nil, e("failed to update task", err) + } + } else { + err = s.CreateTask(ctx, provider, descriptor, status) + if err != nil { + return nil, e("failed to upsert task", err) + } + } + + return s.GetTaskByDescriptor(ctx, provider, descriptor) +} + +func (s *Storage) CreateTask(ctx context.Context, provider models.ConnectorProvider, descriptor models.TaskDescriptor, status models.TaskStatus) error { + connector, err := s.GetConnector(ctx, provider) + if err != nil { + return e("failed to get connector", err) + } + + _, err = s.db.NewInsert().Model(&models.Task{ + ConnectorID: connector.ID, + Descriptor: descriptor.ToMessage(), + Status: status, + }).Exec(ctx) + if err != nil { + return e("failed to create task", err) + } + + return nil +} + +func (s *Storage) ListTasksByStatus(ctx context.Context, provider models.ConnectorProvider, status models.TaskStatus) ([]models.Task, error) { + connector, err := s.GetConnector(ctx, provider) + if err != nil { + return nil, e("failed to get connector", err) + } + + var tasks []models.Task + + err = s.db.NewSelect().Model(&tasks). + Where("connector_id = ?", connector.ID). + Where("status = ?", status). + Scan(ctx) + if err != nil { + return nil, e("failed to get tasks", err) + } + + return tasks, nil +} + +func (s *Storage) ListTasks(ctx context.Context, provider models.ConnectorProvider, pagination Paginator) ([]models.Task, PaginationDetails, error) { + connector, err := s.GetConnector(ctx, provider) + if err != nil { + return nil, PaginationDetails{}, e("failed to get connector", err) + } + + var tasks []models.Task + + query := s.db.NewSelect().Model(&tasks). + Where("connector_id = ?", connector.ID) + + query = pagination.apply(query, "task.created_at") + + err = query.Scan(ctx) + if err != nil { + return nil, PaginationDetails{}, e("failed to get tasks", err) + } + + var ( + hasMore = len(tasks) > pagination.pageSize + firstReference, lastReference string + ) + + if hasMore { + tasks = tasks[:pagination.pageSize] + } + + if len(tasks) > 0 { + firstReference = tasks[0].CreatedAt.Format(time.RFC3339Nano) + lastReference = tasks[len(tasks)-1].CreatedAt.Format(time.RFC3339Nano) + } + + paginationDetails, err := pagination.paginationDetails(hasMore, firstReference, lastReference) + if err != nil { + return nil, PaginationDetails{}, fmt.Errorf("failed to get pagination details: %w", err) + } + + return tasks, paginationDetails, nil +} + +func (s *Storage) ReadOldestPendingTask(ctx context.Context, provider models.ConnectorProvider) (*models.Task, error) { + connector, err := s.GetConnector(ctx, provider) + if err != nil { + return nil, e("failed to get connector", err) + } + + var task models.Task + + err = s.db.NewSelect().Model(&task). + Where("connector_id = ?", connector.ID). + Where("status = ?", models.TaskStatusPending). + Order("created_at ASC"). + Limit(1). + Scan(ctx) + if err != nil { + return nil, e("failed to get task", err) + } + + return &task, nil +} + +func (s *Storage) GetTask(ctx context.Context, id uuid.UUID) (*models.Task, error) { + var task models.Task + + err := s.db.NewSelect().Model(&task). + Where("id = ?", id). + Scan(ctx) + if err != nil { + return nil, e("failed to get task", err) + } + + return &task, nil +} + +func (s *Storage) GetTaskByDescriptor(ctx context.Context, provider models.ConnectorProvider, descriptor models.TaskDescriptor) (*models.Task, error) { + connector, err := s.GetConnector(ctx, provider) + if err != nil { + return nil, e("failed to get connector", err) + } + + var task models.Task + + err = s.db.NewSelect().Model(&task). + Where("connector_id = ?", connector.ID). + Where("descriptor::TEXT = ?::TEXT", descriptor.ToMessage()). + Scan(ctx) + if err != nil { + return nil, e("failed to get task", err) + } + + return &task, nil +} diff --git a/internal/app/task/context.go b/internal/app/task/context.go new file mode 100644 index 00000000..1ed0b776 --- /dev/null +++ b/internal/app/task/context.go @@ -0,0 +1,30 @@ +package task + +import ( + "context" +) + +type ConnectorContext interface { + Context() context.Context + Scheduler() Scheduler +} + +type ConnectorCtx struct { + ctx context.Context + scheduler Scheduler +} + +func (ctx *ConnectorCtx) Context() context.Context { + return ctx.ctx +} + +func (ctx *ConnectorCtx) Scheduler() Scheduler { + return ctx.scheduler +} + +func NewConnectorContext(ctx context.Context, scheduler Scheduler) *ConnectorCtx { + return &ConnectorCtx{ + ctx: ctx, + scheduler: scheduler, + } +} diff --git a/internal/app/task/resolver.go b/internal/app/task/resolver.go new file mode 100644 index 00000000..3752bcdb --- /dev/null +++ b/internal/app/task/resolver.go @@ -0,0 +1,13 @@ +package task + +import "github.com/formancehq/payments/internal/app/models" + +type Resolver interface { + Resolve(descriptor models.TaskDescriptor) Task +} + +type ResolverFn func(descriptor models.TaskDescriptor) Task + +func (fn ResolverFn) Resolve(descriptor models.TaskDescriptor) Task { + return fn(descriptor) +} diff --git a/internal/app/task/scheduler.go b/internal/app/task/scheduler.go new file mode 100644 index 00000000..a51b6a9b --- /dev/null +++ b/internal/app/task/scheduler.go @@ -0,0 +1,367 @@ +package task + +import ( + "context" + "encoding/json" + "fmt" + "runtime/debug" + "sync" + "time" + + "go.uber.org/dig" + + "github.com/google/uuid" + + "github.com/formancehq/payments/internal/app/storage" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/go-libs/logging" + "github.com/pkg/errors" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +var ( + ErrAlreadyScheduled = errors.New("already scheduled") + ErrUnableToResolve = errors.New("unable to resolve task") +) + +type Scheduler interface { + Schedule(p models.TaskDescriptor, restart bool) error +} + +type taskHolder struct { + descriptor models.TaskDescriptor + cancel func() + logger logging.Logger + stopChan StopChan +} + +type ContainerCreateFunc func(ctx context.Context, descriptor models.TaskDescriptor, taskID uuid.UUID) (*dig.Container, error) + +type DefaultTaskScheduler struct { + provider models.ConnectorProvider + logger logging.Logger + store Repository + containerFactory ContainerCreateFunc + tasks map[string]*taskHolder + mu sync.Mutex + maxTasks int + resolver Resolver + stopped bool +} + +func (s *DefaultTaskScheduler) ListTasks(ctx context.Context, pagination storage.Paginator) ([]models.Task, storage.PaginationDetails, error) { + return s.store.ListTasks(ctx, s.provider, pagination) +} + +func (s *DefaultTaskScheduler) ReadTask(ctx context.Context, taskID uuid.UUID) (*models.Task, error) { + return s.store.GetTask(ctx, taskID) +} + +func (s *DefaultTaskScheduler) ReadTaskByDescriptor(ctx context.Context, descriptor models.TaskDescriptor) (*models.Task, error) { + taskDescriptor, err := json.Marshal(descriptor) + if err != nil { + return nil, err + } + + return s.store.GetTaskByDescriptor(ctx, s.provider, taskDescriptor) +} + +func (s *DefaultTaskScheduler) Schedule(descriptor models.TaskDescriptor, restart bool) error { + s.mu.Lock() + defer s.mu.Unlock() + + taskID, err := descriptor.EncodeToString() + if err != nil { + return err + } + + if _, ok := s.tasks[taskID]; ok { + return ErrAlreadyScheduled + } + + if !restart { + _, err := s.ReadTaskByDescriptor(context.Background(), descriptor) + if err == nil { + return nil + } + } + + if s.maxTasks != 0 && len(s.tasks) >= s.maxTasks || s.stopped { + err := s.stackTask(descriptor) + if err != nil { + return errors.Wrap(err, "stacking task") + } + + return nil + } + + if err := s.startTask(descriptor); err != nil { + return errors.Wrap(err, "starting task") + } + + return nil +} + +func (s *DefaultTaskScheduler) Shutdown(ctx context.Context) error { + s.mu.Lock() + s.stopped = true + s.mu.Unlock() + + s.logger.Infof("Stopping scheduler...") + + for name, task := range s.tasks { + task.logger.Debugf("Stopping task") + + if task.stopChan != nil { + errCh := make(chan struct{}) + task.stopChan <- errCh + select { + case <-errCh: + case <-time.After(time.Second): // TODO: Make configurable + task.logger.Debugf("Stopping using stop chan timeout, canceling context") + task.cancel() + } + } else { + task.cancel() + } + + delete(s.tasks, name) + } + + return nil +} + +func (s *DefaultTaskScheduler) Restore(ctx context.Context) error { + tasks, err := s.store.ListTasksByStatus(ctx, s.provider, models.TaskStatusActive) + if err != nil { + return err + } + + for _, task := range tasks { + err = s.startTask(task.GetDescriptor()) + if err != nil { + s.logger.Errorf("Unable to restore task %s: %s", task.ID, err) + } + } + + return nil +} + +func (s *DefaultTaskScheduler) registerTaskError(ctx context.Context, holder *taskHolder, taskErr any) { + var taskError string + + switch v := taskErr.(type) { + case error: + taskError = v.Error() + default: + taskError = fmt.Sprintf("%s", v) + } + + holder.logger.Errorf("Task terminated with error: %s", taskErr) + + err := s.store.UpdateTaskStatus(ctx, s.provider, holder.descriptor, models.TaskStatusFailed, taskError) + if err != nil { + holder.logger.Error("Error updating task status: %s", taskError) + } +} + +func (s *DefaultTaskScheduler) deleteTask(holder *taskHolder) { + s.mu.Lock() + defer s.mu.Unlock() + + taskID, err := holder.descriptor.EncodeToString() + if err != nil { + holder.logger.Errorf("Error encoding task descriptor: %s", err) + + return + } + + delete(s.tasks, taskID) + + if s.stopped { + return + } + + oldestPendingTask, err := s.store.ReadOldestPendingTask(context.Background(), s.provider) + if err != nil { + if errors.Is(err, storage.ErrNotFound) { + return + } + + logging.Error(err) + + return + } + + p := s.resolver.Resolve(oldestPendingTask.GetDescriptor()) + if p == nil { + logging.Errorf("unable to resolve task") + + return + } + + err = s.startTask(oldestPendingTask.GetDescriptor()) + if err != nil { + logging.Error(err) + } +} + +type StopChan chan chan struct{} + +func (s *DefaultTaskScheduler) startTask(descriptor models.TaskDescriptor) error { + task, err := s.store.FindAndUpsertTask(context.Background(), s.provider, descriptor, + models.TaskStatusActive, "") + if err != nil { + return errors.Wrap(err, "finding task and update") + } + + logger := s.logger.WithFields(map[string]interface{}{ + "task-id": task.ID, + }) + + taskResolver := s.resolver.Resolve(task.GetDescriptor()) + if taskResolver == nil { + return ErrUnableToResolve + } + + ctx, cancel := context.WithCancel(context.Background()) + ctx, span := otel.Tracer("com.formance.payments").Start(ctx, "Task", trace.WithAttributes( + attribute.Stringer("id", task.ID), + attribute.Stringer("connector", s.provider), + )) + + holder := &taskHolder{ + cancel: cancel, + logger: logger, + descriptor: descriptor, + } + + container, err := s.containerFactory(ctx, descriptor, task.ID) + if err != nil { + // TODO: Handle error + panic(err) + } + + err = container.Provide(func() context.Context { + return ctx + }) + if err != nil { + panic(err) + } + + err = container.Provide(func() Scheduler { + return s + }) + if err != nil { + panic(err) + } + + err = container.Provide(func() StopChan { + s.mu.Lock() + defer s.mu.Unlock() + + holder.stopChan = make(StopChan, 1) + + return holder.stopChan + }) + if err != nil { + panic(err) + } + + err = container.Provide(func() logging.Logger { + return s.logger + }) + if err != nil { + panic(err) + } + + err = container.Provide(func() StateResolver { + return StateResolverFn(func(ctx context.Context, v any) error { + if task.State == nil || len(task.State) == 0 { + return nil + } + + return json.Unmarshal(task.State, v) + }) + }) + if err != nil { + panic(err) + } + + taskID, err := holder.descriptor.EncodeToString() + if err != nil { + return err + } + + s.tasks[taskID] = holder + + go func() { + logger.Infof("Starting task...") + + defer func() { + defer span.End() + defer s.deleteTask(holder) + + if e := recover(); e != nil { + s.registerTaskError(ctx, holder, e) + debug.PrintStack() + + return + } + }() + + err = container.Invoke(taskResolver) + if err != nil { + s.registerTaskError(ctx, holder, err) + debug.PrintStack() + + return + } + + logger.Infof("Task terminated with success") + + err = s.store.UpdateTaskStatus(ctx, s.provider, descriptor, models.TaskStatusTerminated, "") + if err != nil { + logger.Error("Error updating task status: %s", err) + } + }() + + return nil +} + +func (s *DefaultTaskScheduler) stackTask(descriptor models.TaskDescriptor) error { + s.logger.WithFields(map[string]interface{}{ + "descriptor": string(descriptor), + }).Infof("Stacking task") + + return s.store.UpdateTaskStatus( + context.Background(), s.provider, descriptor, models.TaskStatusPending, "") +} + +var _ Scheduler = &DefaultTaskScheduler{} + +func NewDefaultScheduler( + provider models.ConnectorProvider, + logger logging.Logger, + store Repository, + containerFactory ContainerCreateFunc, + resolver Resolver, + maxTasks int, +) *DefaultTaskScheduler { + return &DefaultTaskScheduler{ + provider: provider, + logger: logger.WithFields(map[string]interface{}{ + "component": "scheduler", + "provider": provider, + }), + store: store, + tasks: map[string]*taskHolder{}, + containerFactory: containerFactory, + maxTasks: maxTasks, + resolver: resolver, + } +} diff --git a/internal/app/task/scheduler_test.go b/internal/app/task/scheduler_test.go new file mode 100644 index 00000000..4e92b011 --- /dev/null +++ b/internal/app/task/scheduler_test.go @@ -0,0 +1,217 @@ +package task + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/google/uuid" + + "go.uber.org/dig" + + "github.com/formancehq/payments/internal/app/models" + + "github.com/formancehq/go-libs/logging/logginglogrus" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +//nolint:gochecknoglobals // allow in tests +var DefaultContainerFactory = ContainerCreateFunc(func(ctx context.Context, descriptor models.TaskDescriptor, taskID uuid.UUID) (*dig.Container, error) { + return dig.New(), nil +}) + +func newDescriptor() models.TaskDescriptor { + return []byte(uuid.New().String()) +} + +func TaskTerminatedWithStatus(store *InMemoryStore, provider models.ConnectorProvider, + descriptor models.TaskDescriptor, expectedStatus models.TaskStatus, errString string, +) func() bool { + return func() bool { + status, resultErr, ok := store.Result(provider, descriptor) + if !ok { + return false + } + + if resultErr != errString { + return false + } + + return status == expectedStatus + } +} + +func TaskTerminated(store *InMemoryStore, provider models.ConnectorProvider, descriptor models.TaskDescriptor) func() bool { + return TaskTerminatedWithStatus(store, provider, descriptor, models.TaskStatusTerminated, "") +} + +func TaskFailed(store *InMemoryStore, provider models.ConnectorProvider, descriptor models.TaskDescriptor, errStr string) func() bool { + return TaskTerminatedWithStatus(store, provider, descriptor, models.TaskStatusFailed, errStr) +} + +func TaskPending(store *InMemoryStore, provider models.ConnectorProvider, descriptor models.TaskDescriptor) func() bool { + return TaskTerminatedWithStatus(store, provider, descriptor, models.TaskStatusPending, "") +} + +func TaskActive(store *InMemoryStore, provider models.ConnectorProvider, descriptor models.TaskDescriptor) func() bool { + return TaskTerminatedWithStatus(store, provider, descriptor, models.TaskStatusActive, "") +} + +func TestTaskScheduler(t *testing.T) { + t.Parallel() + + l := logrus.New() + if testing.Verbose() { + l.SetLevel(logrus.DebugLevel) + } + + logger := logginglogrus.New(l) + + t.Run("Nominal", func(t *testing.T) { + t.Parallel() + + store := NewInMemoryStore() + provider := models.ConnectorProvider(uuid.New().String()) + done := make(chan struct{}) + + scheduler := NewDefaultScheduler(provider, logger, store, + DefaultContainerFactory, ResolverFn(func(descriptor models.TaskDescriptor) Task { + return func(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-done: + return nil + } + } + }), 1) + + descriptor := newDescriptor() + err := scheduler.Schedule(descriptor, false) + require.NoError(t, err) + + require.Eventually(t, TaskActive(store, provider, descriptor), time.Second, 100*time.Millisecond) + close(done) + require.Eventually(t, TaskTerminated(store, provider, descriptor), time.Second, 100*time.Millisecond) + }) + + t.Run("Duplicate task", func(t *testing.T) { + t.Parallel() + + store := NewInMemoryStore() + provider := models.ConnectorProvider(uuid.New().String()) + scheduler := NewDefaultScheduler(provider, logger, store, DefaultContainerFactory, + ResolverFn(func(descriptor models.TaskDescriptor) Task { + return func(ctx context.Context) error { + <-ctx.Done() + + return ctx.Err() + } + }), 1) + + descriptor := newDescriptor() + err := scheduler.Schedule(descriptor, false) + require.NoError(t, err) + require.Eventually(t, TaskActive(store, provider, descriptor), time.Second, 100*time.Millisecond) + + err = scheduler.Schedule(descriptor, false) + require.Equal(t, ErrAlreadyScheduled, err) + }) + + t.Run("Error", func(t *testing.T) { + t.Parallel() + + provider := models.ConnectorProvider(uuid.New().String()) + store := NewInMemoryStore() + scheduler := NewDefaultScheduler(provider, logger, store, DefaultContainerFactory, + ResolverFn(func(descriptor models.TaskDescriptor) Task { + return func() error { + return errors.New("test") + } + }), 1) + + descriptor := newDescriptor() + err := scheduler.Schedule(descriptor, false) + require.NoError(t, err) + require.Eventually(t, TaskFailed(store, provider, descriptor, "test"), time.Second, + 100*time.Millisecond) + }) + + t.Run("Pending", func(t *testing.T) { + t.Parallel() + + provider := models.ConnectorProvider(uuid.New().String()) + store := NewInMemoryStore() + descriptor1 := newDescriptor() + descriptor2 := newDescriptor() + + task1Terminated := make(chan struct{}) + task2Terminated := make(chan struct{}) + + scheduler := NewDefaultScheduler(provider, logger, store, DefaultContainerFactory, + ResolverFn(func(descriptor models.TaskDescriptor) Task { + switch string(descriptor) { + case string(descriptor1): + return func(ctx context.Context) error { + select { + case <-task1Terminated: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + case string(descriptor2): + return func(ctx context.Context) error { + select { + case <-task2Terminated: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + } + + panic("unknown descriptor") + }), 1) + + require.NoError(t, scheduler.Schedule(descriptor1, false)) + require.NoError(t, scheduler.Schedule(descriptor2, false)) + require.Eventually(t, TaskActive(store, provider, descriptor1), time.Second, 100*time.Millisecond) + require.Eventually(t, TaskPending(store, provider, descriptor2), time.Second, 100*time.Millisecond) + close(task1Terminated) + require.Eventually(t, TaskTerminated(store, provider, descriptor1), time.Second, 100*time.Millisecond) + require.Eventually(t, TaskActive(store, provider, descriptor2), time.Second, 100*time.Millisecond) + close(task2Terminated) + require.Eventually(t, TaskTerminated(store, provider, descriptor2), time.Second, 100*time.Millisecond) + }) + + t.Run("Stop scheduler", func(t *testing.T) { + t.Parallel() + + provider := models.ConnectorProvider(uuid.New().String()) + store := NewInMemoryStore() + mainDescriptor := newDescriptor() + workerDescriptor := newDescriptor() + + scheduler := NewDefaultScheduler(provider, logger, store, DefaultContainerFactory, + ResolverFn(func(descriptor models.TaskDescriptor) Task { + switch string(descriptor) { + case string(mainDescriptor): + return func(ctx context.Context, scheduler Scheduler) { + <-ctx.Done() + require.NoError(t, scheduler.Schedule(workerDescriptor, false)) + } + default: + panic("should not be called") + } + }), 1) + + require.NoError(t, scheduler.Schedule(mainDescriptor, false)) + require.Eventually(t, TaskActive(store, provider, mainDescriptor), time.Second, 100*time.Millisecond) + require.NoError(t, scheduler.Shutdown(context.Background())) + require.Eventually(t, TaskTerminated(store, provider, mainDescriptor), time.Second, 100*time.Millisecond) + require.Eventually(t, TaskPending(store, provider, workerDescriptor), time.Second, 100*time.Millisecond) + }) +} diff --git a/internal/app/task/state.go b/internal/app/task/state.go new file mode 100644 index 00000000..bc5722b8 --- /dev/null +++ b/internal/app/task/state.go @@ -0,0 +1,40 @@ +package task + +import ( + "context" + + "github.com/formancehq/payments/internal/app/storage" + + "github.com/pkg/errors" +) + +type StateResolver interface { + ResolveTo(ctx context.Context, v any) error +} +type StateResolverFn func(ctx context.Context, v any) error + +func (fn StateResolverFn) ResolveTo(ctx context.Context, v any) error { + return fn(ctx, v) +} + +func ResolveTo[State any](ctx context.Context, resolver StateResolver, to *State) (*State, error) { + err := resolver.ResolveTo(ctx, to) + if err != nil { + return nil, err + } + + return to, nil +} + +func MustResolveTo[State any](ctx context.Context, resolver StateResolver, to State) State { + state, err := ResolveTo[State](ctx, resolver, &to) + if errors.Is(err, storage.ErrNotFound) { + return to + } + + if err != nil { + panic(err) + } + + return *state +} diff --git a/internal/app/task/store.go b/internal/app/task/store.go new file mode 100644 index 00000000..1791bd88 --- /dev/null +++ b/internal/app/task/store.go @@ -0,0 +1,21 @@ +package task + +import ( + "context" + + "github.com/formancehq/payments/internal/app/storage" + + "github.com/google/uuid" + + "github.com/formancehq/payments/internal/app/models" +) + +type Repository interface { + UpdateTaskStatus(ctx context.Context, provider models.ConnectorProvider, descriptor models.TaskDescriptor, status models.TaskStatus, err string) error + FindAndUpsertTask(ctx context.Context, provider models.ConnectorProvider, descriptor models.TaskDescriptor, status models.TaskStatus, err string) (*models.Task, error) + ListTasksByStatus(ctx context.Context, provider models.ConnectorProvider, status models.TaskStatus) ([]models.Task, error) + ListTasks(ctx context.Context, provider models.ConnectorProvider, pagination storage.Paginator) ([]models.Task, storage.PaginationDetails, error) + ReadOldestPendingTask(ctx context.Context, provider models.ConnectorProvider) (*models.Task, error) + GetTask(ctx context.Context, id uuid.UUID) (*models.Task, error) + GetTaskByDescriptor(ctx context.Context, provider models.ConnectorProvider, descriptor models.TaskDescriptor) (*models.Task, error) +} diff --git a/internal/app/task/storememory.go b/internal/app/task/storememory.go new file mode 100644 index 00000000..ed383df8 --- /dev/null +++ b/internal/app/task/storememory.go @@ -0,0 +1,209 @@ +package task + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/formancehq/payments/internal/app/storage" + + "github.com/formancehq/payments/internal/app/models" +) + +type InMemoryStore struct { + tasks map[uuid.UUID]models.Task + statuses map[string]models.TaskStatus + created map[string]time.Time + errors map[string]string +} + +func (s *InMemoryStore) GetTask(ctx context.Context, id uuid.UUID) (*models.Task, error) { + task, ok := s.tasks[id] + if !ok { + return nil, storage.ErrNotFound + } + + return &task, nil +} + +func (s *InMemoryStore) GetTaskByDescriptor(ctx context.Context, provider models.ConnectorProvider, + descriptor models.TaskDescriptor, +) (*models.Task, error) { + id, err := descriptor.EncodeToString() + if err != nil { + return nil, err + } + + status, ok := s.statuses[id] + if !ok { + return nil, storage.ErrNotFound + } + + return &models.Task{ + Descriptor: descriptor.ToMessage(), + Status: status, + Error: s.errors[id], + State: nil, + CreatedAt: s.created[id], + }, nil +} + +func (s *InMemoryStore) ListTasks(ctx context.Context, + provider models.ConnectorProvider, + pagination storage.Paginator, +) ([]models.Task, storage.PaginationDetails, error) { + ret := make([]models.Task, 0) + + for id, status := range s.statuses { + if !strings.HasPrefix(id, fmt.Sprintf("%s/", provider)) { + continue + } + + var descriptor models.TaskDescriptor + + ret = append(ret, models.Task{ + Descriptor: descriptor.ToMessage(), + Status: status, + Error: s.errors[id], + State: nil, + CreatedAt: s.created[id], + }) + } + + return ret, storage.PaginationDetails{}, nil +} + +func (s *InMemoryStore) ReadOldestPendingTask(ctx context.Context, + provider models.ConnectorProvider, +) (*models.Task, error) { + var ( + oldestDate time.Time + oldestID string + ) + + for id, status := range s.statuses { + if status != models.TaskStatusPending { + continue + } + + if oldestDate.IsZero() || s.created[id].Before(oldestDate) { + oldestDate = s.created[id] + oldestID = id + } + } + + if oldestDate.IsZero() { + return nil, storage.ErrNotFound + } + + descriptorStr := strings.Split(oldestID, "/")[1] + + var descriptor models.TaskDescriptor + + data, err := base64.StdEncoding.DecodeString(descriptorStr) + if err != nil { + return nil, err + } + + err = json.Unmarshal(data, &descriptor) + if err != nil { + return nil, err + } + + return &models.Task{ + Descriptor: descriptor.ToMessage(), + Status: models.TaskStatusPending, + State: nil, + CreatedAt: s.created[oldestID], + }, nil +} + +func (s *InMemoryStore) ListTasksByStatus(ctx context.Context, + provider models.ConnectorProvider, taskStatus models.TaskStatus, +) ([]models.Task, error) { + all, _, err := s.ListTasks(ctx, provider, storage.Paginator{}) + if err != nil { + return nil, err + } + + ret := make([]models.Task, 0) + + for _, v := range all { + if v.Status != taskStatus { + continue + } + + ret = append(ret, v) + } + + return ret, nil +} + +func (s *InMemoryStore) FindAndUpsertTask(ctx context.Context, + provider models.ConnectorProvider, descriptor models.TaskDescriptor, status models.TaskStatus, taskErr string, +) (*models.Task, error) { + err := s.UpdateTaskStatus(ctx, provider, descriptor, status, taskErr) + if err != nil { + return nil, err + } + + return &models.Task{ + Descriptor: descriptor.ToMessage(), + Status: status, + Error: taskErr, + State: nil, + }, nil +} + +func (s *InMemoryStore) UpdateTaskStatus(ctx context.Context, provider models.ConnectorProvider, + descriptor models.TaskDescriptor, status models.TaskStatus, taskError string, +) error { + taskID, err := descriptor.EncodeToString() + if err != nil { + return err + } + + key := fmt.Sprintf("%s/%s", provider, taskID) + + s.statuses[key] = status + + s.errors[key] = taskError + if _, ok := s.created[key]; !ok { + s.created[key] = time.Now() + } + + return nil +} + +func (s *InMemoryStore) Result(provider models.ConnectorProvider, + descriptor models.TaskDescriptor, +) (models.TaskStatus, string, bool) { + taskID, err := descriptor.EncodeToString() + if err != nil { + panic(err) + } + + key := fmt.Sprintf("%s/%s", provider, taskID) + + status, ok := s.statuses[key] + if !ok { + return "", "", false + } + + return status, s.errors[key], true +} + +func NewInMemoryStore() *InMemoryStore { + return &InMemoryStore{ + statuses: make(map[string]models.TaskStatus), + errors: make(map[string]string), + created: make(map[string]time.Time), + } +} + +var _ Repository = &InMemoryStore{} diff --git a/internal/app/task/task.go b/internal/app/task/task.go new file mode 100644 index 00000000..ce267322 --- /dev/null +++ b/internal/app/task/task.go @@ -0,0 +1,3 @@ +package task + +type Task any diff --git a/main.go b/main.go new file mode 100644 index 00000000..4bd7634e --- /dev/null +++ b/main.go @@ -0,0 +1,8 @@ +//go:generate docker run --rm -w /local -v ${PWD}:/local openapitools/openapi-generator-cli:latest validate -i ./openapi.yaml +package main + +import "github.com/formancehq/payments/cmd" + +func main() { + cmd.Execute() +} diff --git a/openapi.yaml b/openapi.yaml new file mode 100644 index 00000000..6eeca419 --- /dev/null +++ b/openapi.yaml @@ -0,0 +1,879 @@ +openapi: 3.0.3 +info: + title: Payments API + version: "PAYMENTS_VERSION" + +# ---------------------- PATHS ---------------------- +paths: + /_info: + get: + summary: Get server info + operationId: getServerInfo + responses: + 200: + $ref: '#/components/responses/ServerInfo' + /payments: + get: + summary: List payments + operationId: listPayments + tags: + - Payments + parameters: + - $ref: '#/components/parameters/PageSize' + - $ref: '#/components/parameters/Cursor' + - $ref: '#/components/parameters/Sort' + responses: + '200': + $ref: '#/components/responses/Payments' + /payments/{paymentId}: + get: + summary: Get a payment + tags: + - Payments + operationId: getPayment + parameters: + - $ref: '#/components/parameters/PaymentId' + responses: + '200': + $ref: '#/components/responses/Payment' + /accounts: + get: + summary: List accounts + operationId: listAccounts + tags: + - Payments + parameters: + - $ref: '#/components/parameters/PageSize' + - $ref: '#/components/parameters/Cursor' + - $ref: '#/components/parameters/Sort' + responses: + '200': + $ref: '#/components/responses/Accounts' + /connectors: + get: + summary: List all installed connectors + operationId: listAllConnectors + tags: + - Payments + description: List all installed connectors. + responses: + '200': + $ref: '#/components/responses/Connectors' + /connectors/configs: + get: + summary: List the configs of each available connector + operationId: listConfigsAvailableConnectors + tags: + - Payments + description: List the configs of each available connector. + responses: + '200': + $ref: '#/components/responses/ConnectorsConfigs' + /connectors/{connector}: + post: + summary: Install a connector + tags: + - Payments + operationId: installConnector + description: Install a connector by its name and config. + parameters: + - $ref: '#/components/parameters/Connector' + requestBody: + $ref: '#/components/requestBodies/ConnectorConfig' + responses: + '204': + $ref: '#/components/responses/NoContent' + delete: + summary: Uninstall a connector + operationId: uninstallConnector + tags: + - Payments + description: Uninstall a connector by its name. + parameters: + - $ref: '#/components/parameters/Connector' + responses: + '204': + $ref: '#/components/responses/NoContent' + /connectors/{connector}/config: + get: + summary: Read the config of a connector + operationId: readConnectorConfig + tags: + - Payments + description: Read connector config + parameters: + - $ref: '#/components/parameters/Connector' + responses: + '200': + $ref: '#/components/responses/ConnectorConfig' + /connectors/{connector}/reset: + post: + summary: Reset a connector + operationId: resetConnector + tags: + - Payments + description: | + Reset a connector by its name. + It will remove the connector and ALL PAYMENTS generated with it. + parameters: + - $ref: '#/components/parameters/Connector' + responses: + '204': + $ref: '#/components/responses/NoContent' + /connectors/{connector}/tasks: + get: + summary: List tasks from a connector + tags: + - Payments + operationId: listConnectorTasks + description: List all tasks associated with this connector. + parameters: + - $ref: '#/components/parameters/Connector' + - $ref: '#/components/parameters/PageSize' + - $ref: '#/components/parameters/Cursor' + responses: + '200': + $ref: '#/components/responses/Tasks' + /connectors/{connector}/tasks/{taskId}: + get: + summary: Read a specific task of the connector + tags: + - Payments + operationId: getConnectorTask + description: Get a specific task associated to the connector. + parameters: + - $ref: '#/components/parameters/Connector' + - $ref: '#/components/parameters/TaskId' + responses: + '200': + $ref: '#/components/responses/Task' + /connectors/stripe/transfer: + post: + summary: Transfer funds between Stripe accounts + tags: + - Payments + operationId: connectorsStripeTransfer + description: Execute a transfer between two Stripe accounts. + requestBody: + $ref: '#/components/requestBodies/StripeTransfer' + responses: + '200': + $ref: '#/components/responses/StripeTransfer' + +# ---------------------- COMPONENTS ---------------------- +components: + # ---------------------- PARAMETERS ---------------------- + parameters: + PageSize: + name: pageSize + in: query + description: | + The maximum number of results to return per page. + example: 100 + schema: + type: integer + format: int64 + minimum: 1 + maximum: 1000 + default: 15 + Cursor: + name: cursor + in: query + description: | + Parameter used in pagination requests. Maximum page size is set to 15. + Set to the value of next for the next page of results. + Set to the value of previous for the previous page of results. + No other parameters can be set when this parameter is set. + schema: + type: string + example: aHR0cHM6Ly9nLnBhZ2UvTmVrby1SYW1lbj9zaGFyZQ== + Sort: + name: sort + in: query + schema: + type: array + items: + type: string + description: Fields used to sort payments (default is date:desc). + example: date:asc,status:desc + PaymentId: + name: paymentId + in: path + schema: + type: string + description: The payment ID. + example: XXX + required: true + Connector: + name: connector + description: The name of the connector. + in: path + schema: + $ref: '#/components/schemas/Connector' + required: true + TaskId: + name: taskId + description: The task ID. + example: task1 + in: path + schema: + type: string + required: true + + # ---------------------- RESPONSES ---------------------- + responses: + NoContent: + description: No content + ServerInfo: + description: Server information + content: + application/json: + schema: + $ref: '#/components/schemas/ServerInfo' + Payments: + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/PaymentsCursor' + Payment: + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/PaymentResponse' + Accounts: + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/AccountsCursor' + Connectors: + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/ConnectorsResponse' + ConnectorsConfigs: + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/ConnectorsConfigsResponse' + ConnectorConfig: + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/ConnectorConfigResponse' + Tasks: + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/TasksCursor' + Task: + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/TaskResponse' + StripeTransfer: + description: OK + content: + application/json: + schema: + $ref: '#/components/schemas/StripeTransferResponse' + # ---------------------- REQUESTS ---------------------- + requestBodies: + ConnectorConfig: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ConnectorConfig' + StripeTransfer: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/StripeTransferRequest' + + # ---------------------- SCHEMAS ---------------------- + schemas: + # ---------------------- CURSORS ---------------------- + CursorBase: + type: object + required: + - pageSize + - hasMore + - data + properties: + pageSize: + type: integer + format: int64 + minimum: 1 + maximum: 1000 + example: 15 + hasMore: + type: boolean + example: false + previous: + type: string + example: "YXVsdCBhbmQgYSBtYXhpbXVtIG1heF9yZXN1bHRzLol=" + next: + type: string + example: "" + PaymentsCursor: + type: object + required: + - cursor + properties: + cursor: + allOf: + - $ref: '#/components/schemas/CursorBase' + - type: object + required: + - data + properties: + data: + type: array + items: + $ref: '#/components/schemas/Payment' + AccountsCursor: + type: object + required: + - cursor + properties: + cursor: + allOf: + - $ref: '#/components/schemas/CursorBase' + - type: object + required: + - data + properties: + data: + type: array + items: + $ref: '#/components/schemas/Account' + TasksCursor: + type: object + required: + - cursor + properties: + cursor: + allOf: + - $ref: '#/components/schemas/CursorBase' + - type: object + required: + - data + properties: + data: + type: array + items: + oneOf: + - $ref: '#/components/schemas/TaskStripe' + - $ref: '#/components/schemas/TaskWise' + - $ref: '#/components/schemas/TaskCurrencyCloud' + - $ref: '#/components/schemas/TaskDummyPay' + - $ref: '#/components/schemas/TaskModulr' + - $ref: '#/components/schemas/TaskBankingCircle' + + # ---------------------- RESPONSES ---------------------- + ConnectorConfigResponse: + type: object + required: + - data + properties: + data: + $ref: '#/components/schemas/ConnectorConfig' + PaymentResponse: + type: object + required: + - data + properties: + data: + $ref: '#/components/schemas/Payment' + ConnectorsResponse: + type: object + required: + - data + properties: + data: + type: array + items: + type: object + properties: + provider: + $ref: '#/components/schemas/Connector' + enabled: + type: boolean + example: true + ConnectorsConfigsResponse: + type: object + required: + - data + properties: + data: + type: object + required: + - connector + properties: + connector: + type: object + required: + - key + properties: + key: + type: object + required: + - dataType + - required + properties: + dataType: + type: string + required: + type: boolean + TaskResponse: + type: object + required: + - data + properties: + data: + oneOf: + - $ref: '#/components/schemas/TaskStripe' + - $ref: '#/components/schemas/TaskWise' + - $ref: '#/components/schemas/TaskCurrencyCloud' + - $ref: '#/components/schemas/TaskDummyPay' + - $ref: '#/components/schemas/TaskModulr' + - $ref: '#/components/schemas/TaskBankingCircle' + + # ---------------------- DATA MODELS ---------------------- + Connector: + type: string + enum: + - STRIPE + - DUMMY-PAY + - WISE + - MODULR + - CURRENCY-CLOUD + - BANKING-CIRCLE + ConnectorConfig: + oneOf: + - $ref: '#/components/schemas/StripeConfig' + - $ref: '#/components/schemas/DummyPayConfig' + - $ref: '#/components/schemas/WiseConfig' + - $ref: '#/components/schemas/ModulrConfig' + - $ref: '#/components/schemas/CurrencyCloudConfig' + - $ref: '#/components/schemas/BankingCircleConfig' + StripeConfig: + type: object + required: + - apiKey + properties: + pollingPeriod: + type: string + example: '60s' + description: | + The frequency at which the connector will try to fetch new BalanceTransaction objects from Stripe API. + default: '120s' + apiKey: + type: string + example: XXX + pageSize: + type: integer + format: int64 + minimum: 0 + description: | + Number of BalanceTransaction to fetch at each polling interval. + default: 10 + example: 50 + DummyPayConfig: + type: object + required: + - directory + properties: + filePollingPeriod: + type: string + example: '60s' + description: The frequency at which the connector will try to fetch new payment objects from the directory + default: '10s' + fileGenerationPeriod: + type: string + example: '60s' + description: The frequency at which the connector will create new payment objects in the directory + default: '10s' + directory: + type: string + example: '/tmp/dummypay' + WiseConfig: + type: object + required: + - apiKey + properties: + apiKey: + type: string + example: 'XXX' + ModulrConfig: + type: object + required: + - apiKey + - apiSecret + properties: + apiKey: + type: string + example: 'XXX' + apiSecret: + type: string + example: 'XXX' + endpoint: + type: string + example: 'XXX' + BankingCircleConfig: + type: object + required: + - username + - password + - endpoint + - authorizationEndpoint + properties: + username: + type: string + example: 'XXX' + password: + type: string + example: 'XXX' + endpoint: + type: string + example: 'XXX' + authorizationEndpoint: + type: string + example: 'XXX' + CurrencyCloudConfig: + type: object + required: + - apiKey + - loginID + properties: + apiKey: + type: string + example: 'XXX' + loginID: + type: string + example: 'XXX' + description: 'Username of the API Key holder' + pollingPeriod: + type: string + example: '60s' + description: The frequency at which the connector will fetch transactions + endpoint: + type: string + example: 'XXX' + description: 'The endpoint to use for the API. Defaults to https://devapi.currencycloud.com' + Payment: + type: object + required: + - id + - reference + - accountID + - type + - provider + - status + - initialAmount + - scheme + - asset + - createdAt + - raw + - adjustments + - metadata + properties: + id: + type: string + example: XXX + reference: + type: string + accountID: + type: string + type: + type: string + enum: + - PAY-IN + - PAYOUT + - TRANSFER + - OTHER + provider: + $ref: '#/components/schemas/Connector' + status: + $ref: '#/components/schemas/PaymentStatus' + initialAmount: + type: integer + format: int64 + minimum: 0 + example: 100 + scheme: + type: string + enum: + - visa + - mastercard + - amex + - diners + - discover + - jcb + - unionpay + - sepa debit + - sepa credit + - sepa + - apple pay + - google pay + - a2a + - ach debit + - ach + - rtp + - unknown + - other + asset: + type: string + example: USD + createdAt: + type: string + format: date-time + raw: + type: object + adjustments: + type: array + items: + $ref: '#/components/schemas/PaymentAdjustment' + metadata: + type: array + items: + $ref: '#/components/schemas/PaymentMetadata' + PaymentAdjustment: + type: object + required: + - status + - amount + - date + - raw + - absolute + properties: + status: + $ref: '#/components/schemas/PaymentStatus' + amount: + type: integer + format: int64 + minimum: 0 + example: 100 + date: + type: string + format: date-time + raw: + type: object + absolute: + type: boolean + PaymentMetadata: + type: object + required: + - key + - value + properties: + key: + type: string + value: + type: string + changelog: + $ref: '#/components/schemas/PaymentMetadataChangelog' + PaymentMetadataChangelog: + type: object + required: + - timestamp + - value + properties: + timestamp: + type: string + format: date-time + value: + type: string + Account: + type: object + required: + - id + - createdAt + - provider + - reference + - type + properties: + id: + type: string + createdAt: + type: string + format: date-time + provider: + $ref: '#/components/schemas/Connector' + reference: + type: string + type: + type: string + enum: + - TARGET + - SOURCE + TaskBase: + type: object + required: + - id + - connectorId + - createdAt + - updatedAt + - descriptor + - status + - state + properties: + id: + type: string + format: uuid + connectorId: + type: string + format: uuid + createdAt: + type: string + format: date-time + updatedAt: + type: string + format: date-time + status: + $ref: '#/components/schemas/PaymentStatus' + state: + type: object + error: + type: string + TaskStripe: + allOf: + - $ref: '#/components/schemas/TaskBase' + - type: object + required: + - descriptor + properties: + descriptor: + type: object + required: + - name + - account + properties: + name: + type: string + main: + type: boolean + account: + type: string + TaskWise: + allOf: + - $ref: '#/components/schemas/TaskBase' + - type: object + required: + - descriptor + properties: + descriptor: + type: object + properties: + name: + type: string + key: + type: string + profileID: + type: integer + format: int64 + minimum: 0 + TaskModulr: + allOf: + - $ref: '#/components/schemas/TaskBase' + - type: object + required: + - descriptor + properties: + descriptor: + type: object + properties: + name: + type: string + key: + type: string + accountID: + type: string + TaskDummyPay: + allOf: + - $ref: '#/components/schemas/TaskBase' + - type: object + required: + - descriptor + properties: + descriptor: + type: object + properties: + name: + type: string + key: + type: string + fileName: + type: string + TaskCurrencyCloud: + allOf: + - $ref: '#/components/schemas/TaskBase' + - type: object + required: + - descriptor + properties: + descriptor: + type: object + properties: + name: + type: string + TaskBankingCircle: + allOf: + - $ref: '#/components/schemas/TaskBase' + - type: object + required: + - descriptor + properties: + descriptor: + type: object + properties: + name: + type: string + key: + type: string + StripeTransferRequest: + type: object + properties: + amount: + type: integer + format: int64 + minimum: 0 + example: 100 + asset: + type: string + example: USD + destination: + type: string + example: acct_1Gqj58KZcSIg2N2q + metadata: + type: object + description: | + A set of key/value pairs that you can attach to a transfer object. + It can be useful for storing additional information about the transfer in a structured format. + example: + order_id: '6735' + StripeTransferResponse: + type: object + PaymentStatus: + type: string + enum: + - PENDING + - ACTIVE + - TERMINATED + - FAILED + ServerInfo: + type: object + required: + - version + properties: + version: + type: string