diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 6c21b051ebaf..bae232092bb9 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1385,7 +1385,7 @@ - name: RKI Covid sourceDefinitionId: d78e5de0-aa44-4744-aa4f-74c818ccfe19 dockerRepository: airbyte/source-rki-covid - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.com/integrations/sources/rki-covid sourceType: api releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 146cbda082c9..919cbb1a7fba 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -12456,7 +12456,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-rki-covid:0.1.1" +- dockerImage: "airbyte/source-rki-covid:0.1.2" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/rki-covid" connectionSpecification: @@ -12465,7 +12465,7 @@ type: "object" required: - "start_date" - additionalProperties: false + additionalProperties: true properties: start_date: type: "string" diff --git a/airbyte-integrations/connectors/source-rki-covid/README.md b/airbyte-integrations/connectors/source-rki-covid/README.md index 833f6bc05c89..e15247de91c8 100644 --- a/airbyte-integrations/connectors/source-rki-covid/README.md +++ b/airbyte-integrations/connectors/source-rki-covid/README.md @@ -15,6 +15,14 @@ Germany: 6. /germany/history/recovered/:days 7. /germany/history/frozen-incidence/:days 8. /germany/history/hospitalization/:days + 9. /germany/states + 10. /germany/states/age-groups + 11. /germany/states/history/cases/:days + 12. /germany/states/history/incidence/:days + 13. /germany/states/history/frozen-incidence/:days + 14. /germany/states/history/deaths/:days + 15. /germany/states/history/recovered/:days + 16. /germany/states/history/hospitalization/:days ``` ### Prerequisites diff --git a/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml b/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml index 749d5d300264..e357e54ef0e9 100644 --- a/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-rki-covid/acceptance-test-config.yml @@ -12,7 +12,7 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [ "germany", "germany_age_groups", "german_history_frozen_incidence"] + empty_streams: [ "germany", "germany_age_groups", "german_history_frozen_incidence", "germany_states_age_groups"] # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file # expect_records: # path: "integration_tests/expected_records.txt" diff --git a/airbyte-integrations/connectors/source-rki-covid/bootstrap.md b/airbyte-integrations/connectors/source-rki-covid/bootstrap.md index 5061ac55ce92..de01a9368897 100644 --- a/airbyte-integrations/connectors/source-rki-covid/bootstrap.md +++ b/airbyte-integrations/connectors/source-rki-covid/bootstrap.md @@ -15,7 +15,19 @@ For production, every developer application can view multiple streams. * [Provides frozen incidence in Germany over days.](https://api.corona-zahlen.org/germany/germany/history/frozen-incidence/:days) \(Incremental\) * [Provides hospitalization rate in Germany over days.](https://api.corona-zahlen.org/germany/germany/history/hospitalization/:days) \(Incremental\) +## Cases In States Of Germany Covid api stream +The basic entry stream is 'GermanyStates'. All other streams are extended version of base stream and passing parameters also result in sliced data. +For production, every developer application can view multiple streams. +## Endpoints +* [Provides covid cases and other information in Germany.](https://api.corona-zahlen.org/state) \(Non-Incremental\ Entry-Stream) +* [Provides covid cases and other information in Germany, group by age.](https://api.corona-zahlen.org/states/age-groupss) \(Non-Incremental\) +* [Provides cases in Germany based on days.](https://api.corona-zahlen.org/germany/states/history/cases/:days) \(Non-Incremental\) +* [Provides incidence rate of covid in Germany based on days.](https://api.corona-zahlen.org/germany/states/history/incidence/:days) \(Non-Incremental\) +* [Provides death rate in Germany over days](https://api.corona-zahlen.org/germany/states/history/deaths/:days) \(Non-Incremental\) +* [Provides recovery rate in Germany over days.](https://api.corona-zahlen.org/germany/states/history/recovered/:days) \(Non-Incremental\) +* [Provides frozen incidence in Germany over days.](https://api.corona-zahlen.org/germany/states/history/frozen-incidence/:days) \(Non-Incremental\) +* [Provides hospitalization rate in Germany over days.](https://api.corona-zahlen.org/germany/states/history/hospitalization/:days) \(Non-Incremental\) Incremental streams have required parameter start-date. Without passing start-date as parameter full-refresh occurs. As cursor field this connector uses "date". \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json index f78d22b27ce4..83b08aa2c4aa 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/abnormal_state.json @@ -16,5 +16,29 @@ }, "german_history_frozen_incidence": { "date": "2024-01-01T00:00:00.000Z" + }, + "germany_states": { + "date": "2024-01-01T00:00:00.000Z" + }, + "germany_states_age_groups": { + "date": "2024-01-01T00:00:00.000Z" + }, + "states_history_cases": { + "date": "2024-01-01T00:00:00.000Z" + }, + "states_history_incidence": { + "date": "2024-01-01T00:00:00.000Z" + }, + "states_history_frozen_incidence": { + "date": "2024-01-01T00:00:00.000Z" + }, + "states_history_deaths": { + "date": "2024-01-01T00:00:00.000Z" + }, + "states_history_recovered": { + "date": "2024-01-01T00:00:00.000Z" + }, + "states_history_hospitalization": { + "date": "2024-01-01T00:00:00.000Z" } } diff --git a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json index de447b2a146b..907efe859cbe 100644 --- a/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-rki-covid/integration_tests/configured_catalog.json @@ -1,5 +1,99 @@ { "streams": [ + { + "stream": { + "name": "germany", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "cases": { "type": "integer" }, + "deaths": { "type": "integer" }, + "recovered": { "type": "integer" }, + "weekIncidence": { "type": "number" }, + "casesPer100k": { "type": "number" }, + "casesPerWeek": { "type": "integer" }, + "delta": { + "type": "object", + "properties": { + "cases": { "type": "integer" }, + "deaths": { "type": "integer" }, + "recovered": { "type": "integer" } + }, + "required": ["cases", "deaths", "recovered"] + }, + "r": { + "type": "object", + "properties": { + "value": { "type": "number" }, + "rValue4Days": { + "type": "object", + "properties": { + "value": { "type": "number" }, + "date": { "type": "string" } + }, + "required": ["date", "value"] + }, + "rValue7Days": { + "type": "object", + "properties": { + "value": { "type": "number" }, + "date": { "type": "string" } + }, + "required": ["date", "value"] + }, + "lastUpdate": { "type": "string" } + }, + "required": ["lastUpdate", "rValue4Days", "rValue7Days", "value"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { "type": "integer" }, + "incidence7Days": { "type": "number" }, + "date": { "type": "string" }, + "lastUpdate": { "type": "string" } + }, + "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"] + }, + "meta": { + "type": "object", + "properties": { + "source": { "type": "string" }, + "contact": { "type": "string" }, + "info": { "type": "string" }, + "lastUpdate": { "type": "string" }, + "lastCheckedForUpdate": { "type": "string" } + }, + "required": [ + "contact", + "info", + "lastCheckedForUpdate", + "lastUpdate", + "source" + ] + } + }, + "required": [ + "cases", + "casesPer100k", + "casesPerWeek", + "deaths", + "delta", + "hospitalization", + "meta", + "r", + "recovered", + "weekIncidence" + ] + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup", + "cursor_field": ["meta", "lastUpdate"], + "primary_key": [["hospitalization"]] + }, { "stream": { "name": "germany_age_groups", @@ -7,7 +101,7 @@ "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append_dedup" }, { "stream": { @@ -76,6 +170,692 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "germany_states", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "population": { + "type": ["integer", "null"] + }, + "cases": { + "type": ["integer", "null"] + }, + "deaths": { + "type": ["integer", "null"] + }, + "casesPerWeek": { + "type": ["integer", "null"] + }, + "deathsPerWeek": { + "type": ["integer", "null"] + }, + "recovered": { + "type": ["integer", "null"] + }, + "abbreviation": { + "type": ["string", "null"] + }, + "weekIncidence": { + "type": ["number", "null"] + }, + "casesPer100k": { + "type": ["number", "null"] + }, + "delta": { + "type": "object", + "properties": { + "cases": { + "type": ["integer", "null"] + }, + "deaths": { + "type": ["integer", "null"] + }, + "recovered": { + "type": ["integer", "null"] + }, + "weekIncidence": { + "type": ["number", "null"] + } + }, + "required": ["cases", "deaths", "recovered", "weekIncidence"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": ["integer", "null"] + }, + "incidence7Days": { + "type": ["number", "null"] + }, + "date": { + "type": ["string", "null"] + }, + "lastUpdate": { + "type": ["string", "null"] + } + }, + "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"] + } + }, + "required": [ + "abbreviation", + "cases", + "casesPer100k", + "casesPerWeek", + "deaths", + "deathsPerWeek", + "delta", + "hospitalization", + "id", + "name", + "population", + "recovered", + "weekIncidence" + ] + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "germany_states_age_groups", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "abbreviation": { + "type": "string" + }, + "A00-A04": { + "type": "object", + "properties": { + "casesMale": { + "type": ["number", "integer"] + }, + "casesFemale": { + "type": ["number", "integer"] + }, + "deathsMale": { + "type": ["number", "integer"] + }, + "deathsFemale": { + "type": ["number", "integer"] + }, + "casesMalePer100k": { + "type": ["number", "integer"] + }, + "casesFemalePer100k": { + "type": ["number", "integer"] + }, + "deathsMalePer100k": { + "type": ["number", "integer"] + }, + "deathsFemalePer100k": { + "type": ["number", "integer"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": ["number", "integer"] + }, + "incidence7Days": { + "type": ["number", "integer"] + }, + "date": { + "type": "string" + } + }, + "required": ["cases7Days", "date", "incidence7Days"] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A05-A14": { + "type": "object", + "properties": { + "casesMale": { + "type": ["number", "integer"] + }, + "casesFemale": { + "type": ["number", "integer"] + }, + "deathsMale": { + "type": ["number", "integer"] + }, + "deathsFemale": { + "type": ["number", "integer"] + }, + "casesMalePer100k": { + "type": ["number", "integer"] + }, + "casesFemalePer100k": { + "type": ["number", "integer"] + }, + "deathsMalePer100k": { + "type": ["number", "integer"] + }, + "deathsFemalePer100k": { + "type": ["number", "integer"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": ["number", "integer"] + }, + "incidence7Days": { + "type": ["number", "integer"] + }, + "date": { + "type": "string" + } + }, + "required": ["cases7Days", "date", "incidence7Days"] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A15-A34": { + "type": "object", + "properties": { + "casesMale": { + "type": ["number", "integer"] + }, + "casesFemale": { + "type": ["number", "integer"] + }, + "deathsMale": { + "type": ["number", "integer"] + }, + "deathsFemale": { + "type": ["number", "integer"] + }, + "casesMalePer100k": { + "type": ["number", "integer"] + }, + "casesFemalePer100k": { + "type": ["number", "integer"] + }, + "deathsMalePer100k": { + "type": ["number", "integer"] + }, + "deathsFemalePer100k": { + "type": ["number", "integer"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": ["number", "integer"] + }, + "incidence7Days": { + "type": ["number", "integer"] + }, + "date": { + "type": ["string", "null"] + } + }, + "required": ["cases7Days", "date", "incidence7Days"] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A35-A59": { + "type": "object", + "properties": { + "casesMale": { + "type": ["number", "integer"] + }, + "casesFemale": { + "type": ["number", "integer"] + }, + "deathsMale": { + "type": ["number", "integer"] + }, + "deathsFemale": { + "type": ["number", "integer"] + }, + "casesMalePer100k": { + "type": ["number", "integer"] + }, + "casesFemalePer100k": { + "type": ["number", "integer"] + }, + "deathsMalePer100k": { + "type": ["number", "integer"] + }, + "deathsFemalePer100k": { + "type": ["number", "integer"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": ["number", "integer"] + }, + "incidence7Days": { + "type": ["number", "integer"] + }, + "date": { + "type": ["string", "null"] + } + }, + "required": ["cases7Days", "date", "incidence7Days"] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A60-A79": { + "type": "object", + "properties": { + "casesMale": { + "type": ["number", "integer"] + }, + "casesFemale": { + "type": ["number", "integer"] + }, + "deathsMale": { + "type": ["number", "integer"] + }, + "deathsFemale": { + "type": ["number", "integer"] + }, + "casesMalePer100k": { + "type": ["number", "integer"] + }, + "casesFemalePer100k": { + "type": ["number", "integer"] + }, + "deathsMalePer100k": { + "type": ["number", "integer"] + }, + "deathsFemalePer100k": { + "type": ["number", "integer"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": ["number", "integer"] + }, + "incidence7Days": { + "type": ["number", "integer"] + }, + "date": { + "type": ["string", "null"] + } + }, + "required": ["cases7Days", "date", "incidence7Days"] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A80+": { + "type": "object", + "properties": { + "casesMale": { + "type": ["number", "integer"] + }, + "casesFemale": { + "type": ["number", "integer"] + }, + "deathsMale": { + "type": ["number", "integer"] + }, + "deathsFemale": { + "type": ["number", "integer"] + }, + "casesMalePer100k": { + "type": ["number", "integer"] + }, + "casesFemalePer100k": { + "type": ["number", "integer"] + }, + "deathsMalePer100k": { + "type": ["number", "integer"] + }, + "deathsFemalePer100k": { + "type": ["number", "integer"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": ["number", "integer"] + }, + "incidence7Days": { + "type": ["number", "integer"] + }, + "date": { + "type": ["string", "null"] + } + }, + "required": ["cases7Days", "date", "incidence7Days"] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + } + }, + "required": [ + "data", + "A00-A04", + "A05-A14", + "A15-A34", + "A35-A59", + "A60-A79", + "A80+" + ] + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "states_history_cases", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "cases": { + "type": ["integer", "null"] + }, + "date": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "abbreviation": { + "type": ["string", "null"] + } + }, + "required": ["abbreviation", "cases", "date", "name"] + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "states_history_incidence", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "weekIncidence": { + "type": ["number", "null"] + }, + "date": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "abbreviation": { + "type": ["string", "null"] + } + }, + "required": ["abbreviation", "date", "name", "weekIncidence"] + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "states_history_frozen_incidence", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "weekIncidence": { + "type": ["number", "null"] + }, + "date": { + "type": ["string", "null"] + }, + "dataSource": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "abbreviation": { + "type": ["string", "null"] + } + }, + "required": [ + "abbreviation", + "dataSource", + "date", + "name", + "weekIncidence" + ] + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "states_history_deaths", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "deaths": { + "type": ["integer", "null"] + }, + "date": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "abbreviation": { + "type": ["string", "null"] + } + }, + "required": ["abbreviation", "date", "deaths", "name"] + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "states_history_recovered", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "recovered": { + "type": ["integer", "null"] + }, + "date": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "abbreviation": { + "type": ["string", "null"] + } + }, + "required": ["abbreviation", "date", "name", "recovered"] + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "states_history_hospitalization", + "json_schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "cases7Days": { + "type": "integer" + }, + "incidence7Days": { + "type": ["number", "null"] + }, + "date": { + "type": ["string", "null"] + }, + "fixedCases7Days": { + "type": ["integer", "null"] + }, + "updatedCases7Days": { + "type": ["integer", "null"] + }, + "adjustedLowerCases7Days": { + "type": ["integer", "null"] + }, + "adjustedCases7Days": { + "type": ["integer", "null"] + }, + "adjustedUpperCases7Days": { + "type": ["integer", "null"] + }, + "fixedIncidence7Days": { + "type": ["number", "null"] + }, + "updatedIncidence7Days": { + "type": ["number", "null"] + }, + "adjustedLowerIncidence7Days": { + "type": ["number", "null"] + }, + "adjustedIncidence7Days": { + "type": ["number", "null"] + }, + "adjustedUpperIncidence7Days": { + "type": ["number", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "abbreviation": { + "type": ["string", "null"] + } + }, + "required": [ + "abbreviation", + "adjustedCases7Days", + "adjustedIncidence7Days", + "adjustedLowerCases7Days", + "adjustedLowerIncidence7Days", + "adjustedUpperCases7Days", + "adjustedUpperIncidence7Days", + "cases7Days", + "date", + "fixedCases7Days", + "fixedIncidence7Days", + "incidence7Days", + "name", + "updatedCases7Days", + "updatedIncidence7Days" + ] + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "default_cursor_field": ["date"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" } ] } diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_states.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_states.json new file mode 100644 index 000000000000..bc3db07daf69 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_states.json @@ -0,0 +1,90 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "population": { + "type": ["integer", "null"] + }, + "cases": { + "type": ["integer", "null"] + }, + "deaths": { + "type": ["integer", "null"] + }, + "casesPerWeek": { + "type": ["integer", "null"] + }, + "deathsPerWeek": { + "type": ["integer", "null"] + }, + "recovered": { + "type": ["integer", "null"] + }, + "abbreviation": { + "type": ["string", "null"] + }, + "weekIncidence": { + "type": ["number", "null"] + }, + "casesPer100k": { + "type": ["number", "null"] + }, + "delta": { + "type": "object", + "properties": { + "cases": { + "type": ["integer", "null"] + }, + "deaths": { + "type": ["integer", "null"] + }, + "recovered": { + "type": ["integer", "null"] + }, + "weekIncidence": { + "type": ["number", "null"] + } + }, + "required": ["cases", "deaths", "recovered", "weekIncidence"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": ["integer", "null"] + }, + "incidence7Days": { + "type": ["number", "null"] + }, + "date": { + "type": ["string", "null"] + }, + "lastUpdate": { + "type": ["string", "null"] + } + }, + "required": ["cases7Days", "date", "incidence7Days", "lastUpdate"] + } + }, + "required": [ + "abbreviation", + "cases", + "casesPer100k", + "casesPerWeek", + "deaths", + "deathsPerWeek", + "delta", + "hospitalization", + "id", + "name", + "population", + "recovered", + "weekIncidence" + ] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_states_age_groups.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_states_age_groups.json new file mode 100644 index 000000000000..3650c6f15f2f --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/germany_states_age_groups.json @@ -0,0 +1,340 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "abbreviation": { + "type": "string" + }, + "A00-A04": { + "type": "object", + "properties": { + "casesMale": { + "type": ["number", "integer"] + }, + "casesFemale": { + "type": ["number", "integer"] + }, + "deathsMale": { + "type": ["number", "integer"] + }, + "deathsFemale": { + "type": ["number", "integer"] + }, + "casesMalePer100k": { + "type": ["number", "integer"] + }, + "casesFemalePer100k": { + "type": ["number", "integer"] + }, + "deathsMalePer100k": { + "type": ["number", "integer"] + }, + "deathsFemalePer100k": { + "type": ["number", "integer"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": ["number", "integer"] + }, + "incidence7Days": { + "type": ["number", "integer"] + }, + "date": { + "type": "string" + } + }, + "required": ["cases7Days", "date", "incidence7Days"] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A05-A14": { + "type": "object", + "properties": { + "casesMale": { + "type": ["number", "integer"] + }, + "casesFemale": { + "type": ["number", "integer"] + }, + "deathsMale": { + "type": ["number", "integer"] + }, + "deathsFemale": { + "type": ["number", "integer"] + }, + "casesMalePer100k": { + "type": ["number", "integer"] + }, + "casesFemalePer100k": { + "type": ["number", "integer"] + }, + "deathsMalePer100k": { + "type": ["number", "integer"] + }, + "deathsFemalePer100k": { + "type": ["number", "integer"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": ["number", "integer"] + }, + "incidence7Days": { + "type": ["number", "integer"] + }, + "date": { + "type": "string" + } + }, + "required": ["cases7Days", "date", "incidence7Days"] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A15-A34": { + "type": "object", + "properties": { + "casesMale": { + "type": ["number", "integer"] + }, + "casesFemale": { + "type": ["number", "integer"] + }, + "deathsMale": { + "type": ["number", "integer"] + }, + "deathsFemale": { + "type": ["number", "integer"] + }, + "casesMalePer100k": { + "type": ["number", "integer"] + }, + "casesFemalePer100k": { + "type": ["number", "integer"] + }, + "deathsMalePer100k": { + "type": ["number", "integer"] + }, + "deathsFemalePer100k": { + "type": ["number", "integer"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": ["number", "integer"] + }, + "incidence7Days": { + "type": ["number", "integer"] + }, + "date": { + "type": ["string", "null"] + } + }, + "required": ["cases7Days", "date", "incidence7Days"] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A35-A59": { + "type": "object", + "properties": { + "casesMale": { + "type": ["number", "integer"] + }, + "casesFemale": { + "type": ["number", "integer"] + }, + "deathsMale": { + "type": ["number", "integer"] + }, + "deathsFemale": { + "type": ["number", "integer"] + }, + "casesMalePer100k": { + "type": ["number", "integer"] + }, + "casesFemalePer100k": { + "type": ["number", "integer"] + }, + "deathsMalePer100k": { + "type": ["number", "integer"] + }, + "deathsFemalePer100k": { + "type": ["number", "integer"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": ["number", "integer"] + }, + "incidence7Days": { + "type": ["number", "integer"] + }, + "date": { + "type": ["string", "null"] + } + }, + "required": ["cases7Days", "date", "incidence7Days"] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A60-A79": { + "type": "object", + "properties": { + "casesMale": { + "type": ["number", "integer"] + }, + "casesFemale": { + "type": ["number", "integer"] + }, + "deathsMale": { + "type": ["number", "integer"] + }, + "deathsFemale": { + "type": ["number", "integer"] + }, + "casesMalePer100k": { + "type": ["number", "integer"] + }, + "casesFemalePer100k": { + "type": ["number", "integer"] + }, + "deathsMalePer100k": { + "type": ["number", "integer"] + }, + "deathsFemalePer100k": { + "type": ["number", "integer"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": ["number", "integer"] + }, + "incidence7Days": { + "type": ["number", "integer"] + }, + "date": { + "type": ["string", "null"] + } + }, + "required": ["cases7Days", "date", "incidence7Days"] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + }, + "A80+": { + "type": "object", + "properties": { + "casesMale": { + "type": ["number", "integer"] + }, + "casesFemale": { + "type": ["number", "integer"] + }, + "deathsMale": { + "type": ["number", "integer"] + }, + "deathsFemale": { + "type": ["number", "integer"] + }, + "casesMalePer100k": { + "type": ["number", "integer"] + }, + "casesFemalePer100k": { + "type": ["number", "integer"] + }, + "deathsMalePer100k": { + "type": ["number", "integer"] + }, + "deathsFemalePer100k": { + "type": ["number", "integer"] + }, + "hospitalization": { + "type": "object", + "properties": { + "cases7Days": { + "type": ["number", "integer"] + }, + "incidence7Days": { + "type": ["number", "integer"] + }, + "date": { + "type": ["string", "null"] + } + }, + "required": ["cases7Days", "date", "incidence7Days"] + } + }, + "required": [ + "casesFemale", + "casesFemalePer100k", + "casesMale", + "casesMalePer100k", + "deathsFemale", + "deathsFemalePer100k", + "deathsMale", + "deathsMalePer100k", + "hospitalization" + ] + } + }, + "required": ["A00-A04", "A05-A14", "A15-A34", "A35-A59", "A60-A79", "A80+"] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_cases.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_cases.json new file mode 100644 index 000000000000..4fa287a2d6f2 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_cases.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "cases": { + "type": ["integer", "null"] + }, + "date": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "abbreviation": { + "type": ["string", "null"] + } + }, + "required": ["abbreviation", "cases", "date", "name"] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_deaths.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_deaths.json new file mode 100644 index 000000000000..acc8238e1bac --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_deaths.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "deaths": { + "type": ["integer", "null"] + }, + "date": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "abbreviation": { + "type": ["string", "null"] + } + }, + "required": ["abbreviation", "date", "deaths", "name"] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_frozen_incidence.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_frozen_incidence.json new file mode 100644 index 000000000000..39b24add5b17 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_frozen_incidence.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "weekIncidence": { + "type": ["number", "null"] + }, + "date": { + "type": ["string", "null"] + }, + "dataSource": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "abbreviation": { + "type": ["string", "null"] + } + }, + "required": ["abbreviation", "dataSource", "date", "name", "weekIncidence"] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_hospitalization.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_hospitalization.json new file mode 100644 index 000000000000..4231538dcab9 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_hospitalization.json @@ -0,0 +1,68 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "cases7Days": { + "type": "integer" + }, + "incidence7Days": { + "type": ["number", "null"] + }, + "date": { + "type": ["string", "null"] + }, + "fixedCases7Days": { + "type": ["integer", "null"] + }, + "updatedCases7Days": { + "type": ["integer", "null"] + }, + "adjustedLowerCases7Days": { + "type": ["integer", "null"] + }, + "adjustedCases7Days": { + "type": ["integer", "null"] + }, + "adjustedUpperCases7Days": { + "type": ["integer", "null"] + }, + "fixedIncidence7Days": { + "type": ["number", "null"] + }, + "updatedIncidence7Days": { + "type": ["number", "null"] + }, + "adjustedLowerIncidence7Days": { + "type": ["number", "null"] + }, + "adjustedIncidence7Days": { + "type": ["number", "null"] + }, + "adjustedUpperIncidence7Days": { + "type": ["number", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "abbreviation": { + "type": ["string", "null"] + } + }, + "required": [ + "abbreviation", + "adjustedCases7Days", + "adjustedIncidence7Days", + "adjustedLowerCases7Days", + "adjustedLowerIncidence7Days", + "adjustedUpperCases7Days", + "adjustedUpperIncidence7Days", + "cases7Days", + "date", + "fixedCases7Days", + "fixedIncidence7Days", + "incidence7Days", + "name", + "updatedCases7Days", + "updatedIncidence7Days" + ] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_incidence.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_incidence.json new file mode 100644 index 000000000000..9c9d40c0adc3 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_incidence.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "weekIncidence": { + "type": ["number", "null"] + }, + "date": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "abbreviation": { + "type": ["string", "null"] + } + }, + "required": ["abbreviation", "date", "name", "weekIncidence"] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_recovered.json b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_recovered.json new file mode 100644 index 000000000000..e827c9588fb8 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/schemas/states_history_recovered.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": { + "recovered": { + "type": ["integer", "null"] + }, + "date": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "abbreviation": { + "type": ["string", "null"] + } + }, + "required": ["abbreviation", "date", "name", "recovered"] +} diff --git a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py index 32b3bd9567f2..0ae6e028a156 100644 --- a/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py +++ b/airbyte-integrations/connectors/source-rki-covid/source_rki_covid/source.py @@ -50,6 +50,24 @@ def path( return "germany/" +# class that contains main source states | full-refresh +class GermanyStates(RkiCovidStream): + """Docs: https://api.corona-zahlen.org/states""" + + primary_key = None + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + if response.json(): + for key, value in response.json().get("data").items(): + yield value + return [{}] + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "states/" + + # class that contains source age-groups in germany. | full-refresh class GermanyAgeGroups(RkiCovidStream): """Docs: https://api.corona-zahlen.org/germany/age-groups""" @@ -65,6 +83,27 @@ def path( return "germany/age-groups" +# class that contains main source states | full-refresh +class GermanyStatesAgeGroups(RkiCovidStream): + """Docs: https://api.corona-zahlen.org/states/age-groups""" + + primary_key = None + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + if response.json(): + for key, value in response.json().get("data").items(): + record = {"abbreviation": key} + for grp, data in value.items(): + record.update({grp: data}) + yield record + return [{}] + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "states/age-groups" + + # Basic incremental stream class IncrementalRkiCovidStream(RkiCovidStream, ABC): @@ -395,6 +434,161 @@ def path( return "germany/history/hospitalization/" +# STATES FULL-REFRESH. +# source: states/history/cases/:days | FULL-REFRESH +class ByStateRkiCovidStream(RkiCovidStream, ABC): + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + if response.json().get("data"): + for key, value in response.json().get("data").items(): + for record in value.get("history"): + record.update({"name": value.get("name"), "abbreviation": key}) + yield record + return [{}] + + +class StatesHistoryCases(ByStateRkiCovidStream): + """Docs: https://api.corona-zahlen.org/germany/states/history/cases/:days""" + + primary_key = None + + def __init__(self, config, **kwargs): + super().__init__(**kwargs) + self.start_date = config.get("start_date") + + def date_to_int(self, start_date) -> int: + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days <= 0: + return 1 + return diff.days + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + if self.start_date: + return "states/history/cases/" + str(self.date_to_int(self.start_date)) + return "states/history/cases/" + + +# source: states/history/incidence/:days | FULL-REFRESH +class StatesHistoryIncidence(ByStateRkiCovidStream): + """Docs: https://api.corona-zahlen.org/germany/states/history/incidence/:days""" + + primary_key = None + + def __init__(self, config, **kwargs): + super().__init__(**kwargs) + self.start_date = config.get("start_date") + + def date_to_int(self, start_date) -> int: + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days <= 0: + return 1 + return diff.days + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + if self.start_date: + return "states/history/incidence/" + str(self.date_to_int(self.start_date)) + return "states/history/incidence/" + + +# source: states/history/frozen-incidence/:days | FULL-REFRESH +class StatesHistoryFrozenIncidence(ByStateRkiCovidStream): + """Docs: https://api.corona-zahlen.org/germany/states/history/frozen-incidence/:days""" + + primary_key = None + + def __init__(self, config, **kwargs): + super().__init__(**kwargs) + self.start_date = config.get("start_date") + + def date_to_int(self, start_date) -> int: + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days <= 0: + return 1 + return diff.days + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + if self.start_date: + return "states/history/frozen-incidence/" + str(self.date_to_int(self.start_date)) + return "states/history/frozen-incidence/" + + +# source: states/history/deaths/:days | FULL-REFRESH +class StatesHistoryDeaths(ByStateRkiCovidStream): + """Docs: https://api.corona-zahlen.org/germany/states/history/deaths/:days""" + + primary_key = None + + def __init__(self, config, **kwargs): + super().__init__(**kwargs) + self.start_date = config.get("start_date") + + def date_to_int(self, start_date) -> int: + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days <= 0: + return 1 + return diff.days + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + if self.start_date: + return "states/history/deaths/" + str(self.date_to_int(self.start_date)) + return "states/history/deaths/" + + +# source: states/history/recovered/:days | FULL-REFRESH +class StatesHistoryRecovered(ByStateRkiCovidStream): + """Docs: https://api.corona-zahlen.org/germany/states/history/recovered/:days""" + + primary_key = None + + def __init__(self, config, **kwargs): + super().__init__(**kwargs) + self.start_date = config.get("start_date") + + def date_to_int(self, start_date) -> int: + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days <= 0: + return 1 + return diff.days + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + if self.start_date: + return "states/history/recovered/" + str(self.date_to_int(self.start_date)) + return "states/history/recovered/" + + +# source: states/history/hospitalization/:days | FULL-REFRESH +class StatesHistoryHospitalization(ByStateRkiCovidStream): + """Docs: https://api.corona-zahlen.org/germany/states/history/hospitalization/:days""" + + primary_key = None + + def __init__(self, config, **kwargs): + super().__init__(**kwargs) + self.start_date = config.get("start_date") + + def date_to_int(self, start_date) -> int: + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days <= 0: + return 1 + return diff.days + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + if self.start_date: + return "states/history/hospitalization/" + str(self.date_to_int(self.start_date)) + return "states/history/hospitalization/" + + # Source class SourceRkiCovid(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: @@ -417,8 +611,8 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: """ :param config: A Mapping of the user input configuration as defined in the connector spec. """ - - return [ + # Streams For Germany + streams = [ Germany(), GermanyAgeGroups(), GermanyHistoryCases(config=config), @@ -428,3 +622,19 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: GermanHistoryFrozenIncidence(config=config), GermanHistoryHospitalization(config=config), ] + + # Streams For States Of Germany + streams.extend( + [ + GermanyStates(), + GermanyStatesAgeGroups(), + StatesHistoryCases(config=config), + StatesHistoryIncidence(config=config), + StatesHistoryFrozenIncidence(config=config), + StatesHistoryDeaths(config=config), + StatesHistoryRecovered(config=config), + StatesHistoryHospitalization(config=config), + ] + ) + + return streams diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py index 94366f8c7416..1ee26991a664 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_source.py @@ -17,5 +17,5 @@ def test_streams(mocker): source = SourceRkiCovid() config_mock = MagicMock() streams = source.streams(config_mock) - expected_streams_number = 8 + expected_streams_number = 16 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistorycases.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistorycases.py new file mode 100644 index 000000000000..496adbf86277 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistorycases.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from datetime import datetime + +from pytest import fixture +from source_rki_covid.source import StatesHistoryCases + + +@fixture +def patch_states_history_cases(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(StatesHistoryCases, "primary_key", None) + + +def check_diff(start_date): + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days <= 0: + return str(1) + return str(diff.days) + + +def test_parse_with_cases(patch_states_history_cases): + config = {"start_date": "2022-04-27"} + stream = StatesHistoryCases(config) + expected_stream_path = "states/history/cases/" + check_diff(config.get("start_date")) + assert stream.path() == expected_stream_path diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistorydeaths.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistorydeaths.py new file mode 100644 index 000000000000..045470947978 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistorydeaths.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from datetime import datetime + +from pytest import fixture +from source_rki_covid.source import StatesHistoryDeaths + + +@fixture +def patch_states_history_deaths(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(StatesHistoryDeaths, "primary_key", None) + + +def check_diff(start_date): + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days <= 0: + return str(1) + return str(diff.days) + + +def test_parse_with_cases(patch_states_history_deaths): + config = {"start_date": "2022-04-27"} + stream = StatesHistoryDeaths(config) + expected_stream_path = "states/history/deaths/" + check_diff(config.get("start_date")) + assert stream.path() == expected_stream_path diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistoryfrozenincidence.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistoryfrozenincidence.py new file mode 100644 index 000000000000..14864a8607ba --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistoryfrozenincidence.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from datetime import datetime + +from pytest import fixture +from source_rki_covid.source import StatesHistoryFrozenIncidence + + +@fixture +def patch_states_history_frozenincidence(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(StatesHistoryFrozenIncidence, "primary_key", None) + + +def check_diff(start_date): + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days <= 0: + return str(1) + return str(diff.days) + + +def test_parse_with_cases(patch_states_history_frozenincidence): + config = {"start_date": "2022-04-27"} + stream = StatesHistoryFrozenIncidence(config) + expected_stream_path = "states/history/frozen-incidence/" + check_diff(config.get("start_date")) + assert stream.path() == expected_stream_path diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistoryhospitalization.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistoryhospitalization.py new file mode 100644 index 000000000000..642191304c9b --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistoryhospitalization.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from datetime import datetime + +from pytest import fixture +from source_rki_covid.source import StatesHistoryHospitalization + + +@fixture +def patch_states_history_hospitalization(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(StatesHistoryHospitalization, "primary_key", None) + + +def check_diff(start_date): + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days <= 0: + return str(1) + return str(diff.days) + + +def test_parse_with_cases(patch_states_history_hospitalization): + config = {"start_date": "2022-04-27"} + stream = StatesHistoryHospitalization(config) + expected_stream_path = "states/history/hospitalization/" + check_diff(config.get("start_date")) + assert stream.path() == expected_stream_path diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistoryincidence.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistoryincidence.py new file mode 100644 index 000000000000..74f0b1b783fa --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistoryincidence.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from datetime import datetime + +from pytest import fixture +from source_rki_covid.source import StatesHistoryIncidence + + +@fixture +def patch_states_history_incidence(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(StatesHistoryIncidence, "primary_key", None) + + +def check_diff(start_date): + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days <= 0: + return str(1) + return str(diff.days) + + +def test_parse_with_cases(patch_states_history_incidence): + config = {"start_date": "2022-04-27"} + stream = StatesHistoryIncidence(config) + expected_stream_path = "states/history/incidence/" + check_diff(config.get("start_date")) + assert stream.path() == expected_stream_path diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistoryrecovered.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistoryrecovered.py new file mode 100644 index 000000000000..c8c2e13b176d --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stateshistoryrecovered.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from datetime import datetime + +from pytest import fixture +from source_rki_covid.source import StatesHistoryRecovered + + +@fixture +def patch_states_history_recovered(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(StatesHistoryRecovered, "primary_key", None) + + +def check_diff(start_date): + diff = datetime.now() - datetime.strptime(start_date, "%Y-%m-%d") + if diff.days <= 0: + return str(1) + return str(diff.days) + + +def test_parse_with_cases(patch_states_history_recovered): + config = {"start_date": "2022-04-27"} + stream = StatesHistoryRecovered(config) + expected_stream_path = "states/history/recovered/" + check_diff(config.get("start_date")) + assert stream.path() == expected_stream_path diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany_states.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany_states.py new file mode 100644 index 000000000000..88e8caf2e738 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_stream_germany_states.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pytest +from source_rki_covid.source import GermanyStates + + +@pytest.fixture +def patch_germany_states_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(GermanyStates, "primary_key", None) + + +def test_path(patch_germany_states_class): + stream = GermanyStates() + expected_params = {"path": "states/"} + assert stream.path() == expected_params.get("path") diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py index d25f2936639f..17d9951cfd90 100644 --- a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams.py @@ -31,15 +31,6 @@ def test_next_page_token(patch_base_class): assert stream.next_page_token(**inputs) == expected_token -# def test_parse_response(patch_base_class): -# stream = RkiCovidStream() -# # TODO: replace this with your input parameters -# inputs = {"response": MagicMock()} -# # TODO: replace this with your expected parced object -# expected_parsed_object = {} -# assert next(stream.parse_response(**inputs)) == expected_parsed_object - - def test_request_headers(patch_base_class): stream = RkiCovidStream() inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} diff --git a/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams_states_agegroup.py b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams_states_agegroup.py new file mode 100644 index 000000000000..d0d606214076 --- /dev/null +++ b/airbyte-integrations/connectors/source-rki-covid/unit_tests/test_streams_states_agegroup.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pytest +from source_rki_covid.source import GermanyStatesAgeGroups + + +@pytest.fixture +def patch_states_age_group(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(GermanyStatesAgeGroups, "primary_key", None) + + +def test_path(patch_states_age_group): + stream = GermanyStatesAgeGroups() + expected_params = {"path": "states/age-groups"} + assert stream.path() == expected_params.get("path")