From a9a1b61ef4ecdf61102e332c98ec5421683816f2 Mon Sep 17 00:00:00 2001 From: letiescanciano <45267095+letiescanciano@users.noreply.github.com> Date: Fri, 2 Sep 2022 13:36:13 +0200 Subject: [PATCH 001/200] fix: Remove Fullstory leftovers (#16223) --- .../docker-compose-migration-test-0-32-0-alpha.yaml | 1 - charts/airbyte/README.md | 1 - charts/airbyte/templates/env-configmap.yaml | 1 - charts/airbyte/values.yaml | 3 --- docker-compose.yaml | 1 - kube/overlays/dev-integration-test/.env | 1 - kube/overlays/dev/.env | 1 - kube/overlays/stable-with-resource-limits/.env | 1 - kube/overlays/stable/.env | 1 - kube/resources/webapp.yaml | 5 ----- 10 files changed, 16 deletions(-) diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-0-32-0-alpha.yaml b/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-0-32-0-alpha.yaml index 10b858274bda..10807814277d 100644 --- a/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-0-32-0-alpha.yaml +++ b/airbyte-tests/src/automaticMigrationAcceptanceTest/resources/docker-compose-migration-test-0-32-0-alpha.yaml @@ -170,7 +170,6 @@ services: - AIRBYTE_ROLE=${AIRBYTE_ROLE:-} - AIRBYTE_VERSION=${VERSION} - API_URL=${API_URL:-} - - FULLSTORY=${FULLSTORY:-} - TRACKING_STRATEGY=${TRACKING_STRATEGY} - INTERNAL_API_HOST=${INTERNAL_API_HOST} - OPENREPLAY=${OPENREPLAY:-} diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index a9740d755018..c96c9ad93150 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -176,7 +176,6 @@ Helm chart to deploy airbyte | webapp.extraEnv | list | `[]` | | | webapp.extraVolumeMounts | list | `[]` | | | webapp.extraVolumes | list | `[]` | | -| webapp.fullstory.enabled | bool | `false` | | | webapp.image.pullPolicy | string | `"IfNotPresent"` | | | webapp.image.repository | string | `"airbyte/webapp"` | | | webapp.image.tag | string | `"0.40.3"` | | diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index 41faaf89542d..1384297e2ad9 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -17,7 +17,6 @@ data: DATABASE_PORT: {{ include "airbyte.database.port" . | quote }} DATABASE_URL: {{ include "airbyte.database.url" . | quote }} DB_DOCKER_MOUNT: airbyte_db - FULLSTORY: {{ ternary "enabled" "disabled" .Values.webapp.fullstory.enabled }} GCS_LOG_BUCKET: {{ .Values.global.logs.gcs.bucket | quote }} GOOGLE_APPLICATION_CREDENTIALS: {{ include "airbyte.gcpLogCredentialsPath" . | quote }} INTERNAL_API_HOST: {{ .Release.Name }}-server-svc:{{ .Values.server.service.port }} diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 144116e426c3..66243bb7d2b5 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -260,9 +260,6 @@ webapp: api: url: /api/v1/ - ## @param webapp.fullstory.enabled Whether or not to enable fullstory - fullstory: - enabled: false ## @param webapp.extraEnv [array] Additional env vars for webapp pod(s). ## Example: diff --git a/docker-compose.yaml b/docker-compose.yaml index a7935832e8f3..70942dd0b37d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -153,7 +153,6 @@ services: - AIRBYTE_ROLE=${AIRBYTE_ROLE:-} - AIRBYTE_VERSION=${VERSION} - API_URL=${API_URL:-} - - FULLSTORY=${FULLSTORY:-} - INTERNAL_API_HOST=${INTERNAL_API_HOST} - OPENREPLAY=${OPENREPLAY:-} - PAPERCUPS_STORYTIME=${PAPERCUPS_STORYTIME:-} diff --git a/kube/overlays/dev-integration-test/.env b/kube/overlays/dev-integration-test/.env index d5277d17fb85..3e7b16ad5273 100644 --- a/kube/overlays/dev-integration-test/.env +++ b/kube/overlays/dev-integration-test/.env @@ -31,7 +31,6 @@ API_URL=/api/v1/ INTERNAL_API_HOST=airbyte-server-svc:8001 WORKER_ENVIRONMENT=kubernetes -FULLSTORY=disabled LOG_LEVEL=INFO # S3/Minio Log Configuration diff --git a/kube/overlays/dev/.env b/kube/overlays/dev/.env index 6ff35c61b12e..49aeeb02dc63 100644 --- a/kube/overlays/dev/.env +++ b/kube/overlays/dev/.env @@ -33,7 +33,6 @@ API_URL=/api/v1/ INTERNAL_API_HOST=airbyte-server-svc:8001 WORKER_ENVIRONMENT=kubernetes -FULLSTORY=disabled LOG_LEVEL=INFO # S3/Minio Log Configuration diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env index a3351fdf9eec..6107741e05fd 100644 --- a/kube/overlays/stable-with-resource-limits/.env +++ b/kube/overlays/stable-with-resource-limits/.env @@ -33,7 +33,6 @@ API_URL=/api/v1/ INTERNAL_API_HOST=airbyte-server-svc:8001 WORKER_ENVIRONMENT=kubernetes -FULLSTORY=enabled LOG_LEVEL=INFO # S3/Minio Log Configuration diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index c4d64ed00c3d..e91081fff5b0 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -33,7 +33,6 @@ API_URL=/api/v1/ INTERNAL_API_HOST=airbyte-server-svc:8001 WORKER_ENVIRONMENT=kubernetes -FULLSTORY=enabled LOG_LEVEL=INFO # S3/Minio Log Configuration diff --git a/kube/resources/webapp.yaml b/kube/resources/webapp.yaml index ae4ce903c05d..9bb5c59ac68a 100644 --- a/kube/resources/webapp.yaml +++ b/kube/resources/webapp.yaml @@ -43,11 +43,6 @@ spec: configMapKeyRef: name: airbyte-env key: TRACKING_STRATEGY - - name: FULLSTORY - valueFrom: - configMapKeyRef: - name: airbyte-env - key: FULLSTORY - name: INTERNAL_API_HOST valueFrom: configMapKeyRef: From a43c098acb687d19bb090d37bc349709924b93f2 Mon Sep 17 00:00:00 2001 From: Miles Armstrong Date: Fri, 2 Sep 2022 13:18:03 +0100 Subject: [PATCH 002/200] Improve airbyte-metrics support in the Helm chart (#16166) * Add a new airbyte-metrics Helm chart * Include metrics Chart as a dependency of the main airbyte Chart * Allow setting METRIC_CLIENT and OTEL_COLLECTOR_ENDPOINT from Helm values * Actually make metrics sub-chart disabled by default --- charts/airbyte-metrics/.gitignore | 2 + charts/airbyte-metrics/.helmignore | 25 ++++ charts/airbyte-metrics/Chart.lock | 6 + charts/airbyte-metrics/Chart.yaml | 31 +++++ charts/airbyte-metrics/README.md | 43 ++++++ charts/airbyte-metrics/templates/_helpers.tpl | 39 ++++++ .../airbyte-metrics/templates/deployment.yaml | 130 ++++++++++++++++++ charts/airbyte-metrics/values.yaml | 98 +++++++++++++ charts/airbyte/Chart.yaml | 4 + charts/airbyte/templates/env-configmap.yaml | 4 +- charts/airbyte/values.yaml | 98 +++++++++++++ 11 files changed, 478 insertions(+), 2 deletions(-) create mode 100644 charts/airbyte-metrics/.gitignore create mode 100644 charts/airbyte-metrics/.helmignore create mode 100644 charts/airbyte-metrics/Chart.lock create mode 100644 charts/airbyte-metrics/Chart.yaml create mode 100644 charts/airbyte-metrics/README.md create mode 100644 charts/airbyte-metrics/templates/_helpers.tpl create mode 100644 charts/airbyte-metrics/templates/deployment.yaml create mode 100644 charts/airbyte-metrics/values.yaml diff --git a/charts/airbyte-metrics/.gitignore b/charts/airbyte-metrics/.gitignore new file mode 100644 index 000000000000..88e91e8a8f34 --- /dev/null +++ b/charts/airbyte-metrics/.gitignore @@ -0,0 +1,2 @@ +# Charts are downloaded at install time with `helm dep build`. +charts diff --git a/charts/airbyte-metrics/.helmignore b/charts/airbyte-metrics/.helmignore new file mode 100644 index 000000000000..f885da3fd491 --- /dev/null +++ b/charts/airbyte-metrics/.helmignore @@ -0,0 +1,25 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +ci.sh diff --git a/charts/airbyte-metrics/Chart.lock b/charts/airbyte-metrics/Chart.lock new file mode 100644 index 000000000000..017faffa076b --- /dev/null +++ b/charts/airbyte-metrics/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://charts.bitnami.com/bitnami + version: 1.17.1 +digest: sha256:dacc73770a5640c011e067ff8840ddf89631fc19016c8d0a9e5ea160e7da8690 +generated: "2022-08-31T12:09:18.473209+01:00" diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml new file mode 100644 index 000000000000..1acc08fea938 --- /dev/null +++ b/charts/airbyte-metrics/Chart.yaml @@ -0,0 +1,31 @@ +apiVersion: v2 +name: metrics +description: Helm chart to deploy airbyte-metrics + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: "0.39.36" + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.40.3" + +dependencies: + - name: common + repository: https://charts.bitnami.com/bitnami + tags: + - bitnami-common + version: 1.x.x diff --git a/charts/airbyte-metrics/README.md b/charts/airbyte-metrics/README.md new file mode 100644 index 000000000000..952cdc3c508f --- /dev/null +++ b/charts/airbyte-metrics/README.md @@ -0,0 +1,43 @@ +# metrics + +![Version: 0.39.36](https://img.shields.io/badge/Version-0.39.36-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.40.3](https://img.shields.io/badge/AppVersion-0.40.3-informational?style=flat-square) + +Helm chart to deploy airbyte-metrics + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://charts.bitnami.com/bitnami | common | 1.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| affinity | object | `{}` | | +| containerSecurityContext | object | `{}` | | +| enabled | bool | `true` | | +| env_vars | object | `{}` | | +| extraContainers | list | `[]` | | +| extraEnv | list | `[]` | | +| extraVolumeMounts | list | `[]` | | +| extraVolumes | list | `[]` | | +| global.database.host | string | `"example.com"` | | +| global.database.port | string | `"5432"` | | +| global.database.secretName | string | `""` | | +| global.database.secretValue | string | `""` | | +| global.deploymentMode | string | `"oss"` | | +| global.extraContainers | list | `[]` | | +| global.serviceAccountName | string | `"placeholderServiceAccount"` | | +| image.pullPolicy | string | `"IfNotPresent"` | | +| image.repository | string | `"airbyte/metrics-reporter"` | | +| nodeSelector | object | `{}` | | +| podAnnotations | object | `{}` | | +| replicaCount | int | `1` | | +| resources.limits | object | `{}` | | +| resources.requests | object | `{}` | | +| secrets | object | `{}` | | +| tolerations | list | `[]` | | + +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.11.0](https://github.com/norwoodj/helm-docs/releases/v1.11.0) diff --git a/charts/airbyte-metrics/templates/_helpers.tpl b/charts/airbyte-metrics/templates/_helpers.tpl new file mode 100644 index 000000000000..e6cc9d05cc14 --- /dev/null +++ b/charts/airbyte-metrics/templates/_helpers.tpl @@ -0,0 +1,39 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "airbyte.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "airbyte.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "airbyte.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Define db secret +*/}} + +{{- define "database.secret.name" -}} +{{- printf "%s-postgresql" .Release.Name }} +{{- end }} diff --git a/charts/airbyte-metrics/templates/deployment.yaml b/charts/airbyte-metrics/templates/deployment.yaml new file mode 100644 index 000000000000..1fe4e8132484 --- /dev/null +++ b/charts/airbyte-metrics/templates/deployment.yaml @@ -0,0 +1,130 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }} + labels: + app.kubernetes.io/name: {{ include "airbyte.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + helm.sh/chart: {{ include "airbyte.chart" . }} + app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.airbyte.io/fullname: {{ include "airbyte.fullname" . }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + airbyte: metrics + strategy: + type: Recreate # Needed due to volume claims + template: + metadata: + labels: + airbyte: metrics + {{- if .Values.podAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ .Values.global.serviceAccountName }} + {{- if .Values.global.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.global.imagePullSecrets }} + {{- printf "- name: %s" .name | nindent 2 }} + {{- end }} + {{- end }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.tolerations }} + tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }} + {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- end }} + containers: + - name: airbyte-metrics-container + image: {{ printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag) }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + {{- if eq .Values.global.deploymentMode "oss" }} + - name: AIRBYTE_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: AIRBYTE_VERSION + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.database.secretName | default (include "airbyte.chart" .) }} + key: {{ .Values.global.database.secretValue | default "postgresql-password" }} + - name: DATABASE_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_URL + - name: DATABASE_USER + valueFrom: + secretKeyRef: + name: {{ .Release.Name }}-airbyte-secrets + key: DATABASE_USER + - name: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION + - name: METRIC_CLIENT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: METRIC_CLIENT + - name: OTEL_COLLECTOR_ENDPOINT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: OTEL_COLLECTOR_ENDPOINT + {{- end }} + # Values from secret + {{- if .Values.secrets }} + {{- range $k, $v := .Values.secrets }} + - name: {{ $k }} + valueFrom: + secretKeyRef: + name: metrics-secrets + key: {{ $k }} + {{- end }} + {{- end }} + + # Values from env + {{- if .Values.env_vars }} + {{- range $k, $v := mergeOverwrite .Values.env_vars .Values.global.env_vars }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- end }} + + # Values from extraEnv for more compability(if you want to use external secret source or other stuff) + {{- if .Values.extraEnv }} + {{- toYaml .Values.extraEnv | nindent 8 }} + {{- end }} + + {{- if .Values.resources }} + resources: {{- toYaml .Values.resources | nindent 10 }} + {{- end }} + {{- if .Values.containerSecurityContext }} + securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} + {{- end }} + volumeMounts: + {{- if .Values.extraVolumeMounts }} + {{ toYaml .Values.extraVolumeMounts | nindent 8 }} + {{- end }} + {{- if .Values.extraContainers }} + {{ toYaml .Values.extraContainers | indent 8 }} + {{- end }} + {{- if .Values.global.extraContainers }} + {{ toYaml .Values.global.extraContainers | indent 8 }} + {{- end }} + volumes: + {{- if .Values.extraVolumes }} +{{ toYaml .Values.extraVolumes | nindent 6 }} + {{- end }} diff --git a/charts/airbyte-metrics/values.yaml b/charts/airbyte-metrics/values.yaml new file mode 100644 index 000000000000..7d79225b9de3 --- /dev/null +++ b/charts/airbyte-metrics/values.yaml @@ -0,0 +1,98 @@ + +global: + serviceAccountName: placeholderServiceAccount + deploymentMode: oss + extraContainers: [] + database: + secretName: "" + secretValue: "" + host: "example.com" + port: "5432" + +enabled: true +## @param metrics.replicaCount Number of metrics-reporter replicas +replicaCount: 1 + +## @param metrics.image.repository The repository to use for the airbyte metrics-reporter image. +## @param metrics.image.pullPolicy the pull policy to use for the airbyte metrics-reporter image +## @param metrics.image.tag The airbyte metrics-reporter image tag. Defaults to the chart's AppVersion +image: + repository: airbyte/metrics-reporter + pullPolicy: IfNotPresent + +## @param metrics.podAnnotations [object] Add extra annotations to the metrics-reporter pod +## +podAnnotations: {} + +## @param metrics.containerSecurityContext Security context for the container +## Examples: +## containerSecurityContext: +## runAsNonRoot: true +## runAsUser: 1000 +## readOnlyRootFilesystem: true +containerSecurityContext: {} + +## metrics-reporter app resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## We usually recommend not to specify default resources and to leave this as a conscious +## choice for the user. This also increases chances charts run on environments with little +## resources, such as Minikube. If you do want to specify resources, uncomment the following +## lines, adjust them as necessary, and remove the curly braces after 'resources:'. +## @param metrics.resources.limits [object] The resources limits for the metrics-reporter container +## @param metrics.resources.requests [object] The requested resources for the metrics-reporter container +resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + +## @param metrics.nodeSelector [object] Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## @param metrics.tolerations [array] Tolerations for metrics-reporter pod assignment. +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] + +## @param metrics.affinity [object] Affinity and anti-affinity for metrics-reporter pod assignment. +## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity +## +affinity: {} + +## @param metrics.extraEnv [array] Additional env vars for metrics-reporter pod(s). +## Example: +## +## extraEnv: +## - name: SAMPLE_ENV_VAR +## value: "key=sample-value" +extraEnv: [] + +## @param metrics.extraVolumeMounts [array] Additional volumeMounts for metrics-reporter container(s). +## Examples (when using `metrics.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumeMounts: +## - name: tmpdir +## mountPath: /tmp +## +extraVolumeMounts: [] + +## @param metrics.extraVolumes [array] Additional volumes for metrics-reporter pod(s). +## Examples (when using `metrics.containerSecurityContext.readOnlyRootFilesystem=true`): +## extraVolumes: +## - name: tmpdir +## emptyDir: {} +## +extraVolumes: [] + +extraContainers: [] + +secrets: {} + +env_vars: {} diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 4fcb003ca1eb..80b6ed7eab51 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -61,3 +61,7 @@ dependencies: name: pod-sweeper repository: "https://airbytehq.github.io/helm-charts/" version: placeholder + - condition: metrics.enabled + name: metrics + repository: "https://airbytehq.github.io/helm-charts/" + version: placeholder diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index 1384297e2ad9..aaaa82bf2c6b 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -52,8 +52,8 @@ data: WORKER_ENVIRONMENT: kubernetes WORKSPACE_DOCKER_MOUNT: airbyte_workspace WORKSPACE_ROOT: /workspace - METRIC_CLIENT: "" - OTEL_COLLECTOR_ENDPOINT: "" + METRIC_CLIENT: {{ .Values.global.metrics.metricClient | default "" | quote }} + OTEL_COLLECTOR_ENDPOINT: {{ .Values.global.metrics.otelCollectorEndpoint | default "" | quote }} ACTIVITY_MAX_ATTEMPT: "" ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS: "" ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS: "" diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 66243bb7d2b5..c423f292f889 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -59,6 +59,13 @@ global: credentials: "" # If credentialsJson is set then credentials auto resolves (to /secrets/gcs-log-creds/gcp.json) credentialsJson: "" + metrics: + ## These parameters are used in the airbyte-env ConfigMap, which is then mounted in deployments in the airbyte-worker and airbyte-metrics Charts. + ## @param global.metrics.metricClient The metric client to configure globally. Supports "otel" + metricClient: "" + + ## @param global.metrics.otelCollectorEndpoint The open-telemetry-collector endpoint that metrics will be sent to. + otelCollectorEndpoint: "" jobs: ## Jobs resource requests and limits ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ @@ -653,6 +660,97 @@ worker: hpa: enabled: false +## @section Metrics parameters +metrics: + enabled: false + + ## @param metrics.replicaCount Number of metrics-reporter replicas + replicaCount: 1 + + ## @param metrics.image.repository The repository to use for the airbyte metrics-reporter image. + ## @param metrics.image.pullPolicy the pull policy to use for the airbyte metrics-reporter image + ## @param metrics.image.tag The airbyte metrics-reporter image tag. Defaults to the chart's AppVersion + image: + repository: airbyte/metrics-reporter + pullPolicy: IfNotPresent + + ## @param metrics.podAnnotations [object] Add extra annotations to the metrics-reporter pod + ## + podAnnotations: {} + + ## @param metrics.containerSecurityContext Security context for the container + ## Examples: + ## containerSecurityContext: + ## runAsNonRoot: true + ## runAsUser: 1000 + ## readOnlyRootFilesystem: true + containerSecurityContext: {} + + ## metrics-reporter app resource requests and limits + ## ref: http://kubernetes.io/docs/user-guide/compute-resources/ + ## We usually recommend not to specify default resources and to leave this as a conscious + ## choice for the user. This also increases chances charts run on environments with little + ## resources, such as Minikube. If you do want to specify resources, uncomment the following + ## lines, adjust them as necessary, and remove the curly braces after 'resources:'. + ## @param metrics.resources.limits [object] The resources limits for the metrics-reporter container + ## @param metrics.resources.requests [object] The requested resources for the metrics-reporter container + resources: + ## Example: + ## limits: + ## cpu: 200m + ## memory: 1Gi + limits: {} + ## Examples: + ## requests: + ## memory: 256Mi + ## cpu: 250m + requests: {} + + ## @param metrics.nodeSelector [object] Node labels for pod assignment + ## Ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## @param metrics.tolerations [array] Tolerations for metrics-reporter pod assignment. + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + + ## @param metrics.affinity [object] Affinity and anti-affinity for metrics-reporter pod assignment. + ## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity + ## + affinity: {} + + ## @param metrics.extraEnv [array] Additional env vars for metrics-reporter pod(s). + ## Example: + ## + ## extraEnv: + ## - name: SAMPLE_ENV_VAR + ## value: "key=sample-value" + extraEnv: [] + + ## @param metrics.extraVolumeMounts [array] Additional volumeMounts for metrics-reporter container(s). + ## Examples (when using `metrics.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumeMounts: + ## - name: tmpdir + ## mountPath: /tmp + ## + extraVolumeMounts: [] + + ## @param metrics.extraVolumes [array] Additional volumes for metrics-reporter pod(s). + ## Examples (when using `metrics.containerSecurityContext.readOnlyRootFilesystem=true`): + ## extraVolumes: + ## - name: tmpdir + ## emptyDir: {} + ## + extraVolumes: [] + + extraContainers: [] + + secrets: {} + + env_vars: {} + ## @section Bootloader Parameters airbyte-bootloader: From c91385deba5d23dcaf0c9f425ec78ee2dec3e3dd Mon Sep 17 00:00:00 2001 From: Edmundo Ruiz Ghanem <168664+edmundito@users.noreply.github.com> Date: Fri, 2 Sep 2022 10:03:34 -0400 Subject: [PATCH 003/200] Improve behavior of password input field (#16011) * Improve behavior of password input field * Show / hide button now focuses back on the text at the previous cursor location * On comonent blur, it hides the information again * Fix logic on input blur * Add hide password on blur test to input * Clear the input selection start on blur * Remove defaultFocus and replace with autoFocus --- .../src/components/base/Input/Input.test.tsx | 64 ++++++++++++++- .../src/components/base/Input/Input.tsx | 77 +++++++++++++------ .../components/ConnectionName.tsx | 2 +- 3 files changed, 115 insertions(+), 28 deletions(-) diff --git a/airbyte-webapp/src/components/base/Input/Input.test.tsx b/airbyte-webapp/src/components/base/Input/Input.test.tsx index 1d7d5cbc586e..13e4ed92f398 100644 --- a/airbyte-webapp/src/components/base/Input/Input.test.tsx +++ b/airbyte-webapp/src/components/base/Input/Input.test.tsx @@ -1,4 +1,5 @@ -import { fireEvent } from "@testing-library/react"; +import { fireEvent, waitFor } from "@testing-library/react"; +import { act } from "react-dom/test-utils"; import { render } from "utils/testutils"; @@ -39,11 +40,68 @@ describe("", () => { getByTestId("toggle-password-visibility-button")?.click(); - expect(getByTestId("input")).toHaveAttribute("type", "text"); - expect(getByTestId("input")).toHaveValue(value); + const inputEl = getByTestId("input") as HTMLInputElement; + + expect(inputEl).toHaveAttribute("type", "text"); + expect(inputEl).toHaveValue(value); + expect(inputEl.selectionStart).toBe(value.length); expect(getByRole("img", { hidden: true })).toHaveAttribute("data-icon", "eye-slash"); }); + test("showing password should remember cursor position", async () => { + const value = "eight888"; + const selectionStart = Math.round(value.length / 2); + + const { getByTestId } = await render(); + const inputEl = getByTestId("input") as HTMLInputElement; + + act(() => { + inputEl.selectionStart = selectionStart; + }); + + getByTestId("toggle-password-visibility-button")?.click(); + + expect(inputEl.selectionStart).toBe(selectionStart); + }); + + test("hides password on blur", async () => { + const value = "eight888"; + const { getByTestId, getByRole } = await render(); + + getByTestId("toggle-password-visibility-button").click(); + + const inputEl = getByTestId("input"); + + expect(inputEl).toHaveFocus(); + act(() => inputEl.blur()); + + await waitFor(() => { + expect(inputEl).toHaveAttribute("type", "password"); + expect(getByRole("img", { hidden: true })).toHaveAttribute("data-icon", "eye"); + }); + }); + + test("cursor position should be at the end after blur and and clicking on show password button", async () => { + const value = "eight888"; + const { getByTestId } = await render(); + const inputEl = getByTestId("input") as HTMLInputElement; + + getByTestId("toggle-password-visibility-button").click(); + expect(inputEl).toHaveFocus(); + act(() => { + inputEl.selectionStart = value.length / 2; + inputEl.blur(); + }); + + await waitFor(() => { + expect(inputEl).toHaveAttribute("type", "password"); + }); + + getByTestId("toggle-password-visibility-button").click(); + expect(inputEl).toHaveFocus(); + expect(inputEl.selectionStart).toBe(value.length); + }); + test("should trigger onChange once", async () => { const onChange = jest.fn(); const { getByTestId } = await render(); diff --git a/airbyte-webapp/src/components/base/Input/Input.tsx b/airbyte-webapp/src/components/base/Input/Input.tsx index 39e00dc62bf1..73eacb136580 100644 --- a/airbyte-webapp/src/components/base/Input/Input.tsx +++ b/airbyte-webapp/src/components/base/Input/Input.tsx @@ -1,7 +1,7 @@ import { faEye, faEyeSlash } from "@fortawesome/free-regular-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import classNames from "classnames"; -import React, { useEffect, useRef, useState } from "react"; +import React, { useCallback, useRef, useState } from "react"; import { useIntl } from "react-intl"; import { useToggle } from "react-use"; import styled from "styled-components"; @@ -24,7 +24,6 @@ const getBackgroundColor = (props: IStyleProps) => { export interface InputProps extends React.InputHTMLAttributes { error?: boolean; light?: boolean; - defaultFocus?: boolean; } const InputContainer = styled.div` @@ -83,46 +82,76 @@ const VisibilityButton = styled(Button)` border: none; `; -const Input: React.FC = ({ defaultFocus = false, onFocus, onBlur, ...props }) => { +const Input: React.FC = ({ ...props }) => { const { formatMessage } = useIntl(); + const inputRef = useRef(null); - const [isContentVisible, setIsContentVisible] = useToggle(false); + const buttonRef = useRef(null); + const inputSelectionStartRef = useRef(null); + + const [isContentVisible, toggleIsContentVisible] = useToggle(false); const [focused, setFocused] = useState(false); const isPassword = props.type === "password"; const isVisibilityButtonVisible = isPassword && !props.disabled; const type = isPassword ? (isContentVisible ? "text" : "password") : props.type; - useEffect(() => { - if (defaultFocus && inputRef.current !== null) { - inputRef.current.focus(); + const focusOnInputElement = useCallback(() => { + if (!inputRef.current) { + return; + } + + const { current: element } = inputRef; + const selectionStart = inputSelectionStartRef.current ?? inputRef.current?.value.length; + + element.focus(); + + if (selectionStart) { + // Update input cursor position to where it was before + window.setTimeout(() => { + element.setSelectionRange(selectionStart, selectionStart); + }, 0); + } + }, []); + + const onContainerFocus: React.FocusEventHandler = () => { + setFocused(true); + }; + + const onContainerBlur: React.FocusEventHandler = (event) => { + if (isVisibilityButtonVisible && event.target === inputRef.current) { + // Save the previous selection + inputSelectionStartRef.current = inputRef.current.selectionStart; + } + + setFocused(false); + + if (isPassword) { + window.setTimeout(() => { + if (document.activeElement !== inputRef.current && document.activeElement !== buttonRef.current) { + toggleIsContentVisible(false); + inputSelectionStartRef.current = null; + } + }, 0); } - }, [inputRef, defaultFocus]); + }; return ( - { - setFocused(true); - onFocus?.(event); - }} - onBlur={(event) => { - setFocused(false); - onBlur?.(event); - }} - /> + {isVisibilityButtonVisible ? ( setIsContentVisible()} + onClick={() => { + toggleIsContentVisible(); + focusOnInputElement(); + }} type="button" aria-label={formatMessage({ id: `ui.input.${isContentVisible ? "hide" : "show"}Password`, diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.tsx index cb1a82ccf98a..dd8692d7a833 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.tsx @@ -79,7 +79,7 @@ const ConnectionName: React.FC = ({ connection }) => { onEscape={onEscape} onEnter={onEnter} disabled={loading} - defaultFocus + autoFocus /> From 6418b32661f82ac001682c6eba92d7fec0b5f607 Mon Sep 17 00:00:00 2001 From: Edmundo Ruiz Ghanem <168664+edmundito@users.noreply.github.com> Date: Fri, 2 Sep 2022 10:17:27 -0400 Subject: [PATCH 004/200] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20?= =?UTF-8?q?=F0=9F=A7=B9=20Migrate=20attempt=20`bytesSynced`=20to=20`totalS?= =?UTF-8?q?tats.bytesEmitted`=20and=20cleanup=20`AttemptDetails`=20compone?= =?UTF-8?q?nt=20(#16126)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Migrate from bytesSynced to bytesEmitted * Cleanup separator in AttemptDetails * Fix notation in AttemptDetails scss --- .../components/AttemptDetails.module.scss | 10 +++++++-- .../JobItem/components/AttemptDetails.tsx | 21 ++++++++----------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.module.scss b/airbyte-webapp/src/components/JobItem/components/AttemptDetails.module.scss index 011bf11f4215..c5dd42b97102 100644 --- a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.module.scss +++ b/airbyte-webapp/src/components/JobItem/components/AttemptDetails.module.scss @@ -1,12 +1,18 @@ @use "../../../scss/colors"; +@use "../../../scss/variables"; -.details { +.container { font-size: 12px; line-height: 15px; color: colors.$grey; } -.truncate { +.details > *:not(:last-child)::after { + content: "|"; + padding: 0 variables.$spacing-sm; +} + +.failedMessage { white-space: nowrap; overflow: hidden; text-overflow: ellipsis; diff --git a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx b/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx index cd22ba49d79d..0a74b3614805 100644 --- a/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx +++ b/airbyte-webapp/src/components/JobItem/components/AttemptDetails.tsx @@ -69,37 +69,34 @@ const AttemptDetails: React.FC = ({ attempt, className, configType }) => const isFailed = attempt.status === Status.FAILED; return ( -
-
- {formatBytes(attempt?.bytesSynced)} | +
+
+ {formatBytes(attempt?.totalStats?.bytesEmitted)} {" "} - |{" "} + /> {" "} - |{" "} + /> {hours ? : null} {hours || minutes ? : null} - {configType ? ( + {configType && ( - {" "} - | + - ) : null} + )}
{isFailed && ( -
+
{formatMessage( { id: "ui.keyValuePairV3", From c8899a121544275923a10fb8bba255dc188e2fdd Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Fri, 2 Sep 2022 09:29:44 -0700 Subject: [PATCH 005/200] Add ProtocolVersion to StandardDefs (#16237) * Add ProtocolVersion to StandardDefs This updates the database read/write operations * Fix letter case --- .../main/resources/types/StandardDestinationDefinition.yaml | 3 +++ .../src/main/resources/types/StandardSourceDefinition.yaml | 3 +++ .../java/io/airbyte/config/persistence/ConfigWriter.java | 4 ++++ .../main/java/io/airbyte/config/persistence/DbConverter.java | 2 ++ .../test/java/io/airbyte/config/persistence/MockData.java | 5 +++++ 5 files changed, 17 insertions(+) diff --git a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml index 375dc2878c15..b69c82a343bd 100644 --- a/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml +++ b/airbyte-config/config-models/src/main/resources/types/StandardDestinationDefinition.yaml @@ -55,3 +55,6 @@ properties: format: date resourceRequirements: "$ref": ActorDefinitionResourceRequirements.yaml + protocolVersion: + type: string + description: the Airbyte Protocol version supported by the connector diff --git a/airbyte-config/config-models/src/main/resources/types/StandardSourceDefinition.yaml b/airbyte-config/config-models/src/main/resources/types/StandardSourceDefinition.yaml index eada43f89766..b08e36599c4e 100644 --- a/airbyte-config/config-models/src/main/resources/types/StandardSourceDefinition.yaml +++ b/airbyte-config/config-models/src/main/resources/types/StandardSourceDefinition.yaml @@ -62,3 +62,6 @@ properties: format: date resourceRequirements: "$ref": ActorDefinitionResourceRequirements.yaml + protocolVersion: + type: string + description: the Airbyte Protocol version supported by the connector diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java index 83969135dcaf..e21b039d3bbd 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/ConfigWriter.java @@ -50,6 +50,7 @@ static void writeStandardSourceDefinition(final List c : Enums.toEnum(standardSourceDefinition.getSourceType().value(), SourceType.class).orElseThrow()) .set(Tables.ACTOR_DEFINITION.SPEC, JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getSpec()))) + .set(Tables.ACTOR_DEFINITION.PROTOCOL_VERSION, standardSourceDefinition.getProtocolVersion()) .set(Tables.ACTOR_DEFINITION.TOMBSTONE, standardSourceDefinition.getTombstone()) .set(Tables.ACTOR_DEFINITION.PUBLIC, standardSourceDefinition.getPublic()) .set(Tables.ACTOR_DEFINITION.CUSTOM, standardSourceDefinition.getCustom()) @@ -79,6 +80,7 @@ static void writeStandardSourceDefinition(final List c : Enums.toEnum(standardSourceDefinition.getSourceType().value(), SourceType.class).orElseThrow()) .set(Tables.ACTOR_DEFINITION.SPEC, JSONB.valueOf(Jsons.serialize(standardSourceDefinition.getSpec()))) + .set(Tables.ACTOR_DEFINITION.PROTOCOL_VERSION, standardSourceDefinition.getProtocolVersion()) .set(Tables.ACTOR_DEFINITION.TOMBSTONE, standardSourceDefinition.getTombstone() != null && standardSourceDefinition.getTombstone()) .set(Tables.ACTOR_DEFINITION.PUBLIC, standardSourceDefinition.getPublic()) .set(Tables.ACTOR_DEFINITION.CUSTOM, standardSourceDefinition.getCustom()) @@ -115,6 +117,7 @@ static void writeStandardDestinationDefinition(final List Date: Fri, 2 Sep 2022 19:40:46 +0300 Subject: [PATCH 006/200] 15700 add tests for PokeAPI (#15701) * add tests for PokeAPI * Update connection.spec.ts add body verification * add page object model for update connection (poke api) test * change structure with using POM * Select sync mode dropdown with a data-testid (#16053) * Fix coments * fix goToDestinationPage signature * move fillEmail method * change structure with using POM * Fix coments * Update connection.spec.ts fix request url and schedule dropdown value Co-authored-by: Alex Birdsall --- .../cypress/commands/common.ts | 44 +------- .../cypress/commands/connection.ts | 30 +++-- .../cypress/commands/connector.ts | 42 +++++++ .../cypress/commands/destination.ts | 16 ++- .../cypress/commands/sidebar.ts | 3 - .../cypress/commands/source.ts | 53 +++++---- .../cypress/integration/connection.spec.ts | 103 +++++++++++++++--- .../cypress/integration/destination.spec.ts | 8 +- .../cypress/integration/onboarding.spec.ts | 2 +- .../cypress/integration/source.spec.ts | 8 +- .../cypress/pages/createConnectorPage.ts | 48 ++++++++ .../cypress/pages/destinationPage.ts | 18 +++ .../cypress/pages/replicationPage.ts | 56 ++++++++++ .../cypress/pages/settingsConnectionPage.ts | 5 + .../cypress/pages/sidebar.ts | 5 + .../cypress/pages/sourcePage.ts | 17 +++ .../components/SyncSettingsDropdown.tsx | 1 + 17 files changed, 352 insertions(+), 107 deletions(-) create mode 100644 airbyte-webapp-e2e-tests/cypress/commands/connector.ts delete mode 100644 airbyte-webapp-e2e-tests/cypress/commands/sidebar.ts create mode 100644 airbyte-webapp-e2e-tests/cypress/pages/createConnectorPage.ts create mode 100644 airbyte-webapp-e2e-tests/cypress/pages/destinationPage.ts create mode 100644 airbyte-webapp-e2e-tests/cypress/pages/replicationPage.ts create mode 100644 airbyte-webapp-e2e-tests/cypress/pages/settingsConnectionPage.ts create mode 100644 airbyte-webapp-e2e-tests/cypress/pages/sidebar.ts create mode 100644 airbyte-webapp-e2e-tests/cypress/pages/sourcePage.ts diff --git a/airbyte-webapp-e2e-tests/cypress/commands/common.ts b/airbyte-webapp-e2e-tests/cypress/commands/common.ts index 54ba9001a2c7..dc2b78c69afd 100644 --- a/airbyte-webapp-e2e-tests/cypress/commands/common.ts +++ b/airbyte-webapp-e2e-tests/cypress/commands/common.ts @@ -2,46 +2,6 @@ export const submitButtonClick = () => { cy.get("button[type=submit]").click(); } -export const fillEmail = (email: string) => { - cy.get("input[name=email]").type(email); -} - -export const fillTestLocalJsonForm = (name: string) => { - cy.intercept("/api/v1/destination_definition_specifications/get").as("getDestinationSpecifications"); - - cy.get("div[data-testid='serviceType']").click(); - cy.get("div").contains("Local JSON").click(); - - cy.wait("@getDestinationSpecifications"); - - cy.get("input[name=name]").clear().type(name); - cy.get("input[name='connectionConfiguration.destination_path']").type("/local"); -} - -export const openSourcePage = () => { - cy.intercept("/api/v1/sources/list").as("getSourcesList"); - cy.visit("/source"); - cy.wait("@getSourcesList"); -} - -export const openDestinationPage = () => { - cy.intercept("/api/v1/destinations/list").as("getDestinationsList"); - cy.visit("/destination"); - cy.wait("@getDestinationsList"); -} - -export const openNewSourceForm = () => { - openSourcePage(); - cy.get("button[data-id='new-source'").click(); - cy.url().should("include", `/source/new-source`); -} - -export const openNewDestinationForm = () => { - openDestinationPage(); - cy.get("button[data-id='new-destination'").click(); - cy.url().should("include", `/destination/new-destination`); -} - export const updateField = (field: string, value: string) => { cy.get("input[name='" + field + "']").clear().type(value); } @@ -61,3 +21,7 @@ export const clearApp = () => { cy.clearLocalStorage(); cy.clearCookies(); } + +export const fillEmail = (email: string) => { + cy.get("input[name=email]").type(email); +} diff --git a/airbyte-webapp-e2e-tests/cypress/commands/connection.ts b/airbyte-webapp-e2e-tests/cypress/commands/connection.ts index e9a7ba8e31f7..55e8493a5c7b 100644 --- a/airbyte-webapp-e2e-tests/cypress/commands/connection.ts +++ b/airbyte-webapp-e2e-tests/cypress/commands/connection.ts @@ -1,26 +1,36 @@ import { submitButtonClick } from "./common"; -import { createTestDestination } from "./destination"; -import { createTestSource } from "./source"; +import { createLocalJsonDestination } from "./destination"; +import { createPokeApiSource, createPostgresSource } from "./source"; +import { openAddSource } from "pages/destinationPage" +import { selectSchedule, setupDestinationNamespaceSourceFormat, enterConnectionName } from "pages/replicationPage" export const createTestConnection = (sourceName: string, destinationName: string) => { cy.intercept("/api/v1/sources/discover_schema").as("discoverSchema"); cy.intercept("/api/v1/web_backend/connections/create").as("createConnection"); - createTestSource(sourceName); - createTestDestination(destinationName); + switch (true) { + case sourceName.includes('PokeAPI'): + createPokeApiSource(sourceName, "luxray") + break; + case sourceName.includes('Postgres'): + createPostgresSource(sourceName); + break; + default: + createPostgresSource(sourceName); + } + + createLocalJsonDestination(destinationName, "/local"); cy.wait(5000); - cy.get("div[data-testid='select-source']").click(); + openAddSource(); cy.get("div").contains(sourceName).click(); cy.wait("@discoverSchema"); - cy.get("input[data-testid='connectionName']").type("Connection name"); - cy.get("div[data-testid='scheduleData.basicSchedule']").click(); - cy.get("div[data-testid='Manual']").click(); + enterConnectionName("Connection name"); + selectSchedule("Manual"); - cy.get("div[data-testid='namespaceDefinition']").click(); - cy.get("div[data-testid='namespaceDefinition-source']").click(); + setupDestinationNamespaceSourceFormat(); submitButtonClick(); cy.wait("@createConnection"); diff --git a/airbyte-webapp-e2e-tests/cypress/commands/connector.ts b/airbyte-webapp-e2e-tests/cypress/commands/connector.ts new file mode 100644 index 000000000000..8abb5fbd12b0 --- /dev/null +++ b/airbyte-webapp-e2e-tests/cypress/commands/connector.ts @@ -0,0 +1,42 @@ +import { enterDestinationPath, selectServiceType, enterName, enterHost, enterPort, enterDatabase, enterUsername, enterPassword, enterPokemonName } from "pages/createConnectorPage" + +export const fillPostgresForm = (name: string, host: string, port: string, database: string, username: string, password: string) => { + cy.intercept("/api/v1/source_definition_specifications/get").as( + "getSourceSpecifications" + ); + + selectServiceType("Postgres"); + + cy.wait("@getSourceSpecifications"); + + enterName(name); + enterHost(host); + enterPort(port); + enterDatabase(database); + enterUsername(username); + enterPassword(password); +}; + +export const fillPokeAPIForm = (name: string, pokeName: string) => { + cy.intercept("/api/v1/source_definition_specifications/get").as( + "getSourceSpecifications" + ); + + selectServiceType("PokeAPI"); + + cy.wait("@getSourceSpecifications"); + + enterName(name); + enterPokemonName(pokeName); +}; + +export const fillLocalJsonForm = (name: string, destinationPath: string) => { + cy.intercept("/api/v1/destination_definition_specifications/get").as("getDestinationSpecifications"); + + selectServiceType("Local JSON"); + + cy.wait("@getDestinationSpecifications"); + + enterName(name); + enterDestinationPath(destinationPath); +} \ No newline at end of file diff --git a/airbyte-webapp-e2e-tests/cypress/commands/destination.ts b/airbyte-webapp-e2e-tests/cypress/commands/destination.ts index a3a6f5514cfd..2f3e54a7cfda 100644 --- a/airbyte-webapp-e2e-tests/cypress/commands/destination.ts +++ b/airbyte-webapp-e2e-tests/cypress/commands/destination.ts @@ -1,13 +1,17 @@ -import { deleteEntity, fillTestLocalJsonForm, openDestinationPage, openNewDestinationForm, openSettingForm, submitButtonClick, updateField } from "./common"; +import { deleteEntity, openSettingForm, submitButtonClick, updateField } from "./common"; +import { fillLocalJsonForm } from "./connector" +import { goToDestinationPage, openNewDestinationForm } from "pages/destinationPage" -export const createTestDestination = (name: string) => { +export const createLocalJsonDestination = (name: string, destinationPath: string) => { cy.intercept("/api/v1/scheduler/destinations/check_connection").as("checkDestinationConnection"); cy.intercept("/api/v1/destinations/create").as("createDestination"); + goToDestinationPage(); openNewDestinationForm(); - fillTestLocalJsonForm(name); + fillLocalJsonForm(name, destinationPath); submitButtonClick(); + cy.wait(3000); cy.wait("@checkDestinationConnection"); cy.wait("@createDestination"); } @@ -16,7 +20,7 @@ export const updateDestination = (name: string, field: string, value: string) => cy.intercept("/api/v1/destinations/check_connection_for_update").as("checkDestinationUpdateConnection"); cy.intercept("/api/v1/destinations/update").as("updateDestination"); - openDestinationPage(); + goToDestinationPage(); openSettingForm(name); updateField(field, value); submitButtonClick(); @@ -26,7 +30,9 @@ export const updateDestination = (name: string, field: string, value: string) => } export const deleteDestination = (name: string) => { - openDestinationPage(); + cy.intercept("/api/v1/destinations/delete").as("deleteDestination"); + goToDestinationPage(); openSettingForm(name); deleteEntity(); + cy.wait("@deleteDestination"); } \ No newline at end of file diff --git a/airbyte-webapp-e2e-tests/cypress/commands/sidebar.ts b/airbyte-webapp-e2e-tests/cypress/commands/sidebar.ts deleted file mode 100644 index 2b4f52015fbf..000000000000 --- a/airbyte-webapp-e2e-tests/cypress/commands/sidebar.ts +++ /dev/null @@ -1,3 +0,0 @@ -export const openSettings = () => { - cy.get("nav a[href*='settings']").click(); -}; diff --git a/airbyte-webapp-e2e-tests/cypress/commands/source.ts b/airbyte-webapp-e2e-tests/cypress/commands/source.ts index d5f59e92de8d..479627e607dd 100644 --- a/airbyte-webapp-e2e-tests/cypress/commands/source.ts +++ b/airbyte-webapp-e2e-tests/cypress/commands/source.ts @@ -1,33 +1,38 @@ -import { deleteEntity, openNewSourceForm, openSettingForm, openSourcePage, submitButtonClick, updateField } from "./common"; - -export const fillPgSourceForm = (name: string) => { - cy.intercept("/api/v1/source_definition_specifications/get").as( - "getSourceSpecifications" - ); - - cy.get("div[data-testid='serviceType']").click(); - cy.get("div").contains("Postgres").click(); - - cy.wait("@getSourceSpecifications"); - - cy.get("input[name=name]").clear().type(name); - cy.get("input[name='connectionConfiguration.host']").type("localhost"); - cy.get("input[name='connectionConfiguration.port']").type("{selectAll}{del}5433"); - cy.get("input[name='connectionConfiguration.database']").type("airbyte_ci"); - cy.get("input[name='connectionConfiguration.username']").type("postgres"); - cy.get("input[name='connectionConfiguration.password']").type( - "secret_password" +import { deleteEntity, openSettingForm, submitButtonClick, updateField } from "./common"; +import { goToSourcePage, openNewSourceForm} from "pages/sourcePage"; +import { fillPostgresForm, fillPokeAPIForm } from "./connector" + +export const createPostgresSource = ( + name: string, + host: string = "localhost", + port: string = "{selectAll}{del}5433", + database: string = "airbyte_ci", + username: string = "postgres", + password: string = "secret_password" +) => { + cy.intercept("/api/v1/scheduler/sources/check_connection").as( + "checkSourceUpdateConnection" ); + cy.intercept("/api/v1/sources/create").as("createSource"); + + goToSourcePage(); + openNewSourceForm(); + fillPostgresForm(name, host, port, database, username, password); + submitButtonClick(); + + cy.wait("@checkSourceUpdateConnection"); + cy.wait("@createSource"); }; -export const createTestSource = (name: string) => { +export const createPokeApiSource = (name: string, pokeName: string) => { cy.intercept("/api/v1/scheduler/sources/check_connection").as( "checkSourceUpdateConnection" ); cy.intercept("/api/v1/sources/create").as("createSource"); + goToSourcePage(); openNewSourceForm(); - fillPgSourceForm(name); + fillPokeAPIForm(name, pokeName); submitButtonClick(); cy.wait("@checkSourceUpdateConnection"); @@ -40,7 +45,7 @@ export const updateSource = (name: string, field: string, value: string) => { ); cy.intercept("/api/v1/sources/update").as("updateSource"); - openSourcePage(); + goToSourcePage(); openSettingForm(name); updateField(field, value); submitButtonClick(); @@ -50,7 +55,9 @@ export const updateSource = (name: string, field: string, value: string) => { } export const deleteSource = (name: string) => { - openSourcePage(); + cy.intercept("/api/v1/sources/delete").as("deleteSource"); + goToSourcePage(); openSettingForm(name); deleteEntity(); + cy.wait("@deleteSource"); } diff --git a/airbyte-webapp-e2e-tests/cypress/integration/connection.spec.ts b/airbyte-webapp-e2e-tests/cypress/integration/connection.spec.ts index e8a05dfdd348..aefc88327194 100644 --- a/airbyte-webapp-e2e-tests/cypress/integration/connection.spec.ts +++ b/airbyte-webapp-e2e-tests/cypress/integration/connection.spec.ts @@ -1,8 +1,11 @@ -import { deleteEntity } from "commands/common"; +import { deleteEntity, submitButtonClick } from "commands/common"; import { createTestConnection } from "commands/connection"; import { deleteDestination } from "commands/destination"; import { deleteSource } from "commands/source"; import { initialSetupCompleted } from "commands/workspaces"; +import { confirmStreamConfigurationChangedPopup, selectSchedule, fillOutDestinationPrefix, goToReplicationTab, setupDestinationNamespaceCustomFormat, selectFullAppendSyncMode, checkSuccessResult} from "pages/replicationPage"; +import { openSourceDestinationFromGrid, goToSourcePage} from "pages/sourcePage"; +import { goToSettingsPage } from "pages/settingsConnectionPage" describe("Connection main actions", () => { beforeEach(() => { @@ -10,10 +13,13 @@ describe("Connection main actions", () => { }); it("Create new connection", () => { - createTestConnection("Test connection source cypress", "Test destination cypress"); + createTestConnection("Test connection source cypress", "Test connection destination cypress"); cy.get("div").contains("Test connection source cypress").should("exist"); - cy.get("div").contains("Test destination cypress").should("exist"); + cy.get("div").contains("Test connection destination cypress").should("exist"); + + deleteSource("Test connection source cypress"); + deleteDestination("Test connection destination cypress"); }); it("Update connection", () => { @@ -21,27 +27,90 @@ describe("Connection main actions", () => { createTestConnection("Test update connection source cypress", "Test update connection destination cypress"); - cy.visit("/source"); - cy.get("div").contains("Test update connection source cypress").click(); - cy.get("div").contains("Test update connection destination cypress").click(); + goToSourcePage(); + openSourceDestinationFromGrid("Test update connection source cypress"); + openSourceDestinationFromGrid("Test update connection destination cypress"); + + goToReplicationTab(); + + selectSchedule('Every hour'); + fillOutDestinationPrefix('auto_test'); + + submitButtonClick(); + + cy.wait("@updateConnection").then((interception) => { + assert.isNotNull(interception.response?.statusCode, '200'); + }); + + checkSuccessResult(); + + deleteSource("Test update connection source cypress"); + deleteDestination("Test update connection destination cypress"); + }); + + it("Update connection (pokeAPI)", () => { + cy.intercept("/api/v1/web_backend/connections/update").as("updateConnection"); - cy.get("div[data-id='replication-step']").click(); + createTestConnection("Test update connection PokeAPI source cypress", "Test update connection Local JSON destination cypress"); - cy.get("div[data-testid='scheduleData.basicSchedule']").click(); - cy.get("div[data-testid='Every hour']").click(); - cy.get("button[type=submit]").first().click(); - cy.wait("@updateConnection"); - cy.get("span[data-id='success-result']").should("exist"); -}); + goToSourcePage(); + openSourceDestinationFromGrid("Test update connection PokeAPI source cypress"); + openSourceDestinationFromGrid("Test update connection Local JSON destination cypress"); + + goToReplicationTab(); + + selectSchedule('Every hour'); + fillOutDestinationPrefix('auto_test'); + setupDestinationNamespaceCustomFormat('_test'); + selectFullAppendSyncMode(); + + submitButtonClick(); + confirmStreamConfigurationChangedPopup(); + + cy.wait("@updateConnection").then((interception) => { + assert.isNotNull(interception.response?.statusCode, '200'); + expect(interception.request.method).to.eq('POST'); + expect(interception.request).property('body').to.contain({ + name: 'Test update connection PokeAPI source cypress <> Test update connection Local JSON destination cypressConnection name', + prefix: 'auto_test', + namespaceDefinition: 'customformat', + namespaceFormat: '${SOURCE_NAMESPACE}_test', + status: 'active', + }); + expect(interception.request.body.scheduleData.basicSchedule).to.contain({ + units: 1, + timeUnit: 'hours' + }); + + const streamToUpdate = interception.request.body.syncCatalog.streams[0]; + + expect(streamToUpdate.config).to.contain({ + aliasName: 'pokemon', + destinationSyncMode: 'append', + selected: true, + }); + + expect(streamToUpdate.stream).to.contain({ + name: "pokemon", + }); + expect(streamToUpdate.stream.supportedSyncModes).to.contain( + 'full_refresh' + ); + }) + checkSuccessResult(); + + deleteSource("Test update connection PokeAPI source cypress"); + deleteDestination("Test update connection Local JSON destination cypress"); + }); it("Delete connection", () => { createTestConnection("Test delete connection source cypress", "Test delete connection destination cypress"); - cy.visit("/source"); - cy.get("div").contains("Test delete connection source cypress").click(); - cy.get("div").contains("Test delete connection destination cypress").click(); + goToSourcePage(); + openSourceDestinationFromGrid("Test delete connection source cypress"); + openSourceDestinationFromGrid("Test delete connection destination cypress"); - cy.get("div[data-id='settings-step']").click(); + goToSettingsPage(); deleteEntity(); diff --git a/airbyte-webapp-e2e-tests/cypress/integration/destination.spec.ts b/airbyte-webapp-e2e-tests/cypress/integration/destination.spec.ts index 5b25066f70d0..a7dbe94c66f5 100644 --- a/airbyte-webapp-e2e-tests/cypress/integration/destination.spec.ts +++ b/airbyte-webapp-e2e-tests/cypress/integration/destination.spec.ts @@ -1,4 +1,4 @@ -import { createTestDestination, deleteDestination, updateDestination } from "commands/destination"; +import { createLocalJsonDestination, deleteDestination, updateDestination } from "commands/destination"; import { initialSetupCompleted } from "commands/workspaces"; describe("Destination main actions", () => { @@ -7,13 +7,13 @@ describe("Destination main actions", () => { }); it("Create new destination", () => { - createTestDestination("Test destination cypress"); + createLocalJsonDestination("Test destination cypress", "/local"); cy.url().should("include", `/destination/`); }); it("Update destination", () => { - createTestDestination("Test destination cypress for update"); + createLocalJsonDestination("Test destination cypress for update", "/local"); updateDestination("Test destination cypress for update", "connectionConfiguration.destination_path", "/local/my-json"); cy.get("div[data-id='success-result']").should("exist"); @@ -21,7 +21,7 @@ describe("Destination main actions", () => { }); it("Delete destination", () => { - createTestDestination("Test destination cypress for delete"); + createLocalJsonDestination("Test destination cypress for delete", "/local"); deleteDestination("Test destination cypress for delete"); cy.visit("/destination"); diff --git a/airbyte-webapp-e2e-tests/cypress/integration/onboarding.spec.ts b/airbyte-webapp-e2e-tests/cypress/integration/onboarding.spec.ts index a5fea357d4a8..3e5d5446273f 100644 --- a/airbyte-webapp-e2e-tests/cypress/integration/onboarding.spec.ts +++ b/airbyte-webapp-e2e-tests/cypress/integration/onboarding.spec.ts @@ -1,4 +1,4 @@ -import { fillEmail, submitButtonClick } from "commands/common"; +import { submitButtonClick, fillEmail } from "commands/common"; import { initialSetupCompleted } from "commands/workspaces"; describe("Preferences actions", () => { diff --git a/airbyte-webapp-e2e-tests/cypress/integration/source.spec.ts b/airbyte-webapp-e2e-tests/cypress/integration/source.spec.ts index 8f532ed10921..12bd0f38b206 100644 --- a/airbyte-webapp-e2e-tests/cypress/integration/source.spec.ts +++ b/airbyte-webapp-e2e-tests/cypress/integration/source.spec.ts @@ -1,4 +1,4 @@ -import { createTestSource, deleteSource, updateSource } from "commands/source"; +import { createPostgresSource, deleteSource, updateSource } from "commands/source"; import { initialSetupCompleted } from "commands/workspaces"; describe("Source main actions", () => { @@ -7,14 +7,14 @@ describe("Source main actions", () => { }); it("Create new source", () => { - createTestSource("Test source cypress"); + createPostgresSource("Test source cypress"); cy.url().should("include", `/source/`); }); //TODO: add update source on some other connector or create 1 more user for pg it.skip("Update source", () => { - createTestSource("Test source cypress for update"); + createPostgresSource("Test source cypress for update"); updateSource("Test source cypress for update", "connectionConfiguration.start_date", "2020-11-11"); cy.get("div[data-id='success-result']").should("exist"); @@ -22,7 +22,7 @@ describe("Source main actions", () => { }); it("Delete source", () => { - createTestSource("Test source cypress for delete"); + createPostgresSource("Test source cypress for delete"); deleteSource("Test source cypress for delete"); cy.visit("/"); diff --git a/airbyte-webapp-e2e-tests/cypress/pages/createConnectorPage.ts b/airbyte-webapp-e2e-tests/cypress/pages/createConnectorPage.ts new file mode 100644 index 000000000000..3915fe347879 --- /dev/null +++ b/airbyte-webapp-e2e-tests/cypress/pages/createConnectorPage.ts @@ -0,0 +1,48 @@ +const selectTypeDropdown = "div[data-testid='serviceType']"; +const nameInput = "input[name=name]"; +const hostInput = "input[name='connectionConfiguration.host']"; +const portInput = "input[name='connectionConfiguration.port']"; +const databaseInput = "input[name='connectionConfiguration.database']"; +const usernameInput = "input[name='connectionConfiguration.username']"; +const passwordInput = "input[name='connectionConfiguration.password']"; +const pokemonNameInput = "input[name='connectionConfiguration.pokemon_name']"; +const destinationPathInput = "input[name='connectionConfiguration.destination_path']"; + +export const selectServiceType = (type: string) => { + cy.get(selectTypeDropdown).click(); + cy.get("div").contains(type).click(); +} + +export const enterName = (name: string) => { + cy.get(nameInput).clear().type(name); +} + +export const enterHost = (host: string) => { + cy.get(hostInput).type(host); +} + +export const enterPort = (port: string) => { + cy.get(portInput).type(port); +} + +export const enterDatabase = (database: string) => { + cy.get(databaseInput).type(database); +} + +export const enterUsername = (username: string) => { + cy.get(usernameInput).type(username); +} + +export const enterPassword = (password: string) => { + cy.get(passwordInput).type(password); +} + +export const enterPokemonName = (pokeName: string) => { + cy.get(pokemonNameInput).type(pokeName); +} + +export const enterDestinationPath = (destinationPath: string) => { + cy.get(destinationPathInput).type(destinationPath); + +} + diff --git a/airbyte-webapp-e2e-tests/cypress/pages/destinationPage.ts b/airbyte-webapp-e2e-tests/cypress/pages/destinationPage.ts new file mode 100644 index 000000000000..ebb253600dcd --- /dev/null +++ b/airbyte-webapp-e2e-tests/cypress/pages/destinationPage.ts @@ -0,0 +1,18 @@ +const newDestination = "button[data-id='new-destination'"; +const addSourceButton = "div[data-testid='select-source']"; + +export const goToDestinationPage = () => { + cy.intercept("/api/v1/destinations/list").as("getDestinationsList"); + cy.visit("/destination"); + cy.wait(3000); + cy.wait("@getDestinationsList"); + } + + export const openNewDestinationForm = () => { + cy.get(newDestination).click(); + cy.url().should("include", `/destination/new-destination`); + } + + export const openAddSource = () => { + cy.get(addSourceButton).click(); + } \ No newline at end of file diff --git a/airbyte-webapp-e2e-tests/cypress/pages/replicationPage.ts b/airbyte-webapp-e2e-tests/cypress/pages/replicationPage.ts new file mode 100644 index 000000000000..093ef4c32d34 --- /dev/null +++ b/airbyte-webapp-e2e-tests/cypress/pages/replicationPage.ts @@ -0,0 +1,56 @@ +const scheduleDropdown = "div[data-testid='scheduleData.basicSchedule']"; +const scheduleValue = (value: string) => `div[data-testid='${value}']`; +const destinationPrefix = "input[data-testid='prefixInput']"; +const replicationTab = "div[data-id='replication-step']"; +const destinationNamespace = "div[data-testid='namespaceDefinition']"; +const destinationNamespaceCustom = "div[data-testid='namespaceDefinition-customformat']"; +const destinationNamespaceSource = "div[data-testid='namespaceDefinition-source']"; +const destinationNamespaceCustomInput = "input[data-testid='input']"; +const syncModeDropdown = "div[data-testid='syncSettingsDropdown'] input"; +const successResult = "span[data-id='success-result']"; +const saveStreamChangesButton = "button[data-testid='resetModal-save']"; +const connectionNameInput = "input[data-testid='connectionName']"; + +export const goToReplicationTab = () => { + cy.get(replicationTab).click(); +} + +export const enterConnectionName = (name: string) => { + cy.get(connectionNameInput).type(name); +} + +export const selectSchedule = (value: string) => { + cy.get(scheduleDropdown).click(); + cy.get(scheduleValue(value)).click(); +} + +export const fillOutDestinationPrefix = (value: string) => { + cy.get(destinationPrefix).clear().type(value).should('have.value', value);; +} + +export const setupDestinationNamespaceCustomFormat = (value: string) => { + cy.get(destinationNamespace).click(); + cy.get(destinationNamespaceCustom).click(); + cy.get(destinationNamespaceCustomInput).first().type(value).should('have.value', '${SOURCE_NAMESPACE}' + value); +} + +export const setupDestinationNamespaceSourceFormat = () => { + cy.get(destinationNamespace).click(); + cy.get(destinationNamespaceSource).click(); +} + +export const selectFullAppendSyncMode = () => { + cy.get(syncModeDropdown).first().click({ force: true }); + + cy.get(`.react-select__menu`) + .contains("Append") // it would be nice to select for "Full refresh" is there too + .click(); +}; + +export const checkSuccessResult = () => { + cy.get(successResult).should("exist"); +} + +export const confirmStreamConfigurationChangedPopup = () => { + cy.get(saveStreamChangesButton).click(); +} diff --git a/airbyte-webapp-e2e-tests/cypress/pages/settingsConnectionPage.ts b/airbyte-webapp-e2e-tests/cypress/pages/settingsConnectionPage.ts new file mode 100644 index 000000000000..283659169d7b --- /dev/null +++ b/airbyte-webapp-e2e-tests/cypress/pages/settingsConnectionPage.ts @@ -0,0 +1,5 @@ +const settingsTab = "div[data-id='settings-step']"; + +export const goToSettingsPage = () => { + cy.get(settingsTab).click(); +} diff --git a/airbyte-webapp-e2e-tests/cypress/pages/sidebar.ts b/airbyte-webapp-e2e-tests/cypress/pages/sidebar.ts new file mode 100644 index 000000000000..31ff3cb2742f --- /dev/null +++ b/airbyte-webapp-e2e-tests/cypress/pages/sidebar.ts @@ -0,0 +1,5 @@ +const setting = "nav a[href*='settings']"; + +export const openSettings = () => { + cy.get(setting).click(); +}; diff --git a/airbyte-webapp-e2e-tests/cypress/pages/sourcePage.ts b/airbyte-webapp-e2e-tests/cypress/pages/sourcePage.ts new file mode 100644 index 000000000000..dcd2719c6965 --- /dev/null +++ b/airbyte-webapp-e2e-tests/cypress/pages/sourcePage.ts @@ -0,0 +1,17 @@ +const newSource = "button[data-id='new-source'"; + +export const goToSourcePage = () => { + cy.intercept("/api/v1/sources/list").as("getSourcesList"); + cy.visit("/source"); + cy.wait(3000); + cy.wait("@getSourcesList"); +} + +export const openSourceDestinationFromGrid = (value: string) => { + cy.get("div").contains(value).click(); +} + +export const openNewSourceForm = () => { + cy.get(newSource).click(); + cy.url().should("include", `/source/new-source`); + } diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/components/SyncSettingsDropdown.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/components/SyncSettingsDropdown.tsx index e435b8ca7c8f..f19ade854af6 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/components/SyncSettingsDropdown.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/components/SyncSettingsDropdown.tsx @@ -106,6 +106,7 @@ const SyncSettingsDropdown: React.FC = (props) => ( Option, Control: DropdownControl, }} + data-testid="syncSettingsDropdown" $withBorder /> ); From 68897bab3a07ac8cc7238c846b1109b53fa18101 Mon Sep 17 00:00:00 2001 From: Greg Solovyev Date: Fri, 2 Sep 2022 11:05:10 -0700 Subject: [PATCH 007/200] Hide ES and Redis destination connectors from Cloud (#16276) --- airbyte-webapp/src/core/domain/connector/constants.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-webapp/src/core/domain/connector/constants.ts b/airbyte-webapp/src/core/domain/connector/constants.ts index 8e2e4e87ea44..31d323498c23 100644 --- a/airbyte-webapp/src/core/domain/connector/constants.ts +++ b/airbyte-webapp/src/core/domain/connector/constants.ts @@ -18,6 +18,8 @@ export const DEV_IMAGE_TAG = "dev"; export const getExcludedConnectorIds = (workspaceId: string) => isCloudApp() ? [ + "68f351a7-2745-4bef-ad7f-996b8e51bb8c", // hide ElasticSearch Destination https://github.com/airbytehq/airbyte-cloud/issues/2594 + "d4d3fef9-e319-45c2-881a-bd02ce44cc9f", // hide Redis Destination https://github.com/airbytehq/airbyte-cloud/issues/2593 "2470e835-feaf-4db6-96f3-70fd645acc77", // Salesforce Singer ...(workspaceId !== "54135667-ce73-4820-a93c-29fe1510d348" // Shopify workspace for review ? ["9da77001-af33-4bcd-be46-6252bf9342b9"] // Shopify From abc01d3c499d9566583fcbae401d7c825c304257 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III <90398440+octavia-squidington-iii@users.noreply.github.com> Date: Fri, 2 Sep 2022 20:17:00 +0200 Subject: [PATCH 008/200] Bump Airbyte version from 0.40.3 to 0.40.4 (#16275) Co-authored-by: timroes --- .bumpversion.cfg | 2 +- .env | 2 +- airbyte-bootloader/Dockerfile | 2 +- airbyte-container-orchestrator/Dockerfile | 2 +- airbyte-metrics/reporter/Dockerfile | 2 +- airbyte-server/Dockerfile | 2 +- airbyte-webapp/package-lock.json | 4 ++-- airbyte-webapp/package.json | 2 +- airbyte-workers/Dockerfile | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte/Chart.yaml | 2 +- charts/airbyte/README.md | 10 +++++----- charts/airbyte/values.yaml | 8 ++++---- docs/operator-guides/upgrading-airbyte.md | 2 +- kube/overlays/stable-with-resource-limits/.env | 2 +- .../stable-with-resource-limits/kustomization.yaml | 12 ++++++------ kube/overlays/stable/.env | 2 +- kube/overlays/stable/kustomization.yaml | 12 ++++++------ octavia-cli/Dockerfile | 2 +- octavia-cli/README.md | 4 ++-- octavia-cli/install.sh | 2 +- octavia-cli/setup.py | 2 +- 26 files changed, 45 insertions(+), 45 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index bbc158631598..941b8f7e1bf6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.40.3 +current_version = 0.40.4 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? diff --git a/.env b/.env index c03dab62734e..70b3dfe88ec6 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ ### SHARED ### -VERSION=0.40.3 +VERSION=0.40.4 # When using the airbyte-db via default docker image CONFIG_ROOT=/data diff --git a/airbyte-bootloader/Dockerfile b/airbyte-bootloader/Dockerfile index 9d80829241ec..b90a80d10416 100644 --- a/airbyte-bootloader/Dockerfile +++ b/airbyte-bootloader/Dockerfile @@ -2,7 +2,7 @@ ARG JDK_VERSION=19-slim-bullseye ARG JDK_IMAGE=openjdk:${JDK_VERSION} FROM ${JDK_IMAGE} -ARG VERSION=0.40.3 +ARG VERSION=0.40.4 ENV APPLICATION airbyte-bootloader ENV VERSION ${VERSION} diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index 21ffe13a97fa..0794fd545e33 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -28,7 +28,7 @@ RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] htt RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y kubectl # Don't change this manually. Bump version expects to make moves based on this string -ARG VERSION=0.40.3 +ARG VERSION=0.40.4 ENV APPLICATION airbyte-container-orchestrator ENV VERSION=${VERSION} diff --git a/airbyte-metrics/reporter/Dockerfile b/airbyte-metrics/reporter/Dockerfile index 9c709b3f33cd..0102e7bc3163 100644 --- a/airbyte-metrics/reporter/Dockerfile +++ b/airbyte-metrics/reporter/Dockerfile @@ -2,7 +2,7 @@ ARG JDK_VERSION=19-slim-bullseye ARG JDK_IMAGE=openjdk:${JDK_VERSION} FROM ${JDK_IMAGE} AS metrics-reporter -ARG VERSION=0.40.3 +ARG VERSION=0.40.4 ENV APPLICATION airbyte-metrics-reporter ENV VERSION ${VERSION} diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index e22ed79209fb..729d4889ac78 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -4,7 +4,7 @@ FROM ${JDK_IMAGE} AS server EXPOSE 8000 -ARG VERSION=0.40.3 +ARG VERSION=0.40.4 ENV APPLICATION airbyte-server ENV VERSION ${VERSION} diff --git a/airbyte-webapp/package-lock.json b/airbyte-webapp/package-lock.json index 3fdf4956dccb..20c35e94e4c3 100644 --- a/airbyte-webapp/package-lock.json +++ b/airbyte-webapp/package-lock.json @@ -1,12 +1,12 @@ { "name": "airbyte-webapp", - "version": "0.40.3", + "version": "0.40.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "airbyte-webapp", - "version": "0.40.3", + "version": "0.40.4", "dependencies": { "@floating-ui/react-dom": "^1.0.0", "@fortawesome/fontawesome-svg-core": "^6.1.1", diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index ea74b0593cce..ea091e12e7dc 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -1,6 +1,6 @@ { "name": "airbyte-webapp", - "version": "0.40.3", + "version": "0.40.4", "private": true, "engines": { "node": ">=16.0.0" diff --git a/airbyte-workers/Dockerfile b/airbyte-workers/Dockerfile index 192a34a60ed6..93433d9653b2 100644 --- a/airbyte-workers/Dockerfile +++ b/airbyte-workers/Dockerfile @@ -27,7 +27,7 @@ RUN wget -O /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages. RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list RUN apt-get update && apt-get install -y kubectl -ARG VERSION=0.40.3 +ARG VERSION=0.40.4 ENV APPLICATION airbyte-workers ENV VERSION ${VERSION} diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 14f5553196a7..f44c022022de 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -21,7 +21,7 @@ version: "0.39.36" # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.40.3" +appVersion: "0.40.4" dependencies: - name: common diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index 87e9049bce88..85e1ec832c91 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -21,7 +21,7 @@ version: "0.39.36" # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.40.3" +appVersion: "0.40.4" dependencies: - name: common diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 51b79f4dcc1a..a97696197ae4 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -21,7 +21,7 @@ version: "0.39.36" # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.40.3" +appVersion: "0.40.4" dependencies: - name: common diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index 3b72f7d8937a..8128e24886ee 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -21,7 +21,7 @@ version: "0.39.36" # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.40.3" +appVersion: "0.40.4" dependencies: - name: common diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 2297042cbe0b..783f6dd2eabe 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -21,7 +21,7 @@ version: "0.39.36" # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.40.3" +appVersion: "0.40.4" dependencies: - name: common diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 80b6ed7eab51..851299a5a440 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -21,7 +21,7 @@ version: 0.39.36 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.40.3" +appVersion: "0.40.4" dependencies: - name: common diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index c96c9ad93150..309fbb67764b 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -1,6 +1,6 @@ # airbyte -![Version: 0.39.36](https://img.shields.io/badge/Version-0.39.36-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.40.3](https://img.shields.io/badge/AppVersion-0.39.41--alpha-informational?style=flat-square) +![Version: 0.39.36](https://img.shields.io/badge/Version-0.39.36-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.40.4](https://img.shields.io/badge/AppVersion-0.39.41--alpha-informational?style=flat-square) Helm chart to deploy airbyte @@ -26,7 +26,7 @@ Helm chart to deploy airbyte | airbyte-bootloader.enabled | bool | `true` | | | airbyte-bootloader.image.pullPolicy | string | `"IfNotPresent"` | | | airbyte-bootloader.image.repository | string | `"airbyte/bootloader"` | | -| airbyte-bootloader.image.tag | string | `"0.40.3"` | | +| airbyte-bootloader.image.tag | string | `"0.40.4"` | | | airbyte-bootloader.nodeSelector | object | `{}` | | | airbyte-bootloader.podAnnotations | object | `{}` | | | airbyte-bootloader.resources.limits | object | `{}` | | @@ -113,7 +113,7 @@ Helm chart to deploy airbyte | server.extraVolumes | list | `[]` | | | server.image.pullPolicy | string | `"IfNotPresent"` | | | server.image.repository | string | `"airbyte/server"` | | -| server.image.tag | string | `"0.40.3"` | | +| server.image.tag | string | `"0.40.4"` | | | server.livenessProbe.enabled | bool | `true` | | | server.livenessProbe.failureThreshold | int | `3` | | | server.livenessProbe.initialDelaySeconds | int | `30` | | @@ -178,7 +178,7 @@ Helm chart to deploy airbyte | webapp.extraVolumes | list | `[]` | | | webapp.image.pullPolicy | string | `"IfNotPresent"` | | | webapp.image.repository | string | `"airbyte/webapp"` | | -| webapp.image.tag | string | `"0.40.3"` | | +| webapp.image.tag | string | `"0.40.4"` | | | webapp.ingress.annotations | object | `{}` | | | webapp.ingress.className | string | `""` | | | webapp.ingress.enabled | bool | `false` | | @@ -213,7 +213,7 @@ Helm chart to deploy airbyte | worker.extraVolumes | list | `[]` | | | worker.image.pullPolicy | string | `"IfNotPresent"` | | | worker.image.repository | string | `"airbyte/worker"` | | -| worker.image.tag | string | `"0.40.3"` | | +| worker.image.tag | string | `"0.40.4"` | | | worker.livenessProbe.enabled | bool | `true` | | | worker.livenessProbe.failureThreshold | int | `3` | | | worker.livenessProbe.initialDelaySeconds | int | `30` | | diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index c423f292f889..52c2a3c94a60 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -150,7 +150,7 @@ webapp: image: repository: airbyte/webapp pullPolicy: IfNotPresent - tag: 0.40.3 + tag: 0.40.4 ## @param webapp.podAnnotations [object] Add extra annotations to the webapp pod(s) ## @@ -420,7 +420,7 @@ server: image: repository: airbyte/server pullPolicy: IfNotPresent - tag: 0.40.3 + tag: 0.40.4 ## @param server.podAnnotations [object] Add extra annotations to the server pod ## @@ -548,7 +548,7 @@ worker: image: repository: airbyte/worker pullPolicy: IfNotPresent - tag: 0.40.3 + tag: 0.40.4 ## @param worker.podAnnotations [object] Add extra annotations to the worker pod(s) ## @@ -761,7 +761,7 @@ airbyte-bootloader: image: repository: airbyte/bootloader pullPolicy: IfNotPresent - tag: 0.40.3 + tag: 0.40.4 ## @param bootloader.podAnnotations [object] Add extra annotations to the bootloader pod ## diff --git a/docs/operator-guides/upgrading-airbyte.md b/docs/operator-guides/upgrading-airbyte.md index 67d4ec3e4064..f8033410dac4 100644 --- a/docs/operator-guides/upgrading-airbyte.md +++ b/docs/operator-guides/upgrading-airbyte.md @@ -103,7 +103,7 @@ If you are upgrading from (i.e. your current version of Airbyte is) Airbyte vers Here's an example of what it might look like with the values filled in. It assumes that the downloaded `airbyte_archive.tar.gz` is in `/tmp`. ```bash - docker run --rm -v /tmp:/config airbyte/migration:0.40.3 --\ + docker run --rm -v /tmp:/config airbyte/migration:0.40.4 --\ --input /config/airbyte_archive.tar.gz\ --output /config/airbyte_archive_migrated.tar.gz ``` diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env index 6107741e05fd..0b9e521414e1 100644 --- a/kube/overlays/stable-with-resource-limits/.env +++ b/kube/overlays/stable-with-resource-limits/.env @@ -1,4 +1,4 @@ -AIRBYTE_VERSION=0.40.3 +AIRBYTE_VERSION=0.40.4 # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_HOST=airbyte-db-svc diff --git a/kube/overlays/stable-with-resource-limits/kustomization.yaml b/kube/overlays/stable-with-resource-limits/kustomization.yaml index 2aaa5aeda840..6c3c6d3442b5 100644 --- a/kube/overlays/stable-with-resource-limits/kustomization.yaml +++ b/kube/overlays/stable-with-resource-limits/kustomization.yaml @@ -8,19 +8,19 @@ bases: images: - name: airbyte/db - newTag: 0.40.3 + newTag: 0.40.4 - name: airbyte/bootloader - newTag: 0.40.3 + newTag: 0.40.4 - name: airbyte/server - newTag: 0.40.3 + newTag: 0.40.4 - name: airbyte/webapp - newTag: 0.40.3 + newTag: 0.40.4 - name: airbyte/worker - newTag: 0.40.3 + newTag: 0.40.4 - name: temporalio/auto-setup newTag: 1.7.0 - name: airbyte/cron - newTag: 0.40.3 + newTag: 0.40.4 configMapGenerator: - name: airbyte-env diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index e91081fff5b0..08b88fc50ef9 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -1,4 +1,4 @@ -AIRBYTE_VERSION=0.40.3 +AIRBYTE_VERSION=0.40.4 # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_HOST=airbyte-db-svc diff --git a/kube/overlays/stable/kustomization.yaml b/kube/overlays/stable/kustomization.yaml index 2223df69fe75..57db11eedaf6 100644 --- a/kube/overlays/stable/kustomization.yaml +++ b/kube/overlays/stable/kustomization.yaml @@ -8,19 +8,19 @@ bases: images: - name: airbyte/db - newTag: 0.40.3 + newTag: 0.40.4 - name: airbyte/bootloader - newTag: 0.40.3 + newTag: 0.40.4 - name: airbyte/server - newTag: 0.40.3 + newTag: 0.40.4 - name: airbyte/webapp - newTag: 0.40.3 + newTag: 0.40.4 - name: airbyte/worker - newTag: 0.40.3 + newTag: 0.40.4 - name: temporalio/auto-setup newTag: 1.7.0 - name: airbyte/cron - newTag: 0.40.3 + newTag: 0.40.4 configMapGenerator: - name: airbyte-env diff --git a/octavia-cli/Dockerfile b/octavia-cli/Dockerfile index 78afe36db1fc..2ca2f0a9703a 100644 --- a/octavia-cli/Dockerfile +++ b/octavia-cli/Dockerfile @@ -14,5 +14,5 @@ USER octavia-cli WORKDIR /home/octavia-project ENTRYPOINT ["octavia"] -LABEL io.airbyte.version=0.40.3 +LABEL io.airbyte.version=0.40.4 LABEL io.airbyte.name=airbyte/octavia-cli diff --git a/octavia-cli/README.md b/octavia-cli/README.md index f291efbc06bf..1591b503d7ae 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -104,7 +104,7 @@ This script: ```bash touch ~/.octavia # Create a file to store env variables that will be mapped the octavia-cli container mkdir my_octavia_project_directory # Create your octavia project directory where YAML configurations will be stored. -docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.40.3 +docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.40.4 ``` ### Using `docker-compose` @@ -709,7 +709,7 @@ You can disable telemetry by setting the `OCTAVIA_ENABLE_TELEMETRY` environment | Version | Date | Description | PR | | ------- | ---------- | ------------------------------------------------------------------------------------- | ----------------------------------------------------------- | -| 0.40.3 | 2022-08-10 | Enable cron and basic scheduling | [#15253](https://github.com/airbytehq/airbyte/pull/15253) | +| 0.40.4 | 2022-08-10 | Enable cron and basic scheduling | [#15253](https://github.com/airbytehq/airbyte/pull/15253) | | 0.39.33 | 2022-07-05 | Add `octavia import all` command | [#14374](https://github.com/airbytehq/airbyte/pull/14374) | | 0.39.32 | 2022-06-30 | Create import command to import and manage existing Airbyte resource from octavia-cli | [#14137](https://github.com/airbytehq/airbyte/pull/14137) | | 0.39.27 | 2022-06-24 | Create get command to retrieve resources JSON representation | [#13254](https://github.com/airbytehq/airbyte/pull/13254) | diff --git a/octavia-cli/install.sh b/octavia-cli/install.sh index 0f1cabe30846..c45c2504039e 100755 --- a/octavia-cli/install.sh +++ b/octavia-cli/install.sh @@ -3,7 +3,7 @@ # This install scripts currently only works for ZSH and Bash profiles. # It creates an octavia alias in your profile bound to a docker run command and your current user. -VERSION=0.40.3 +VERSION=0.40.4 OCTAVIA_ENV_FILE=${HOME}/.octavia detect_profile() { diff --git a/octavia-cli/setup.py b/octavia-cli/setup.py index babc9f25961a..9765df041fff 100644 --- a/octavia-cli/setup.py +++ b/octavia-cli/setup.py @@ -15,7 +15,7 @@ setup( name="octavia-cli", - version="0.40.3", + version="0.40.4", description="A command line interface to manage Airbyte configurations", long_description=README, author="Airbyte", From 7664115c2cf664f51668605d06e213db7241635c Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi <53845333+lazebnyi@users.noreply.github.com> Date: Fri, 2 Sep 2022 20:48:39 +0200 Subject: [PATCH 009/200] Re-name google analytics cionnectors (#16287) --- .../init/src/main/resources/seed/source_definitions.yaml | 4 ++-- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 4 ++-- .../source_google_analytics_data_api/spec.json | 2 +- .../source_google_analytics_v4/spec.json | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index f63def833a2c..9ac86a0dafae 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -352,7 +352,7 @@ icon: google-adwords.svg sourceType: api releaseStage: generally_available -- name: Google Analytics +- name: Google Analytics (Universal Analytics) sourceDefinitionId: eff3616a-f9c3-11eb-9a03-0242ac130003 dockerRepository: airbyte/source-google-analytics-v4 dockerImageTag: 0.1.25 @@ -360,7 +360,7 @@ icon: google-analytics.svg sourceType: api releaseStage: generally_available -- name: Google Analytics Data API +- name: Google Analytics (v4) sourceDefinitionId: 3cc2eafd-84aa-4dca-93af-322d9dfeec1a dockerRepository: airbyte/source-google-analytics-data-api dockerImageTag: 0.0.3 diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 3e73d3773071..cf4c7f5d8371 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3063,7 +3063,7 @@ documentationUrl: "https://docs.airbyte.com/integrations/sources/google-analytics-universal-analytics" connectionSpecification: $schema: "http://json-schema.org/draft-07/schema#" - title: "Google Analytics V4 Spec" + title: "Google Analytics (Universal Analytics) Spec" type: "object" required: - "view_id" @@ -3193,7 +3193,7 @@ documentationUrl: "https://docs.airbyte.com/integrations/sources/google-analytics-v4" connectionSpecification: $schema: "http://json-schema.org/draft-07/schema#" - title: "Google Analytics Data API Spec" + title: "Google Analytics (v4) Spec" type: "object" required: - "property_id" diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json index f735d9a53805..5c40921695ed 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json @@ -2,7 +2,7 @@ "documentationUrl": "https://docs.airbyte.com/integrations/sources/google-analytics-v4", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google Analytics Data API Spec", + "title": "Google Analytics (v4) Spec", "type": "object", "required": ["property_id", "date_ranges_start_date"], "additionalProperties": true, diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json index 481801580858..a83d8b3a77b4 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/spec.json @@ -2,7 +2,7 @@ "documentationUrl": "https://docs.airbyte.com/integrations/sources/google-analytics-universal-analytics", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google Analytics V4 Spec", + "title": "Google Analytics (Universal Analytics) Spec", "type": "object", "required": ["view_id", "start_date"], "additionalProperties": true, From f8f06febc69b255acc00aeefdad71cbe9a93a1df Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Fri, 2 Sep 2022 11:58:37 -0700 Subject: [PATCH 010/200] Fix github action syntax (#16277) --- .github/workflows/gke-kube-test-command.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gke-kube-test-command.yml b/.github/workflows/gke-kube-test-command.yml index e69b9916b649..68a7aba4362b 100644 --- a/.github/workflows/gke-kube-test-command.yml +++ b/.github/workflows/gke-kube-test-command.yml @@ -1,7 +1,7 @@ name: GKE Kube Acceptance Test on: schedule: - - cron '0 0 * * 0' # runs at midnight UTC every Sunday + - cron: '0 0 * * 0' # runs at midnight UTC every Sunday workflow_dispatch: inputs: repo: From 89ae9e05e7621adf5ba131c55f3d3a9ae323578d Mon Sep 17 00:00:00 2001 From: Serhii Chvaliuk Date: Fri, 2 Sep 2022 22:55:33 +0300 Subject: [PATCH 011/200] =?UTF-8?q?=F0=9F=90=9B=20Source=20Facebook=20Mark?= =?UTF-8?q?eting:=20remove=20"end=5Fdate"=20from=20config=20if=20empty=20v?= =?UTF-8?q?alue=20(re-implement=20#16096)=20(#16222)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergey Chvalyuk --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 13 +++-- .../source-facebook-marketing/Dockerfile | 2 +- .../integration_tests/spec.json | 8 +-- .../source_facebook_marketing/source.py | 28 ++++------ .../source_facebook_marketing/spec.py | 8 +-- .../source_facebook_marketing/utils.py | 27 +++++----- .../unit_tests/__init__.py | 0 .../unit_tests/test_source.py | 51 ++++++++++++++++--- .../unit_tests/utils.py | 19 +++++++ .../sources/facebook-marketing.md | 1 + 11 files changed, 106 insertions(+), 53 deletions(-) create mode 100644 airbyte-integrations/connectors/source-facebook-marketing/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-facebook-marketing/unit_tests/utils.py diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 9ac86a0dafae..32731214b6c3 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -256,7 +256,7 @@ - name: Facebook Marketing sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c dockerRepository: airbyte/source-facebook-marketing - dockerImageTag: 0.2.61 + dockerImageTag: 0.2.62 documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing icon: facebook.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index cf4c7f5d8371..bd64a801bc8b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1857,7 +1857,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-facebook-marketing:0.2.61" +- dockerImage: "airbyte/source-facebook-marketing:0.2.62" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" @@ -1891,7 +1891,7 @@ \ between start_date and this date will be replicated. Not setting this\ \ option will result in always syncing the latest data." order: 2 - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + pattern: "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" examples: - "2017-01-26T00:00:00Z" type: "string" @@ -1939,7 +1939,8 @@ type: "array" items: title: "ValidEnums" - description: "An enumeration." + description: "Generic enumeration.\n\nDerive from this class to\ + \ define new enumerations." enum: - "account_currency" - "account_id" @@ -2078,7 +2079,8 @@ type: "array" items: title: "ValidBreakdowns" - description: "An enumeration." + description: "Generic enumeration.\n\nDerive from this class to\ + \ define new enumerations." enum: - "ad_format_asset" - "age" @@ -2111,7 +2113,8 @@ type: "array" items: title: "ValidActionBreakdowns" - description: "An enumeration." + description: "Generic enumeration.\n\nDerive from this class to\ + \ define new enumerations." enum: - "action_canvas_component_name" - "action_carousel_card_id" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index 76a5737d7ac0..985a728288be 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.61 +LABEL io.airbyte.version=0.2.62 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json index 1f7fbfd3942b..706eae4c9f02 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json +++ b/airbyte-integrations/connectors/source-facebook-marketing/integration_tests/spec.json @@ -25,7 +25,7 @@ "title": "End Date", "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DDT00:00:00Z. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the latest data.", "order": 2, - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", "examples": ["2017-01-26T00:00:00Z"], "type": "string", "format": "date-time" @@ -73,7 +73,7 @@ "type": "array", "items": { "title": "ValidEnums", - "description": "An enumeration.", + "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.", "enum": [ "account_currency", "account_id", @@ -215,7 +215,7 @@ "type": "array", "items": { "title": "ValidBreakdowns", - "description": "An enumeration.", + "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.", "enum": [ "ad_format_asset", "age", @@ -251,7 +251,7 @@ "type": "array", "items": { "title": "ValidActionBreakdowns", - "description": "An enumeration.", + "description": "Generic enumeration.\n\nDerive from this class to define new enumerations.", "enum": [ "action_canvas_component_name", "action_carousel_card_id", diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 0400ffdeb698..9a53bc42ae76 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -3,12 +3,10 @@ # import logging -import os from typing import Any, List, Mapping, Optional, Tuple, Type import pendulum import requests -from airbyte_cdk.connector import _WriteConfigProtocol from airbyte_cdk.models import AuthSpecification, ConnectorSpecification, DestinationSyncMode, OAuth2Specification from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -38,16 +36,12 @@ class SourceFacebookMarketing(AbstractSource): - def configure(self: _WriteConfigProtocol, config: Mapping[str, Any], temp_dir: str) -> Mapping[str, Any]: - source_spec = self.spec(logging.getLogger("airbyte")) - end_date = source_spec.connectionSpecification["properties"]["end_date"] - # We highlight here that "end_date" is not a simple "string" field it's an extended type with "format" modifier. - # If "end_date" is provided as an empty string we can treat this case as missed value. - if end_date["type"] == "string" and "format" in end_date: - if config.get("end_date") == "": - config.pop("end_date") - config_path = os.path.join(temp_dir, "config.json") - self.write_config(config, config_path) + def _validate_and_transform(self, config: Mapping[str, Any]): + if config.get("end_date") == "": + config.pop("end_date") + config = ConnectorConfig.parse_obj(config) + config.start_date = pendulum.instance(config.start_date) + config.end_date = pendulum.instance(config.end_date) return config def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: @@ -57,9 +51,10 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> :param config: the user-input config object conforming to the connector's spec.json :return Tuple[bool, Any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - config = ConnectorConfig.parse_obj(config) - if pendulum.instance(config.end_date) < pendulum.instance(config.start_date): - raise ValueError("end_date must be equal or after start_date.") + config = self._validate_and_transform(config) + if config.end_date < config.start_date: + return False, "end_date must be equal or after start_date." + try: api = API(account_id=config.account_id, access_token=config.access_token) logger.info(f"Select account {api.account}") @@ -73,8 +68,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: :param config: A Mapping of the user input configuration as defined in the connector spec. :return: list of the stream instances """ - config: ConnectorConfig = ConnectorConfig.parse_obj(config) - + config = self._validate_and_transform(config) config.start_date = validate_start_date(config.start_date) config.end_date = validate_end_date(config.start_date, config.end_date) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py index 5b7f435dc41f..397787ca8ad5 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py @@ -3,11 +3,10 @@ # import logging -from datetime import datetime +from datetime import datetime, timezone from enum import Enum from typing import List, Optional -import pendulum from airbyte_cdk.sources.config import BaseConfig from facebook_business.adobjects.adsinsights import AdsInsights from pydantic import BaseModel, Field, PositiveInt @@ -19,6 +18,7 @@ ValidBreakdowns = Enum("ValidBreakdowns", AdsInsights.Breakdowns.__dict__) ValidActionBreakdowns = Enum("ValidActionBreakdowns", AdsInsights.ActionBreakdowns.__dict__) DATE_TIME_PATTERN = "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" +EMPTY_PATTERN = "^$" class InsightConfig(BaseModel): @@ -118,9 +118,9 @@ class Config: "All data generated between start_date and this date will be replicated. " "Not setting this option will result in always syncing the latest data." ), - pattern=DATE_TIME_PATTERN, + pattern=EMPTY_PATTERN + "|" + DATE_TIME_PATTERN, examples=["2017-01-26T00:00:00Z"], - default_factory=pendulum.now, + default_factory=lambda: datetime.now(tz=timezone.utc), ) access_token: str = Field( diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py index 60bef095fa67..392658ef1825 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py @@ -3,9 +3,9 @@ # import logging -from datetime import datetime import pendulum +from pendulum import DateTime logger = logging.getLogger("airbyte") @@ -16,26 +16,25 @@ DATA_RETENTION_PERIOD = 37 -def validate_start_date(start_date: datetime) -> datetime: - pendulum_date = pendulum.instance(start_date) - time_zone = start_date.tzinfo - current_date = pendulum.today(time_zone) - if pendulum_date.timestamp() > pendulum.now().timestamp(): - message = f"The start date cannot be in the future. Set start date to today's date - {current_date}." +def validate_start_date(start_date: DateTime) -> DateTime: + now = pendulum.now(tz=start_date.tzinfo) + today = now.replace(microsecond=0, second=0, minute=0, hour=0) + retention_date = today.subtract(months=DATA_RETENTION_PERIOD) + + if start_date > now: + message = f"The start date cannot be in the future. Set start date to today's date - {today}." logger.warning(message) - return current_date - elif pendulum_date.timestamp() < current_date.subtract(months=DATA_RETENTION_PERIOD).timestamp(): - current_date = pendulum.today(time_zone) + return today + elif start_date < retention_date: message = ( - f"The start date cannot be beyond {DATA_RETENTION_PERIOD} months from the current date. " - f"Set start date to {current_date.subtract(months=DATA_RETENTION_PERIOD)}." + f"The start date cannot be beyond {DATA_RETENTION_PERIOD} months from the current date. Set start date to {retention_date}." ) logger.warning(message) - return current_date.subtract(months=DATA_RETENTION_PERIOD) + return retention_date return start_date -def validate_end_date(start_date: datetime, end_date: datetime) -> datetime: +def validate_end_date(start_date: DateTime, end_date: DateTime) -> DateTime: if start_date > end_date: message = f"The end date must be after start date. Set end date to {start_date}." logger.warning(message) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/__init__.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py index e4b2d2af6a6b..ed8ba0068dfa 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py @@ -2,25 +2,42 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + +from copy import deepcopy + import pydantic import pytest -from airbyte_cdk.models import ConnectorSpecification +from airbyte_cdk.models import AirbyteConnectionStatus, ConnectorSpecification, Status +from facebook_business import FacebookAdsApi, FacebookSession from source_facebook_marketing import SourceFacebookMarketing from source_facebook_marketing.spec import ConnectorConfig +from .utils import command_check + @pytest.fixture(name="config") def config_fixture(): config = { - "account_id": 123, + "account_id": "123", "access_token": "TOKEN", - "start_date": "2019-10-10T00:00:00", - "end_date": "2020-10-10T00:00:00", + "start_date": "2019-10-10T00:00:00Z", + "end_date": "2020-10-10T00:00:00Z", } return config +@pytest.fixture +def config_gen(config): + def inner(**kwargs): + new_config = deepcopy(config) + # WARNING, no support deep dictionaries + new_config.update(kwargs) + return {k: v for k, v in new_config.items() if v is not ...} + + return inner + + @pytest.fixture(name="api") def api_fixture(mocker): api_mock = mocker.patch("source_facebook_marketing.source.API") @@ -45,9 +62,10 @@ def test_check_connection_ok(self, api, config, logger_mock): def test_check_connection_end_date_before_start_date(self, api, config, logger_mock): config["start_date"] = "2019-10-10T00:00:00" config["end_date"] = "2019-10-09T00:00:00" - - with pytest.raises(ValueError, match="end_date must be equal or after start_date."): - SourceFacebookMarketing().check_connection(logger_mock, config=config) + assert SourceFacebookMarketing().check_connection(logger_mock, config=config) == ( + False, + "end_date must be equal or after start_date.", + ) def test_check_connection_empty_config(self, api, logger_mock): config = {} @@ -95,3 +113,22 @@ def test_update_insights_streams(self, api, config): assert SourceFacebookMarketing()._update_insights_streams( insights=config.custom_insights, default_args=insights_args, streams=streams ) + + +def test_check_config(config_gen, requests_mock): + requests_mock.register_uri("GET", FacebookSession.GRAPH + f"/{FacebookAdsApi.API_VERSION}/act_123/", {}) + + source = SourceFacebookMarketing() + assert command_check(source, config_gen()) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) + + status = command_check(source, config_gen(start_date="2019-99-10T00:00:00Z")) + assert status.status == Status.FAILED + + status = command_check(source, config_gen(end_date="2019-99-10T00:00:00Z")) + assert status.status == Status.FAILED + + with pytest.raises(Exception): + assert command_check(source, config_gen(start_date=...)) + + assert command_check(source, config_gen(end_date=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) + assert command_check(source, config_gen(end_date="")) == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/utils.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/utils.py new file mode 100644 index 000000000000..776315e717e6 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/utils.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from unittest import mock + +from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification +from airbyte_cdk.sources import Source +from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config + + +def command_check(source: Source, config): + logger = mock.MagicMock() + connector_config, _ = split_config(config) + if source.check_config_against_spec: + source_spec: ConnectorSpecification = source.spec(logger) + check_config_against_spec_or_exit(connector_config, source_spec) + return source.check(logger, config) diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index a3423281b006..093ec63e142b 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -120,6 +120,7 @@ Please be informed that the connector uses the `lookback_window` parameter to pe | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.2.62 | 2022-09-01 | [16222](https://github.com/airbytehq/airbyte/pull/16222) | Remove `end_date` from config if empty value (re-implement #16096) | | 0.2.61 | 2022-08-29 | [16096](https://github.com/airbytehq/airbyte/pull/16096) | Remove `end_date` from config if empty value | | 0.2.60 | 2022-08-19 | [15788](https://github.com/airbytehq/airbyte/pull/15788) | Retry FacebookBadObjectError | | 0.2.59 | 2022-08-04 | [15327](https://github.com/airbytehq/airbyte/pull/15327) | Shift date validation from config validation to stream method | From e831a712192588f2c014a6601f7e7a7102b67064 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi <53845333+lazebnyi@users.noreply.github.com> Date: Fri, 2 Sep 2022 21:59:44 +0200 Subject: [PATCH 012/200] Source Google Analytics v4: Re-name google analytics connector (#16306) * Re-name google analytics cionnectors * Re-named google analytics v4 --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- .../source_google_analytics_data_api/spec.json | 2 +- docs/integrations/sources/google-analytics-v4.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 32731214b6c3..7eefa8398a06 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -360,7 +360,7 @@ icon: google-analytics.svg sourceType: api releaseStage: generally_available -- name: Google Analytics (v4) +- name: Google Analytics 4 (GA4) sourceDefinitionId: 3cc2eafd-84aa-4dca-93af-322d9dfeec1a dockerRepository: airbyte/source-google-analytics-data-api dockerImageTag: 0.0.3 diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index bd64a801bc8b..43387628eb05 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3196,7 +3196,7 @@ documentationUrl: "https://docs.airbyte.com/integrations/sources/google-analytics-v4" connectionSpecification: $schema: "http://json-schema.org/draft-07/schema#" - title: "Google Analytics (v4) Spec" + title: "Google Analytics 4 (GA4) Spec" type: "object" required: - "property_id" diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json index 5c40921695ed..cc683ceccd2e 100644 --- a/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json +++ b/airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/spec.json @@ -2,7 +2,7 @@ "documentationUrl": "https://docs.airbyte.com/integrations/sources/google-analytics-v4", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Google Analytics (v4) Spec", + "title": "Google Analytics 4 (GA4) Spec", "type": "object", "required": ["property_id", "date_ranges_start_date"], "additionalProperties": true, diff --git a/docs/integrations/sources/google-analytics-v4.md b/docs/integrations/sources/google-analytics-v4.md index b017428d80e0..44255df8b399 100644 --- a/docs/integrations/sources/google-analytics-v4.md +++ b/docs/integrations/sources/google-analytics-v4.md @@ -1,4 +1,4 @@ -# Google Analytics (v4) +# Google Analytics 4 (GA4) This page guides you through the process of setting up the Google Analytics source connector. From bf487913d2ae9f09f4cc9719d1dad1ff976d8daf Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Fri, 2 Sep 2022 13:21:17 -0700 Subject: [PATCH 013/200] Add scheduled task to clean up old files from workspace (#16247) * Add airbyte-cron to bumpversion * Update airbyte-cron version to current * Add workspace clean up job * Add missing env var to docker-compose * Update file deletion logging --- .bumpversion.cfg | 2 + airbyte-cron/Dockerfile | 2 +- airbyte-cron/build.gradle | 1 + .../cron/selfhealing/WorkspaceCleaner.java | 70 +++++++++++++++++++ docker-compose.yaml | 4 ++ 5 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 airbyte-cron/src/main/java/io/airbyte/cron/selfhealing/WorkspaceCleaner.java diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 941b8f7e1bf6..0642342ce4d4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -14,6 +14,8 @@ serialize = [bumpversion:file:airbyte-container-orchestrator/Dockerfile] +[bumpversion:file:airbyte-cron/Dockerfile] + [bumpversion:file:airbyte-metrics/reporter/Dockerfile] [bumpversion:file:airbyte-server/Dockerfile] diff --git a/airbyte-cron/Dockerfile b/airbyte-cron/Dockerfile index 01398125e71b..9a60b94e5c28 100644 --- a/airbyte-cron/Dockerfile +++ b/airbyte-cron/Dockerfile @@ -2,7 +2,7 @@ ARG JDK_VERSION=19-slim-bullseye ARG JDK_IMAGE=openjdk:${JDK_VERSION} FROM ${JDK_IMAGE} AS cron -ARG VERSION=0.40.0-alpha +ARG VERSION=0.40.3 ENV APPLICATION airbyte-cron ENV VERSION ${VERSION} diff --git a/airbyte-cron/build.gradle b/airbyte-cron/build.gradle index df9d61e10154..74c250068159 100644 --- a/airbyte-cron/build.gradle +++ b/airbyte-cron/build.gradle @@ -3,6 +3,7 @@ plugins { } dependencies { + implementation project(':airbyte-config:config-models') implementation project(':airbyte-workers') runtimeOnly 'io.micronaut:micronaut-http-server-netty:3.6.0' diff --git a/airbyte-cron/src/main/java/io/airbyte/cron/selfhealing/WorkspaceCleaner.java b/airbyte-cron/src/main/java/io/airbyte/cron/selfhealing/WorkspaceCleaner.java new file mode 100644 index 000000000000..574574fe5982 --- /dev/null +++ b/airbyte-cron/src/main/java/io/airbyte/cron/selfhealing/WorkspaceCleaner.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.cron.selfhealing; + +import io.airbyte.config.Configs; +import io.airbyte.config.EnvConfigs; +import io.micronaut.scheduling.annotation.Scheduled; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.util.Date; +import java.util.concurrent.atomic.AtomicInteger; +import javax.inject.Singleton; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.AgeFileFilter; + +@Singleton +@Slf4j +public class WorkspaceCleaner { + + private final Path workspaceRoot; + private final long maxAgeFilesInDays; + + WorkspaceCleaner() { + // TODO Configs should get injected through micronaut + final Configs configs = new EnvConfigs(); + + this.workspaceRoot = configs.getWorkspaceRoot(); + // We align max file age on temporal for history consistency + // It might make sense configure this independently in the future + this.maxAgeFilesInDays = configs.getTemporalRetentionInDays(); + } + + /* + * Delete files older than maxAgeFilesInDays from the workspace + * + * NOTE: this is currently only intended to work for docker + */ + @Scheduled(fixedRate = "1d") + public void deleteOldFiles() throws IOException { + final Date oldestAllowed = getDateFromDaysAgo(maxAgeFilesInDays); + log.info("Deleting files older than {} days ({})", maxAgeFilesInDays, oldestAllowed); + + final AtomicInteger counter = new AtomicInteger(0); + Files.walk(workspaceRoot) + .map(Path::toFile) + .filter(f -> new AgeFileFilter(oldestAllowed).accept(f)) + .forEach(file -> { + log.debug("Deleting file: " + file.toString()); + FileUtils.deleteQuietly(file); + counter.incrementAndGet(); + final File parentDir = file.getParentFile(); + if (parentDir.isDirectory() && parentDir.listFiles().length == 0) { + FileUtils.deleteQuietly(parentDir); + } + }); + log.info("deleted {} files", counter.get()); + } + + private static Date getDateFromDaysAgo(final long daysAgo) { + return Date.from(LocalDateTime.now().minusDays(daysAgo).toInstant(OffsetDateTime.now().getOffset())); + } + +} diff --git a/docker-compose.yaml b/docker-compose.yaml index 70942dd0b37d..d57f7be17540 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -186,6 +186,10 @@ services: - POSTGRES_PWD=${DATABASE_PASSWORD} - POSTGRES_SEEDS=${DATABASE_HOST} - POSTGRES_USER=${DATABASE_USER} + - TEMPORAL_HISTORY_RETENTION_IN_DAYS=${TEMPORAL_HISTORY_RETENTION_IN_DAYS} + - WORKSPACE_ROOT=${WORKSPACE_ROOT} + volumes: + - workspace:${WORKSPACE_ROOT} volumes: workspace: name: ${WORKSPACE_DOCKER_MOUNT} From 82ceb8a75da49554c375d8edd054a86eafa3be92 Mon Sep 17 00:00:00 2001 From: Ryan Fu Date: Fri, 2 Sep 2022 14:04:19 -0700 Subject: [PATCH 014/200] Hide Destination connections from UI (#16310) --- .../src/core/domain/connector/constants.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/airbyte-webapp/src/core/domain/connector/constants.ts b/airbyte-webapp/src/core/domain/connector/constants.ts index 31d323498c23..32ee6a72974e 100644 --- a/airbyte-webapp/src/core/domain/connector/constants.ts +++ b/airbyte-webapp/src/core/domain/connector/constants.ts @@ -18,9 +18,21 @@ export const DEV_IMAGE_TAG = "dev"; export const getExcludedConnectorIds = (workspaceId: string) => isCloudApp() ? [ + "707456df-6f4f-4ced-b5c6-03f73bcad1c5", // hide Cassandra Destination https://github.com/airbytehq/airbyte-cloud/issues/2606 + "072d5540-f236-4294-ba7c-ade8fd918496", // hide Databricks Destination https://github.com/airbytehq/airbyte-cloud/issues/2607 + "8ccd8909-4e99-4141-b48d-4984b70b2d89", // hide DynamoDB Destination https://github.com/airbytehq/airbyte-cloud/issues/2608 "68f351a7-2745-4bef-ad7f-996b8e51bb8c", // hide ElasticSearch Destination https://github.com/airbytehq/airbyte-cloud/issues/2594 + "ca8f6566-e555-4b40-943a-545bf123117a", // hide GCS Destination https://github.com/airbytehq/airbyte-cloud/issues/2609 + "9f760101-60ae-462f-9ee6-b7a9dafd454d", // hide Kafka Destination https://github.com/airbytehq/airbyte-cloud/issues/2610 + "294a4790-429b-40ae-9516-49826b9702e1", // hide MariaDB Destination https://github.com/airbytehq/airbyte-cloud/issues/2611 + "8b746512-8c2e-6ac1-4adc-b59faafd473c", // hide MongoDB Destination https://github.com/airbytehq/airbyte-cloud/issues/2612 + "f3802bc4-5406-4752-9e8d-01e504ca8194", // hide MQTT Destination https://github.com/airbytehq/airbyte-cloud/issues/2613 + "2340cbba-358e-11ec-8d3d-0242ac130203", // hide Pular Destination https://github.com/airbytehq/airbyte-cloud/issues/2614 "d4d3fef9-e319-45c2-881a-bd02ce44cc9f", // hide Redis Destination https://github.com/airbytehq/airbyte-cloud/issues/2593 + "2c9d93a7-9a17-4789-9de9-f46f0097eb70", // hide Rockset Destination https://github.com/airbytehq/airbyte-cloud/issues/2615 + "4816b78f-1489-44c1-9060-4b19d5fa9362", // hide S3 Destination https://github.com/airbytehq/airbyte-cloud/issues/2616 "2470e835-feaf-4db6-96f3-70fd645acc77", // Salesforce Singer + "3dc6f384-cd6b-4be3-ad16-a41450899bf0", // hide Scylla Destination https://github.com/airbytehq/airbyte-cloud/issues/2617 ...(workspaceId !== "54135667-ce73-4820-a93c-29fe1510d348" // Shopify workspace for review ? ["9da77001-af33-4bcd-be46-6252bf9342b9"] // Shopify : []), From 59cba5082e2774bb637fd90921431c7313e98376 Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Fri, 2 Sep 2022 15:12:58 -0700 Subject: [PATCH 015/200] Skip unit tests when run-tests is false (#16267) --- tools/integrations/manage.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/integrations/manage.sh b/tools/integrations/manage.sh index 6b34d6693fad..a9f2875b9024 100755 --- a/tools/integrations/manage.sh +++ b/tools/integrations/manage.sh @@ -48,12 +48,13 @@ cmd_build() { # Note that we are only building (and testing) once on this build machine's architecture # Learn more @ https://github.com/airbytehq/airbyte/pull/13004 ./gradlew --no-daemon "$(_to_gradle_path "$path" clean)" - ./gradlew --no-daemon "$(_to_gradle_path "$path" build)" if [ "$run_tests" = false ] ; then - echo "Skipping integration tests..." + echo "Building and skipping unit tests + integration tests..." + ./gradlew --no-daemon "$(_to_gradle_path "$path" build)" -x test else - echo "Running integration tests..." + echo "Building and running unit tests + integration tests..." + ./gradlew --no-daemon "$(_to_gradle_path "$path" build)" if test "$path" == "airbyte-integrations/bases/base-normalization"; then ./gradlew --no-daemon --scan :airbyte-integrations:bases:base-normalization:airbyteDocker From b9a7df17816262bef60df8112d0d7cf7126529f0 Mon Sep 17 00:00:00 2001 From: Rodi Reich Zilberman <867491+rodireich@users.noreply.github.com> Date: Fri, 2 Sep 2022 18:48:01 -0700 Subject: [PATCH 016/200] Hide a bunch more destination with potential unsecure API access (#16320) * Hide a bunch more destination with potential unsecure API access * Hide also Exchange rate API source --- airbyte-webapp/src/core/domain/connector/constants.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-webapp/src/core/domain/connector/constants.ts b/airbyte-webapp/src/core/domain/connector/constants.ts index 32ee6a72974e..457e58b7f74c 100644 --- a/airbyte-webapp/src/core/domain/connector/constants.ts +++ b/airbyte-webapp/src/core/domain/connector/constants.ts @@ -33,6 +33,10 @@ export const getExcludedConnectorIds = (workspaceId: string) => "4816b78f-1489-44c1-9060-4b19d5fa9362", // hide S3 Destination https://github.com/airbytehq/airbyte-cloud/issues/2616 "2470e835-feaf-4db6-96f3-70fd645acc77", // Salesforce Singer "3dc6f384-cd6b-4be3-ad16-a41450899bf0", // hide Scylla Destination https://github.com/airbytehq/airbyte-cloud/issues/2617 + "af7c921e-5892-4ff2-b6c1-4a5ab258fb7e", // hide MeiliSearch Destination https://github.com/airbytehq/airbyte/issues/16313 + "e06ad785-ad6f-4647-b2e8-3027a5c59454", // hide RabbitMQ Destination https://github.com/airbytehq/airbyte/issues/16315 + "0eeee7fb-518f-4045-bacc-9619e31c43ea", // hide Amazon SQS Destination https://github.com/airbytehq/airbyte/issues/16316 + "e2b40e36-aa0e-4bed-b41b-bcea6fa348b1", // hide Exchange rate Source https://github.com/airbytehq/airbyte/issues/16311 ...(workspaceId !== "54135667-ce73-4820-a93c-29fe1510d348" // Shopify workspace for review ? ["9da77001-af33-4bcd-be46-6252bf9342b9"] // Shopify : []), From 28a6adde643d226ee6002eccc167c29928953d77 Mon Sep 17 00:00:00 2001 From: VitaliiMaltsev <39538064+VitaliiMaltsev@users.noreply.github.com> Date: Sat, 3 Sep 2022 21:55:54 +0300 Subject: [PATCH 017/200] MSSQL Source : Standardize spec.json for DB connectors that support log-based CDC replication (#16215) * Fixed bucket naming for S3 * removed redundant configs * MSSQL Source : Standardize spec.json for DB connectors that support log-based CDC replication * bump version * bump version --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 12 ++++++------ .../source-mssql-strict-encrypt/Dockerfile | 2 +- .../src/test/resources/expected_spec.json | 10 +++++----- .../connectors/source-mssql/Dockerfile | 2 +- .../source/mssql/MssqlCdcHelper.java | 14 ++++++++++---- .../source-mssql/src/main/resources/spec.json | 10 +++++----- .../mssql/CdcMssqlSourceAcceptanceTest.java | 4 ++-- .../mssql/CdcMssqlSourceDatatypeTest.java | 4 ++-- .../mssql/FillMsSqlTestDbScriptTest.java | 4 ++-- .../source/mssql/CdcMssqlSourceTest.java | 6 +++--- .../source/mssql/MssqlCdcHelperTest.java | 19 +++++++++++-------- docs/integrations/sources/mssql.md | 1 + 13 files changed, 50 insertions(+), 40 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 7eefa8398a06..3996c7fe6fc2 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -599,7 +599,7 @@ - name: Microsoft SQL Server (MSSQL) sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 dockerRepository: airbyte/source-mssql - dockerImageTag: 0.4.17 + dockerImageTag: 0.4.18 documentationUrl: https://docs.airbyte.io/integrations/sources/mssql icon: mssql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 43387628eb05..1a558151a514 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5273,7 +5273,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mssql:0.4.17" +- dockerImage: "airbyte/source-mssql:0.4.18" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql" connectionSpecification: @@ -5374,7 +5374,7 @@ description: "Specifies the host name of the server. The value of\ \ this property must match the subject property of the certificate." order: 7 - replication: + replication_method: type: "object" title: "Replication Method" description: "The replication method used for extracting data from the database.\ @@ -5389,9 +5389,9 @@ description: "Standard replication requires no setup on the DB side but\ \ will not be able to represent deletions incrementally." required: - - "replication_type" + - "method" properties: - replication_type: + method: type: "string" const: "STANDARD" enum: @@ -5402,9 +5402,9 @@ description: "CDC uses {TBC} to detect inserts, updates, and deletes.\ \ This needs to be configured on the source database itself." required: - - "replication_type" + - "method" properties: - replication_type: + method: type: "string" const: "CDC" enum: diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile index f7c29543a386..8c1bf75d5c42 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.17 +LABEL io.airbyte.version=0.4.18 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json index af6cc24193fd..641e888a8b4b 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json @@ -87,7 +87,7 @@ } ] }, - "replication": { + "replication_method": { "type": "object", "title": "Replication Method", "description": "The replication method used for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", @@ -97,9 +97,9 @@ { "title": "Standard", "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["replication_type"], + "required": ["method"], "properties": { - "replication_type": { + "method": { "type": "string", "const": "STANDARD", "enum": ["STANDARD"], @@ -111,9 +111,9 @@ { "title": "Logical Replication (CDC)", "description": "CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "required": ["replication_type"], + "required": ["method"], "properties": { - "replication_type": { + "method": { "type": "string", "const": "CDC", "enum": ["CDC"], diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index 3522a1b3604f..0ee5ccbc01a8 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.17 +LABEL io.airbyte.version=0.4.18 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java index 51672a826ab8..63c814edef63 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcHelper.java @@ -17,6 +17,7 @@ public class MssqlCdcHelper { // it is an oneOf object private static final String REPLICATION_FIELD = "replication"; private static final String REPLICATION_TYPE_FIELD = "replication_type"; + private static final String METHOD_FIELD = "method"; private static final String CDC_SNAPSHOT_ISOLATION_FIELD = "snapshot_isolation"; private static final String CDC_DATA_TO_SYNC_FIELD = "data_to_sync"; @@ -91,14 +92,19 @@ public static DataToSync from(final String value) { @VisibleForTesting static boolean isCdc(final JsonNode config) { // new replication method config since version 0.4.0 - if (config.hasNonNull(REPLICATION_FIELD)) { - final JsonNode replicationConfig = config.get(REPLICATION_FIELD); - return ReplicationMethod.valueOf(replicationConfig.get(REPLICATION_TYPE_FIELD).asText()) == ReplicationMethod.CDC; + if (config.hasNonNull(LEGACY_REPLICATION_FIELD) && config.get(LEGACY_REPLICATION_FIELD).isObject()) { + final JsonNode replicationConfig = config.get(LEGACY_REPLICATION_FIELD); + return ReplicationMethod.valueOf(replicationConfig.get(METHOD_FIELD).asText()) == ReplicationMethod.CDC; } // legacy replication method config before version 0.4.0 - if (config.hasNonNull(LEGACY_REPLICATION_FIELD)) { + if (config.hasNonNull(LEGACY_REPLICATION_FIELD) && config.get(LEGACY_REPLICATION_FIELD).isTextual()) { return ReplicationMethod.valueOf(config.get(LEGACY_REPLICATION_FIELD).asText()) == ReplicationMethod.CDC; } + if (config.hasNonNull(REPLICATION_FIELD)) { + final JsonNode replicationConfig = config.get(REPLICATION_FIELD); + return ReplicationMethod.valueOf(replicationConfig.get(REPLICATION_TYPE_FIELD).asText()) == ReplicationMethod.CDC; + } + return false; } diff --git a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json index aba59c81aa7f..f599164e9579 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json @@ -100,7 +100,7 @@ } ] }, - "replication": { + "replication_method": { "type": "object", "title": "Replication Method", "description": "The replication method used for extracting data from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", @@ -110,9 +110,9 @@ { "title": "Standard", "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", - "required": ["replication_type"], + "required": ["method"], "properties": { - "replication_type": { + "method": { "type": "string", "const": "STANDARD", "enum": ["STANDARD"], @@ -124,9 +124,9 @@ { "title": "Logical Replication (CDC)", "description": "CDC uses {TBC} to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", - "required": ["replication_type"], + "required": ["method"], "properties": { - "replication_type": { + "method": { "type": "string", "const": "CDC", "enum": ["CDC"], diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java index 9961d5753309..c2dd276fc26f 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceAcceptanceTest.java @@ -97,7 +97,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Int container.start(); final JsonNode replicationConfig = Jsons.jsonNode(Map.of( - "replication_type", "CDC", + "method", "CDC", "data_to_sync", "Existing and New", "snapshot_isolation", "Snapshot")); @@ -107,7 +107,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Int .put(JdbcUtils.DATABASE_KEY, DB_NAME) .put(JdbcUtils.USERNAME_KEY, TEST_USER_NAME) .put(JdbcUtils.PASSWORD_KEY, TEST_USER_PASSWORD) - .put("replication", replicationConfig) + .put("replication_method", replicationConfig) .build()); dslContext = DSLContextFactory.create( diff --git a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java index f232d81b503e..0a202c36910d 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-integration/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceDatatypeTest.java @@ -32,7 +32,7 @@ protected Database setupDatabase() throws Exception { container.start(); final JsonNode replicationConfig = Jsons.jsonNode(Map.of( - "replication_type", "CDC", + "method", "CDC", "data_to_sync", "Existing and New", "snapshot_isolation", "Snapshot")); @@ -42,7 +42,7 @@ protected Database setupDatabase() throws Exception { .put(JdbcUtils.DATABASE_KEY, DB_NAME) .put(JdbcUtils.USERNAME_KEY, container.getUsername()) .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication", replicationConfig) + .put("replication_method", replicationConfig) .build()); dslContext = DSLContextFactory.create( diff --git a/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java b/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java index 162a6db04615..b22020307068 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test-performance/java/io/airbyte/integrations/source/mssql/FillMsSqlTestDbScriptTest.java @@ -40,7 +40,7 @@ protected String getImageName() { @Override protected Database setupDatabase(final String dbName) { final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() - .put("replication_type", "Standard") + .put("method", "Standard") .build()); config = Jsons.jsonNode(ImmutableMap.builder() @@ -49,7 +49,7 @@ protected Database setupDatabase(final String dbName) { .put(JdbcUtils.DATABASE_KEY, dbName) // set your db name .put(JdbcUtils.USERNAME_KEY, "your_username") .put(JdbcUtils.PASSWORD_KEY, "your_pass") - .put("replication", replicationMethod) + .put("replication_method", replicationMethod) .build()); dslContext = DSLContextFactory.create( diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java index b04e49bd536a..5badf3afcbe0 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/CdcMssqlSourceTest.java @@ -86,7 +86,7 @@ private void init() { source = new MssqlSource(); final JsonNode replicationConfig = Jsons.jsonNode(Map.of( - "replication_type", "CDC", + "method", "CDC", "data_to_sync", "Existing and New", "snapshot_isolation", "Snapshot")); config = Jsons.jsonNode(ImmutableMap.builder() @@ -96,7 +96,7 @@ private void init() { .put(JdbcUtils.SCHEMAS_KEY, List.of(MODELS_SCHEMA, MODELS_SCHEMA + "_random")) .put(JdbcUtils.USERNAME_KEY, TEST_USER_NAME) .put(JdbcUtils.PASSWORD_KEY, TEST_USER_PASSWORD) - .put("replication", replicationConfig) + .put("replication_method", replicationConfig) .build()); dataSource = DataSourceFactory.create( @@ -279,7 +279,7 @@ void testAssertSnapshotIsolationAllowed() { @Test void testAssertSnapshotIsolationDisabled() { final JsonNode replicationConfig = Jsons.jsonNode(ImmutableMap.builder() - .put("replication_type", "CDC") + .put("method", "CDC") .put("data_to_sync", "New Changes Only") // set snapshot_isolation level to "Read Committed" to disable snapshot .put("snapshot_isolation", "Read Committed") diff --git a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java index 79193cc8169c..eec4115076a2 100644 --- a/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java +++ b/airbyte-integrations/connectors/source-mssql/src/test/java/io/airbyte/integrations/source/mssql/MssqlCdcHelperTest.java @@ -27,27 +27,30 @@ public void testIsCdc() { assertTrue(MssqlCdcHelper.isCdc(LEGACY_CDC_CONFIG)); // new replication method config since version 0.4.0 - final JsonNode newNonCdc = Jsons.jsonNode(Map.of("replication", - Jsons.jsonNode(Map.of("replication_type", "STANDARD")))); + final JsonNode newNonCdc = Jsons.jsonNode(Map.of("replication_method", + Jsons.jsonNode(Map.of("method", "STANDARD")))); assertFalse(MssqlCdcHelper.isCdc(newNonCdc)); - final JsonNode newCdc = Jsons.jsonNode(Map.of("replication", + final JsonNode newCdc = Jsons.jsonNode(Map.of("replication_method", Jsons.jsonNode(Map.of( - "replication_type", "CDC", + "method", "CDC", "data_to_sync", "Existing and New", "snapshot_isolation", "Snapshot")))); assertTrue(MssqlCdcHelper.isCdc(newCdc)); // migration from legacy to new config final JsonNode mixNonCdc = Jsons.jsonNode(Map.of( - "replication_method", "CDC", - "replication", Jsons.jsonNode(Map.of("replication_type", "STANDARD")))); + "replication_method", Jsons.jsonNode(Map.of("method", "STANDARD")), + "replication", Jsons.jsonNode(Map.of("replication_type", "CDC")))); assertFalse(MssqlCdcHelper.isCdc(mixNonCdc)); final JsonNode mixCdc = Jsons.jsonNode(Map.of( - "replication_method", "Standard", "replication", Jsons.jsonNode(Map.of( - "replication_type", "CDC", + "replication_type", "Standard", + "data_to_sync", "Existing and New", + "snapshot_isolation", "Snapshot")), + "replication_method", Jsons.jsonNode(Map.of( + "method", "CDC", "data_to_sync", "Existing and New", "snapshot_isolation", "Snapshot")))); assertTrue(MssqlCdcHelper.isCdc(mixCdc)); diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index 65052d2c57d2..a1139f48d9a5 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -306,6 +306,7 @@ If you do not see a type in this list, assume that it is coerced into a string. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----------------------------------------------------- |:-------------------------------------------------------------------------------------------------------| +| 0.4.18 | 2022-09-03 | [14910](https://github.com/airbytehq/airbyte/pull/14910) | Standardize spec for CDC replication. Replace the `replication_method` enum with a config object with a `method` enum field. | | 0.4.17 | 2022-09-01 | [16261](https://github.com/airbytehq/airbyte/pull/16261) | Emit state messages more frequently | | 0.4.16 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | | 0.4.15 | 2022-08-11 | [15538](https://github.com/airbytehq/airbyte/pull/15538) | Allow additional properties in db stream state | From e9a8a052677027cc3c88c41f36206bcf7d59e7f0 Mon Sep 17 00:00:00 2001 From: VitaliiMaltsev <39538064+VitaliiMaltsev@users.noreply.github.com> Date: Sun, 4 Sep 2022 15:30:04 +0300 Subject: [PATCH 018/200] MySQL Source : Standardize spec.json for DB connectors that support log-based CDC replication (#16216) * Fixed bucket naming for S3 * removed redundant configs * MySQL Source : Standardize spec.json for DB connectors that support log-based CDC replication * fixed strict encrypt tests * fixed mysql tests * bump version * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 41 ++++++++++++++----- .../source-mysql-strict-encrypt/Dockerfile | 2 +- ...cateStrictEncryptSourceAcceptanceTest.java | 7 +++- ...ySqlStrictEncryptSourceAcceptanceTest.java | 11 ++--- .../src/test/resources/expected_spec.json | 36 ++++++++++++++-- .../connectors/source-mysql/Dockerfile | 2 +- .../source/mysql/MySqlSource.java | 13 ++++-- .../source-mysql/src/main/resources/spec.json | 36 ++++++++++++++-- ...SqlSslCertificateSourceAcceptanceTest.java | 7 +++- .../CdcBinlogsMySqlSourceDatatypeTest.java | 6 ++- ...nitialSnapshotMySqlSourceDatatypeTest.java | 7 +++- .../mysql/CdcMySqlSourceAcceptanceTest.java | 7 ++-- ...lSslCaCertificateSourceAcceptanceTest.java | 5 ++- .../mysql/MySqlSourceAcceptanceTest.java | 7 ++-- .../source/mysql/MySqlSourceDatatypeTest.java | 8 ++-- .../mysql/MySqlSslSourceAcceptanceTest.java | 9 +++- .../SshKeyMySqlSourceAcceptanceTest.java | 2 +- .../SshPasswordMySqlSourceAcceptanceTest.java | 2 +- .../mysql/FillMySqlTestDbScriptTest.java | 7 +++- docs/integrations/sources/mysql.md | 3 +- 21 files changed, 165 insertions(+), 55 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 3996c7fe6fc2..c730776d3fd1 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -647,7 +647,7 @@ - name: MySQL sourceDefinitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad dockerRepository: airbyte/source-mysql - dockerImageTag: 0.6.8 + dockerImageTag: 0.6.9 documentationUrl: https://docs.airbyte.io/integrations/sources/mysql icon: mysql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 1a558151a514..95c732a60e9b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -6048,7 +6048,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mysql:0.6.8" +- dockerImage: "airbyte/source-mysql:0.6.9" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/mysql" connectionSpecification: @@ -6236,18 +6236,37 @@ airbyte_secret: true order: 4 replication_method: - type: "string" + type: "object" title: "Replication Method" - description: "Replication method which is used for data extraction from\ - \ the database. STANDARD replication requires no setup on the DB side\ - \ but will not be able to represent deletions incrementally. CDC uses\ - \ the Binlog to detect inserts, updates, and deletes. This needs to be\ - \ configured on the source database itself." + description: "Replication method to use for extracting data from the database." order: 8 - default: "STANDARD" - enum: - - "STANDARD" - - "CDC" + oneOf: + - title: "STANDARD" + description: "Standard replication requires no setup on the DB side but\ + \ will not be able to represent deletions incrementally." + required: + - "method" + properties: + method: + type: "string" + const: "STANDARD" + enum: + - "STANDARD" + default: "STANDARD" + order: 0 + - title: "Logical Replication (CDC)" + description: "CDC uses the Binlog to detect inserts, updates, and deletes.\ + \ This needs to be configured on the source database itself." + required: + - "method" + properties: + method: + type: "string" + const: "CDC" + enum: + - "CDC" + default: "CDC" + order: 0 tunnel_method: type: "object" title: "SSH Tunnel Method" diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile index bea4a4b304d6..b385f3d6571a 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.8 +LABEL io.airbyte.version=0.6.9 LABEL io.airbyte.name=airbyte/source-mysql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java index ecfa9e92953a..b637cf07a011 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/AbstractMySqlSslCertificateStrictEncryptSourceAcceptanceTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.source.mysql_strict_encrypt; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; @@ -31,7 +32,9 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc certs = MySqlUtils.getCertificate(container, true); var sslMode = getSslConfig(); - + final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() + .put("method", "STANDARD") + .build()); config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) @@ -40,7 +43,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) .put(JdbcUtils.SSL_KEY, true) .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", ReplicationMethod.STANDARD) + .put("replication_method", replicationMethod) .build()); } diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java index 43493837f006..6a5ef2c3e2b5 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test-integration/java/io/airbyte/integrations/source/mysql_strict_encrypt/MySqlStrictEncryptSourceAcceptanceTest.java @@ -16,7 +16,6 @@ import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.source.mysql.MySqlSource; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; @@ -46,9 +45,11 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc container.start(); var sslMode = ImmutableMap.builder() - .put(JdbcUtils.MODE_KEY, "required") - .build(); - + .put(JdbcUtils.MODE_KEY, "required") + .build(); + final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() + .put("method", "STANDARD") + .build()); config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) @@ -56,7 +57,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc .put(JdbcUtils.USERNAME_KEY, container.getUsername()) .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", MySqlSource.ReplicationMethod.STANDARD) + .put("replication_method", replicationMethod) .build()); try (final DSLContext dslContext = DSLContextFactory.create( diff --git a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json index 2c6da3511475..c964d26ecad9 100644 --- a/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mysql-strict-encrypt/src/test/resources/expected_spec.json @@ -174,12 +174,40 @@ ] }, "replication_method": { - "type": "string", + "type": "object", "title": "Replication Method", - "description": "Replication method which is used for data extraction from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", + "description": "Replication method to use for extracting data from the database.", "order": 8, - "default": "STANDARD", - "enum": ["STANDARD", "CDC"] + "oneOf": [ + { + "title": "STANDARD", + "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "STANDARD", + "enum": ["STANDARD"], + "default": "STANDARD", + "order": 0 + } + } + }, + { + "title": "Logical Replication (CDC)", + "description": "CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "CDC", + "enum": ["CDC"], + "default": "CDC", + "order": 0 + } + } + } + ] } } } diff --git a/airbyte-integrations/connectors/source-mysql/Dockerfile b/airbyte-integrations/connectors/source-mysql/Dockerfile index 49e9e93fb676..92d7a9ff378f 100644 --- a/airbyte-integrations/connectors/source-mysql/Dockerfile +++ b/airbyte-integrations/connectors/source-mysql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.6.8 +LABEL io.airbyte.version=0.6.9 LABEL io.airbyte.name=airbyte/source-mysql diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index 73351a988a5e..0d7da1036287 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -172,9 +172,16 @@ public JsonNode toDatabaseConfig(final JsonNode config) { } private static boolean isCdc(final JsonNode config) { - return config.hasNonNull("replication_method") - && ReplicationMethod.valueOf(config.get("replication_method").asText()) - .equals(ReplicationMethod.CDC); + if (config.hasNonNull("replication_method")) { + if (config.get("replication_method").isTextual()) { + return ReplicationMethod.valueOf(config.get("replication_method").asText()) + .equals(ReplicationMethod.CDC); + } else if (config.get("replication_method").isObject()) { + return config.get("replication_method").get("method").asText() + .equals(ReplicationMethod.CDC.name()); + } + } + return false; } @Override diff --git a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json index eefb37c035ba..c09509ff9498 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mysql/src/main/resources/spec.json @@ -181,12 +181,40 @@ ] }, "replication_method": { - "type": "string", + "type": "object", "title": "Replication Method", - "description": "Replication method which is used for data extraction from the database. STANDARD replication requires no setup on the DB side but will not be able to represent deletions incrementally. CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", + "description": "Replication method to use for extracting data from the database.", "order": 8, - "default": "STANDARD", - "enum": ["STANDARD", "CDC"] + "oneOf": [ + { + "title": "STANDARD", + "description": "Standard replication requires no setup on the DB side but will not be able to represent deletions incrementally.", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "STANDARD", + "enum": ["STANDARD"], + "default": "STANDARD", + "order": 0 + } + } + }, + { + "title": "Logical Replication (CDC)", + "description": "CDC uses the Binlog to detect inserts, updates, and deletes. This needs to be configured on the source database itself.", + "required": ["method"], + "properties": { + "method": { + "type": "string", + "const": "CDC", + "enum": ["CDC"], + "default": "CDC", + "order": 0 + } + } + } + ] } } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java index 9f615d331b23..68efdaa1be3a 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/AbstractMySqlSslCertificateSourceAcceptanceTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.source.mysql; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; @@ -32,7 +33,9 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc certs = getCertificates(); var sslMode = getSslConfig(); - + final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() + .put("method", "STANDARD") + .build()); config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) @@ -41,7 +44,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) .put(JdbcUtils.SSL_KEY, true) .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", ReplicationMethod.STANDARD) + .put("replication_method", replicationMethod) .build()); } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcBinlogsMySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcBinlogsMySqlSourceDatatypeTest.java index 0867feaf2b32..045b78b828b9 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcBinlogsMySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcBinlogsMySqlSourceDatatypeTest.java @@ -81,14 +81,16 @@ protected void setupEnvironment(TestDestinationEnv environment) throws Exception protected Database setupDatabase() throws Exception { container = new MySQLContainer<>("mysql:8.0"); container.start(); - + final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() + .put("method", "CDC") + .build()); config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) .put(JdbcUtils.USERNAME_KEY, container.getUsername()) .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", MySqlSource.ReplicationMethod.CDC) + .put("replication_method", replicationMethod) .build()); dslContext = DSLContextFactory.create( diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcInitialSnapshotMySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcInitialSnapshotMySqlSourceDatatypeTest.java index 566d5b6cecab..d52defbdaa99 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcInitialSnapshotMySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcInitialSnapshotMySqlSourceDatatypeTest.java @@ -4,6 +4,7 @@ package io.airbyte.integrations.source.mysql; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; @@ -29,14 +30,16 @@ protected void tearDown(final TestDestinationEnv testEnv) { protected Database setupDatabase() throws Exception { container = new MySQLContainer<>("mysql:8.0"); container.start(); - + final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() + .put("method", "CDC") + .build()); config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) .put(JdbcUtils.USERNAME_KEY, container.getUsername()) .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", MySqlSource.ReplicationMethod.CDC) + .put("replication_method", replicationMethod) .put("snapshot_mode", "initial_only") .build()); diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java index be71ec84fc2d..fb64826100b8 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java @@ -18,7 +18,6 @@ import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.AirbyteMessage; @@ -99,14 +98,16 @@ protected JsonNode getState() { protected void setupEnvironment(final TestDestinationEnv environment) { container = new MySQLContainer<>("mysql:8.0"); container.start(); - + final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() + .put("method", "CDC") + .build()); config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) .put(JdbcUtils.USERNAME_KEY, container.getUsername()) .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", ReplicationMethod.CDC) + .put("replication_method", replicationMethod) .build()); revokeAllPermissions(); diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSslCaCertificateSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSslCaCertificateSourceAcceptanceTest.java index 53a9cfa1ef29..3141173e8b86 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSslCaCertificateSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSslCaCertificateSourceAcceptanceTest.java @@ -110,6 +110,9 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc .put("client_key", certs.getClientKey()) .put("client_key_password", "Passw0rd") .build(); + final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() + .put("method", "CDC") + .build()); config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) @@ -119,7 +122,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) .put(JdbcUtils.SSL_KEY, true) .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", ReplicationMethod.CDC) + .put("replication_method", replicationMethod) .build()); revokeAllPermissions(); diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java index 0c276801038a..5aaf3979c4ab 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceAcceptanceTest.java @@ -13,7 +13,6 @@ import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; import io.airbyte.integrations.base.ssh.SshHelpers; -import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; import io.airbyte.integrations.standardtest.source.SourceAcceptanceTest; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.protocol.models.CatalogHelpers; @@ -41,14 +40,16 @@ public class MySqlSourceAcceptanceTest extends SourceAcceptanceTest { protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { container = new MySQLContainer<>("mysql:8.0"); container.start(); - + final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() + .put("method", "STANDARD") + .build()); config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) .put(JdbcUtils.USERNAME_KEY, container.getUsername()) .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", ReplicationMethod.STANDARD) + .put("replication_method", replicationMethod) .build()); try (final DSLContext dslContext = DSLContextFactory.create( diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceDatatypeTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceDatatypeTest.java index 584565b3b6d3..a77f81dbeb63 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceDatatypeTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSourceDatatypeTest.java @@ -4,13 +4,13 @@ package io.airbyte.integrations.source.mysql; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; import io.airbyte.db.factory.DSLContextFactory; import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import java.util.Map; import org.jooq.SQLDialect; @@ -27,14 +27,16 @@ protected void tearDown(final TestDestinationEnv testEnv) { protected Database setupDatabase() throws Exception { container = new MySQLContainer<>("mysql:8.0"); container.start(); - + final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() + .put("method", "STANDARD") + .build()); config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, container.getHost()) .put(JdbcUtils.PORT_KEY, container.getFirstMappedPort()) .put(JdbcUtils.DATABASE_KEY, container.getDatabaseName()) .put(JdbcUtils.USERNAME_KEY, container.getUsername()) .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) - .put("replication_method", ReplicationMethod.STANDARD) + .put("replication_method", replicationMethod) .build()); final Database database = new Database( diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java index 9d8dfdb6bd86..0c7d79c48d41 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/MySqlSslSourceAcceptanceTest.java @@ -4,13 +4,15 @@ package io.airbyte.integrations.source.mysql; +import static io.airbyte.integrations.source.mysql.MySqlSource.SSL_PARAMETERS; + +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; import io.airbyte.db.factory.DSLContextFactory; import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import org.jooq.DSLContext; import org.jooq.SQLDialect; @@ -22,6 +24,9 @@ public class MySqlSslSourceAcceptanceTest extends MySqlSourceAcceptanceTest { protected void setupEnvironment(final TestDestinationEnv environment) throws Exception { container = new MySQLContainer<>("mysql:8.0"); container.start(); + final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() + .put("method", "STANDARD") + .build()); var sslMode = ImmutableMap.builder() .put(JdbcUtils.MODE_KEY, "required") @@ -35,7 +40,7 @@ protected void setupEnvironment(final TestDestinationEnv environment) throws Exc .put(JdbcUtils.PASSWORD_KEY, container.getPassword()) .put(JdbcUtils.SSL_KEY, true) .put(JdbcUtils.SSL_MODE_KEY, sslMode) - .put("replication_method", ReplicationMethod.STANDARD) + .put("replication_method", replicationMethod) .build()); try (final DSLContext dslContext = DSLContextFactory.create( diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/SshKeyMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/SshKeyMySqlSourceAcceptanceTest.java index 9da81da967b2..9c975b65ffcb 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/SshKeyMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/SshKeyMySqlSourceAcceptanceTest.java @@ -10,7 +10,7 @@ public class SshKeyMySqlSourceAcceptanceTest extends AbstractSshMySqlSourceAccep @Override public Path getConfigFilePath() { - return Path.of("secrets/ssh-key-config.json"); + return Path.of("secrets/ssh-key-repl-config.json"); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/SshPasswordMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/SshPasswordMySqlSourceAcceptanceTest.java index 2a7ac07cf677..fb32a73a8bc3 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/SshPasswordMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/SshPasswordMySqlSourceAcceptanceTest.java @@ -10,7 +10,7 @@ public class SshPasswordMySqlSourceAcceptanceTest extends AbstractSshMySqlSource @Override public Path getConfigFilePath() { - return Path.of("secrets/ssh-pwd-config.json"); + return Path.of("secrets/ssh-pwd-repl-config.json"); } } diff --git a/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java b/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java index 9bedc8f6af48..3af06df21c26 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-performance/java/io/airbyte/integrations/source/mysql/FillMySqlTestDbScriptTest.java @@ -11,7 +11,6 @@ import io.airbyte.db.factory.DSLContextFactory; import io.airbyte.db.factory.DatabaseDriver; import io.airbyte.db.jdbc.JdbcUtils; -import io.airbyte.integrations.source.mysql.MySqlSource.ReplicationMethod; import io.airbyte.integrations.standardtest.source.TestDestinationEnv; import io.airbyte.integrations.standardtest.source.performancetest.AbstractSourceFillDbWithTestData; import java.util.Map; @@ -38,13 +37,17 @@ protected String getImageName() { @Override protected Database setupDatabase(final String dbName) throws Exception { + + final JsonNode replicationMethod = Jsons.jsonNode(ImmutableMap.builder() + .put("method", "STANDARD") + .build()); config = Jsons.jsonNode(ImmutableMap.builder() .put(JdbcUtils.HOST_KEY, "your_host") .put(JdbcUtils.PORT_KEY, 3306) .put(JdbcUtils.DATABASE_KEY, dbName) // set your db name .put(JdbcUtils.USERNAME_KEY, "your_username") .put(JdbcUtils.PASSWORD_KEY, "your_pass") - .put("replication_method", ReplicationMethod.STANDARD) + .put("replication_method", replicationMethod) .build()); final Database database = new Database( diff --git a/docs/integrations/sources/mysql.md b/docs/integrations/sources/mysql.md index 0b1226c2d200..b73fd2e5ee72 100644 --- a/docs/integrations/sources/mysql.md +++ b/docs/integrations/sources/mysql.md @@ -188,7 +188,8 @@ If you do not see a type in this list, assume that it is coerced into a string. | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------| -| 0.6.8 | 2022-09-01 | [16259](https://github.com/airbytehq/airbyte/pull/16259) | Emit state messages more frequently | +| 0.6.9 | 2022-09-03 | [16216](https://github.com/airbytehq/airbyte/pull/16216) | Standardize spec for CDC replication. Replace the `replication_method` enum with a config object with a `method` enum field. | +| 0.6.8 | 2022-09-01 | [16259](https://github.com/airbytehq/airbyte/pull/16259) | Emit state messages more frequently | | 0.6.7 | 2022-08-30 | [16114](https://github.com/airbytehq/airbyte/pull/16114) | Prevent traffic going on an unsecured channel in strict-encryption version of source mysql | | 0.6.6 | 2022-08-25 | [15993](https://github.com/airbytehq/airbyte/pull/15993) | Improved support for connecting over SSL | | 0.6.5 | 2022-08-25 | [15917](https://github.com/airbytehq/airbyte/pull/15917) | Fix temporal data type default value bug | From 63bc3230d4b5df9d117e87a4eddd56f80502c2fb Mon Sep 17 00:00:00 2001 From: Liren Tu Date: Sun, 4 Sep 2022 11:59:02 -0700 Subject: [PATCH 019/200] Add comments about intermediate state emission (#16262) * Add comments about intermediate state emission * Adjust wording * Format code --- .../relationaldb/StateDecoratingIterator.java | 31 ++++++++++++++----- .../StateDecoratingIteratorTest.java | 17 ++++++---- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java index 12476b3b0306..605c3aca8eba 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java @@ -27,15 +27,30 @@ public class StateDecoratingIterator extends AbstractIterator im private final AirbyteStreamNameNamespacePair pair; private final String cursorField; private final JsonSchemaPrimitive cursorType; - private final int stateEmissionFrequency; private final String initialCursor; private String maxCursor; private boolean hasEmittedFinalState; - // The intermediateStateMessage is set to the latest state message. - // For every stateEmissionFrequency messages, emitIntermediateState is set to true and - // the latest intermediateStateMessage will be emitted. + /** + * These parameters are for intermediate state message emission. We can emit an intermediate state + * when the following two conditions are met. + *

+ * 1. The records are sorted by the cursor field. This is true when {@code stateEmissionFrequency} > + * 0. This logic is guaranteed in {@code AbstractJdbcSource#queryTableIncremental}, in which an + * "ORDER BY" clause is appended to the SQL query if {@code stateEmissionFrequency} > 0. + *

+ * 2. There is a cursor value that is ready for emission. A cursor value is "ready" if there is no + * more record with the same value. We cannot emit a cursor at will, because there may be multiple + * records with the same cursor value. If we emit a cursor ignoring this condition, should the sync + * fail right after the emission, the next sync may skip some records with the same cursor value due + * to "WHERE cursor_field > cursor" in {@code AbstractJdbcSource#queryTableIncremental}. + *

+ * The {@code intermediateStateMessage} is set to the latest state message that is ready for + * emission. For every {@code stateEmissionFrequency} messages, {@code emitIntermediateState} is set + * to true and the latest "ready" state will be emitted in the next {@code computeNext} call. + */ + private final int stateEmissionFrequency; private int totalRecordCount = 0; private boolean emitIntermediateState = false; private AirbyteMessage intermediateStateMessage = null; @@ -47,9 +62,11 @@ public class StateDecoratingIterator extends AbstractIterator im * @param cursorField Path to the comparator field used to track the records read so far * @param initialCursor name of the initial cursor column * @param cursorType ENUM type of primitive values that can be used as a cursor for checkpointing - * @param stateEmissionFrequency If larger than 0, intermediate states will be emitted for every - * stateEmissionFrequency records. Only emit intermediate states if the records are sorted by - * the cursor field. + * @param stateEmissionFrequency If larger than 0, the records are sorted by the cursor field, and + * intermediate states will be emitted for every {@code stateEmissionFrequency} records. The + * order of the records is guaranteed in {@code AbstractJdbcSource#queryTableIncremental}, in + * which an "ORDER BY" clause is appended to the SQL query if {@code stateEmissionFrequency} + * > 0. */ public StateDecoratingIterator(final Iterator messageIterator, final StateManager stateManager, diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java index c67f537f76a8..7532ca2517b5 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java @@ -7,7 +7,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -75,6 +74,7 @@ private static AirbyteMessage createStateMessage(final String recordValue) { private Iterator createExceptionIterator() { return new Iterator() { + final Iterator internalMessageIterator = MoreIterators.of(RECORD_MESSAGE_1, RECORD_MESSAGE_2, RECORD_MESSAGE_2, RECORD_MESSAGE_3); @@ -88,7 +88,8 @@ public AirbyteMessage next() { if (internalMessageIterator.hasNext()) { return internalMessageIterator.next(); } else { - // this line throws a RunTimeException wrapped around a SQLException to mimic the flow of when a SQLException is thrown and wrapped in + // this line throws a RunTimeException wrapped around a SQLException to mimic the flow of when a + // SQLException is thrown and wrapped in // StreamingJdbcDatabase#tryAdvance throw new RuntimeException(new SQLException("Connection marked broken because of SQLSTATE(080006)", "08006")); } @@ -186,10 +187,12 @@ void testIteratorCatchesExceptionWhenEmissionFrequencyNonZero() { 1); assertEquals(RECORD_MESSAGE_1, iterator.next()); assertEquals(RECORD_MESSAGE_2, iterator.next()); - // continues to emit RECORD_MESSAGE_2 since cursorField has not changed thus not satisfying the condition of "ready" + // continues to emit RECORD_MESSAGE_2 since cursorField has not changed thus not satisfying the + // condition of "ready" assertEquals(RECORD_MESSAGE_2, iterator.next()); assertEquals(RECORD_MESSAGE_3, iterator.next()); - // emits the first state message since the iterator has changed cursorFields (2 -> 3) and met the frequency minimum of 1 record + // emits the first state message since the iterator has changed cursorFields (2 -> 3) and met the + // frequency minimum of 1 record assertEquals(STATE_MESSAGE_2, iterator.next()); // no further records to read since Exception was caught above and marked iterator as endOfData() assertFalse(iterator.hasNext()); @@ -210,8 +213,10 @@ void testIteratorCatchesExceptionWhenEmissionFrequencyZero() { assertEquals(RECORD_MESSAGE_2, iterator.next()); assertEquals(RECORD_MESSAGE_2, iterator.next()); assertEquals(RECORD_MESSAGE_3, iterator.next()); - // since stateEmission is not set to emit frequently, this will catch the error but not emit state message since it wasn't in a ready state - // of having a frequency > 0 but will prevent an exception from causing the iterator to fail by marking iterator as endOfData() + // since stateEmission is not set to emit frequently, this will catch the error but not emit state + // message since it wasn't in a ready state + // of having a frequency > 0 but will prevent an exception from causing the iterator to fail by + // marking iterator as endOfData() assertFalse(iterator.hasNext()); } From 13a238c2d53ef88d73cacc590cadc86f369317b4 Mon Sep 17 00:00:00 2001 From: Serhii Chvaliuk Date: Mon, 5 Sep 2022 13:03:35 +0300 Subject: [PATCH 020/200] =?UTF-8?q?=F0=9F=8E=89=20Source=20Amazon=20Ads:?= =?UTF-8?q?=20improve=20`config.start=5Fdate`=20validation=20(#16191)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergey Chvalyuk --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-amazon-ads/Dockerfile | 2 +- .../source_amazon_ads/source.py | 21 ++++++++++----- .../streams/report_streams/report_streams.py | 6 +---- .../source-amazon-ads/unit_tests/conftest.py | 14 +++++++++- .../unit_tests/test_report_streams.py | 12 +++++---- .../unit_tests/test_source.py | 27 +++++++++++++++++-- .../source-amazon-ads/unit_tests/utils.py | 20 ++++++++++++++ docs/integrations/sources/amazon-ads.md | 1 + 10 files changed, 84 insertions(+), 23 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index c730776d3fd1..b5b71c15558f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -17,7 +17,7 @@ - name: Amazon Ads sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246 dockerRepository: airbyte/source-amazon-ads - dockerImageTag: 0.1.18 + dockerImageTag: 0.1.19 documentationUrl: https://docs.airbyte.io/integrations/sources/amazon-ads icon: amazonads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 95c732a60e9b..c1e96e54aec1 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -87,7 +87,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-amazon-ads:0.1.18" +- dockerImage: "airbyte/source-amazon-ads:0.1.19" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile index ed940909275c..ab4778154832 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-amazon-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.18 +LABEL io.airbyte.version=0.1.19 LABEL io.airbyte.name=airbyte/source-amazon-ads diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py index 302d5925ec90..a03c781355e3 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py @@ -4,10 +4,9 @@ import logging -import os from typing import Any, List, Mapping, Optional, Tuple -from airbyte_cdk.connector import _WriteConfigProtocol +import pendulum from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.auth import Oauth2Authenticator @@ -36,16 +35,19 @@ # Oauth 2.0 authentication URL for amazon TOKEN_URL = "https://api.amazon.com/auth/o2/token" +CONFIG_DATE_FORMAT = "YYYY-MM-DD" class SourceAmazonAds(AbstractSource): - def configure(self: _WriteConfigProtocol, config: Mapping[str, Any], temp_dir: str) -> Mapping[str, Any]: + def _validate_and_transform(self, config: Mapping[str, Any]): + start_date = config.get("start_date") + if start_date: + config["start_date"] = pendulum.from_format(start_date, CONFIG_DATE_FORMAT).date() + else: + config["start_date"] = None if not config.get("region"): source_spec = self.spec(logging.getLogger("airbyte")) - default_region = source_spec.connectionSpecification["properties"]["region"]["default"] - config["region"] = default_region - config_path = os.path.join(temp_dir, "config.json") - self.write_config(config, config_path) + config["region"] = source_spec.connectionSpecification["properties"]["region"]["default"] return config def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: @@ -54,6 +56,10 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ + try: + config = self._validate_and_transform(config) + except Exception as e: + return False, str(e) # Check connection by sending list of profiles request. Its most simple # request, not require additional parameters and usually has few data # in response body. @@ -67,6 +73,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: :param config: A Mapping of the user input configuration as defined in the connector spec. :return list of streams for current source """ + config = self._validate_and_transform(config) auth = self._make_authenticator(config) stream_args = {"config": config, "authenticator": auth} # All data for individual Amazon Ads stream divided into sets of data for diff --git a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py index 594d5c413fd3..35c8fab7d269 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py @@ -96,7 +96,6 @@ class ReportStream(BasicAmazonAdsStream, ABC): # (Service limits section) # Format used to specify metric generation date over Amazon Ads API. REPORT_DATE_FORMAT = "YYYYMMDD" - CONFIG_DATE_FORMAT = "YYYY-MM-DD" cursor_field = "reportDate" def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authenticator: Oauth2Authenticator): @@ -106,10 +105,7 @@ def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authentic self._model = self._generate_model() self.report_wait_timeout = config.get("report_wait_timeout", 30) self.report_generation_maximum_retries = config.get("report_generation_max_retries", 5) - # Set start date from config file - self._start_date = config.get("start_date") - if self._start_date: - self._start_date = pendulum.from_format(self._start_date, self.CONFIG_DATE_FORMAT).date() + self._start_date: Optional[Date] = config.get("start_date") super().__init__(config, profiles) @property diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py index 727f81d7001f..54cfe29dfefb 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/conftest.py @@ -2,6 +2,8 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from copy import deepcopy + from pytest import fixture @@ -10,12 +12,22 @@ def config(): return { "client_id": "test_client_id", "client_secret": "test_client_secret", - "scope": "test_scope", "refresh_token": "test_refresh", "region": "NA", } +@fixture +def config_gen(config): + def inner(**kwargs): + new_config = deepcopy(config) + # WARNING, no support deep dictionaries + new_config.update(kwargs) + return {k: v for k, v in new_config.items() if v is not ...} + + return inner + + @fixture def profiles_response(): return """ diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py index 9c106407b25c..985f79f66552 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py @@ -8,6 +8,7 @@ from functools import partial from unittest import mock +import pendulum import pytest import responses from airbyte_cdk.models import SyncMode @@ -16,6 +17,7 @@ from pytest import raises from requests.exceptions import ConnectionError from source_amazon_ads.schemas.profile import AccountInfo, Profile +from source_amazon_ads.source import CONFIG_DATE_FORMAT from source_amazon_ads.streams import ( SponsoredBrandsReportStream, SponsoredBrandsVideoReportStream, @@ -339,10 +341,10 @@ def test_display_report_stream_slices_incremental(config): def test_get_start_date(config): profiles = make_profiles() - config["start_date"] = "2021-07-10" + config["start_date"] = pendulum.from_format("2021-07-10", CONFIG_DATE_FORMAT).date() stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) assert stream.get_start_date(profiles[0], {}) == Date(2021, 7, 10) - config["start_date"] = "2021-05-10" + config["start_date"] = pendulum.from_format("2021-05-10", CONFIG_DATE_FORMAT).date() stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock()) assert stream.get_start_date(profiles[0], {}) == Date(2021, 6, 1) @@ -368,7 +370,7 @@ def test_stream_slices_different_timezones(config): def test_stream_slices_lazy_evaluation(config): with freeze_time("2022-06-01T23:50:00+00:00") as frozen_datetime: - config["start_date"] = "2021-05-10" + config["start_date"] = pendulum.from_format("2021-05-10", CONFIG_DATE_FORMAT).date() profile1 = Profile(profileId=1, timezone="UTC", accountInfo=AccountInfo(marketplaceStringId="", id="", type="seller")) profile2 = Profile(profileId=2, timezone="UTC", accountInfo=AccountInfo(marketplaceStringId="", id="", type="seller")) @@ -491,7 +493,7 @@ def test_read_incremental_without_records_start_date(config): ) profiles = make_profiles() - config["start_date"] = "2020-12-25" + config["start_date"] = pendulum.from_format("2020-12-25", CONFIG_DATE_FORMAT).date() stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock()) with freeze_time("2021-01-02 12:00:00") as frozen_datetime: @@ -514,7 +516,7 @@ def test_read_incremental_with_records_start_date(config): ) profiles = make_profiles() - config["start_date"] = "2020-12-25" + config["start_date"] = pendulum.from_format("2020-12-25", CONFIG_DATE_FORMAT).date() stream = SponsoredDisplayReportStream(config, profiles, authenticator=mock.MagicMock()) with freeze_time("2021-01-02 12:00:00") as frozen_datetime: diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py index 52876c71b9f7..4637ad670fa4 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py @@ -7,6 +7,8 @@ from jsonschema import Draft4Validator from source_amazon_ads import SourceAmazonAds +from .utils import command_check, url_strip_query + def setup_responses(): responses.add( @@ -39,12 +41,33 @@ def test_spec(): @responses.activate -def test_check(config): +def test_check(config_gen): setup_responses() source = SourceAmazonAds() - assert source.check(None, config) == AirbyteConnectionStatus(status=Status.SUCCEEDED) + + assert command_check(source, config_gen(start_date=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED) assert len(responses.calls) == 2 + assert command_check(source, config_gen(start_date="")) == AirbyteConnectionStatus(status=Status.SUCCEEDED) + assert len(responses.calls) == 4 + + assert source.check(None, config_gen(start_date="2022-02-20")) == AirbyteConnectionStatus(status=Status.SUCCEEDED) + assert len(responses.calls) == 6 + + assert command_check(source, config_gen(start_date="2022-20-02")) == AirbyteConnectionStatus( + status=Status.FAILED, message="'month must be in 1..12'" + ) + assert len(responses.calls) == 6 + + assert command_check(source, config_gen(start_date="no date")) == AirbyteConnectionStatus( + status=Status.FAILED, message="'String does not match format YYYY-MM-DD'" + ) + assert len(responses.calls) == 6 + + assert command_check(source, config_gen(region=...)) == AirbyteConnectionStatus(status=Status.SUCCEEDED) + assert len(responses.calls) == 8 + assert url_strip_query(responses.calls[7].request.url) == "https://advertising-api.amazon.com/v2/profiles" + @responses.activate def test_source_streams(config): diff --git a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py index 6c1c110d35df..71f5d8566f31 100644 --- a/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py +++ b/airbyte-integrations/connectors/source-amazon-ads/unit_tests/utils.py @@ -3,9 +3,14 @@ # from typing import Any, Iterator, MutableMapping +from unittest import mock +from urllib.parse import urlparse, urlunparse from airbyte_cdk.models import SyncMode +from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification +from airbyte_cdk.sources import Source from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, Any]) -> Iterator[dict]: @@ -23,3 +28,18 @@ def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str, if hasattr(stream_instance, "state"): stream_state.clear() stream_state.update(stream_instance.state) + + +def command_check(source: Source, config): + logger = mock.MagicMock() + connector_config, _ = split_config(config) + if source.check_config_against_spec: + source_spec: ConnectorSpecification = source.spec(logger) + check_config_against_spec_or_exit(connector_config, source_spec) + return source.check(logger, config) + + +def url_strip_query(url): + parsed_result = urlparse(url) + parsed_result = parsed_result._replace(query="") + return urlunparse(parsed_result) diff --git a/docs/integrations/sources/amazon-ads.md b/docs/integrations/sources/amazon-ads.md index d2be14f8eb0a..3f19f9795a57 100644 --- a/docs/integrations/sources/amazon-ads.md +++ b/docs/integrations/sources/amazon-ads.md @@ -90,6 +90,7 @@ Information about expected report generation waiting time you may find [here](ht | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------| +| 0.1.19 | 2022-08-31 | [16191](https://github.com/airbytehq/airbyte/pull/16191) | Improved connector's input configuration validation | | 0.1.18 | 2022-08-25 | [15951](https://github.com/airbytehq/airbyte/pull/15951) | Skip API error "Tactic T00020 is not supported for report API in marketplace A1C3SOZRARQ6R3." | | 0.1.17 | 2022-08-24 | [15921](https://github.com/airbytehq/airbyte/pull/15921) | Skip API error "Report date is too far in the past." | | 0.1.16 | 2022-08-23 | [15822](https://github.com/airbytehq/airbyte/pull/15822) | Set default value for 'region' if needed | From d49c3063e6445ad388a64ff8fc56576dd2410dbb Mon Sep 17 00:00:00 2001 From: letiescanciano <45267095+letiescanciano@users.noreply.github.com> Date: Mon, 5 Sep 2022 12:23:20 +0200 Subject: [PATCH 021/200] Add email to identify users analytics call (#16327) --- airbyte-webapp/src/packages/cloud/cloudRoutes.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx b/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx index 137b04702833..1ee4a26122f6 100644 --- a/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx +++ b/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx @@ -152,7 +152,7 @@ export const Routing: React.FC = () => { [user] ); useAnalyticsRegisterValues(analyticsContext); - useAnalyticsIdentifyUser(user?.userId, { providers }); + useAnalyticsIdentifyUser(user?.userId, { providers, email: user?.email }); useTrackPageAnalytics(); if (!inited) { From bdfafe5547febbf3eb77a4d4505eecfbe00e111b Mon Sep 17 00:00:00 2001 From: letiescanciano <45267095+letiescanciano@users.noreply.github.com> Date: Mon, 5 Sep 2022 13:28:18 +0200 Subject: [PATCH 022/200] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20Add=20pro?= =?UTF-8?q?per=20page=20view=20events=20for=20Segment=20(#16220)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🪟 🔧 Add proper page view events for Segment --- airbyte-webapp/package-lock.json | 11 ++++ airbyte-webapp/package.json | 1 + airbyte-webapp/src/config/types.ts | 4 +- .../core/analytics/AnalyticsService.test.ts | 12 ++++ .../src/core/analytics/AnalyticsService.ts | 4 +- airbyte-webapp/src/core/analytics/types.ts | 9 --- .../src/hooks/services/Analytics/index.tsx | 1 + .../services/Analytics/pageNameUtils.tsx | 61 ------------------- .../services/Analytics/pageTrackingCodes.tsx | 33 ++++++++++ .../Analytics/useTrackPageAnalytics.tsx | 18 ------ .../src/packages/cloud/cloudRoutes.tsx | 2 - .../cloud/views/FirebaseActionRoute.tsx | 2 + .../cloud/views/auth/LoginPage/LoginPage.tsx | 2 + .../ResetPasswordPage/ResetPasswordPage.tsx | 2 + .../views/auth/SignupPage/SignupPage.tsx | 3 + .../views/credits/CreditsPage/CreditsPage.tsx | 3 +- .../AccountSettingsView.tsx | 2 + .../UsersSettingsView/UsersSettingsView.tsx | 3 + .../WorkspaceSettingsView.tsx | 3 +- .../WorkspacesPage/WorkspacesPage.tsx | 3 + .../AllConnectionsPage/AllConnectionsPage.tsx | 2 + .../ConnectionItemPage/ConnectionItemPage.tsx | 3 +- .../components/ReplicationView.tsx | 3 + .../components/SettingsView.tsx | 2 + .../components/StatusView.tsx | 4 +- .../components/TransformationView.tsx | 2 + .../CreationFormPage/CreationFormPage.tsx | 2 + .../AllDestinationsPage.tsx | 2 + .../CreateDestinationPage.tsx | 4 +- .../DestinationItemPage.tsx | 2 + .../components/DestinationSettings.tsx | 3 + .../pages/OnboardingPage/OnboardingPage.tsx | 4 +- .../pages/PreferencesPage/PreferencesPage.tsx | 4 +- .../pages/ConnectorsPage/DestinationsPage.tsx | 3 + .../pages/ConnectorsPage/SourcesPage.tsx | 3 + .../pages/MetricsPage/MetricsPage.tsx | 2 + .../NotificationPage/NotificationPage.tsx | 3 + .../pages/AllSourcesPage/AllSourcesPage.tsx | 3 +- .../CreateSourcePage/CreateSourcePage.tsx | 2 + .../pages/SourceItemPage/SourceItemPage.tsx | 2 + .../components/SourceSettings.tsx | 2 + airbyte-webapp/src/pages/routes.tsx | 3 - 42 files changed, 131 insertions(+), 108 deletions(-) delete mode 100644 airbyte-webapp/src/hooks/services/Analytics/pageNameUtils.tsx create mode 100644 airbyte-webapp/src/hooks/services/Analytics/pageTrackingCodes.tsx delete mode 100644 airbyte-webapp/src/hooks/services/Analytics/useTrackPageAnalytics.tsx diff --git a/airbyte-webapp/package-lock.json b/airbyte-webapp/package-lock.json index 20c35e94e4c3..8b788d2035ac 100644 --- a/airbyte-webapp/package-lock.json +++ b/airbyte-webapp/package-lock.json @@ -18,6 +18,7 @@ "@monaco-editor/react": "^4.4.5", "@sentry/react": "^6.19.6", "@sentry/tracing": "^6.19.6", + "@types/segment-analytics": "^0.0.34", "classnames": "^2.3.1", "dayjs": "^1.11.3", "firebase": "^9.8.2", @@ -14690,6 +14691,11 @@ "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==" }, + "node_modules/@types/segment-analytics": { + "version": "0.0.34", + "resolved": "https://registry.npmjs.org/@types/segment-analytics/-/segment-analytics-0.0.34.tgz", + "integrity": "sha512-fiOyEgyqJY2Mv9k72WG4XoY4fVE31byiSUrEFcNh+MgHcH3HuJmoz2J7ktO3YizBrN6/RuaH1tY5J/5I5BJHJQ==" + }, "node_modules/@types/serve-index": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/@types/serve-index/-/serve-index-1.9.1.tgz", @@ -58880,6 +58886,11 @@ "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", "integrity": "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==" }, + "@types/segment-analytics": { + "version": "0.0.34", + "resolved": "https://registry.npmjs.org/@types/segment-analytics/-/segment-analytics-0.0.34.tgz", + "integrity": "sha512-fiOyEgyqJY2Mv9k72WG4XoY4fVE31byiSUrEFcNh+MgHcH3HuJmoz2J7ktO3YizBrN6/RuaH1tY5J/5I5BJHJQ==" + }, "@types/serve-index": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/@types/serve-index/-/serve-index-1.9.1.tgz", diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index ea091e12e7dc..c1b9eaf54348 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -34,6 +34,7 @@ "@monaco-editor/react": "^4.4.5", "@sentry/react": "^6.19.6", "@sentry/tracing": "^6.19.6", + "@types/segment-analytics": "^0.0.34", "classnames": "^2.3.1", "dayjs": "^1.11.3", "firebase": "^9.8.2", diff --git a/airbyte-webapp/src/config/types.ts b/airbyte-webapp/src/config/types.ts index 350268cdcde6..cb97cee2f51a 100644 --- a/airbyte-webapp/src/config/types.ts +++ b/airbyte-webapp/src/config/types.ts @@ -1,5 +1,3 @@ -import { SegmentAnalytics } from "core/analytics/types"; - import { OutboundLinks } from "./links"; declare global { @@ -14,7 +12,7 @@ declare global { REACT_APP_INTEGRATION_DOCS_URLS?: string; SEGMENT_TOKEN?: string; LAUNCHDARKLY_KEY?: string; - analytics: SegmentAnalytics; + analytics: SegmentAnalytics.AnalyticsJS; } } diff --git a/airbyte-webapp/src/core/analytics/AnalyticsService.test.ts b/airbyte-webapp/src/core/analytics/AnalyticsService.test.ts index 2edf85729ad3..de3ab037a67c 100644 --- a/airbyte-webapp/src/core/analytics/AnalyticsService.test.ts +++ b/airbyte-webapp/src/core/analytics/AnalyticsService.test.ts @@ -10,6 +10,18 @@ describe("AnalyticsService", () => { identify: jest.fn(), page: jest.fn(), reset: jest.fn(), + user: jest.fn(), + setAnonymousId: jest.fn(), + init: jest.fn(), + use: jest.fn(), + addIntegration: jest.fn(), + load: jest.fn(), + trackLink: jest.fn(), + trackForm: jest.fn(), + ready: jest.fn(), + debug: jest.fn(), + on: jest.fn(), + timeout: jest.fn(), }; }); diff --git a/airbyte-webapp/src/core/analytics/AnalyticsService.ts b/airbyte-webapp/src/core/analytics/AnalyticsService.ts index 9388330bad63..7dc1dc75dd4d 100644 --- a/airbyte-webapp/src/core/analytics/AnalyticsService.ts +++ b/airbyte-webapp/src/core/analytics/AnalyticsService.ts @@ -1,9 +1,9 @@ -import { Action, EventParams, Namespace, SegmentAnalytics } from "./types"; +import { Action, EventParams, Namespace } from "./types"; export class AnalyticsService { constructor(private context: Record, private version?: string) {} - private getSegmentAnalytics = (): SegmentAnalytics | undefined => window.analytics; + private getSegmentAnalytics = (): SegmentAnalytics.AnalyticsJS | undefined => window.analytics; alias = (newId: string): void => this.getSegmentAnalytics()?.alias?.(newId); diff --git a/airbyte-webapp/src/core/analytics/types.ts b/airbyte-webapp/src/core/analytics/types.ts index 0150e570b307..92aa3994135f 100644 --- a/airbyte-webapp/src/core/analytics/types.ts +++ b/airbyte-webapp/src/core/analytics/types.ts @@ -1,12 +1,3 @@ -export interface SegmentAnalytics { - page: (name?: string) => void; - reset: () => void; - alias: (newId: string) => void; - track: (name: string, properties: Record) => void; - identify: (userId?: string, traits?: Record) => void; - group: (organisationId: string, traits: Record) => void; -} - export const enum Namespace { SOURCE = "Source", DESTINATION = "Destination", diff --git a/airbyte-webapp/src/hooks/services/Analytics/index.tsx b/airbyte-webapp/src/hooks/services/Analytics/index.tsx index 003a962f4d8b..b46338e5090c 100644 --- a/airbyte-webapp/src/hooks/services/Analytics/index.tsx +++ b/airbyte-webapp/src/hooks/services/Analytics/index.tsx @@ -1 +1,2 @@ export * from "./useAnalyticsService"; +export * from "./pageTrackingCodes"; diff --git a/airbyte-webapp/src/hooks/services/Analytics/pageNameUtils.tsx b/airbyte-webapp/src/hooks/services/Analytics/pageNameUtils.tsx deleted file mode 100644 index 26f402b9fd7c..000000000000 --- a/airbyte-webapp/src/hooks/services/Analytics/pageNameUtils.tsx +++ /dev/null @@ -1,61 +0,0 @@ -import { SettingsRoute } from "pages/SettingsPage/SettingsPage"; - -import { RoutePaths } from "../../../pages/routePaths"; - -const getPageName = (pathname: string): string => { - const itemSourcePageRegex = new RegExp(`${RoutePaths.Source}/.*`); - const itemDestinationPageRegex = new RegExp(`${RoutePaths.Destination}/.*`); - const itemSourceToDestinationPageRegex = new RegExp( - `(${RoutePaths.Source}|${RoutePaths.Destination})${RoutePaths.Connection}/.*` - ); - - if (pathname === RoutePaths.Destination) { - return "Destinations Page"; - } - if (pathname === RoutePaths.Source) { - return "Sources Page"; - } - if (pathname === `${RoutePaths.Source}/${RoutePaths.SourceNew}`) { - return "Create Source Page"; - } - if (pathname === `${RoutePaths.Destination}/${RoutePaths.DestinationNew}`) { - return "Create Destination Page"; - } - if ( - pathname === `${RoutePaths.Source}/${RoutePaths.ConnectionNew}` || - pathname === `${RoutePaths.Destination}/${RoutePaths.ConnectionNew}` - ) { - return "Create Connection Page"; - } - if (pathname.match(itemSourceToDestinationPageRegex)) { - return "Source to Destination Page"; - } - if (pathname.match(itemDestinationPageRegex)) { - return "Destination Item Page"; - } - if (pathname.match(itemSourcePageRegex)) { - return "Source Item Page"; - } - if (pathname === `${RoutePaths.Settings}/${SettingsRoute.Source}`) { - return "Settings Sources Connectors Page"; - } - if (pathname === `${RoutePaths.Settings}/${SettingsRoute.Destination}`) { - return "Settings Destinations Connectors Page"; - } - if (pathname === `${RoutePaths.Settings}/${SettingsRoute.Configuration}`) { - return "Settings Configuration Page"; - } - if (pathname === `${RoutePaths.Settings}/${SettingsRoute.Notifications}`) { - return "Settings Notifications Page"; - } - if (pathname === `${RoutePaths.Settings}/${SettingsRoute.Metrics}`) { - return "Settings Metrics Page"; - } - if (pathname === RoutePaths.Connections) { - return "Connections Page"; - } - - return ""; -}; - -export { getPageName }; diff --git a/airbyte-webapp/src/hooks/services/Analytics/pageTrackingCodes.tsx b/airbyte-webapp/src/hooks/services/Analytics/pageTrackingCodes.tsx new file mode 100644 index 000000000000..0099472cfd1b --- /dev/null +++ b/airbyte-webapp/src/hooks/services/Analytics/pageTrackingCodes.tsx @@ -0,0 +1,33 @@ +export enum PageTrackingCodes { + SIGNUP = "Auth.Signup", + LOGIN = "Auth.Login", + RESET_PASSWORD = "Auth.ResetPassword", + VERIFY_EMAIL = "Auth.VerifyEmail", + ONBOARDING = "Onboarding", + SOURCE_NEW = "Source.New", + SOURCE_LIST = "Source.List", + SOURCE_ITEM = "Source.Item", + SOURCE_ITEM_SETTINGS = "Source.Item.Settings", + DESTINATION_NEW = "Destination.New", + DESTINATION_LIST = "Destination.List", + DESTINATION_ITEM = "Destination.Item", + DESTINATION_ITEM_SETTINGS = "Destination.Item.Settings", + CONNECTIONS_NEW = "Connections.New", + CONNECTIONS_LIST = "Connections.List", + CONNECTIONS_ITEM = "Connections.Item", + CONNECTIONS_ITEM_STATUS = "Connections.Item.Status", + CONNECTIONS_ITEM_TRANSFORMATION = "Connections.Item.TransformationView", + CONNECTIONS_ITEM_REPLICATION = "Connections.Item.ReplicationView", + CONNECTIONS_ITEM_SETTINGS = "Connections.Item.Settings", + SETTINGS_ACCOUNT = "Settings.Account", + SETTINGS_WORKSPACE = "Settings.Workspace", + SETTINGS_DESTINATION = "Settings.Destination", + SETTINGS_SOURCE = "Settings.Source", + SETTINGS_CONFIGURATION = "Settings.Configuration", + SETTINGS_NOTIFICATION = "Settings.Notifications", + SETTINGS_ACCESS_MANAGEMENT = "Settings.AccessManagement", + SETTINGS_METRICS = "Settings.Metrics", + CREDITS = "Credits", + WORKSPACES = "Workspaces", + PREFERENCES = "Preferences", +} diff --git a/airbyte-webapp/src/hooks/services/Analytics/useTrackPageAnalytics.tsx b/airbyte-webapp/src/hooks/services/Analytics/useTrackPageAnalytics.tsx deleted file mode 100644 index 434ad12ef243..000000000000 --- a/airbyte-webapp/src/hooks/services/Analytics/useTrackPageAnalytics.tsx +++ /dev/null @@ -1,18 +0,0 @@ -import { useEffect } from "react"; - -import useRouter from "hooks/useRouter"; - -import { getPageName } from "./pageNameUtils"; -import { useAnalyticsService } from "./useAnalyticsService"; - -export const useTrackPageAnalytics = () => { - const { pathname } = useRouter(); - const analyticsService = useAnalyticsService(); - useEffect(() => { - const pathWithoutWorkspaceId = pathname.split("/").splice(2).join("."); - const pageName = getPageName(pathWithoutWorkspaceId); - if (pageName) { - analyticsService.page(pageName); - } - }, [analyticsService, pathname]); -}; diff --git a/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx b/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx index 1ee4a26122f6..f1c85db4d459 100644 --- a/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx +++ b/airbyte-webapp/src/packages/cloud/cloudRoutes.tsx @@ -6,7 +6,6 @@ import ApiErrorBoundary from "components/ApiErrorBoundary"; import LoadingPage from "components/LoadingPage"; import { useAnalyticsIdentifyUser, useAnalyticsRegisterValues } from "hooks/services/Analytics/useAnalyticsService"; -import { useTrackPageAnalytics } from "hooks/services/Analytics/useTrackPageAnalytics"; import { FeatureItem, FeatureSet, useFeatureService } from "hooks/services/Feature"; import { useApiHealthPoll } from "hooks/services/Health"; import { OnboardingServiceProvider } from "hooks/services/Onboarding"; @@ -153,7 +152,6 @@ export const Routing: React.FC = () => { ); useAnalyticsRegisterValues(analyticsContext); useAnalyticsIdentifyUser(user?.userId, { providers, email: user?.email }); - useTrackPageAnalytics(); if (!inited) { return ; diff --git a/airbyte-webapp/src/packages/cloud/views/FirebaseActionRoute.tsx b/airbyte-webapp/src/packages/cloud/views/FirebaseActionRoute.tsx index f440bdb47e5e..de44efa7d0e2 100644 --- a/airbyte-webapp/src/packages/cloud/views/FirebaseActionRoute.tsx +++ b/airbyte-webapp/src/packages/cloud/views/FirebaseActionRoute.tsx @@ -5,6 +5,7 @@ import { useAsync } from "react-use"; import LoadingPage from "components/LoadingPage"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useNotificationService } from "hooks/services/Notification"; import useRouter from "hooks/useRouter"; import { useAuthService } from "packages/cloud/services/auth/AuthService"; @@ -26,6 +27,7 @@ export const VerifyEmailAction: React.FC = () => { const { formatMessage } = useIntl(); const { registerNotification } = useNotificationService(); + useTrackPage(PageTrackingCodes.VERIFY_EMAIL); useAsync(async () => { if (query.mode === FirebaseActionMode.VERIFY_EMAIL) { // Send verification code to authentication service diff --git a/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx index ca6275378ba2..654645e82ed2 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx @@ -6,6 +6,7 @@ import * as yup from "yup"; import { LabeledInput, Link, LoadingButton } from "components"; import HeadTitle from "components/HeadTitle"; +import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; import useRouter from "hooks/useRouter"; import { CloudRoutes } from "packages/cloud/cloudRoutes"; import { FieldError } from "packages/cloud/lib/errors/FieldError"; @@ -26,6 +27,7 @@ const LoginPage: React.FC = () => { const { formatMessage } = useIntl(); const { login } = useAuthService(); const { query, replace } = useRouter(); + useTrackPage(PageTrackingCodes.LOGIN); return (

diff --git a/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx index 67e557c108f3..c1bef301b897 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx @@ -6,6 +6,7 @@ import * as yup from "yup"; import { LoadingButton, LabeledInput, Link } from "components"; import HeadTitle from "components/HeadTitle"; +import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; import { useNotificationService } from "hooks/services/Notification/NotificationService"; import { useAuthService } from "packages/cloud/services/auth/AuthService"; @@ -22,6 +23,7 @@ const ResetPasswordPage: React.FC = () => { const { registerNotification } = useNotificationService(); const { formatMessage } = useIntl(); + useTrackPage(PageTrackingCodes.RESET_PASSWORD); return (
diff --git a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/SignupPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/SignupPage.tsx index 3464b868fb33..ac266480f2ad 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/SignupPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/SignupPage.tsx @@ -4,6 +4,8 @@ import { FormattedMessage } from "react-intl"; import { Text } from "components/base/Text"; import HeadTitle from "components/HeadTitle"; +import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; + import { OAuthLogin } from "../OAuthLogin"; import { Disclaimer, SignupForm } from "./components/SignupForm"; import SpecialBlock from "./components/SpecialBlock"; @@ -14,6 +16,7 @@ interface SignupPageProps { } const SignupPage: React.FC = ({ highlightStyle }) => { + useTrackPage(PageTrackingCodes.SIGNUP); return (
diff --git a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/CreditsPage.tsx b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/CreditsPage.tsx index 9b098dfac48d..00bcf92a30d4 100644 --- a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/CreditsPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/CreditsPage.tsx @@ -6,6 +6,7 @@ import { PageTitle } from "components"; import HeadTitle from "components/HeadTitle"; import MainPageWithScroll from "components/MainPageWithScroll"; +import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; import { useAuthService } from "packages/cloud/services/auth/AuthService"; import CreditsUsage from "./components/CreditsUsage"; @@ -23,7 +24,7 @@ const EmailVerificationHintWithMargin = styled(EmailVerificationHint)` const CreditsPage: React.FC = () => { const { emailVerified } = useAuthService(); - + useTrackPage(PageTrackingCodes.CREDITS); return ( } diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx index 25ef797ee9c3..e73690a7eff4 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx @@ -5,6 +5,7 @@ import styled from "styled-components"; import { LoadingButton } from "components"; +import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; import { useAuthService } from "packages/cloud/services/auth/AuthService"; import { SettingsCard } from "pages/SettingsPage/pages/SettingsComponents"; @@ -19,6 +20,7 @@ const AccountSettingsView: React.FC = () => { const authService = useAuthService(); const { mutateAsync: logout, isLoading: isLoggingOut } = useMutation(() => authService.logout()); + useTrackPage(PageTrackingCodes.SETTINGS_ACCOUNT); return ( <> diff --git a/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx index 4fba32a03b70..5dab216dd7b4 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/UsersSettingsView/UsersSettingsView.tsx @@ -6,6 +6,7 @@ import { useToggle } from "react-use"; import { Button, H5, LoadingButton } from "components"; import Table from "components/Table"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; import { User } from "packages/cloud/lib/domain/users"; @@ -41,6 +42,8 @@ const RemoveUserSection: React.FC<{ workspaceId: string; email: string }> = ({ w }; export const UsersSettingsView: React.FC = () => { + useTrackPage(PageTrackingCodes.SETTINGS_ACCESS_MANAGEMENT); + const [modalIsOpen, toggleModal] = useToggle(false); const { workspaceId } = useCurrentWorkspace(); diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx index 451ebeef8bdc..855d9e7a6149 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx @@ -5,6 +5,7 @@ import styled from "styled-components"; import { Button, LabeledInput, LoadingButton } from "components"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; import { useRemoveWorkspace, @@ -33,7 +34,7 @@ const Buttons = styled.div` export const WorkspaceSettingsView: React.FC = () => { const { formatMessage } = useIntl(); - + useTrackPage(PageTrackingCodes.SETTINGS_WORKSPACE); const { exitWorkspace } = useWorkspaceService(); const workspace = useCurrentWorkspace(); const removeWorkspace = useRemoveWorkspace(); diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/WorkspacesPage.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/WorkspacesPage.tsx index a421abd8b628..f0cb410b5640 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/WorkspacesPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/WorkspacesPage.tsx @@ -4,6 +4,8 @@ import styled from "styled-components"; import { H1, H3 } from "components"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; + import WorkspacesList from "./components/WorkspacesList"; const MainContent = styled.div` @@ -25,6 +27,7 @@ const Subtitle = styled(H3)` `; const WorkspacesPage: React.FC = () => { + useTrackPage(PageTrackingCodes.WORKSPACES); return ( diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/AllConnectionsPage.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/AllConnectionsPage.tsx index 624eccd483c6..042fe7473e25 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/AllConnectionsPage.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/AllConnectionsPage/AllConnectionsPage.tsx @@ -5,6 +5,7 @@ import { Button, LoadingPage, MainPageWithScroll, PageTitle } from "components"; import { EmptyResourceListView } from "components/EmptyResourceListView"; import HeadTitle from "components/HeadTitle"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { FeatureItem, useFeature } from "hooks/services/Feature"; import { useConnectionList } from "hooks/services/useConnectionHook"; import useRouter from "hooks/useRouter"; @@ -15,6 +16,7 @@ import ConnectionsTable from "./components/ConnectionsTable"; const AllConnectionsPage: React.FC = () => { const { push } = useRouter(); + useTrackPage(PageTrackingCodes.CONNECTIONS_LIST); const { connections } = useConnectionList(); const allowCreateConnection = useFeature(FeatureItem.AllowCreateConnection); diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx index 15586bfd2428..646253cc47a3 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx @@ -7,7 +7,7 @@ import HeadTitle from "components/HeadTitle"; import { getFrequencyType } from "config/utils"; import { Action, Namespace } from "core/analytics"; import { ConnectionStatus } from "core/request/AirbyteClient"; -import { useAnalyticsService } from "hooks/services/Analytics"; +import { useAnalyticsService, useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useGetConnection } from "hooks/services/useConnectionHook"; import TransformationView from "pages/ConnectionPage/pages/ConnectionItemPage/components/TransformationView"; @@ -28,6 +28,7 @@ const ConnectionItemPage: React.FC = () => { const [isStatusUpdating, setStatusUpdating] = useState(false); const analyticsService = useAnalyticsService(); + useTrackPage(PageTrackingCodes.CONNECTIONS_ITEM); const { source, destination } = connection; const onAfterSaveSchema = () => { diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx index 367433ef0ece..665fa532c968 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx @@ -10,6 +10,7 @@ import LoadingSchema from "components/LoadingSchema"; import { toWebBackendConnectionUpdate } from "core/domain/connection"; import { ConnectionStateType, ConnectionStatus } from "core/request/AirbyteClient"; +import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { useModalService } from "hooks/services/Modal"; import { @@ -89,6 +90,8 @@ export const ReplicationView: React.FC = ({ onAfterSaveSch const [activeUpdatingSchemaMode, setActiveUpdatingSchemaMode] = useState(false); const [saved, setSaved] = useState(false); const connectionService = useConnectionService(); + useTrackPage(PageTrackingCodes.CONNECTIONS_ITEM_REPLICATION); + const { mutateAsync: updateConnection } = useUpdateConnection(); const { connection: initialConnection, refreshConnectionCatalog } = useConnectionLoad(connectionId); diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx index d882015ccc88..866b0d65cc41 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx @@ -2,6 +2,7 @@ import React from "react"; import DeleteBlock from "components/DeleteBlock"; +import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; import { useDeleteConnection } from "hooks/services/useConnectionHook"; import { WebBackendConnectionRead } from "../../../../../core/request/AirbyteClient"; @@ -15,6 +16,7 @@ interface SettingsViewProps { const SettingsView: React.FC = ({ connection }) => { const { mutateAsync: deleteConnection } = useDeleteConnection(); + useTrackPage(PageTrackingCodes.CONNECTIONS_ITEM_SETTINGS); const onDelete = () => deleteConnection(connection); return ( diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx index c4cba57124ef..4b07bfa9fed7 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx @@ -12,7 +12,7 @@ import { getFrequencyType } from "config/utils"; import { Action, Namespace } from "core/analytics"; import { ConnectionStatus, JobWithAttemptsRead, WebBackendConnectionRead } from "core/request/AirbyteClient"; import Status from "core/statuses"; -import { useAnalyticsService } from "hooks/services/Analytics"; +import { useTrackPage, PageTrackingCodes, useAnalyticsService } from "hooks/services/Analytics"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { FeatureItem, useFeature } from "hooks/services/Feature"; import { useResetConnection, useSyncConnection } from "hooks/services/useConnectionHook"; @@ -47,10 +47,10 @@ const getJobRunningOrPending = (jobs: JobWithAttemptsRead[]) => { }; const StatusView: React.FC = ({ connection }) => { + useTrackPage(PageTrackingCodes.CONNECTIONS_ITEM_STATUS); const [activeJob, setActiveJob] = useState(); const [jobPageSize, setJobPageSize] = useState(JOB_PAGE_SIZE_INCREMENT); const analyticsService = useAnalyticsService(); - const { jobs, isPreviousData: isJobPageLoading } = useListJobs({ configId: connection.connectionId, configTypes: ["sync", "reset_connection"], diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/TransformationView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/TransformationView.tsx index f3f211b8bae6..9733bafe175d 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/TransformationView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/TransformationView.tsx @@ -7,6 +7,7 @@ import styled from "styled-components"; import { ContentCard, H4 } from "components"; import { buildConnectionUpdate, NormalizationType } from "core/domain/connection"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { FeatureItem, useFeature } from "hooks/services/Feature"; import { useUpdateConnection } from "hooks/services/useConnectionHook"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; @@ -125,6 +126,7 @@ const TransformationView: React.FC = ({ connection }) = const { mutateAsync: updateConnection } = useUpdateConnection(); const workspace = useCurrentWorkspace(); + useTrackPage(PageTrackingCodes.CONNECTIONS_ITEM_TRANSFORMATION); const { supportsNormalization } = definition; const supportsDbt = useFeature(FeatureItem.AllowCustomDBT) && definition.supportsDbt; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/CreationFormPage.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/CreationFormPage.tsx index ece8c4bba86e..5ac60770b562 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/CreationFormPage.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/CreationFormPage.tsx @@ -8,6 +8,7 @@ import CreateConnectionContent from "components/CreateConnectionContent"; import HeadTitle from "components/HeadTitle"; import StepsMenu from "components/StepsMenu"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; import { useGetDestination } from "hooks/services/useDestinationHook"; import { useGetSource } from "hooks/services/useSourceHook"; @@ -70,6 +71,7 @@ function usePreloadData(): { } export const CreationFormPage: React.FC = () => { + useTrackPage(PageTrackingCodes.CONNECTIONS_NEW); const { location, push } = useRouter(); const { clearAllFormChanges } = useFormChangeTrackerService(); diff --git a/airbyte-webapp/src/pages/DestinationPage/pages/AllDestinationsPage/AllDestinationsPage.tsx b/airbyte-webapp/src/pages/DestinationPage/pages/AllDestinationsPage/AllDestinationsPage.tsx index f7bbd69a2487..f83304301a3a 100644 --- a/airbyte-webapp/src/pages/DestinationPage/pages/AllDestinationsPage/AllDestinationsPage.tsx +++ b/airbyte-webapp/src/pages/DestinationPage/pages/AllDestinationsPage/AllDestinationsPage.tsx @@ -6,6 +6,7 @@ import { EmptyResourceListView } from "components/EmptyResourceListView"; import HeadTitle from "components/HeadTitle"; import PageTitle from "components/PageTitle"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useDestinationList } from "hooks/services/useDestinationHook"; import useRouter from "hooks/useRouter"; @@ -15,6 +16,7 @@ import DestinationsTable from "./components/DestinationsTable"; const AllDestinationsPage: React.FC = () => { const { push } = useRouter(); const { destinations } = useDestinationList(); + useTrackPage(PageTrackingCodes.DESTINATION_LIST); const onCreateDestination = () => push(`${RoutePaths.DestinationNew}`); diff --git a/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/CreateDestinationPage.tsx b/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/CreateDestinationPage.tsx index 8376cf6c6489..9bf79734eece 100644 --- a/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/CreateDestinationPage.tsx +++ b/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/CreateDestinationPage.tsx @@ -6,6 +6,7 @@ import HeadTitle from "components/HeadTitle"; import PageTitle from "components/PageTitle"; import { ConnectionConfiguration } from "core/domain/connection"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useCreateDestination } from "hooks/services/useDestinationHook"; import useRouter from "hooks/useRouter"; import { useDestinationDefinitionList } from "services/connector/DestinationDefinitionService"; @@ -14,9 +15,10 @@ import { ConnectorDocumentationWrapper } from "views/Connector/ConnectorDocument import { DestinationForm } from "./components/DestinationForm"; export const CreateDestinationPage: React.FC = () => { + useTrackPage(PageTrackingCodes.DESTINATION_NEW); + const { push } = useRouter(); const [successRequest, setSuccessRequest] = useState(false); - const { destinationDefinitions } = useDestinationDefinitionList(); const { mutateAsync: createDestination } = useCreateDestination(); diff --git a/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/DestinationItemPage.tsx b/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/DestinationItemPage.tsx index 4624ebcc3ee2..2bb7c4720e12 100644 --- a/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/DestinationItemPage.tsx +++ b/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/DestinationItemPage.tsx @@ -10,6 +10,7 @@ import { ConnectorIcon } from "components/ConnectorIcon"; import HeadTitle from "components/HeadTitle"; import Placeholder, { ResourceTypes } from "components/Placeholder"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useConnectionList } from "hooks/services/useConnectionHook"; import { useSourceList } from "hooks/services/useSourceHook"; import useRouter from "hooks/useRouter"; @@ -24,6 +25,7 @@ import DestinationConnectionTable from "./components/DestinationConnectionTable" import DestinationSettings from "./components/DestinationSettings"; const DestinationItemPage: React.FC = () => { + useTrackPage(PageTrackingCodes.DESTINATION_ITEM); const { params, push } = useRouter(); const currentStep = useMemo(() => (params["*"] === "" ? StepsTypes.OVERVIEW : params["*"]), [params]); diff --git a/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationSettings.tsx b/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationSettings.tsx index 8dc229acd031..39e2e9ac61fe 100644 --- a/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationSettings.tsx +++ b/airbyte-webapp/src/pages/DestinationPage/pages/DestinationItemPage/components/DestinationSettings.tsx @@ -6,6 +6,7 @@ import DeleteBlock from "components/DeleteBlock"; import { ConnectionConfiguration } from "core/domain/connection"; import { Connector } from "core/domain/connector"; import { DestinationRead, WebBackendConnectionRead } from "core/request/AirbyteClient"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; import { useDeleteDestination, useUpdateDestination } from "hooks/services/useDestinationHook"; import { useDestinationDefinition } from "services/connector/DestinationDefinitionService"; @@ -30,6 +31,8 @@ const DestinationsSettings: React.FC = ({ const formId = useUniqueFormId(); const { clearFormChange } = useFormChangeTrackerService(); + useTrackPage(PageTrackingCodes.DESTINATION_ITEM_SETTINGS); + const onSubmitForm = async (values: { name: string; serviceType: string; diff --git a/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx b/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx index be82a8631f74..86147dd59c91 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx @@ -8,7 +8,7 @@ import ApiErrorBoundary from "components/ApiErrorBoundary"; import HeadTitle from "components/HeadTitle"; import LoadingPage from "components/LoadingPage"; -import { useAnalyticsService } from "hooks/services/Analytics/useAnalyticsService"; +import { useAnalyticsService, useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import useWorkspace from "hooks/services/useWorkspace"; import useRouterHook from "hooks/useRouter"; import { useCurrentWorkspaceState } from "services/workspaces/WorkspacesService"; @@ -63,6 +63,8 @@ const TITLE_BY_STEP: Partial> = { const OnboardingPage: React.FC = () => { const analyticsService = useAnalyticsService(); + useTrackPage(PageTrackingCodes.ONBOARDING); + const { push } = useRouterHook(); useEffectOnce(() => { diff --git a/airbyte-webapp/src/pages/PreferencesPage/PreferencesPage.tsx b/airbyte-webapp/src/pages/PreferencesPage/PreferencesPage.tsx index a1b2e049bd27..c0d52a8ed6e4 100644 --- a/airbyte-webapp/src/pages/PreferencesPage/PreferencesPage.tsx +++ b/airbyte-webapp/src/pages/PreferencesPage/PreferencesPage.tsx @@ -6,7 +6,7 @@ import { H1 } from "components"; import { PageViewContainer } from "components/CenteredPageComponents"; import HeadTitle from "components/HeadTitle"; -import { useTrackPage } from "hooks/services/Analytics/useAnalyticsService"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import useWorkspace from "hooks/services/useWorkspace"; import { PreferencesForm } from "views/Settings/PreferencesForm"; @@ -15,7 +15,7 @@ const Title = styled(H1)` `; const PreferencesPage: React.FC = () => { - useTrackPage("Preferences Page"); + useTrackPage(PageTrackingCodes.PREFERENCES); const { setInitialSetupConfig } = useWorkspace(); diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx index 2052c5f7c981..08e228f7741e 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx @@ -3,6 +3,7 @@ import { useIntl } from "react-intl"; import { useAsyncFn } from "react-use"; import { DestinationDefinitionRead } from "core/request/AirbyteClient"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import useConnector from "hooks/services/useConnector"; import { useDestinationDefinitionList, @@ -13,6 +14,8 @@ import { useDestinationList } from "../../../../hooks/services/useDestinationHoo import ConnectorsView from "./components/ConnectorsView"; const DestinationsPage: React.FC = () => { + useTrackPage(PageTrackingCodes.SETTINGS_DESTINATION); + const [isUpdateSuccess, setIsUpdateSuccess] = useState(false); const { formatMessage } = useIntl(); const { destinationDefinitions } = useDestinationDefinitionList(); diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx index 775b0900cabb..8caceba30a50 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx @@ -3,6 +3,7 @@ import { useIntl } from "react-intl"; import { useAsyncFn } from "react-use"; import { SourceDefinitionRead } from "core/request/AirbyteClient"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import useConnector from "hooks/services/useConnector"; import { useSourceList } from "hooks/services/useSourceHook"; import { useSourceDefinitionList, useUpdateSourceDefinition } from "services/connector/SourceDefinitionService"; @@ -10,6 +11,8 @@ import { useSourceDefinitionList, useUpdateSourceDefinition } from "services/con import ConnectorsView from "./components/ConnectorsView"; const SourcesPage: React.FC = () => { + useTrackPage(PageTrackingCodes.SETTINGS_SOURCE); + const [isUpdateSuccess, setIsUpdateSucces] = useState(false); const [feedbackList, setFeedbackList] = useState>({}); diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/MetricsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/MetricsPage.tsx index 04efca5b7ea6..957261bd6405 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/MetricsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/MetricsPage.tsx @@ -3,6 +3,7 @@ import { FormattedMessage } from "react-intl"; import HeadTitle from "components/HeadTitle"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useCurrentWorkspace } from "services/workspaces/WorkspacesService"; import useWorkspaceEditor from "../../components/useWorkspaceEditor"; @@ -13,6 +14,7 @@ const MetricsPage: React.FC = () => { const workspace = useCurrentWorkspace(); const { errorMessage, successMessage, loading, updateData } = useWorkspaceEditor(); + useTrackPage(PageTrackingCodes.SETTINGS_METRICS); const onChange = async (data: { anonymousDataCollection: boolean }) => { await updateData({ ...workspace, ...data, news: !!workspace.news, securityUpdates: !!workspace.securityUpdates }); }; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/NotificationPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/NotificationPage.tsx index 0c2513409239..10e81d65a9a7 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/NotificationPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/NotificationPage.tsx @@ -3,6 +3,7 @@ import { FormattedMessage } from "react-intl"; import HeadTitle from "components/HeadTitle"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import useWorkspace, { useCurrentWorkspace, WebhookPayload } from "hooks/services/useWorkspace"; import { Content, SettingsCard } from "../SettingsComponents"; @@ -41,6 +42,8 @@ function useAsyncWithTimeout(f: (data: K) => Promise) { } const NotificationPage: React.FC = () => { + useTrackPage(PageTrackingCodes.SETTINGS_NOTIFICATION); + const { updateWebhook, testWebhook } = useWorkspace(); const workspace = useCurrentWorkspace(); diff --git a/airbyte-webapp/src/pages/SourcesPage/pages/AllSourcesPage/AllSourcesPage.tsx b/airbyte-webapp/src/pages/SourcesPage/pages/AllSourcesPage/AllSourcesPage.tsx index 32d75a296dc0..9872aaad7eab 100644 --- a/airbyte-webapp/src/pages/SourcesPage/pages/AllSourcesPage/AllSourcesPage.tsx +++ b/airbyte-webapp/src/pages/SourcesPage/pages/AllSourcesPage/AllSourcesPage.tsx @@ -6,6 +6,7 @@ import { EmptyResourceListView } from "components/EmptyResourceListView"; import HeadTitle from "components/HeadTitle"; import PageTitle from "components/PageTitle"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useSourceList } from "hooks/services/useSourceHook"; import useRouter from "hooks/useRouter"; @@ -15,7 +16,7 @@ import SourcesTable from "./components/SourcesTable"; const AllSourcesPage: React.FC = () => { const { push } = useRouter(); const { sources } = useSourceList(); - + useTrackPage(PageTrackingCodes.SOURCE_LIST); const onCreateSource = () => push(`${RoutePaths.SourceNew}`); return sources.length ? ( { + useTrackPage(PageTrackingCodes.SOURCE_NEW); const { push } = useRouter(); const [successRequest, setSuccessRequest] = useState(false); diff --git a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx index cbdd1f628211..00fbdf1ad9fe 100644 --- a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx +++ b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx @@ -12,6 +12,7 @@ import LoadingPage from "components/LoadingPage"; import PageTitle from "components/PageTitle"; import Placeholder, { ResourceTypes } from "components/Placeholder"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useConnectionList } from "hooks/services/useConnectionHook"; import { useGetSource } from "hooks/services/useSourceHook"; import useRouter from "hooks/useRouter"; @@ -26,6 +27,7 @@ import SourceConnectionTable from "./components/SourceConnectionTable"; import SourceSettings from "./components/SourceSettings"; const SourceItemPage: React.FC = () => { + useTrackPage(PageTrackingCodes.SOURCE_ITEM); const { query, params, push } = useRouter<{ id: string }, { id: string; "*": string }>(); const currentStep = useMemo(() => (params["*"] === "" ? StepsTypes.OVERVIEW : params["*"]), [params]); diff --git a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx index 1cc4fbae223a..75d66f656807 100644 --- a/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx +++ b/airbyte-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SourceSettings.tsx @@ -5,6 +5,7 @@ import DeleteBlock from "components/DeleteBlock"; import { ConnectionConfiguration } from "core/domain/connection"; import { SourceRead, WebBackendConnectionRead } from "core/request/AirbyteClient"; +import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; import { useDeleteSource, useUpdateSource } from "hooks/services/useSourceHook"; import { useSourceDefinition } from "services/connector/SourceDefinitionService"; @@ -26,6 +27,7 @@ const SourceSettings: React.FC = ({ currentSource, connecti const formId = useUniqueFormId(); const { clearFormChange } = useFormChangeTrackerService(); + useTrackPage(PageTrackingCodes.SOURCE_ITEM_SETTINGS); useEffect(() => { return () => { setDocumentationPanelOpen(false); diff --git a/airbyte-webapp/src/pages/routes.tsx b/airbyte-webapp/src/pages/routes.tsx index 260bcdc11896..bb3b9205ed84 100644 --- a/airbyte-webapp/src/pages/routes.tsx +++ b/airbyte-webapp/src/pages/routes.tsx @@ -5,7 +5,6 @@ import { useEffectOnce } from "react-use"; import ApiErrorBoundary from "components/ApiErrorBoundary"; import { useAnalyticsIdentifyUser, useAnalyticsRegisterValues } from "hooks/services/Analytics"; -import { useTrackPageAnalytics } from "hooks/services/Analytics/useTrackPageAnalytics"; import { useApiHealthPoll } from "hooks/services/Health"; import { OnboardingServiceProvider } from "hooks/services/Onboarding"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; @@ -33,7 +32,6 @@ const useAddAnalyticsContextForWorkspace = (workspace: WorkspaceRead): void => { ); useAnalyticsRegisterValues(analyticsContext); useAnalyticsIdentifyUser(workspace.workspaceId); - useTrackPageAnalytics(); }; const MainViewRoutes: React.FC<{ workspace: WorkspaceRead }> = ({ workspace }) => { @@ -81,7 +79,6 @@ export const AutoSelectFirstWorkspace: React.FC<{ includePath?: boolean }> = ({ const RoutingWithWorkspace: React.FC = () => { const workspace = useCurrentWorkspace(); useAddAnalyticsContextForWorkspace(workspace); - useTrackPageAnalytics(); useApiHealthPoll(); return ( From be36bb2e1c3655903a19b41ff554288fadb2d0e9 Mon Sep 17 00:00:00 2001 From: oneshcheret <33333155+sashaNeshcheret@users.noreply.github.com> Date: Mon, 5 Sep 2022 15:03:11 +0300 Subject: [PATCH 023/200] Mssql source: add schemas for discovery during set up (#16002) * Mssql source: add schemas during discovery * Source mssql: temp changes for testing ci * Source mssql: update expected version for strict encrypt version * Source mssql: update order in spec * Source mssql: added filter by requested schemas * Source mssql: bump versions * Source mssql: format * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 21 +++++++++--- .../source-mssql-strict-encrypt/Dockerfile | 2 +- .../src/test/resources/expected_spec.json | 20 ++++++++--- .../connectors/source-mssql/Dockerfile | 2 +- .../source/mssql/MssqlSource.java | 33 +++++++++++++++++++ .../source-mssql/src/main/resources/spec.json | 20 ++++++++--- docs/integrations/sources/mssql.md | 1 + 8 files changed, 85 insertions(+), 16 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index b5b71c15558f..e4e8dcc8782f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -599,7 +599,7 @@ - name: Microsoft SQL Server (MSSQL) sourceDefinitionId: b5ea17b1-f170-46dc-bc31-cc744ca984c1 dockerRepository: airbyte/source-mssql - dockerImageTag: 0.4.18 + dockerImageTag: 0.4.19 documentationUrl: https://docs.airbyte.io/integrations/sources/mssql icon: mssql.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index c1e96e54aec1..e405c3726145 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5273,7 +5273,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-mssql:0.4.18" +- dockerImage: "airbyte/source-mssql:0.4.19" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mssql" connectionSpecification: @@ -5307,30 +5307,41 @@ examples: - "master" order: 2 + schemas: + title: "Schemas" + description: "The list of schemas to sync from. Defaults to user. Case sensitive." + type: "array" + items: + type: "string" + minItems: 0 + uniqueItems: true + default: + - "dbo" + order: 3 username: description: "The username which is used to access the database." title: "Username" type: "string" - order: 3 + order: 4 password: description: "The password associated with the username." title: "Password" type: "string" airbyte_secret: true - order: 4 + order: 5 jdbc_url_params: title: "JDBC URL Params" description: "Additional properties to pass to the JDBC URL string when\ \ connecting to the database formatted as 'key=value' pairs separated\ \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." type: "string" - order: 5 + order: 6 ssl_method: title: "SSL Method" type: "object" description: "The encryption method which is used when communicating with\ \ the database." - order: 6 + order: 7 oneOf: - title: "Unencrypted" description: "Data transfer will not be encrypted." diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile index 8c1bf75d5c42..2db8feec8df6 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.18 +LABEL io.airbyte.version=0.4.19 LABEL io.airbyte.name=airbyte/source-mssql-strict-encrypt diff --git a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json index 641e888a8b4b..bcfb76b07eca 100644 --- a/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-mssql-strict-encrypt/src/test/resources/expected_spec.json @@ -28,30 +28,42 @@ "examples": ["master"], "order": 2 }, + "schemas": { + "title": "Schemas", + "description": "The list of schemas to sync from. Defaults to user. Case sensitive.", + "type": "array", + "items": { + "type": "string" + }, + "minItems": 0, + "uniqueItems": true, + "default": ["dbo"], + "order": 3 + }, "username": { "description": "The username which is used to access the database.", "title": "Username", "type": "string", - "order": 3 + "order": 4 }, "password": { "description": "The password associated with the username.", "title": "Password", "type": "string", "airbyte_secret": true, - "order": 4 + "order": 5 }, "jdbc_url_params": { "title": "JDBC URL Params", "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "type": "string", - "order": 5 + "order": 6 }, "ssl_method": { "title": "SSL Method", "type": "object", "description": "The encryption method which is used when communicating with the database.", - "order": 6, + "order": 7, "oneOf": [ { "title": "Encrypted (trust server certificate)", diff --git a/airbyte-integrations/connectors/source-mssql/Dockerfile b/airbyte-integrations/connectors/source-mssql/Dockerfile index 0ee5ccbc01a8..51da741743fa 100644 --- a/airbyte-integrations/connectors/source-mssql/Dockerfile +++ b/airbyte-integrations/connectors/source-mssql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-mssql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.4.18 +LABEL io.airbyte.version=0.4.19 LABEL io.airbyte.name=airbyte/source-mssql diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java index f1969722ac56..84d440940d6e 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java @@ -63,6 +63,7 @@ public class MssqlSource extends AbstractJdbcSource implements Source public static final String CDC_LSN = "_ab_cdc_lsn"; private static final String HIERARCHYID = "hierarchyid"; private static final int INTERMEDIATE_STATE_EMISSION_FREQUENCY = 10_000; + private List schemas; public static Source sshWrappedSource() { return new SshWrappedSource(new MssqlSource(), JdbcUtils.HOST_LIST_KEY, JdbcUtils.PORT_LIST_KEY); @@ -184,6 +185,13 @@ public JsonNode toDatabaseConfig(final JsonNode mssqlConfig) { mssqlConfig.get(JdbcUtils.PORT_KEY).asText(), mssqlConfig.get(JdbcUtils.DATABASE_KEY).asText())); + if (mssqlConfig.has("schemas") && mssqlConfig.get("schemas").isArray()) { + schemas = new ArrayList<>(); + for (final JsonNode schema : mssqlConfig.get("schemas")) { + schemas.add(schema.asText()); + } + } + if (mssqlConfig.has("ssl_method")) { readSsl(mssqlConfig, additionalParameters); } @@ -236,6 +244,31 @@ public AirbyteCatalog discover(final JsonNode config) throws Exception { return catalog; } + @Override + public List>> discoverInternal(JdbcDatabase database) throws Exception { + final List>> internals = super.discoverInternal(database); + if (schemas != null && !schemas.isEmpty()) { + // process explicitly filtered (from UI) schemas + List>> resultInternals = internals + .stream() + .filter(this::isTableInRequestedSchema) + .toList(); + for (TableInfo> info : resultInternals) { + LOGGER.debug("Found table (schema: {}): {}", info.getNameSpace(), info.getName()); + } + return resultInternals; + } else { + LOGGER.info("No schemas explicitly set on UI to process, so will process all of existing schemas in DB"); + return internals; + } + } + + private boolean isTableInRequestedSchema(TableInfo> tableInfo) { + return schemas + .stream() + .anyMatch(schema -> schema.equals(tableInfo.getNameSpace())); + } + @Override public List> getCheckOperations(final JsonNode config) throws Exception { diff --git a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json index f599164e9579..8b8acf4b2630 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-mssql/src/main/resources/spec.json @@ -28,30 +28,42 @@ "examples": ["master"], "order": 2 }, + "schemas": { + "title": "Schemas", + "description": "The list of schemas to sync from. Defaults to user. Case sensitive.", + "type": "array", + "items": { + "type": "string" + }, + "minItems": 0, + "uniqueItems": true, + "default": ["dbo"], + "order": 3 + }, "username": { "description": "The username which is used to access the database.", "title": "Username", "type": "string", - "order": 3 + "order": 4 }, "password": { "description": "The password associated with the username.", "title": "Password", "type": "string", "airbyte_secret": true, - "order": 4 + "order": 5 }, "jdbc_url_params": { "title": "JDBC URL Params", "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", "type": "string", - "order": 5 + "order": 6 }, "ssl_method": { "title": "SSL Method", "type": "object", "description": "The encryption method which is used when communicating with the database.", - "order": 6, + "order": 7, "oneOf": [ { "title": "Unencrypted", diff --git a/docs/integrations/sources/mssql.md b/docs/integrations/sources/mssql.md index a1139f48d9a5..491b6db48ba0 100644 --- a/docs/integrations/sources/mssql.md +++ b/docs/integrations/sources/mssql.md @@ -306,6 +306,7 @@ If you do not see a type in this list, assume that it is coerced into a string. | Version | Date | Pull Request | Subject | |:--------|:-----------| :----------------------------------------------------- |:-------------------------------------------------------------------------------------------------------| +| 0.4.19 | 2022-09-05 | [16002](https://github.com/airbytehq/airbyte/pull/16002) | Added ability to specify schemas for discovery during setting connector up | | 0.4.18 | 2022-09-03 | [14910](https://github.com/airbytehq/airbyte/pull/14910) | Standardize spec for CDC replication. Replace the `replication_method` enum with a config object with a `method` enum field. | | 0.4.17 | 2022-09-01 | [16261](https://github.com/airbytehq/airbyte/pull/16261) | Emit state messages more frequently | | 0.4.16 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | From 76d23d43d67b01d1ceaafa2a0af7827cad2294c1 Mon Sep 17 00:00:00 2001 From: letiescanciano <45267095+letiescanciano@users.noreply.github.com> Date: Mon, 5 Sep 2022 14:56:50 +0200 Subject: [PATCH 024/200] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20Add=20exp?= =?UTF-8?q?eriment=20information=20to=20page=20views=20events=20(Segment)?= =?UTF-8?q?=20(#16329)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/src/core/analytics/AnalyticsService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-webapp/src/core/analytics/AnalyticsService.ts b/airbyte-webapp/src/core/analytics/AnalyticsService.ts index 7dc1dc75dd4d..3995d78bfe6f 100644 --- a/airbyte-webapp/src/core/analytics/AnalyticsService.ts +++ b/airbyte-webapp/src/core/analytics/AnalyticsService.ts @@ -7,7 +7,7 @@ export class AnalyticsService { alias = (newId: string): void => this.getSegmentAnalytics()?.alias?.(newId); - page = (name: string): void => this.getSegmentAnalytics()?.page?.(name); + page = (name: string): void => this.getSegmentAnalytics()?.page?.(name, { ...this.context }); reset = (): void => this.getSegmentAnalytics()?.reset?.(); From 5853d52ff2c22f6f935b8389bf7349e37c456f19 Mon Sep 17 00:00:00 2001 From: "Roman Yermilov [GL]" <86300758+roman-yermilov-gl@users.noreply.github.com> Date: Mon, 5 Sep 2022 20:14:08 +0400 Subject: [PATCH 025/200] Sourse Zendesk Talk: OAuth2.0 and unittests (#15764) * Sourse Zendesk Talk: - Oauth2.0 support - unittest coverage 90% * Source Zendesk Talk: update documentation * Source Zendesk Talk: update changelog * Source Zendesk Talk: update field description * Source Zendesk Talk: fix documentation caption * Source Zendesk Talk: fix spec, improve tests * Source Zendesk Talk: remove Makefile * Source Zendesk Talk: fix tests * Source Zendesk Talk: fix tests * Source Zendesk Talk: acceptance-test-config.yml test commit * Source Zendesk Talk: acceptance-test-config.yml test commit * Source Zendesk Talk: fix spec, add config to acceptance tests * added oauth java test * auto-bump connector version [ci skip] Co-authored-by: Oleksandr Bazarnov Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 142 ++++++++++++++---- .../connectors/source-zendesk-talk/Dockerfile | 2 +- .../acceptance-test-config.yml | 17 ++- .../acceptance-test-docker.sh | 2 +- .../integration_tests/invalid_config.json | 12 +- .../integration_tests/spec.json | 40 ----- .../source_zendesk_talk/source.py | 67 +++------ .../source_zendesk_talk/spec.json | 125 +++++++++++++++ .../source_zendesk_talk/streams.py | 24 +-- .../unit_tests/test_source.py | 97 ++++++++++++ .../unit_tests/test_streams.py | 43 +++++- .../oauth/OAuthImplementationFactory.java | 1 + .../oauth/flows/ZendeskTalkOAuthFlow.java | 99 ++++++++++++ .../oauth/flows/ZendeskTalkOAuthFlowTest.java | 101 +++++++++++++ docs/integrations/sources/zendesk-talk.md | 76 +++++----- 16 files changed, 674 insertions(+), 176 deletions(-) mode change 100644 => 100755 airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-docker.sh delete mode 100644 airbyte-integrations/connectors/source-zendesk-talk/integration_tests/spec.json create mode 100644 airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/spec.json create mode 100644 airbyte-integrations/connectors/source-zendesk-talk/unit_tests/test_source.py create mode 100644 airbyte-oauth/src/main/java/io/airbyte/oauth/flows/ZendeskTalkOAuthFlow.java create mode 100644 airbyte-oauth/src/test/java/io/airbyte/oauth/flows/ZendeskTalkOAuthFlowTest.java diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e4e8dcc8782f..a0c5bd9ca094 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1116,7 +1116,7 @@ - name: Zendesk Talk sourceDefinitionId: c8630570-086d-4a40-99ae-ea5b18673071 dockerRepository: airbyte/source-zendesk-talk - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-talk icon: zendesk.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index e405c3726145..9ed58b181082 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -10979,49 +10979,125 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-zendesk-talk:0.1.3" +- dockerImage: "airbyte/source-zendesk-talk:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-talk" - changelogUrl: "https://docs.airbyte.io/integrations/sources/zendesk-talk" connectionSpecification: - title: "Zendesk Talk Spec" + $schema: "http://json-schema.org/draft-07/schema#" + title: "Source Zendesk Talk Spec" type: "object" + required: + - "start_date" + - "subdomain" properties: - subdomain: - title: "Subdomain" - description: "The subdomain for your Zendesk Talk." - type: "string" - access_token: - title: "Access Token" - description: "The value of the API token generated. See the docs for more information." - airbyte_secret: true - type: "string" - email: - title: "Email" - description: "The user email for your Zendesk account." - type: "string" start_date: - title: "Replication Start Date" - description: "The date/datetime from which you'd like to replicate data\ - \ for Zendesk Talk API, in the format YYYY-MM-DDT00:00:00Z. The time part\ - \ is optional." - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$" + type: "string" + title: "Start Date" + description: "The date from which you'd like to replicate data for Zendesk\ + \ Talk API, in the format YYYY-MM-DDT00:00:00Z. All data generated after\ + \ this date will be replicated." examples: - - "2017-01-25T00:00:00Z" - - "2017-01-25" + - "2020-10-15T00:00:00Z" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + subdomain: type: "string" - format: "date-time" - required: - - "subdomain" - - "access_token" - - "email" - - "start_date" - supportsIncremental: true + title: "Subdomain" + description: "This is your Zendesk subdomain that can be found in your account\ + \ URL. For example, in https://{MY_SUBDOMAIN}.zendesk.com/, where MY_SUBDOMAIN\ + \ is the value of your subdomain." + credentials: + title: "Authentication" + type: "object" + description: "Zendesk service provides two authentication methods. Choose\ + \ between: `OAuth2.0` or `API token`." + oneOf: + - title: "API Token" + type: "object" + required: + - "email" + - "api_token" + additionalProperties: true + properties: + auth_type: + type: "string" + const: "api_token" + email: + title: "Email" + type: "string" + description: "The user email for your Zendesk account." + api_token: + title: "API Token" + type: "string" + description: "The value of the API token generated. See the docs\ + \ for more information." + airbyte_secret: true + - title: "OAuth2.0" + type: "object" + required: + - "access_token" + additionalProperties: true + properties: + auth_type: + type: "string" + const: "oauth2.0" + order: 0 + access_token: + type: "string" + title: "Access Token" + description: "The value of the API token generated. See the docs\ + \ for more information." + airbyte_secret: true supportsNormalization: false supportsDBT: false - supported_destination_sync_modes: - - "append" + supported_destination_sync_modes: [] + advanced_auth: + auth_flow_type: "oauth2.0" + predicate_key: + - "credentials" + - "auth_type" + predicate_value: "oauth2.0" + oauth_config_specification: + oauth_user_input_from_connector_config_specification: + type: "object" + additionalProperties: false + properties: + subdomain: + type: "string" + path_in_connector_config: + - "subdomain" + complete_oauth_output_specification: + type: "object" + additionalProperties: false + properties: + access_token: + type: "string" + path_in_connector_config: + - "credentials" + - "access_token" + complete_oauth_server_input_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + client_secret: + type: "string" + complete_oauth_server_output_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + path_in_connector_config: + - "credentials" + - "client_id" + client_secret: + type: "string" + path_in_connector_config: + - "credentials" + - "client_secret" - dockerImage: "airbyte/source-zenloop:0.1.1" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/zenloop" diff --git a/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile b/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile index 06a9154b7226..cb02e69e2589 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile +++ b/airbyte-integrations/connectors/source-zendesk-talk/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-zendesk-talk diff --git a/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-config.yml b/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-config.yml index ee4de5d3440e..bc80a5ed6add 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-config.yml @@ -4,14 +4,21 @@ connector_image: airbyte/source-zendesk-talk:dev tests: spec: - - spec_path: "integration_tests/spec.json" + - spec_path: "source_zendesk_talk/spec.json" connection: - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" + - config_path: "secrets/config_old.json" + status: "succeed" discovery: - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.3" + - config_path: "secrets/config_old.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.3" basic_read: - config_path: "secrets/config.json" incremental: @@ -19,8 +26,8 @@ tests: future_state_path: "integration_tests/abnormal_state.json" full_refresh: - config_path: "secrets/config.json" - # Statistics streams (with single record) have artificial PK that changes everytime + configured_catalog_path: "integration_tests/configured_catalog.json" ignored_fields: - "account_overview": ["current_timestamp"] - "agents_overview": ["current_timestamp"] - "current_queue_activity": ["current_timestamp"] + account_overview: ["current_timestamp"] + agents_overview: ["current_timestamp"] + current_queue_activity: ["current_timestamp"] diff --git a/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-docker.sh old mode 100644 new mode 100755 index e4d8b1cef896..c51577d10690 --- a/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-docker.sh +++ b/airbyte-integrations/connectors/source-zendesk-talk/acceptance-test-docker.sh @@ -1,7 +1,7 @@ #!/usr/bin/env sh # Build latest connector image -docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) # Pull latest acctest image docker pull airbyte/source-acceptance-test:latest diff --git a/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/invalid_config.json index 53a3ef003b81..0bae92b8eca8 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/invalid_config.json @@ -1,6 +1,8 @@ { - "email": "integration-test@airbyte.io", - "access_token": "some_token", - "subdomain": "domain", - "start_date": "2021-04-01T00:00:00Z" -} + "credentials": { + "auth_type": "oauth2.0", + "access_token": "sometoken" + }, + "subdomain": "d3v-airbyte", + "start_date": "2221-04-01T00:00:00Z" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/spec.json b/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/spec.json deleted file mode 100644 index cf6171f0ad81..000000000000 --- a/airbyte-integrations/connectors/source-zendesk-talk/integration_tests/spec.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk-talk", - "changelogUrl": "https://docs.airbyte.io/integrations/sources/zendesk-talk", - "connectionSpecification": { - "title": "Zendesk Talk Spec", - "type": "object", - "properties": { - "subdomain": { - "title": "Subdomain", - "description": "The subdomain for your Zendesk Talk.", - "type": "string" - }, - "access_token": { - "title": "Access Token", - "description": "The value of the API token generated. See the docs for more information.", - "airbyte_secret": true, - "type": "string" - }, - "email": { - "title": "Email", - "description": "The user email for your Zendesk account.", - "type": "string" - }, - "start_date": { - "title": "Replication Start Date", - "description": "The date/datetime from which you'd like to replicate data for Zendesk Talk API, in the format YYYY-MM-DDT00:00:00Z. The time part is optional.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$", - "examples": ["2017-01-25T00:00:00Z", "2017-01-25"], - "type": "string", - "format": "date-time" - } - }, - "required": ["subdomain", "access_token", "email", "start_date"] - }, - "supportsIncremental": true, - "supportsNormalization": false, - "supportsDBT": false, - "supported_destination_sync_modes": ["append"], - "authSpecification": null -} diff --git a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/source.py b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/source.py index fe9b7ef833a0..d734a53e7f73 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/source.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/source.py @@ -2,14 +2,15 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from datetime import datetime from typing import Any, List, Mapping, Tuple +import pendulum +import requests from airbyte_cdk import AirbyteLogger -from airbyte_cdk.models import ConnectorSpecification, DestinationSyncMode, SyncMode +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from pydantic import BaseModel, Field +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from requests.auth import HTTPBasicAuth from source_zendesk_talk.streams import ( AccountOverview, @@ -28,31 +29,25 @@ ) -class ConnectorConfig(BaseModel): - class Config: - title = "Zendesk Talk Spec" - - subdomain: str = Field( - description="The subdomain for your Zendesk Talk.", - ) - access_token: str = Field( - description='The value of the API token generated. See the docs for more information.', - airbyte_secret=True, - ) - email: str = Field(description="The user email for your Zendesk account.") - start_date: datetime = Field( - title="Replication Start Date", - description="The date/datetime from which you'd like to replicate data for Zendesk Talk API, in the format YYYY-MM-DDT00:00:00Z. The time part is optional.", - pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$", - examples=["2017-01-25T00:00:00Z", "2017-01-25"], - ) - - class SourceZendeskTalk(AbstractSource): + @classmethod + def get_authenticator(cls, config: Mapping[str, Any]) -> requests.auth.AuthBase: + # old authentication flow support + if "access_token" in config and "email" in config: + return HTTPBasicAuth(username=f'{config["email"]}/token', password=config["access_token"]) + # new authentication flow + auth = config["credentials"] + if auth: + if auth["auth_type"] == "oauth2.0": + return TokenAuthenticator(token=auth["access_token"]) + elif auth["auth_type"] == "api_token": + return HTTPBasicAuth(username=f'{auth["email"]}/token', password=auth["api_token"]) + else: + raise Exception(f"Not implemented authorization method: {auth['auth_type']}") + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: - parsed_config = ConnectorConfig.parse_obj(config) - authenticator = HTTPBasicAuth(username=f"{parsed_config.email}/token", password=parsed_config.access_token) - stream = AccountOverview(authenticator=authenticator, subdomain=parsed_config.subdomain) + authenticator = self.get_authenticator(config) + stream = AccountOverview(authenticator=authenticator, subdomain=config["subdomain"]) account_info = next(iter(stream.read_records(sync_mode=SyncMode.full_refresh)), None) if not account_info: @@ -61,10 +56,9 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - parsed_config = ConnectorConfig.parse_obj(config) - authenticator = HTTPBasicAuth(username=f"{parsed_config.email}/token", password=parsed_config.access_token) - common_kwargs = dict(authenticator=authenticator, subdomain=parsed_config.subdomain) - incremental_kwargs = dict(**common_kwargs, start_date=parsed_config.start_date) + authenticator = self.get_authenticator(config) + common_kwargs = {"authenticator": authenticator, "subdomain": config["subdomain"]} + incremental_kwargs = {**common_kwargs, **{"start_date": pendulum.parse(config["start_date"])}} return [ AccountOverview(**common_kwargs), @@ -81,16 +75,3 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: IVRs(**common_kwargs), PhoneNumbers(**common_kwargs), ] - - def spec(self, *args, **kwargs) -> ConnectorSpecification: - """ - Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username and password) - required to run this integration. - """ - return ConnectorSpecification( - documentationUrl="https://docs.airbyte.io/integrations/sources/zendesk-talk", - changelogUrl="https://docs.airbyte.io/integrations/sources/zendesk-talk", - supportsIncremental=True, - supported_destination_sync_modes=[DestinationSyncMode.append], - connectionSpecification=ConnectorConfig.schema(), - ) diff --git a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/spec.json b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/spec.json new file mode 100644 index 000000000000..c4c0e368a3f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/spec.json @@ -0,0 +1,125 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/zendesk-talk", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Source Zendesk Talk Spec", + "type": "object", + "required": ["start_date", "subdomain"], + "properties": { + "start_date": { + "type": "string", + "title": "Start Date", + "description": "The date from which you'd like to replicate data for Zendesk Talk API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", + "examples": ["2020-10-15T00:00:00Z"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + }, + "subdomain": { + "type": "string", + "title": "Subdomain", + "description": "This is your Zendesk subdomain that can be found in your account URL. For example, in https://{MY_SUBDOMAIN}.zendesk.com/, where MY_SUBDOMAIN is the value of your subdomain." + }, + "credentials": { + "title": "Authentication", + "type": "object", + "description": "Zendesk service provides two authentication methods. Choose between: `OAuth2.0` or `API token`.", + "oneOf": [ + { + "title": "API Token", + "type": "object", + "required": ["email", "api_token"], + "additionalProperties": true, + "properties": { + "auth_type": { + "type": "string", + "const": "api_token" + }, + "email": { + "title": "Email", + "type": "string", + "description": "The user email for your Zendesk account." + }, + "api_token": { + "title": "API Token", + "type": "string", + "description": "The value of the API token generated. See the docs for more information.", + "airbyte_secret": true + } + } + }, + { + "title": "OAuth2.0", + "type": "object", + "required": ["access_token"], + "additionalProperties": true, + "properties": { + "auth_type": { + "type": "string", + "const": "oauth2.0", + "order": 0 + }, + "access_token": { + "type": "string", + "title": "Access Token", + "description": "The value of the API token generated. See the docs for more information.", + "airbyte_secret": true + } + } + } + ] + } + } + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_type"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "access_token": { + "type": "string", + "path_in_connector_config": ["credentials", "access_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } + }, + "oauth_user_input_from_connector_config_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "subdomain": { + "type": "string", + "path_in_connector_config": ["subdomain"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py index 656acf18f815..14b6f25e5d3a 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/source_zendesk_talk/streams.py @@ -31,7 +31,7 @@ def data_field(self) -> str: @property def url_base(self) -> str: """API base url based on configured subdomain""" - return f"https://{self._subdomain}.zendesk.com/api/v2/channels/voice" + return f"https://{self._subdomain}.zendesk.com/api/v2/channels/voice/" def backoff_time(self, response: requests.Response) -> Optional[float]: """ @@ -162,7 +162,7 @@ class PhoneNumbers(ZendeskTalkStream): data_field = "phone_numbers" def path(self, **kwargs) -> str: - return "/phone_numbers" + return "phone_numbers" class Addresses(ZendeskTalkStream): @@ -173,7 +173,7 @@ class Addresses(ZendeskTalkStream): data_field = "addresses" def path(self, **kwargs) -> str: - return "/addresses" + return "addresses" class GreetingCategories(ZendeskTalkStream): @@ -184,7 +184,7 @@ class GreetingCategories(ZendeskTalkStream): data_field = "greeting_categories" def path(self, **kwargs) -> str: - return "/greeting_categories" + return "greeting_categories" class Greetings(ZendeskTalkStream): @@ -195,7 +195,7 @@ class Greetings(ZendeskTalkStream): data_field = "greetings" def path(self, **kwargs) -> str: - return "/greetings" + return "greetings" class IVRs(ZendeskTalkStream): @@ -209,7 +209,7 @@ class IVRs(ZendeskTalkStream): cache_filename = "ivrs.yml" def path(self, **kwargs) -> str: - return "/ivr.json" + return "ivr.json" class IVRMenus(IVRs): @@ -251,7 +251,7 @@ class AccountOverview(ZendeskTalkSingleRecordStream): data_field = "account_overview" def path(self, **kwargs) -> str: - return "/stats/account_overview" + return "stats/account_overview" class AgentsActivity(ZendeskTalkStream): @@ -263,7 +263,7 @@ class AgentsActivity(ZendeskTalkStream): primary_key = "agent_id" def path(self, **kwargs) -> str: - return "/stats/agents_activity" + return "stats/agents_activity" class AgentsOverview(ZendeskTalkSingleRecordStream): @@ -274,7 +274,7 @@ class AgentsOverview(ZendeskTalkSingleRecordStream): data_field = "agents_overview" def path(self, **kwargs) -> str: - return "/stats/agents_overview" + return "stats/agents_overview" class CurrentQueueActivity(ZendeskTalkSingleRecordStream): @@ -285,7 +285,7 @@ class CurrentQueueActivity(ZendeskTalkSingleRecordStream): data_field = "current_queue_activity" def path(self, **kwargs) -> str: - return "/stats/current_queue_activity" + return "stats/current_queue_activity" class Calls(ZendeskTalkIncrementalStream): @@ -297,7 +297,7 @@ class Calls(ZendeskTalkIncrementalStream): cursor_field = "updated_at" def path(self, **kwargs) -> str: - return "/stats/incremental/calls" + return "stats/incremental/calls" class CallLegs(ZendeskTalkIncrementalStream): @@ -309,4 +309,4 @@ class CallLegs(ZendeskTalkIncrementalStream): cursor_field = "updated_at" def path(self, **kwargs) -> str: - return "/stats/incremental/legs" + return "stats/incremental/legs" diff --git a/airbyte-integrations/connectors/source-zendesk-talk/unit_tests/test_source.py b/airbyte-integrations/connectors/source-zendesk-talk/unit_tests/test_source.py new file mode 100644 index 000000000000..64e57cd03ddb --- /dev/null +++ b/airbyte-integrations/connectors/source-zendesk-talk/unit_tests/test_source.py @@ -0,0 +1,97 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pendulum +import pytest +from airbyte_cdk.models import AirbyteConnectionStatus, Status +from airbyte_cdk.sources.streams.http import HttpStream +from source_zendesk_talk import SourceZendeskTalk + + +@pytest.fixture +def patch_base_class_oauth20(mocker): + return { + "config": { + "credentials": {"auth_type": "oauth2.0", "access_token": "accesstoken"}, + "subdomain": "airbyte-subdomain", + "start_date": "2021-04-01T00:00:00Z", + } + } + + +@pytest.fixture +def patch_base_class_api_token(mocker): + return { + "config": { + "credentials": {"auth_type": "api_token", "api_token": "accesstoken", "email": "email@example.com"}, + "subdomain": "airbyte-subdomain", + "start_date": "2021-04-01T00:00:00Z", + } + } + + +def test_check_connection_oauth20(mocker, patch_base_class_oauth20): + source = SourceZendeskTalk() + + logger_mock, config_mock = mocker.MagicMock(), mocker.MagicMock() + config_mock.__getitem__.side_effect = patch_base_class_oauth20["config"].__getitem__ + + mocker.patch.object(HttpStream, "read_records", return_value=[mocker.MagicMock()]) + assert source.check(logger_mock, config_mock) == AirbyteConnectionStatus(status=Status.SUCCEEDED) + + +def test_check_connection_api_token(mocker, patch_base_class_api_token): + source = SourceZendeskTalk() + + logger_mock, config_mock = mocker.MagicMock(), mocker.MagicMock() + config_mock.__getitem__.side_effect = patch_base_class_api_token["config"].__getitem__ + + mocker.patch.object(HttpStream, "read_records", return_value=[mocker.MagicMock()]) + assert source.check(logger_mock, config_mock) == AirbyteConnectionStatus(status=Status.SUCCEEDED) + + +def test_streams(mocker, patch_base_class_oauth20): + source = SourceZendeskTalk() + + config_mock = mocker.MagicMock() + config_mock.__getitem__.side_effect = patch_base_class_oauth20["config"].__getitem__ + + all_streams = source.streams(config_mock) + + expected_streams_number = 13 + streams = filter( + lambda s: s.__class__.__name__ + in [ + "AccountOverview", + "Addresses", + "AgentsActivity", + "AgentsOverview", + "CurrentQueueActivity", + "Greetings", + "GreetingCategories", + "IVRMenus", + "IVRRoutes", + "IVRs", + "PhoneNumbers", + ], + all_streams, + ) + + incremental_streams = filter( + lambda s: s.__class__.__name__ + in [ + "Calls", + "CallLegs", + ], + all_streams, + ) + + assert len(all_streams) == expected_streams_number + + for s in incremental_streams: + assert s._start_date == pendulum.parse(patch_base_class_oauth20["config"]["start_date"]) + assert s._subdomain == "airbyte-subdomain" + + for s in streams: + assert s._subdomain == "airbyte-subdomain" diff --git a/airbyte-integrations/connectors/source-zendesk-talk/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-zendesk-talk/unit_tests/test_streams.py index bbaad3fa0d32..26bb37f57467 100644 --- a/airbyte-integrations/connectors/source-zendesk-talk/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-zendesk-talk/unit_tests/test_streams.py @@ -2,12 +2,13 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +import random from urllib.parse import urlparse import pendulum import pytest import requests -from source_zendesk_talk.streams import ZendeskTalkIncrementalStream, ZendeskTalkSingleRecordStream, ZendeskTalkStream +from source_zendesk_talk.streams import IVRMenus, IVRRoutes, ZendeskTalkIncrementalStream, ZendeskTalkSingleRecordStream, ZendeskTalkStream class NonIncrementalStream(ZendeskTalkStream): @@ -233,3 +234,43 @@ def test_parse_response(self, mocker, now): result = list(stream.parse_response(response=response)) assert result == [{"field1": "value", "field2": 3, stream.primary_key: int(now().timestamp())}] + + +class TestIVRMenusStream: + def test_ivr_menus_parse_response(self, mocker): + stream = IVRMenus(subdomain="test-domain", authenticator=mocker.MagicMock()) + ivrs = [ + {"id": random.randint(10000, 99999), "menus": [dict(key="value")]}, + {"id": random.randint(10000, 99999), "menus": [dict(key="value")]}, + {"id": random.randint(10000, 99999), "menus": [dict(key="value")]}, + {"id": random.randint(10000, 99999), "menus": [dict(key="value")]}, + ] + response_data = {"ivrs": ivrs} + response = mocker.MagicMock() + response.json.side_effect = [response_data] + for i, menu in enumerate(stream.parse_response(response)): + assert menu == {"ivr_id": ivrs[i]["id"], **ivrs[i]["menus"][0]} + assert i + 1 == 4 + + +class TestIVRRoutesStream: + def test_ivr_menus_parse_response(self, mocker): + stream = IVRRoutes(subdomain="test-domain", authenticator=mocker.MagicMock()) + ivr_routes = [ + { + "id": 1, + "menus": [ + {"id": 1.1, "routes": [{"route": "1.1.1 route"}, {"route": "1.1.2 route"}]}, + {"id": 1.2, "routes": [{"route": "1.2 route"}]}, + ], + }, + ] + response = mocker.MagicMock() + response.json.side_effect = [{"ivrs": ivr_routes}] + + assert [record for record in stream.parse_response(response)] == [ + {"ivr_id": 1, "ivr_menu_id": 1.1, "id": 1.1, "routes": [{"route": "1.1.1 route"}, {"route": "1.1.2 route"}]}, + {"ivr_id": 1, "ivr_menu_id": 1.1, "id": 1.2, "routes": [{"route": "1.2 route"}]}, + {"ivr_id": 1, "ivr_menu_id": 1.2, "id": 1.1, "routes": [{"route": "1.1.1 route"}, {"route": "1.1.2 route"}]}, + {"ivr_id": 1, "ivr_menu_id": 1.2, "id": 1.2, "routes": [{"route": "1.2 route"}]}, + ] diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index 449b7f240a21..8438d59e4788 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -55,6 +55,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository, final .put("airbyte/source-drift", new DriftOAuthFlow(configRepository, httpClient)) .put("airbyte/source-zendesk-chat", new ZendeskChatOAuthFlow(configRepository, httpClient)) .put("airbyte/source-zendesk-support", new ZendeskSupportOAuthFlow(configRepository, httpClient)) + .put("airbyte/source-zendesk-talk", new ZendeskTalkOAuthFlow(configRepository, httpClient)) .put("airbyte/source-monday", new MondayOAuthFlow(configRepository, httpClient)) .put("airbyte/source-zendesk-sunshine", new ZendeskSunshineOAuthFlow(configRepository, httpClient)) .put("airbyte/source-mailchimp", new MailchimpOAuthFlow(configRepository, httpClient)) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/ZendeskTalkOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/ZendeskTalkOAuthFlow.java new file mode 100644 index 000000000000..0a3e79346769 --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/ZendeskTalkOAuthFlow.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuth2Flow; +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.http.client.utils.URIBuilder; + +/** + * Following docs from + * https://support.zendesk.com/hc/en-us/articles/4408845965210-Using-OAuth-authentication-with-your-application + */ +public class ZendeskTalkOAuthFlow extends BaseOAuth2Flow { + + public ZendeskTalkOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) { + super(configRepository, httpClient); + } + + @VisibleForTesting + public ZendeskTalkOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + @Override + protected String formatConsentUrl(final UUID definitionId, + final String clientId, + final String redirectUrl, + final JsonNode inputOAuthConfiguration) + throws IOException { + + // getting subdomain value from user's config + final String subdomain = getConfigValueUnsafe(inputOAuthConfiguration, "subdomain"); + + final URIBuilder builder = new URIBuilder() + .setScheme("https") + .setHost(subdomain + ".zendesk.com") + .setPath("oauth/authorizations/new") + // required + .addParameter("client_id", clientId) + .addParameter("redirect_uri", redirectUrl) + .addParameter("response_type", "code") + .addParameter("scope", "read") + .addParameter("state", getState()); + + try { + return builder.build().toString(); + } catch (URISyntaxException e) { + throw new IOException("Failed to format Consent URL for OAuth flow", e); + } + } + + @Override + protected Map getAccessTokenQueryParameters(String clientId, + String clientSecret, + String authCode, + String redirectUrl) { + return ImmutableMap.builder() + // required + .put("grant_type", "authorization_code") + .put("code", authCode) + .put("client_id", clientId) + .put("client_secret", clientSecret) + .put("redirect_uri", redirectUrl) + .put("scope", "read") + .build(); + } + + @Override + protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) { + // getting subdomain value from user's config + final String subdomain = getConfigValueUnsafe(inputOAuthConfiguration, "subdomain"); + return "https://" + subdomain + ".zendesk.com/oauth/tokens"; + } + + @Override + protected Map extractOAuthOutput(final JsonNode data, final String accessTokenUrl) throws IOException { + final Map result = new HashMap<>(); + // getting out access_token + if (data.has("access_token")) { + result.put("access_token", data.get("access_token").asText()); + } else { + throw new IOException(String.format("Missing 'access_token' in query params from %s", accessTokenUrl)); + } + return result; + } + +} diff --git a/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/ZendeskTalkOAuthFlowTest.java b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/ZendeskTalkOAuthFlowTest.java new file mode 100644 index 000000000000..edfcc5258626 --- /dev/null +++ b/airbyte-oauth/src/test/java/io/airbyte/oauth/flows/ZendeskTalkOAuthFlowTest.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import io.airbyte.commons.json.Jsons; +import io.airbyte.oauth.BaseOAuthFlow; +import io.airbyte.oauth.MoreOAuthParameters; +import java.util.Map; +import org.junit.jupiter.api.Test; + +@SuppressWarnings("PMD.JUnitTestsShouldIncludeAssert") +class ZendeskTalkOAuthFlowTest extends BaseOAuthFlowTest { + + @Override + protected BaseOAuthFlow getOAuthFlow() { + return new ZendeskTalkOAuthFlow(getConfigRepository(), getHttpClient(), this::getConstantState); + } + + @Override + protected String getExpectedConsentUrl() { + return "https://test_subdomain.zendesk.com/oauth/authorizations/new?client_id=test_client_id&redirect_uri=https%3A%2F%2Fairbyte.io&response_type=code&scope=read&state=state"; + } + + @Override + protected JsonNode getInputOAuthConfiguration() { + return Jsons.jsonNode(Map.of("subdomain", "test_subdomain")); + } + + @Override + protected JsonNode getUserInputFromConnectorConfigSpecification() { + return getJsonSchema(Map.of("subdomain", Map.of("type", "string"))); + } + + @Test + @Override + void testEmptyOutputCompleteSourceOAuth() {} + + @Test + @Override + void testGetSourceConsentUrlEmptyOAuthSpec() {} + + @Test + @Override + void testValidateOAuthOutputFailure() {} + + @Test + @Override + void testCompleteSourceOAuth() {} + + @Test + @Override + void testEmptyInputCompleteDestinationOAuth() {} + + @Test + @Override + void testDeprecatedCompleteDestinationOAuth() {} + + @Test + @Override + void testDeprecatedCompleteSourceOAuth() {} + + @Test + @Override + void testEmptyOutputCompleteDestinationOAuth() {} + + @Test + @Override + void testCompleteDestinationOAuth() {} + + @Test + @Override + void testGetDestinationConsentUrlEmptyOAuthSpec() {} + + @Test + @Override + void testEmptyInputCompleteSourceOAuth() {} + + @Override + protected Map getExpectedOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK, + "client_secret", MoreOAuthParameters.SECRET_MASK); + } + + @Override + protected JsonNode getCompleteOAuthOutputSpecification() { + return getJsonSchema(Map.of("access_token", Map.of("type", "string"))); + } + + @Override + protected Map getExpectedFilteredOutput() { + return Map.of( + "access_token", "access_token_response", + "client_id", MoreOAuthParameters.SECRET_MASK); + } + +} diff --git a/docs/integrations/sources/zendesk-talk.md b/docs/integrations/sources/zendesk-talk.md index f8b4a3d0db4a..1a634a197742 100644 --- a/docs/integrations/sources/zendesk-talk.md +++ b/docs/integrations/sources/zendesk-talk.md @@ -1,12 +1,39 @@ # Zendesk Talk -## Sync overview +## Prerequisites -The Zendesk Talk source supports Full Refresh syncs. +* Zendesk API Token or Zendesk OAuth Client +* Zendesk Email (For API Token authentication) +* Zendesk Subdomain -This source can sync data for the [Zendesk Talk API](https://developer.zendesk.com/rest_api/docs/voice-api/introduction). +## Setup guide -### Output schema +Generate a API access token as described in [Zendesk docs](https://support.zendesk.com/hc/en-us/articles/226022787-Generating-a-new-API-token-) + +We recommend creating a restricted, read-only key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access. + +Another option is to use OAuth2.0 for authentication. See [Zendesk docs](https://support.zendesk.com/hc/en-us/articles/4408845965210-Using-OAuth-authentication-with-your-application) for details. + +## Step 2: Set up the connector in Airbyte + +### For Airbyte Cloud: + +1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. +2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. +3. On the Set up the source page, enter the name for the connector and select **Zendesk Talk** from the Source type dropdown. +4. Fill in the rest of the fields: + * *Subdomain* + * *Start Date* + * *Authentication (API Token / OAuth2.0)* +5. Click **Set up source** + +## Supported sync modes + +The **Zendesk Talk** source connector supports the following[ sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): +* Full Refresh +* Incremental Sync + +## Supported Streams This Source is capable of syncing the following core Streams: @@ -24,45 +51,26 @@ This Source is capable of syncing the following core Streams: * [IVR Routes](https://developer.zendesk.com/rest_api/docs/voice-api/ivr_routes#list-ivr-routes) * [Phone Numbers](https://developer.zendesk.com/rest_api/docs/voice-api/phone_numbers#list-phone-numbers) -### Data type mapping - -| Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `string` | `string` | | -| `number` | `number` | | -| `array` | `array` | | -| `object` | `object` | | - -### Features - -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Yes | | -| Namespaces | No | | - -### Performance considerations +## Performance considerations The connector is restricted by normal Zendesk [requests limitation](https://developer.zendesk.com/rest_api/docs/voice-api/introduction#rate-limits). The Zendesk connector should not run into Zendesk API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. -## Getting started +## Data type map -### Requirements - -* Zendesk API Token -* Zendesk Email -* Zendesk Subdomain +| Integration Type | Airbyte Type | Notes | +| :------- | :------- | :--- | +| `string` | `string` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | -### Setup guide -Generate a API access token as described in [Zendesk docs](https://support.zendesk.com/hc/en-us/articles/226022787-Generating-a-new-API-token-) - -We recommend creating a restricted, read-only key specifically for Airbyte access. This will allow you to control which resources Airbyte should be able to access. +## Changelog -### CHANGELOG | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | -| `0.1.3` | 2021-11-11 | [7173](https://github.com/airbytehq/airbyte/pull/7173) | Fix pagination and migrate to CDK | +| `0.1.4` | 2022-08-19 | [15764](https://github.com/airbytehq/airbyte/pull/15764) | Support OAuth2.0 | +| `0.1.3` | 2021-11-11 | [7173](https://github.com/airbytehq/airbyte/pull/7173) | Fix pagination and migrate to CDK | From f182492cdf6ab7bac25cc62ac8a1fb23b47ad67a Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi <53845333+lazebnyi@users.noreply.github.com> Date: Mon, 5 Sep 2022 18:59:26 +0200 Subject: [PATCH 026/200] =?UTF-8?q?=F0=9F=8E=89Source=20Sendgrid:=20increa?= =?UTF-8?q?se=20unit=20test=20coverage=20at=20least=2090%=20(#16332)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added unit tests * Updated test name * Updated setup * Fix requirements * Updated release stage * Updated expected_records --- .../resources/seed/source_definitions.yaml | 2 +- .../connectors/source-sendgrid/README.md | 71 ++++++++++++----- .../integration_tests/expected_records.txt | 22 ------ .../source-sendgrid/requirements.txt | 2 +- .../connectors/source-sendgrid/setup.py | 19 ++++- .../source-sendgrid/unit_tests/unit_test.py | 76 ++++++++++++++++++- 6 files changed, 143 insertions(+), 49 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index a0c5bd9ca094..c9ca23ac3c9d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -904,7 +904,7 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/sendgrid icon: sendgrid.svg sourceType: api - releaseStage: alpha + releaseStage: beta - name: Shopify sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9 dockerRepository: airbyte/source-shopify diff --git a/airbyte-integrations/connectors/source-sendgrid/README.md b/airbyte-integrations/connectors/source-sendgrid/README.md index bb171e8b10cd..a394bde397c4 100644 --- a/airbyte-integrations/connectors/source-sendgrid/README.md +++ b/airbyte-integrations/connectors/source-sendgrid/README.md @@ -1,6 +1,6 @@ # Sendgrid Source -This is the repository for the Sendgrid source connector, written in Python. +This is the repository for the Sendgrid source connector, written in Python. For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/sendgrid). ## Local development @@ -21,6 +21,7 @@ development environment of choice. To activate it from the terminal, run: ``` source .venv/bin/activate pip install -r requirements.txt +pip install '.[tests]' ``` If you are in an IDE, follow your IDE's instructions to activate the virtualenv. @@ -30,7 +31,9 @@ If this is mumbo jumbo to you, don't worry about it, just put your deps in `setu should work as you expect. #### Building via Gradle -From the Airbyte repository root, run: +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: ``` ./gradlew :airbyte-integrations:connectors:source-sendgrid:build ``` @@ -38,13 +41,12 @@ From the Airbyte repository root, run: #### Create credentials **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/sendgrid) to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_sendgrid/spec.json` file. -Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. See `integration_tests/sample_config.json` for a sample config file. **If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source sendgrid test creds` and place them into `secrets/config.json`. - ### Locally running the connector ``` python main.py spec @@ -53,12 +55,6 @@ python main.py discover --config secrets/config.json python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json ``` -### Unit Tests -To run unit tests locally, from the connector directory run: -``` -python -m pytest unit_tests -``` - ### Locally running the connector docker image #### Build @@ -82,22 +78,55 @@ docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sendgrid:dev check --c docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-sendgrid:dev discover --config /secrets/config.json docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-sendgrid:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json ``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` ### Integration Tests -1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-sendgrid:integrationTest` to run the standard integration test suite. -1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`. - Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-sendgrid:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-sendgrid:integrationTest +``` ## Dependency Management All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list ### Publishing a new version of the connector You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? -1. Make sure your changes are passing unit and integration tests -1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer). -1. Create a Pull Request -1. Pat yourself on the back for being an awesome contributor -1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master - -### Changelog -See the [docs](https://docs.airbyte.io/integrations/sources/sendgrid#changelog) for the changelog. +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.txt index f5e598018cab..765cd206c150 100644 --- a/airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-sendgrid/integration_tests/expected_records.txt @@ -1,25 +1,3 @@ -{"stream": "campaigns", "data": {"created_at": "2021-09-08T09:07:48Z", "id": "3c5a9fa6-1084-11ec-ac32-4228d699bad5", "name": "Untitled Single Send", "status": "triggered", "updated_at": "2021-09-08T09:11:08Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-09-08T09:04:36Z", "id": "c9f286fb-1083-11ec-ae03-ca0fc7f28419", "name": "Copy of Untitled Single Send", "status": "triggered", "updated_at": "2021-09-08T09:09:08Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-09-08T08:53:59Z", "id": "4e5be6a3-1082-11ec-8512-9afd40c324e6", "name": "Untitled Single Send", "status": "triggered", "updated_at": "2021-09-08T08:57:08Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-09-08T08:51:59Z", "id": "06ee105f-1082-11ec-8245-86a627812e3d", "name": "Untitled Single Send", "status": "triggered", "updated_at": "2021-09-08T08:55:08Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:13:02Z", "id": "d497b877-6486-11eb-be53-b2a243c7228c", "name": "Campaign 18", "status": "draft", "updated_at": "2021-02-01T12:13:02Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:13:00Z", "id": "d36a06e7-6486-11eb-bb4f-823d082c01b8", "name": "Campaign 17", "status": "draft", "updated_at": "2021-02-01T12:13:00Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:58Z", "id": "d258b5a0-6486-11eb-8b51-8aa6caa37fdd", "name": "Campaign 16", "status": "draft", "updated_at": "2021-02-01T12:12:58Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:56Z", "id": "d12b87ec-6486-11eb-be53-b2a243c7228c", "name": "Campaign 15", "status": "draft", "updated_at": "2021-02-01T12:12:56Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:54Z", "id": "d0072250-6486-11eb-bd19-4680e22af4b6", "name": "Campaign 14", "status": "draft", "updated_at": "2021-02-01T12:12:54Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:53Z", "id": "cf204596-6486-11eb-bd19-4680e22af4b6", "name": "Campaign 13", "status": "draft", "updated_at": "2021-02-01T12:12:53Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:51Z", "id": "cdfe8cde-6486-11eb-be53-b2a243c7228c", "name": "Campaign 12", "status": "draft", "updated_at": "2021-02-01T12:12:51Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:49Z", "id": "ccecefcb-6486-11eb-bd77-2a301ccc59da", "name": "Campaign 11", "status": "draft", "updated_at": "2021-02-01T12:12:49Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:46Z", "id": "caeabedf-6486-11eb-bd19-4680e22af4b6", "name": "Campaign 10", "status": "draft", "updated_at": "2021-02-01T12:12:46Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:43Z", "id": "c93431e7-6486-11eb-8b51-8aa6caa37fdd", "name": "Campaign 9", "status": "draft", "updated_at": "2021-02-01T12:12:43Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:41Z", "id": "c7eb9306-6486-11eb-bd19-4680e22af4b6", "name": "Campaign 8", "status": "draft", "updated_at": "2021-02-01T12:12:41Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:38Z", "id": "c646b948-6486-11eb-bd77-2a301ccc59da", "name": "Campaign 7", "status": "draft", "updated_at": "2021-02-01T12:12:38Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:35Z", "id": "c47ccad9-6486-11eb-8b51-8aa6caa37fdd", "name": "Campaign 6", "status": "draft", "updated_at": "2021-02-01T12:12:35Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:33Z", "id": "c36c66f3-6486-11eb-bd19-4680e22af4b6", "name": "Campaign 5", "status": "draft", "updated_at": "2021-02-01T12:12:33Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:31Z", "id": "c24fdc68-6486-11eb-bd77-2a301ccc59da", "name": "Campaign 4", "status": "draft", "updated_at": "2021-02-01T12:12:31Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:25Z", "id": "be9d147f-6486-11eb-8b51-8aa6caa37fdd", "name": "Third Campaign", "status": "draft", "updated_at": "2021-02-01T12:12:25Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:12:18Z", "id": "ba43f256-6486-11eb-bb4f-823d082c01b8", "name": "Second Campaign", "status": "draft", "updated_at": "2021-02-01T12:12:18Z", "is_abtest": false}, "emitted_at": 1631093369000} -{"stream": "campaigns", "data": {"created_at": "2021-02-01T12:10:59Z", "id": "8b17a7b7-6486-11eb-bd77-2a301ccc59da", "name": "First Campaign", "status": "draft", "updated_at": "2021-02-01T12:10:59Z", "is_abtest": false}, "emitted_at": 1631093369000} {"stream": "lists", "data": {"name": "Test List: 19", "id": "0236d6d2-75d2-42c5-962d-603e0deaf8d1", "contact_count": 20, "_metadata": {"self": "https://api.sendgrid.com/v3/marketing/lists/0236d6d2-75d2-42c5-962d-603e0deaf8d1"}}, "emitted_at": 1631093370000} {"stream": "lists", "data": {"name": "List for CI tests, number 30", "id": "041ee031-005e-41e7-ad3b-a427f90f54af", "contact_count": 0, "_metadata": {"self": "https://api.sendgrid.com/v3/marketing/lists/041ee031-005e-41e7-ad3b-a427f90f54af"}}, "emitted_at": 1631093370000} {"stream": "lists", "data": {"name": "my_list", "id": "07315be4-500c-4f30-8217-22cb8e39dd37", "contact_count": 0, "_metadata": {"self": "https://api.sendgrid.com/v3/marketing/lists/07315be4-500c-4f30-8217-22cb8e39dd37"}}, "emitted_at": 1631093370000} diff --git a/airbyte-integrations/connectors/source-sendgrid/requirements.txt b/airbyte-integrations/connectors/source-sendgrid/requirements.txt index 7b9114ed5867..0411042aa091 100644 --- a/airbyte-integrations/connectors/source-sendgrid/requirements.txt +++ b/airbyte-integrations/connectors/source-sendgrid/requirements.txt @@ -1,2 +1,2 @@ -# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. +-e ../../bases/source-acceptance-test -e . diff --git a/airbyte-integrations/connectors/source-sendgrid/setup.py b/airbyte-integrations/connectors/source-sendgrid/setup.py index c5826e3411d8..2a287c1387a6 100644 --- a/airbyte-integrations/connectors/source-sendgrid/setup.py +++ b/airbyte-integrations/connectors/source-sendgrid/setup.py @@ -5,12 +5,27 @@ from setuptools import find_packages, setup +MAIN_REQUIREMENTS = [ + "airbyte-cdk", + "backoff", + "requests", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "source-acceptance-test", + "requests-mock", +] + setup( name="source_sendgrid", description="Source implementation for Sendgrid.", author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-cdk~=0.1", "backoff", "requests", "pytest==6.1.2", "pytest-mock"], - package_data={"": ["*.json", "schemas/*.json"]}, + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, ) diff --git a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py index 69ede2de4116..f44774be7614 100644 --- a/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-sendgrid/unit_tests/unit_test.py @@ -3,14 +3,29 @@ # import unittest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pendulum import pytest import requests from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import SyncMode from source_sendgrid.source import SourceSendgrid -from source_sendgrid.streams import Messages, SendgridStream +from source_sendgrid.streams import ( + Blocks, + Campaigns, + Contacts, + GlobalSuppressions, + Lists, + Messages, + Segments, + SendgridStream, + SendgridStreamIncrementalMixin, + SendgridStreamOffsetPagination, + SuppressionGroupMembers, + SuppressionGroups, + Templates, +) FAKE_NOW = pendulum.DateTime(2022, 1, 1, tzinfo=pendulum.timezone("utc")) @@ -52,3 +67,60 @@ def test_messages_stream_request_params(mock_pendulum_now): request_params == "query=last_event_time%20BETWEEN%20TIMESTAMP%20%222019-05-20T13%3A30%3A00Z%22%20AND%20TIMESTAMP%20%222022-01-01T00%3A00%3A00Z%22&limit=1000" ) + + +def test_streams(): + streams = SourceSendgrid().streams(config={"apikey": "wrong.api.key123", "start_time": FAKE_NOW}) + + assert len(streams) == 15 + + +@patch.multiple(SendgridStreamOffsetPagination, __abstractmethods__=set()) +def test_pagination(mocker): + stream = SendgridStreamOffsetPagination() + state = {} + response = requests.Response() + mocker.patch.object(response, "json", return_value={None: 1}) + mocker.patch.object(response, "request", return_value=MagicMock()) + next_page_token = stream.next_page_token(response) + request_params = stream.request_params(stream_state=state, next_page_token=next_page_token) + assert request_params == {"limit": 50} + + +@patch.multiple(SendgridStreamIncrementalMixin, __abstractmethods__=set()) +def test_stream_state(): + stream = SendgridStreamIncrementalMixin(start_time=FAKE_NOW) + state = {} + request_params = stream.request_params(stream_state=state) + assert request_params == {"end_time": pendulum.now().int_timestamp, "start_time": FAKE_NOW} + + +@pytest.mark.parametrize( + "stream_class, url , expected", + ( + [Templates, "https://api.sendgrid.com/v3/templates", []], + [Lists, "https://api.sendgrid.com/v3/marketing/lists", []], + [Campaigns, "https://api.sendgrid.com/v3/marketing/campaigns", []], + [Contacts, "https://api.sendgrid.com/v3/marketing/contacts", []], + [Segments, "https://api.sendgrid.com/v3/marketing/segments", []], + [Blocks, "https://api.sendgrid.com/v3/suppression/blocks", ["name", "id", "contact_count", "_metadata"]], + [SuppressionGroupMembers, "https://api.sendgrid.com/v3/asm/suppressions", ["name", "id", "contact_count", "_metadata"]], + [SuppressionGroups, "https://api.sendgrid.com/v3/asm/groups", ["name", "id", "contact_count", "_metadata"]], + [GlobalSuppressions, "https://api.sendgrid.com/v3/suppression/unsubscribes", ["name", "id", "contact_count", "_metadata"]], + ), +) +def test_read_records( + stream_class, + url, + expected, + requests_mock, +): + try: + stream = stream_class(start_time=FAKE_NOW) + except TypeError: + stream = stream_class() + requests_mock.get("https://api.sendgrid.com/v3/marketing", json={}) + requests_mock.get(url, json={"name": "test", "id": "id", "contact_count": 20, "_metadata": {"self": "self"}}) + records = list(stream.read_records(sync_mode=SyncMode)) + + assert records == expected From 04f513570d05b4c28add2bca7e674b9968bb9734 Mon Sep 17 00:00:00 2001 From: Serhii Lazebnyi <53845333+lazebnyi@users.noreply.github.com> Date: Mon, 5 Sep 2022 19:25:10 +0200 Subject: [PATCH 027/200] Bumped release stage (#16337) --- .../init/src/main/resources/seed/source_definitions.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index c9ca23ac3c9d..14546013e66e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1120,7 +1120,7 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-talk icon: zendesk.svg sourceType: api - releaseStage: alpha + releaseStage: beta - name: Zenloop sourceDefinitionId: f1e4c7f6-db5c-4035-981f-d35ab4998794 dockerRepository: airbyte/source-zenloop @@ -1135,7 +1135,7 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/sentry icon: sentry.svg sourceType: api - releaseStage: alpha + releaseStage: beta - name: Zoom sourceDefinitionId: aea2fd0d-377d-465e-86c0-4fdc4f688e51 dockerRepository: airbyte/source-zoom-singer From 19f4af8d28ee5cfb18a2e2d242f64d1eb2a75fe4 Mon Sep 17 00:00:00 2001 From: Serhii Chvaliuk Date: Mon, 5 Sep 2022 21:48:07 +0300 Subject: [PATCH 028/200] Source Bing Ads: backoff socket.timeout (#16335) Signed-off-by: Sergey Chvalyuk --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 2 +- airbyte-integrations/connectors/source-bing-ads/Dockerfile | 2 +- .../connectors/source-bing-ads/source_bing_ads/client.py | 7 ++++++- docs/integrations/sources/bing-ads.md | 3 ++- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 14546013e66e..694abbda8bb5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -112,7 +112,7 @@ - name: Bing Ads sourceDefinitionId: 47f25999-dd5e-4636-8c39-e7cea2453331 dockerRepository: airbyte/source-bing-ads - dockerImageTag: 0.1.11 + dockerImageTag: 0.1.12 documentationUrl: https://docs.airbyte.io/integrations/sources/bing-ads icon: bingads.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 9ed58b181082..f90e1338d01d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -900,7 +900,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/source-bing-ads:0.1.11" +- dockerImage: "airbyte/source-bing-ads:0.1.12" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/bing-ads" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-bing-ads/Dockerfile b/airbyte-integrations/connectors/source-bing-ads/Dockerfile index 269fa40236d0..6e35e1ee0056 100644 --- a/airbyte-integrations/connectors/source-bing-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-bing-ads/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.11 +LABEL io.airbyte.version=0.1.12 LABEL io.airbyte.name=airbyte/source-bing-ads diff --git a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py index e1b13afec54c..b6d06ca1a1db 100644 --- a/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py +++ b/airbyte-integrations/connectors/source-bing-ads/source_bing_ads/client.py @@ -2,10 +2,12 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +import socket import sys from datetime import datetime, timedelta, timezone from functools import lru_cache from typing import Any, Iterator, Mapping, Optional +from urllib.error import URLError import backoff import pendulum @@ -94,6 +96,9 @@ def is_token_expiring(self) -> bool: return False if token_updated_expires_in > self.refresh_token_safe_delta else True def should_retry(self, error: WebFault) -> bool: + if isinstance(error, URLError) and isinstance(error.reason, socket.timeout): + return False + error_code = str(errorcode_of_exception(error)) give_up = error_code not in self.retry_on_codes if give_up: @@ -112,7 +117,7 @@ def log_retry_attempt(self, details: Mapping[str, Any]) -> None: def request(self, **kwargs: Mapping[str, Any]) -> Mapping[str, Any]: return backoff.on_exception( backoff.expo, - WebFault, + (WebFault, URLError), max_tries=self.max_retries, factor=self.retry_factor, jitter=None, diff --git a/docs/integrations/sources/bing-ads.md b/docs/integrations/sources/bing-ads.md index dfcd8c831557..302937e3d9b1 100644 --- a/docs/integrations/sources/bing-ads.md +++ b/docs/integrations/sources/bing-ads.md @@ -105,7 +105,8 @@ API limits number of requests for all Microsoft Advertising clients. You can fin | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------- | -| 0.1.11 | 2022-08-25 | [15684](https://github.com/airbytehq/airbyte/pull/15684) (published in [15987](https://github.com/airbytehq/airbyte/pull/15987))| Fixed log messages being unreadable | +| 0.1.12 | 2022-09-05 | [16335](https://github.com/airbytehq/airbyte/pull/16335) | Added backoff for socket.timeout | +| 0.1.11 | 2022-08-25 | [15684](https://github.com/airbytehq/airbyte/pull/15684) (published in [15987](https://github.com/airbytehq/airbyte/pull/15987))| Fixed log messages being unreadable | | 0.1.10 | 2022-08-12 | [15602](https://github.com/airbytehq/airbyte/pull/15602) | Fixed bug caused Hourly Reports to crash due to invalid fields set | | 0.1.9 | 2022-08-02 | [14862](https://github.com/airbytehq/airbyte/pull/14862) | Added missing columns | | 0.1.8 | 2022-06-15 | [13801](https://github.com/airbytehq/airbyte/pull/13801) | All reports `hourly/daily/weekly/monthly` will be generated by default, these options are removed from input configuration | From 7ae67d1b0dcba157c8c0fc5f80aa1e2fec5f15d4 Mon Sep 17 00:00:00 2001 From: midavadim Date: Mon, 5 Sep 2022 23:00:13 +0300 Subject: [PATCH 029/200] :tada: Source Iterable - added new events streams (#16067) * increased unit test coverage * added additional events streams * updated tests * bumped connector version, update changelog * fixed indentations * api.py and iterable_streams.py merged into one stream.py file * Fix unit tests * updated release stage * fixed import * Updated version in seed * auto-bump connector version [ci skip] * updated source_specs.yaml Co-authored-by: Serhii Lazebnyi Co-authored-by: Serhii Lazebnyi <53845333+lazebnyi@users.noreply.github.com> Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 4 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-iterable/Dockerfile | 2 +- .../configured_catalog_additional_events.json | 291 ++++++++ .../source-iterable/source_iterable/api.py | 269 -------- .../source_iterable/iterable_streams.py | 265 -------- .../source-iterable/source_iterable/source.py | 54 +- .../source_iterable/streams.py | 628 ++++++++++++++++++ .../source-iterable/unit_tests/conftest.py | 5 + .../test_export_adjustable_range.py | 24 +- .../unit_tests/test_exports_stream.py | 4 +- .../source-iterable/unit_tests/test_source.py | 18 +- .../unit_tests/test_stream_events.py | 203 ++++++ .../unit_tests/test_streams.py | 346 +++++----- docs/integrations/sources/iterable.md | 51 +- 15 files changed, 1402 insertions(+), 764 deletions(-) create mode 100644 airbyte-integrations/connectors/source-iterable/integration_tests/configured_catalog_additional_events.json delete mode 100755 airbyte-integrations/connectors/source-iterable/source_iterable/api.py delete mode 100644 airbyte-integrations/connectors/source-iterable/source_iterable/iterable_streams.py create mode 100644 airbyte-integrations/connectors/source-iterable/source_iterable/streams.py create mode 100644 airbyte-integrations/connectors/source-iterable/unit_tests/test_stream_events.py diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 694abbda8bb5..f16263d7c9d2 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -473,11 +473,11 @@ - name: Iterable sourceDefinitionId: 2e875208-0c0b-4ee4-9e92-1cb3156ea799 dockerRepository: airbyte/source-iterable - dockerImageTag: 0.1.16 + dockerImageTag: 0.1.17 documentationUrl: https://docs.airbyte.io/integrations/sources/iterable icon: iterable.svg sourceType: api - releaseStage: alpha + releaseStage: beta - name: Jenkins sourceDefinitionId: d6f73702-d7a0-4e95-9758-b0fb1af0bfba dockerRepository: farosai/airbyte-jenkins-source diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index f90e1338d01d..b03d04def912 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4181,7 +4181,7 @@ oauthFlowInitParameters: [] oauthFlowOutputParameters: - - "access_token" -- dockerImage: "airbyte/source-iterable:0.1.16" +- dockerImage: "airbyte/source-iterable:0.1.17" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/iterable" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-iterable/Dockerfile b/airbyte-integrations/connectors/source-iterable/Dockerfile index d3ea1538299d..9aa55203fc03 100644 --- a/airbyte-integrations/connectors/source-iterable/Dockerfile +++ b/airbyte-integrations/connectors/source-iterable/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.16 +LABEL io.airbyte.version=0.1.17 LABEL io.airbyte.name=airbyte/source-iterable diff --git a/airbyte-integrations/connectors/source-iterable/integration_tests/configured_catalog_additional_events.json b/airbyte-integrations/connectors/source-iterable/integration_tests/configured_catalog_additional_events.json new file mode 100644 index 000000000000..62896085f18b --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/integration_tests/configured_catalog_additional_events.json @@ -0,0 +1,291 @@ +{ + "streams": [ + { + "stream": { + "name": "push_send", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "push_send_skip", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "push_open", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + + { + "stream": { + "name": "push_uninstall", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "push_bounce", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "web_push_send", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "web_push_click", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "web_push_send_skip", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "in_app_send", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "in_app_open", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "in_app_click", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "in_app_close", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "in_app_delete", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "in_app_delivery", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "in_app_send_skip", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "inbox_session", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "inbox_message_impression", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "sms_send", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "sms_bounce", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "sms_click", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "sms_received", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "sms_send_skip", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "sms_usage_info", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "purchase", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "custom_event", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "hosted_unsubscribe_click", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["createdAt"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/api.py b/airbyte-integrations/connectors/source-iterable/source_iterable/api.py deleted file mode 100755 index b59b1e804361..000000000000 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/api.py +++ /dev/null @@ -1,269 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -import csv -import json -import urllib.parse as urlparse -from io import StringIO -from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional - -import requests -from airbyte_cdk.models import SyncMode -from source_iterable.iterable_streams import IterableExportStreamAdjustableRange, IterableExportStreamRanged, IterableStream - -EVENT_ROWS_LIMIT = 200 -CAMPAIGNS_PER_REQUEST = 20 - - -class Lists(IterableStream): - data_field = "lists" - - def path(self, **kwargs) -> str: - return "lists" - - -class ListUsers(IterableStream): - primary_key = "listId" - data_field = "getUsers" - name = "list_users" - - def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str: - return f"lists/{self.data_field}?listId={stream_slice['list_id']}" - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - lists = Lists(authenticator=self._cred) - for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh)): - yield {"list_id": list_record["id"]} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - list_id = self._get_list_id(response.url) - for user in response.iter_lines(): - yield {"email": user.decode(), "listId": list_id} - - @staticmethod - def _get_list_id(url: str) -> int: - parsed_url = urlparse.urlparse(url) - for q in parsed_url.query.split("&"): - key, value = q.split("=") - if key == "listId": - return int(value) - - -class Campaigns(IterableStream): - data_field = "campaigns" - - def path(self, **kwargs) -> str: - return "campaigns" - - -class CampaignsMetrics(IterableStream): - name = "campaigns_metrics" - primary_key = None - data_field = None - - def __init__(self, start_date: str, **kwargs): - """ - https://api.iterable.com/api/docs#campaigns_metrics - """ - super().__init__(**kwargs) - self.start_date = start_date - - def path(self, **kwargs) -> str: - return "campaigns/metrics" - - def request_params(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(**kwargs) - params["campaignId"] = stream_slice.get("campaign_ids") - params["startDateTime"] = self.start_date - - return params - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - lists = Campaigns(authenticator=self._cred) - campaign_ids = [] - for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh)): - campaign_ids.append(list_record["id"]) - - if len(campaign_ids) == CAMPAIGNS_PER_REQUEST: - yield {"campaign_ids": campaign_ids} - campaign_ids = [] - - if campaign_ids: - yield {"campaign_ids": campaign_ids} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - content = response.content.decode() - records = self._parse_csv_string_to_dict(content) - - for record in records: - yield {"data": record} - - @staticmethod - def _parse_csv_string_to_dict(csv_string: str) -> List[Dict[str, Any]]: - """ - Parse a response with a csv type to dict object - Example: - csv_string = "a,b,c,d - 1,2,,3 - 6,,1,2" - - output = [{"a": 1, "b": 2, "d": 3}, - {"a": 6, "c": 1, "d": 2}] - - - :param csv_string: API endpoint response with csv format - :return: parsed API response - - """ - - reader = csv.DictReader(StringIO(csv_string), delimiter=",") - result = [] - - for row in reader: - for key, value in row.items(): - if value == "": - continue - try: - row[key] = int(value) - except ValueError: - row[key] = float(value) - row = {k: v for k, v in row.items() if v != ""} - - result.append(row) - - return result - - -class Channels(IterableStream): - data_field = "channels" - - def path(self, **kwargs) -> str: - return "channels" - - -class EmailBounce(IterableExportStreamAdjustableRange): - name = "email_bounce" - data_field = "emailBounce" - - -class EmailClick(IterableExportStreamAdjustableRange): - name = "email_click" - data_field = "emailClick" - - -class EmailComplaint(IterableExportStreamAdjustableRange): - name = "email_complaint" - data_field = "emailComplaint" - - -class EmailOpen(IterableExportStreamAdjustableRange): - name = "email_open" - data_field = "emailOpen" - - -class EmailSend(IterableExportStreamAdjustableRange): - name = "email_send" - data_field = "emailSend" - - -class EmailSendSkip(IterableExportStreamAdjustableRange): - name = "email_send_skip" - data_field = "emailSendSkip" - - -class EmailSubscribe(IterableExportStreamAdjustableRange): - name = "email_subscribe" - data_field = "emailSubscribe" - - -class EmailUnsubscribe(IterableExportStreamAdjustableRange): - name = "email_unsubscribe" - data_field = "emailUnsubscribe" - - -class Events(IterableStream): - """ - https://api.iterable.com/api/docs#export_exportUserEvents - """ - - primary_key = None - data_field = "events" - common_fields = ("itblInternal", "_type", "createdAt", "email") - - def path(self, **kwargs) -> str: - return "export/userEvents" - - def request_params(self, stream_slice: Optional[Mapping[str, Any]], **kwargs) -> MutableMapping[str, Any]: - params = super().request_params(**kwargs) - params.update({"email": stream_slice["email"], "includeCustomEvents": "true"}) - - return params - - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - lists = ListUsers(authenticator=self._cred) - stream_slices = lists.stream_slices() - - for stream_slice in stream_slices: - for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh), stream_slice=stream_slice): - yield {"email": list_record["email"]} - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - Parse jsonl response body. - Put common event fields at the top level. - Put the rest of the fields in the `data` subobject. - """ - - jsonl_records = StringIO(response.text) - for record in jsonl_records: - record_dict = json.loads(record) - record_dict_common_fields = {} - for field in self.common_fields: - record_dict_common_fields[field] = record_dict.pop(field, None) - - yield {**record_dict_common_fields, "data": record_dict} - - -class MessageTypes(IterableStream): - data_field = "messageTypes" - name = "message_types" - - def path(self, **kwargs) -> str: - return "messageTypes" - - -class Metadata(IterableStream): - primary_key = None - data_field = "results" - - def path(self, **kwargs) -> str: - return "metadata" - - -class Templates(IterableExportStreamRanged): - data_field = "templates" - template_types = ["Base", "Blast", "Triggered", "Workflow"] - message_types = ["Email", "Push", "InApp", "SMS"] - - def path(self, **kwargs) -> str: - return "templates" - - def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: - for template in self.template_types: - for message in self.message_types: - self.stream_params = {"templateType": template, "messageMedium": message} - yield from super().read_records(stream_slice=stream_slice, **kwargs) - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_json = response.json() - records = response_json.get(self.data_field, []) - - for record in records: - record[self.cursor_field] = self._field_to_datetime(record[self.cursor_field]) - yield record - - -class Users(IterableExportStreamRanged): - data_field = "user" - cursor_field = "profileUpdatedAt" diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/iterable_streams.py b/airbyte-integrations/connectors/source-iterable/source_iterable/iterable_streams.py deleted file mode 100644 index 92a8a32af786..000000000000 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/iterable_streams.py +++ /dev/null @@ -1,265 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -import json -from abc import ABC, abstractmethod -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union - -import pendulum -import requests -from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream -from pendulum.datetime import DateTime -from requests.exceptions import ChunkedEncodingError -from source_iterable.slice_generators import AdjustableSliceGenerator, RangeSliceGenerator, StreamSlice - - -class IterableStream(HttpStream, ABC): - - # Hardcode the value because it is not returned from the API - BACKOFF_TIME_CONSTANT = 10.0 - # define date-time fields with potential wrong format - - url_base = "https://api.iterable.com/api/" - primary_key = "id" - - def __init__(self, authenticator): - self._cred = authenticator - super().__init__(authenticator) - - @property - @abstractmethod - def data_field(self) -> str: - """ - :return: Default field name to get data from response - """ - - def backoff_time(self, response: requests.Response) -> Optional[float]: - return self.BACKOFF_TIME_CONSTANT - - def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - Iterable API does not support pagination - """ - return None - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - response_json = response.json() - records = response_json.get(self.data_field, []) - - for record in records: - yield record - - -class IterableExportStream(IterableStream, ABC): - """ - This stream utilize "export" Iterable api for getting large amount of data. - It can return data in form of new line separater strings each of each - representing json object. - Data could be windowed by date ranges by applying startDateTime and - endDateTime parameters. Single request could return large volumes of data - and request rate is limited by 4 requests per minute. - - Details: https://api.iterable.com/api/docs#export_exportDataJson - """ - - cursor_field = "createdAt" - primary_key = None - - def __init__(self, start_date=None, **kwargs): - super().__init__(**kwargs) - self._start_date = pendulum.parse(start_date) - self.stream_params = {"dataTypeName": self.data_field} - - def path(self, **kwargs) -> str: - return "export/data.json" - - def backoff_time(self, response: requests.Response) -> Optional[float]: - # Use default exponential backoff - return None - - # For python backoff package expo backoff delays calculated according to formula: - # delay = factor * base ** n where base is 2 - # With default factor equal to 5 and 5 retries delays would be 5, 10, 20, 40 and 80 seconds. - # For exports stream there is a limit of 4 requests per minute. - # Tune up factor and retries to send a lot of excessive requests before timeout exceed. - @property - def retry_factor(self) -> int: - return 20 - - # With factor 20 it woud be 20, 40, 80 and 160 seconds delays. - @property - def max_retries(self) -> Union[int, None]: - return 4 - - @staticmethod - def _field_to_datetime(value: Union[int, str]) -> pendulum.datetime: - if isinstance(value, int): - value = pendulum.from_timestamp(value / 1000.0) - elif isinstance(value, str): - value = pendulum.parse(value, strict=False) - else: - raise ValueError(f"Unsupported type of datetime field {type(value)}") - return value - - def get_updated_state( - self, - current_stream_state: MutableMapping[str, Any], - latest_record: Mapping[str, Any], - ) -> Mapping[str, Any]: - """ - Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object - and returning an updated state object. - """ - latest_benchmark = latest_record[self.cursor_field] - if current_stream_state.get(self.cursor_field): - return { - self.cursor_field: str( - max( - latest_benchmark, - self._field_to_datetime(current_stream_state[self.cursor_field]), - ) - ) - } - return {self.cursor_field: str(latest_benchmark)} - - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: StreamSlice, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - - params = super().request_params(stream_state=stream_state) - params.update( - { - "startDateTime": stream_slice.start_date.strftime("%Y-%m-%d %H:%M:%S"), - "endDateTime": stream_slice.end_date.strftime("%Y-%m-%d %H:%M:%S"), - }, - **self.stream_params, - ) - return params - - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - for obj in response.iter_lines(): - record = json.loads(obj) - record[self.cursor_field] = self._field_to_datetime(record[self.cursor_field]) - yield record - - def request_kwargs( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Mapping[str, Any]: - """ - https://api.iterable.com/api/docs#export_exportDataJson - Sending those type of requests could download large piece of json - objects splitted with newline character. - Passing stream=True argument to requests.session.send method to avoid - loading whole analytics report content into memory. - """ - return {"stream": True} - - def get_start_date(self, stream_state: Mapping[str, Any]) -> DateTime: - stream_state = stream_state or {} - start_datetime = self._start_date - if stream_state.get(self.cursor_field): - start_datetime = pendulum.parse(stream_state[self.cursor_field]) - return start_datetime - - def stream_slices( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Optional[StreamSlice]]: - - start_datetime = self.get_start_date(stream_state) - return [StreamSlice(start_datetime, pendulum.now("UTC"))] - - -class IterableExportStreamRanged(IterableExportStream): - """ - This class use RangeSliceGenerator class to break single request into - ranges with same (or less for final range) number of days. By default it 90 - days. - """ - - def stream_slices( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Optional[StreamSlice]]: - - start_datetime = self.get_start_date(stream_state) - - return RangeSliceGenerator(start_datetime) - - -class IterableExportStreamAdjustableRange(IterableExportStream): - """ - For streams that could produce large amount of data in single request so we - cant just use IterableExportStreamRanged to split it in even ranges. If - request processing takes a lot of time API server could just close - connection and connector code would fail with ChunkedEncodingError. - - To solve this problem we use AdjustableSliceGenerator that able to adjust - next slice range based on two factor: - 1. Previous slice range / time to process ratio. - 2. Had previous request failed with ChunkedEncodingError - - In case of slice processing request failed with ChunkedEncodingError (which - means that API server closed connection cause of request takes to much - time) make CHUNKED_ENCODING_ERROR_RETRIES (3) retries each time reducing - slice length. - - See AdjustableSliceGenerator description for more details on next slice length adjustment alghorithm. - """ - - _adjustable_generator: AdjustableSliceGenerator = None - CHUNKED_ENCODING_ERROR_RETRIES = 3 - - def stream_slices( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Optional[StreamSlice]]: - - start_datetime = self.get_start_date(stream_state) - self._adjustable_generator = AdjustableSliceGenerator(start_datetime) - return self._adjustable_generator - - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str], - stream_slice: StreamSlice, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - start_time = pendulum.now() - for _ in range(self.CHUNKED_ENCODING_ERROR_RETRIES): - try: - - self.logger.info( - f"Processing slice of {(stream_slice.end_date - stream_slice.start_date).total_days()} days for stream {self.name}" - ) - for record in super().read_records( - sync_mode=sync_mode, - cursor_field=cursor_field, - stream_slice=stream_slice, - stream_state=stream_state, - ): - now = pendulum.now() - self._adjustable_generator.adjust_range(now - start_time) - yield record - start_time = now - break - except ChunkedEncodingError: - self.logger.warn("ChunkedEncodingError occured, decrease days range and try again") - stream_slice = self._adjustable_generator.reduce_range() - else: - raise Exception(f"ChunkedEncodingError: Reached maximum number of retires: {self.CHUNKED_ENCODING_ERROR_RETRIES}") diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/source.py b/airbyte-integrations/connectors/source-iterable/source_iterable/source.py index fc255b4ad0ba..9789e9f5727f 100644 --- a/airbyte-integrations/connectors/source-iterable/source_iterable/source.py +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/source.py @@ -9,10 +9,11 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from .api import ( +from .streams import ( Campaigns, CampaignsMetrics, Channels, + CustomEvent, EmailBounce, EmailClick, EmailComplaint, @@ -22,12 +23,37 @@ EmailSubscribe, EmailUnsubscribe, Events, + HostedUnsubscribeClick, + InAppClick, + InAppClose, + InAppDelete, + InAppDelivery, + InAppOpen, + InAppSend, + InAppSendSkip, + InboxMessageImpression, + InboxSession, Lists, ListUsers, MessageTypes, Metadata, + Purchase, + PushBounce, + PushOpen, + PushSend, + PushSendSkip, + PushUninstall, + SmsBounce, + SmsClick, + SmsReceived, + SmsSend, + SmsSendSkip, + SmsUsageInfo, Templates, Users, + WebPushClick, + WebPushSend, + WebPushSendSkip, ) @@ -62,6 +88,32 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: EmailSendSkip(authenticator=authenticator, start_date=config["start_date"]), EmailSubscribe(authenticator=authenticator, start_date=config["start_date"]), EmailUnsubscribe(authenticator=authenticator, start_date=config["start_date"]), + PushSend(authenticator=authenticator, start_date=config["start_date"]), + PushSendSkip(authenticator=authenticator, start_date=config["start_date"]), + PushOpen(authenticator=authenticator, start_date=config["start_date"]), + PushUninstall(authenticator=authenticator, start_date=config["start_date"]), + PushBounce(authenticator=authenticator, start_date=config["start_date"]), + WebPushSend(authenticator=authenticator, start_date=config["start_date"]), + WebPushClick(authenticator=authenticator, start_date=config["start_date"]), + WebPushSendSkip(authenticator=authenticator, start_date=config["start_date"]), + InAppSend(authenticator=authenticator, start_date=config["start_date"]), + InAppOpen(authenticator=authenticator, start_date=config["start_date"]), + InAppClick(authenticator=authenticator, start_date=config["start_date"]), + InAppClose(authenticator=authenticator, start_date=config["start_date"]), + InAppDelete(authenticator=authenticator, start_date=config["start_date"]), + InAppDelivery(authenticator=authenticator, start_date=config["start_date"]), + InAppSendSkip(authenticator=authenticator, start_date=config["start_date"]), + InboxSession(authenticator=authenticator, start_date=config["start_date"]), + InboxMessageImpression(authenticator=authenticator, start_date=config["start_date"]), + SmsSend(authenticator=authenticator, start_date=config["start_date"]), + SmsBounce(authenticator=authenticator, start_date=config["start_date"]), + SmsClick(authenticator=authenticator, start_date=config["start_date"]), + SmsReceived(authenticator=authenticator, start_date=config["start_date"]), + SmsSendSkip(authenticator=authenticator, start_date=config["start_date"]), + SmsUsageInfo(authenticator=authenticator, start_date=config["start_date"]), + Purchase(authenticator=authenticator, start_date=config["start_date"]), + CustomEvent(authenticator=authenticator, start_date=config["start_date"]), + HostedUnsubscribeClick(authenticator=authenticator, start_date=config["start_date"]), Events(authenticator=authenticator), Lists(authenticator=authenticator), ListUsers(authenticator=authenticator), diff --git a/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py new file mode 100644 index 000000000000..244f080023d0 --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/source_iterable/streams.py @@ -0,0 +1,628 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import csv +import json +import urllib.parse as urlparse +from abc import ABC, abstractmethod +from io import StringIO +from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Union + +import pendulum +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams.core import package_name_from_class +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.utils.schema_helpers import ResourceSchemaLoader +from pendulum.datetime import DateTime +from requests.exceptions import ChunkedEncodingError +from source_iterable.slice_generators import AdjustableSliceGenerator, RangeSliceGenerator, StreamSlice + +EVENT_ROWS_LIMIT = 200 +CAMPAIGNS_PER_REQUEST = 20 + + +class IterableStream(HttpStream, ABC): + + # Hardcode the value because it is not returned from the API + BACKOFF_TIME_CONSTANT = 10.0 + # define date-time fields with potential wrong format + + url_base = "https://api.iterable.com/api/" + primary_key = "id" + + def __init__(self, authenticator): + self._cred = authenticator + super().__init__(authenticator) + + @property + @abstractmethod + def data_field(self) -> str: + """ + :return: Default field name to get data from response + """ + + def backoff_time(self, response: requests.Response) -> Optional[float]: + return self.BACKOFF_TIME_CONSTANT + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + Iterable API does not support pagination + """ + return None + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_json = response.json() + records = response_json.get(self.data_field, []) + + for record in records: + yield record + + +class IterableExportStream(IterableStream, ABC): + """ + This stream utilize "export" Iterable api for getting large amount of data. + It can return data in form of new line separater strings each of each + representing json object. + Data could be windowed by date ranges by applying startDateTime and + endDateTime parameters. Single request could return large volumes of data + and request rate is limited by 4 requests per minute. + + Details: https://api.iterable.com/api/docs#export_exportDataJson + """ + + cursor_field = "createdAt" + primary_key = None + + def __init__(self, start_date=None, **kwargs): + super().__init__(**kwargs) + self._start_date = pendulum.parse(start_date) + self.stream_params = {"dataTypeName": self.data_field} + + def path(self, **kwargs) -> str: + return "export/data.json" + + def backoff_time(self, response: requests.Response) -> Optional[float]: + # Use default exponential backoff + return None + + # For python backoff package expo backoff delays calculated according to formula: + # delay = factor * base ** n where base is 2 + # With default factor equal to 5 and 5 retries delays would be 5, 10, 20, 40 and 80 seconds. + # For exports stream there is a limit of 4 requests per minute. + # Tune up factor and retries to send a lot of excessive requests before timeout exceed. + @property + def retry_factor(self) -> int: + return 20 + + # With factor 20 it woud be 20, 40, 80 and 160 seconds delays. + @property + def max_retries(self) -> Union[int, None]: + return 4 + + @staticmethod + def _field_to_datetime(value: Union[int, str]) -> pendulum.datetime: + if isinstance(value, int): + value = pendulum.from_timestamp(value / 1000.0) + elif isinstance(value, str): + value = pendulum.parse(value, strict=False) + else: + raise ValueError(f"Unsupported type of datetime field {type(value)}") + return value + + def get_updated_state( + self, + current_stream_state: MutableMapping[str, Any], + latest_record: Mapping[str, Any], + ) -> Mapping[str, Any]: + """ + Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object + and returning an updated state object. + """ + latest_benchmark = latest_record[self.cursor_field] + if current_stream_state.get(self.cursor_field): + return { + self.cursor_field: str( + max( + latest_benchmark, + self._field_to_datetime(current_stream_state[self.cursor_field]), + ) + ) + } + return {self.cursor_field: str(latest_benchmark)} + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: StreamSlice, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + + params = super().request_params(stream_state=stream_state) + params.update( + { + "startDateTime": stream_slice.start_date.strftime("%Y-%m-%d %H:%M:%S"), + "endDateTime": stream_slice.end_date.strftime("%Y-%m-%d %H:%M:%S"), + }, + **self.stream_params, + ) + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + for obj in response.iter_lines(): + record = json.loads(obj) + record[self.cursor_field] = self._field_to_datetime(record[self.cursor_field]) + yield record + + def request_kwargs( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Mapping[str, Any]: + """ + https://api.iterable.com/api/docs#export_exportDataJson + Sending those type of requests could download large piece of json + objects splitted with newline character. + Passing stream=True argument to requests.session.send method to avoid + loading whole analytics report content into memory. + """ + return {"stream": True} + + def get_start_date(self, stream_state: Mapping[str, Any]) -> DateTime: + stream_state = stream_state or {} + start_datetime = self._start_date + if stream_state.get(self.cursor_field): + start_datetime = pendulum.parse(stream_state[self.cursor_field]) + return start_datetime + + def stream_slices( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Optional[StreamSlice]]: + + start_datetime = self.get_start_date(stream_state) + return [StreamSlice(start_datetime, pendulum.now("UTC"))] + + +class IterableExportStreamRanged(IterableExportStream, ABC): + """ + This class use RangeSliceGenerator class to break single request into + ranges with same (or less for final range) number of days. By default it 90 + days. + """ + + def stream_slices( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Optional[StreamSlice]]: + + start_datetime = self.get_start_date(stream_state) + + return RangeSliceGenerator(start_datetime) + + +class IterableExportStreamAdjustableRange(IterableExportStream, ABC): + """ + For streams that could produce large amount of data in single request so we + cant just use IterableExportStreamRanged to split it in even ranges. If + request processing takes a lot of time API server could just close + connection and connector code would fail with ChunkedEncodingError. + + To solve this problem we use AdjustableSliceGenerator that able to adjust + next slice range based on two factor: + 1. Previous slice range / time to process ratio. + 2. Had previous request failed with ChunkedEncodingError + + In case of slice processing request failed with ChunkedEncodingError (which + means that API server closed connection cause of request takes to much + time) make CHUNKED_ENCODING_ERROR_RETRIES (3) retries each time reducing + slice length. + + See AdjustableSliceGenerator description for more details on next slice length adjustment alghorithm. + """ + + _adjustable_generator: AdjustableSliceGenerator = None + CHUNKED_ENCODING_ERROR_RETRIES = 3 + + def stream_slices( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Optional[StreamSlice]]: + + start_datetime = self.get_start_date(stream_state) + self._adjustable_generator = AdjustableSliceGenerator(start_datetime) + return self._adjustable_generator + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str], + stream_slice: StreamSlice, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + start_time = pendulum.now() + for _ in range(self.CHUNKED_ENCODING_ERROR_RETRIES): + try: + + self.logger.info( + f"Processing slice of {(stream_slice.end_date - stream_slice.start_date).total_days()} days for stream {self.name}" + ) + for record in super().read_records( + sync_mode=sync_mode, + cursor_field=cursor_field, + stream_slice=stream_slice, + stream_state=stream_state, + ): + now = pendulum.now() + self._adjustable_generator.adjust_range(now - start_time) + yield record + start_time = now + break + except ChunkedEncodingError: + self.logger.warn("ChunkedEncodingError occurred, decrease days range and try again") + stream_slice = self._adjustable_generator.reduce_range() + else: + raise Exception(f"ChunkedEncodingError: Reached maximum number of retires: {self.CHUNKED_ENCODING_ERROR_RETRIES}") + + +class IterableExportEventsStreamAdjustableRange(IterableExportStreamAdjustableRange, ABC): + def get_json_schema(self) -> Mapping[str, Any]: + """All child stream share the same 'events' schema""" + return ResourceSchemaLoader(package_name_from_class(self.__class__)).get_schema("events") + + +class Lists(IterableStream): + data_field = "lists" + + def path(self, **kwargs) -> str: + return "lists" + + +class ListUsers(IterableStream): + primary_key = "listId" + data_field = "getUsers" + name = "list_users" + + def path(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> str: + return f"lists/{self.data_field}?listId={stream_slice['list_id']}" + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + lists = Lists(authenticator=self._cred) + for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh)): + yield {"list_id": list_record["id"]} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + list_id = self._get_list_id(response.url) + for user in response.iter_lines(): + yield {"email": user.decode(), "listId": list_id} + + @staticmethod + def _get_list_id(url: str) -> int: + parsed_url = urlparse.urlparse(url) + for q in parsed_url.query.split("&"): + key, value = q.split("=") + if key == "listId": + return int(value) + + +class Campaigns(IterableStream): + data_field = "campaigns" + + def path(self, **kwargs) -> str: + return "campaigns" + + +class CampaignsMetrics(IterableStream): + name = "campaigns_metrics" + primary_key = None + data_field = None + + def __init__(self, start_date: str, **kwargs): + """ + https://api.iterable.com/api/docs#campaigns_metrics + """ + super().__init__(**kwargs) + self.start_date = start_date + + def path(self, **kwargs) -> str: + return "campaigns/metrics" + + def request_params(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(**kwargs) + params["campaignId"] = stream_slice.get("campaign_ids") + params["startDateTime"] = self.start_date + + return params + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + lists = Campaigns(authenticator=self._cred) + campaign_ids = [] + for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh)): + campaign_ids.append(list_record["id"]) + + if len(campaign_ids) == CAMPAIGNS_PER_REQUEST: + yield {"campaign_ids": campaign_ids} + campaign_ids = [] + + if campaign_ids: + yield {"campaign_ids": campaign_ids} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + content = response.content.decode() + records = self._parse_csv_string_to_dict(content) + + for record in records: + yield {"data": record} + + @staticmethod + def _parse_csv_string_to_dict(csv_string: str) -> List[Dict[str, Any]]: + """ + Parse a response with a csv type to dict object + Example: + csv_string = "a,b,c,d + 1,2,,3 + 6,,1,2" + + output = [{"a": 1, "b": 2, "d": 3}, + {"a": 6, "c": 1, "d": 2}] + + + :param csv_string: API endpoint response with csv format + :return: parsed API response + + """ + + reader = csv.DictReader(StringIO(csv_string), delimiter=",") + result = [] + + for row in reader: + for key, value in row.items(): + if value == "": + continue + try: + row[key] = int(value) + except ValueError: + row[key] = float(value) + row = {k: v for k, v in row.items() if v != ""} + + result.append(row) + + return result + + +class Channels(IterableStream): + data_field = "channels" + + def path(self, **kwargs) -> str: + return "channels" + + +class MessageTypes(IterableStream): + data_field = "messageTypes" + name = "message_types" + + def path(self, **kwargs) -> str: + return "messageTypes" + + +class Metadata(IterableStream): + primary_key = None + data_field = "results" + + def path(self, **kwargs) -> str: + return "metadata" + + +class Events(IterableStream): + """ + https://api.iterable.com/api/docs#export_exportUserEvents + """ + + primary_key = None + data_field = "events" + common_fields = ("itblInternal", "_type", "createdAt", "email") + + def path(self, **kwargs) -> str: + return "export/userEvents" + + def request_params(self, stream_slice: Optional[Mapping[str, Any]], **kwargs) -> MutableMapping[str, Any]: + params = super().request_params(**kwargs) + params.update({"email": stream_slice["email"], "includeCustomEvents": "true"}) + + return params + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + lists = ListUsers(authenticator=self._cred) + stream_slices = lists.stream_slices() + + for stream_slice in stream_slices: + for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh), stream_slice=stream_slice): + yield {"email": list_record["email"]} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + Parse jsonl response body. + Put common event fields at the top level. + Put the rest of the fields in the `data` subobject. + """ + + jsonl_records = StringIO(response.text) + for record in jsonl_records: + record_dict = json.loads(record) + record_dict_common_fields = {} + for field in self.common_fields: + record_dict_common_fields[field] = record_dict.pop(field, None) + + yield {**record_dict_common_fields, "data": record_dict} + + +class EmailBounce(IterableExportStreamAdjustableRange): + data_field = "emailBounce" + + +class EmailClick(IterableExportStreamAdjustableRange): + data_field = "emailClick" + + +class EmailComplaint(IterableExportStreamAdjustableRange): + data_field = "emailComplaint" + + +class EmailOpen(IterableExportStreamAdjustableRange): + data_field = "emailOpen" + + +class EmailSend(IterableExportStreamAdjustableRange): + data_field = "emailSend" + + +class EmailSendSkip(IterableExportStreamAdjustableRange): + data_field = "emailSendSkip" + + +class EmailSubscribe(IterableExportStreamAdjustableRange): + data_field = "emailSubscribe" + + +class EmailUnsubscribe(IterableExportStreamAdjustableRange): + data_field = "emailUnsubscribe" + + +class PushSend(IterableExportEventsStreamAdjustableRange): + data_field = "pushSend" + + +class PushSendSkip(IterableExportEventsStreamAdjustableRange): + data_field = "pushSendSkip" + + +class PushOpen(IterableExportEventsStreamAdjustableRange): + data_field = "pushOpen" + + +class PushUninstall(IterableExportEventsStreamAdjustableRange): + data_field = "pushUninstall" + + +class PushBounce(IterableExportEventsStreamAdjustableRange): + data_field = "pushBounce" + + +class WebPushSend(IterableExportEventsStreamAdjustableRange): + data_field = "webPushSend" + + +class WebPushClick(IterableExportEventsStreamAdjustableRange): + data_field = "webPushClick" + + +class WebPushSendSkip(IterableExportEventsStreamAdjustableRange): + data_field = "webPushSendSkip" + + +class InAppSend(IterableExportEventsStreamAdjustableRange): + data_field = "inAppSend" + + +class InAppOpen(IterableExportEventsStreamAdjustableRange): + data_field = "inAppOpen" + + +class InAppClick(IterableExportEventsStreamAdjustableRange): + data_field = "inAppClick" + + +class InAppClose(IterableExportEventsStreamAdjustableRange): + data_field = "inAppClose" + + +class InAppDelete(IterableExportEventsStreamAdjustableRange): + data_field = "inAppDelete" + + +class InAppDelivery(IterableExportEventsStreamAdjustableRange): + data_field = "inAppDelivery" + + +class InAppSendSkip(IterableExportEventsStreamAdjustableRange): + data_field = "inAppSendSkip" + + +class InboxSession(IterableExportEventsStreamAdjustableRange): + data_field = "inboxSession" + + +class InboxMessageImpression(IterableExportEventsStreamAdjustableRange): + data_field = "inboxMessageImpression" + + +class SmsSend(IterableExportEventsStreamAdjustableRange): + data_field = "smsSend" + + +class SmsBounce(IterableExportEventsStreamAdjustableRange): + data_field = "smsBounce" + + +class SmsClick(IterableExportEventsStreamAdjustableRange): + data_field = "smsClick" + + +class SmsReceived(IterableExportEventsStreamAdjustableRange): + data_field = "smsReceived" + + +class SmsSendSkip(IterableExportEventsStreamAdjustableRange): + data_field = "smsSendSkip" + + +class SmsUsageInfo(IterableExportEventsStreamAdjustableRange): + data_field = "smsUsageInfo" + + +class Purchase(IterableExportEventsStreamAdjustableRange): + data_field = "purchase" + + +class CustomEvent(IterableExportEventsStreamAdjustableRange): + data_field = "customEvent" + + +class HostedUnsubscribeClick(IterableExportEventsStreamAdjustableRange): + data_field = "hostedUnsubscribeClick" + + +class Templates(IterableExportStreamRanged): + data_field = "templates" + template_types = ["Base", "Blast", "Triggered", "Workflow"] + message_types = ["Email", "Push", "InApp", "SMS"] + + def path(self, **kwargs) -> str: + return "templates" + + def read_records(self, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + for template in self.template_types: + for message in self.message_types: + self.stream_params = {"templateType": template, "messageMedium": message} + yield from super().read_records(stream_slice=stream_slice, **kwargs) + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_json = response.json() + records = response_json.get(self.data_field, []) + + for record in records: + record[self.cursor_field] = self._field_to_datetime(record[self.cursor_field]) + yield record + + +class Users(IterableExportStreamRanged): + data_field = "user" + cursor_field = "profileUpdatedAt" diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/conftest.py b/airbyte-integrations/connectors/source-iterable/unit_tests/conftest.py index 55c126d4d7ec..3453f15fdf3b 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/conftest.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/conftest.py @@ -17,3 +17,8 @@ def catalog(request): ) ] ) + + +@pytest.fixture(name="config") +def config_fixture(): + return {"api_key": 123, "start_date": "2019-10-10T00:00:00"} diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_export_adjustable_range.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_export_adjustable_range.py index c5d8a66aa29d..22f39c9174dc 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_export_adjustable_range.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_export_adjustable_range.py @@ -76,12 +76,13 @@ def response_cb(req): "catalog, days_duration, days_per_minute_rate", [ ("email_send", 10, 200), + # tests are commented because they take a lot of time for completion # ("email_send", 100, 200000), # ("email_send", 10000, 200000), # ("email_click", 1000, 20), # ("email_open", 1000, 1), - ("email_open", 1, 1000), - ("email_open", 0, 1000000), + # ("email_open", 1, 1000), + # ("email_open", 0, 1000000), ], indirect=["catalog"], ) @@ -109,22 +110,3 @@ def response_cb(req): assert sum(ranges) == days_duration assert len(ranges) == len(records) assert len(responses.calls) == 3 * len(ranges) - - -@responses.activate -@pytest.mark.parametrize("catalog", (["email_send"]), indirect=True) -def test_email_stream_chunked_encoding_exception(catalog, time_mock): - TEST_START_DATE = "2020" - DAYS_DURATION = 100 - - time_mock.move_to(pendulum.parse(TEST_START_DATE) + pendulum.Duration(days=DAYS_DURATION)) - - responses.add( - "GET", - "https://api.iterable.com/api/export/data.json", - body=ChunkedEncodingError(), - ) - - with pytest.raises(Exception, match="ChunkedEncodingError: Reached maximum number of retires: 3"): - read_from_source(catalog) - assert len(responses.calls) == 15 diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py index a0f9784c2499..80555f358ab4 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_exports_stream.py @@ -10,8 +10,8 @@ import responses from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.streams.http.auth import NoAuth -from source_iterable.api import Users -from source_iterable.iterable_streams import StreamSlice +from source_iterable.slice_generators import StreamSlice +from source_iterable.streams import Users @pytest.fixture diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_source.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_source.py index d2aa4253d19f..4c11c1b855a1 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_source.py @@ -2,10 +2,22 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # +from unittest.mock import MagicMock, patch + from source_iterable.source import SourceIterable +from source_iterable.streams import Lists -def test_source_streams(): - config = {"start_date": "2021-01-01", "api_key": "api_key"} +def test_source_streams(config): streams = SourceIterable().streams(config=config) - assert len(streams) == 18 + assert len(streams) == 44 + + +def test_source_check_connection_ok(config): + with patch.object(Lists, "read_records", return_value=iter([1])): + assert SourceIterable().check_connection(MagicMock(), config=config) == (True, None) + + +def test_source_check_connection_failed(config): + with patch.object(Lists, "read_records", return_value=0): + assert SourceIterable().check_connection(MagicMock(), config=config)[0] is False diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_stream_events.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_stream_events.py new file mode 100644 index 000000000000..00557f742f7b --- /dev/null +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_stream_events.py @@ -0,0 +1,203 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import json + +import pytest +import requests +import responses +from airbyte_cdk.sources.streams.http.auth import NoAuth +from source_iterable.streams import Events + + +@responses.activate +@pytest.mark.parametrize( + "response_objects,expected_objects,jsonl_body", + [ + ( + [ + { + "createdAt": "2021", + "signupSource": "str", + "emailListIds": [1], + "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"}, + "_type": "str", + "messageTypeIds": [], + "channelIds": [], + "email": "test@mail.com", + "profileUpdatedAt": "2021", + }, + { + "productRecommendationCount": 1, + "campaignId": 1, + "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"}, + "contentId": 1, + "_type": "1", + "messageId": "1", + "messageBusId": "1", + "templateId": 1, + "createdAt": "2021", + "messageTypeId": 1, + "catalogCollectionCount": 1, + "catalogLookupCount": 0, + "email": "test@mail.com", + "channelId": 1, + }, + { + "createdAt": "2021", + "campaignId": 1, + "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"}, + "_type": "str", + "messageId": "1", + "templateId": 1, + "recipientState": "str", + "email": "test@mail.com", + }, + { + "unsubSource": "str", + "createdAt": "2021", + "emailListIds": [], + "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"}, + "_type": "str", + "messageId": "1", + "messageTypeIds": [], + "channelIds": [1], + "templateId": 1, + "recipientState": "str", + "email": "test@mail.com", + }, + ], + [], + False, + ), + ( + [ + { + "createdAt": "2021", + "signupSource": "str", + "emailListIds": [1], + "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"}, + "_type": "str", + "messageTypeIds": [], + "channelIds": [], + "email": "test@mail.com", + "profileUpdatedAt": "2021", + } + ], + [], + False, + ), + ( + [ + { + "createdAt": "2021", + "signupSource": "str", + "emailListIds": [1], + "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"}, + "_type": "str", + "messageTypeIds": [], + "channelIds": [], + "email": "test@mail.com", + "profileUpdatedAt": "2021", + }, + { + "productRecommendationCount": 1, + "campaignId": 1, + "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"}, + "contentId": 1, + "_type": "1", + "messageId": "1", + "messageBusId": "1", + "templateId": 1, + "createdAt": "2021", + "messageTypeId": 1, + "catalogCollectionCount": 1, + "catalogLookupCount": 0, + "email": "test@mail.com", + "channelId": 1, + }, + ], + [ + { + "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"}, + "_type": "str", + "createdAt": "2021", + "email": "test@mail.com", + "data": { + "signupSource": "str", + "emailListIds": [1], + "messageTypeIds": [], + "channelIds": [], + "profileUpdatedAt": "2021", + }, + }, + { + "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"}, + "_type": "1", + "createdAt": "2021", + "email": "test@mail.com", + "data": { + "productRecommendationCount": 1, + "campaignId": 1, + "contentId": 1, + "messageId": "1", + "messageBusId": "1", + "templateId": 1, + "messageTypeId": 1, + "catalogCollectionCount": 1, + "catalogLookupCount": 0, + "channelId": 1, + }, + }, + ], + True, + ), + ( + [ + { + "createdAt": "2021", + "signupSource": "str", + "emailListIds": [1], + "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"}, + "_type": "str", + "messageTypeIds": [], + "channelIds": [], + "email": "test@mail.com", + "profileUpdatedAt": "2021", + } + ], + [ + { + "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"}, + "_type": "str", + "createdAt": "2021", + "email": "test@mail.com", + "data": { + "signupSource": "str", + "emailListIds": [1], + "messageTypeIds": [], + "channelIds": [], + "profileUpdatedAt": "2021", + }, + } + ], + True, + ), + ], +) +def test_events_parse_response(response_objects, expected_objects, jsonl_body): + if jsonl_body: + response_body = "\n".join([json.dumps(obj) for obj in response_objects]) + else: + response_body = json.dumps(response_objects) + responses.add(responses.GET, "https://example.com", body=response_body) + response = requests.get("https://example.com") + stream = Events(authenticator=NoAuth()) + + if jsonl_body: + records = [record for record in stream.parse_response(response)] + assert records == expected_objects + else: + with pytest.raises(TypeError): + [record for record in stream.parse_response(response)] diff --git a/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py index d74f6f69738f..9806990a019e 100644 --- a/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-iterable/unit_tests/test_streams.py @@ -2,202 +2,174 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -import json - +import pendulum import pytest import requests import responses from airbyte_cdk.sources.streams.http.auth import NoAuth -from source_iterable.api import Events +from source_iterable.streams import ( + Campaigns, + CampaignsMetrics, + Channels, + Events, + Lists, + ListUsers, + MessageTypes, + Metadata, + Templates, + Users, +) -@responses.activate @pytest.mark.parametrize( - "response_objects,expected_objects,jsonl_body", + "stream,date,slice,expected_path", [ - ( - [ - { - "createdAt": "2021", - "signupSource": "str", - "emailListIds": [1], - "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"}, - "_type": "str", - "messageTypeIds": [], - "channelIds": [], - "email": "test@mail.com", - "profileUpdatedAt": "2021", - }, - { - "productRecommendationCount": 1, - "campaignId": 1, - "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"}, - "contentId": 1, - "_type": "1", - "messageId": "1", - "messageBusId": "1", - "templateId": 1, - "createdAt": "2021", - "messageTypeId": 1, - "catalogCollectionCount": 1, - "catalogLookupCount": 0, - "email": "test@mail.com", - "channelId": 1, - }, - { - "createdAt": "2021", - "campaignId": 1, - "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"}, - "_type": "str", - "messageId": "1", - "templateId": 1, - "recipientState": "str", - "email": "test@mail.com", - }, - { - "unsubSource": "str", - "createdAt": "2021", - "emailListIds": [], - "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"}, - "_type": "str", - "messageId": "1", - "messageTypeIds": [], - "channelIds": [1], - "templateId": 1, - "recipientState": "str", - "email": "test@mail.com", - }, - ], - [], - False, - ), - ( - [ - { - "createdAt": "2021", - "signupSource": "str", - "emailListIds": [1], - "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"}, - "_type": "str", - "messageTypeIds": [], - "channelIds": [], - "email": "test@mail.com", - "profileUpdatedAt": "2021", - } - ], - [], - False, - ), - ( - [ - { - "createdAt": "2021", - "signupSource": "str", - "emailListIds": [1], - "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"}, - "_type": "str", - "messageTypeIds": [], - "channelIds": [], - "email": "test@mail.com", - "profileUpdatedAt": "2021", - }, - { - "productRecommendationCount": 1, - "campaignId": 1, - "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"}, - "contentId": 1, - "_type": "1", - "messageId": "1", - "messageBusId": "1", - "templateId": 1, - "createdAt": "2021", - "messageTypeId": 1, - "catalogCollectionCount": 1, - "catalogLookupCount": 0, - "email": "test@mail.com", - "channelId": 1, - }, - ], - [ - { - "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"}, - "_type": "str", - "createdAt": "2021", - "email": "test@mail.com", - "data": { - "signupSource": "str", - "emailListIds": [1], - "messageTypeIds": [], - "channelIds": [], - "profileUpdatedAt": "2021", - }, - }, - { - "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "2021"}, - "_type": "1", - "createdAt": "2021", - "email": "test@mail.com", - "data": { - "productRecommendationCount": 1, - "campaignId": 1, - "contentId": 1, - "messageId": "1", - "messageBusId": "1", - "templateId": 1, - "messageTypeId": 1, - "catalogCollectionCount": 1, - "catalogLookupCount": 0, - "channelId": 1, - }, - }, - ], - True, - ), - ( - [ - { - "createdAt": "2021", - "signupSource": "str", - "emailListIds": [1], - "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"}, - "_type": "str", - "messageTypeIds": [], - "channelIds": [], - "email": "test@mail.com", - "profileUpdatedAt": "2021", - } - ], - [ - { - "itblInternal": {"documentUpdatedAt": "2021", "documentCreatedAt": "202"}, - "_type": "str", - "createdAt": "2021", - "email": "test@mail.com", - "data": { - "signupSource": "str", - "emailListIds": [1], - "messageTypeIds": [], - "channelIds": [], - "profileUpdatedAt": "2021", - }, - } - ], - True, - ), + (Lists, False, {}, "lists"), + (Campaigns, False, {}, "campaigns"), + (Channels, False, {}, "channels"), + (Events, False, {}, "export/userEvents"), + (MessageTypes, False, {}, "messageTypes"), + (Metadata, False, {}, "metadata"), + (ListUsers, False, {"list_id": 1}, "lists/getUsers?listId=1"), + (CampaignsMetrics, True, {}, "campaigns/metrics"), + (Templates, True, {}, "templates"), ], ) -def test_events_parse_response(response_objects, expected_objects, jsonl_body): - if jsonl_body: - response_body = "\n".join([json.dumps(obj) for obj in response_objects]) - else: - response_body = json.dumps(response_objects) - responses.add(responses.GET, "https://example.com", body=response_body) - response = requests.get("https://example.com") +def test_path(config, stream, date, slice, expected_path): + args = {"authenticator": NoAuth()} + if date: + args["start_date"] = "2019-10-10T00:00:00" + + assert stream(**args).path(stream_slice=slice) == expected_path + + +def test_campaigns_metrics_csv(): + csv_string = "a,b,c,d\n1, 2,,3\n6,,1, 2\n" + output = [{"a": 1, "b": 2, "d": 3}, {"a": 6, "c": 1, "d": 2}] + + assert CampaignsMetrics._parse_csv_string_to_dict(csv_string) == output + + +@pytest.mark.parametrize( + "url,id", + [ + ("http://google.com?listId=1&another=another", 1), + ("http://google.com?another=another", None), + ], +) +def test_list_users_get_list_id(url, id): + assert ListUsers._get_list_id(url) == id + + +def test_campaigns_metrics_request_params(): + stream = CampaignsMetrics(authenticator=NoAuth(), start_date="2019-10-10T00:00:00") + params = stream.request_params(stream_slice={"campaign_ids": "c101"}, stream_state=None) + assert params == {"campaignId": "c101", "startDateTime": "2019-10-10T00:00:00"} + + +def test_events_request_params(): stream = Events(authenticator=NoAuth()) + params = stream.request_params(stream_slice={"email": "a@a.a"}, stream_state=None) + assert params == {"email": "a@a.a", "includeCustomEvents": "true"} + + +def test_templates_parse_response(): + stream = Templates(authenticator=NoAuth(), start_date="2019-10-10T00:00:00") + with responses.RequestsMock() as rsps: + rsps.add( + responses.GET, + "https://api.iterable.com/api/1/foobar", + json={"templates": [{"createdAt": "2022", "id": 1}]}, + status=200, + content_type="application/json", + ) + resp = requests.get("https://api.iterable.com/api/1/foobar") + + records = stream.parse_response(response=resp) + + assert list(records) == [{"id": 1, "createdAt": pendulum.parse("2022", strict=False)}] + - if jsonl_body: - records = [record for record in stream.parse_response(response)] - assert records == expected_objects - else: - with pytest.raises(TypeError): - [record for record in stream.parse_response(response)] +def test_list_users_parse_response(): + stream = ListUsers(authenticator=NoAuth()) + with responses.RequestsMock() as rsps: + rsps.add( + responses.GET, + "https://api.iterable.com/lists/getUsers?listId=100", + body="user100", + status=200, + content_type="application/json", + ) + resp = requests.get("https://api.iterable.com/lists/getUsers?listId=100") + + records = stream.parse_response(response=resp) + + assert list(records) == [{"email": "user100", "listId": 100}] + + +def test_campaigns_metrics_parse_response(): + + stream = CampaignsMetrics(authenticator=NoAuth(), start_date="2019-10-10T00:00:00") + with responses.RequestsMock() as rsps: + rsps.add( + responses.GET, + "https://api.iterable.com/lists/getUsers?listId=100", + body="""a,b,c,d +1, 2,, 3 +6,, 1, 2 +""", + status=200, + content_type="application/json", + ) + resp = requests.get("https://api.iterable.com/lists/getUsers?listId=100") + + records = stream.parse_response(response=resp) + + assert list(records) == [ + {"data": {"a": 1, "b": 2, "d": 3}}, + {"data": {"a": 6, "c": 1, "d": 2}}, + ] + + +def test_iterable_stream_parse_response(): + stream = Lists(authenticator=NoAuth()) + with responses.RequestsMock() as rsps: + rsps.add( + responses.GET, + "https://api.iterable.com/lists/getUsers?listId=100", + json={"lists": [{"id": 1}, {"id": 2}]}, + status=200, + content_type="application/json", + ) + resp = requests.get("https://api.iterable.com/lists/getUsers?listId=100") + + records = stream.parse_response(response=resp) + + assert list(records) == [{"id": 1}, {"id": 2}] + + +def test_iterable_stream_backoff_time(): + stream = Lists(authenticator=NoAuth()) + assert stream.backoff_time(response=None) == stream.BACKOFF_TIME_CONSTANT + + +def test_iterable_export_stream_backoff_time(): + stream = Users(authenticator=NoAuth(), start_date="2019-10-10T00:00:00") + assert stream.backoff_time(response=None) is None + + +@pytest.mark.parametrize( + "current_state,record_date,expected_state", + [ + ({}, "2022", {"profileUpdatedAt": "2022-01-01T00:00:00+00:00"}), + ({"profileUpdatedAt": "2020-01-01T00:00:00+00:00"}, "2022", {"profileUpdatedAt": "2022-01-01T00:00:00+00:00"}), + ({"profileUpdatedAt": "2022-01-01T00:00:00+00:00"}, "2020", {"profileUpdatedAt": "2022-01-01T00:00:00+00:00"}), + ], +) +def test_get_updated_state(current_state, record_date, expected_state): + stream = Users(authenticator=NoAuth(), start_date="2019-10-10T00:00:00") + state = stream.get_updated_state( + current_stream_state=current_state, + latest_record={"profileUpdatedAt": pendulum.parse(record_date)}, + ) + assert state == expected_state diff --git a/docs/integrations/sources/iterable.md b/docs/integrations/sources/iterable.md index cc22157f7a7f..aee1db7dae42 100644 --- a/docs/integrations/sources/iterable.md +++ b/docs/integrations/sources/iterable.md @@ -60,19 +60,46 @@ The Iterable source connector supports the following [sync modes](https://docs.a * [Metadata](https://api.iterable.com/api/docs#metadata_list_tables) * [Templates](https://api.iterable.com/api/docs#templates_getTemplates) * [Users](https://api.iterable.com/api/docs#export_exportDataJson) +* [PushSend](https://api.iterable.com/api/docs#export_exportDataJson) +* [PushSendSkip](https://api.iterable.com/api/docs#export_exportDataJson) +* [PushOpen](https://api.iterable.com/api/docs#export_exportDataJson) +* [PushUninstall](https://api.iterable.com/api/docs#export_exportDataJson) +* [PushBounce](https://api.iterable.com/api/docs#export_exportDataJson) +* [WebPushSend](https://api.iterable.com/api/docs#export_exportDataJson) +* [WebPushClick](https://api.iterable.com/api/docs#export_exportDataJson) +* [WebPushSendSkip](https://api.iterable.com/api/docs#export_exportDataJson) +* [InAppSend](https://api.iterable.com/api/docs#export_exportDataJson) +* [InAppOpen](https://api.iterable.com/api/docs#export_exportDataJson) +* [InAppClick](https://api.iterable.com/api/docs#export_exportDataJson) +* [InAppClose](https://api.iterable.com/api/docs#export_exportDataJson) +* [InAppDelete](https://api.iterable.com/api/docs#export_exportDataJson) +* [InAppDelivery](https://api.iterable.com/api/docs#export_exportDataJson) +* [InAppSendSkip](https://api.iterable.com/api/docs#export_exportDataJson) +* [InboxSession](https://api.iterable.com/api/docs#export_exportDataJson) +* [InboxMessageImpression](https://api.iterable.com/api/docs#export_exportDataJson) +* [SmsSend](https://api.iterable.com/api/docs#export_exportDataJson) +* [SmsBounce](https://api.iterable.com/api/docs#export_exportDataJson) +* [SmsClick](https://api.iterable.com/api/docs#export_exportDataJson) +* [SmsReceived](https://api.iterable.com/api/docs#export_exportDataJson) +* [SmsSendSkip](https://api.iterable.com/api/docs#export_exportDataJson) +* [SmsUsageInfo](https://api.iterable.com/api/docs#export_exportDataJson) +* [Purchase](https://api.iterable.com/api/docs#export_exportDataJson) +* [CustomEvent](https://api.iterable.com/api/docs#export_exportDataJson) +* [HostedUnsubscribeClick](https://api.iterable.com/api/docs#export_exportDataJson) ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------| :----- |:---------------------------------------------------------------------------| -| 0.1.16 | 2022-08-15 | [15670](https://github.com/airbytehq/airbyte/pull/15670) | Api key is passed via header | -| 0.1.15 | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description | -| 0.1.14 | 2021-12-01 | [8380](https://github.com/airbytehq/airbyte/pull/8380) | Update `Events` stream to use `export/userEvents` endpoint | -| 0.1.13 | 2021-11-22 | [8091](https://github.com/airbytehq/airbyte/pull/8091) | Adjust slice ranges for email streams | -| 0.1.12 | 2021-11-09 | [7780](https://github.com/airbytehq/airbyte/pull/7780) | Split EmailSend stream into slices to fix premature connection close error | -| 0.1.11 | 2021-11-03 | [7619](https://github.com/airbytehq/airbyte/pull/7619) | Bugfix type error while incrementally loading the `Templates` stream | -| 0.1.10 | 2021-11-03 | [7591](https://github.com/airbytehq/airbyte/pull/7591) | Optimize export streams memory consumption for large requests | -| 0.1.9 | 2021-10-06 | [5915](https://github.com/airbytehq/airbyte/pull/5915) | Enable campaign_metrics stream | -| 0.1.8 | 2021-09-20 | [5915](https://github.com/airbytehq/airbyte/pull/5915) | Add new streams: campaign_metrics, events | -| 0.1.7 | 2021-09-20 | [6242](https://github.com/airbytehq/airbyte/pull/6242) | Updated schema for: campaigns, lists, templates, metadata | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------| +| 0.1.17 | 2022-09-02 | [16067](https://github.com/airbytehq/airbyte/pull/16067) | added new events streams | +| 0.1.16 | 2022-08-15 | [15670](https://github.com/airbytehq/airbyte/pull/15670) | Api key is passed via header | +| 0.1.15 | 2021-12-06 | [8524](https://github.com/airbytehq/airbyte/pull/8524) | Update connector fields title/description | +| 0.1.14 | 2021-12-01 | [8380](https://github.com/airbytehq/airbyte/pull/8380) | Update `Events` stream to use `export/userEvents` endpoint | +| 0.1.13 | 2021-11-22 | [8091](https://github.com/airbytehq/airbyte/pull/8091) | Adjust slice ranges for email streams | +| 0.1.12 | 2021-11-09 | [7780](https://github.com/airbytehq/airbyte/pull/7780) | Split EmailSend stream into slices to fix premature connection close error | +| 0.1.11 | 2021-11-03 | [7619](https://github.com/airbytehq/airbyte/pull/7619) | Bugfix type error while incrementally loading the `Templates` stream | +| 0.1.10 | 2021-11-03 | [7591](https://github.com/airbytehq/airbyte/pull/7591) | Optimize export streams memory consumption for large requests | +| 0.1.9 | 2021-10-06 | [5915](https://github.com/airbytehq/airbyte/pull/5915) | Enable campaign_metrics stream | +| 0.1.8 | 2021-09-20 | [5915](https://github.com/airbytehq/airbyte/pull/5915) | Add new streams: campaign_metrics, events | +| 0.1.7 | 2021-09-20 | [6242](https://github.com/airbytehq/airbyte/pull/6242) | Updated schema for: campaigns, lists, templates, metadata | From bf22c2b7127a343e2e4c72f51b7572d1dd15884d Mon Sep 17 00:00:00 2001 From: JJ Nilbodee Date: Mon, 5 Sep 2022 21:44:55 +0100 Subject: [PATCH 030/200] :bug: Helm: airbyte-server deployment values error (#16334) --- charts/airbyte-server/templates/deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/airbyte-server/templates/deployment.yaml b/charts/airbyte-server/templates/deployment.yaml index 2961c8a0d463..d4da2d9a8c41 100644 --- a/charts/airbyte-server/templates/deployment.yaml +++ b/charts/airbyte-server/templates/deployment.yaml @@ -242,7 +242,7 @@ spec: securityContext: {{- toYaml .Values.containerSecurityContext | nindent 10 }} {{- end }} volumeMounts: - {{- if eq .Values.deploymentMode "oss" }} + {{- if eq .Values.global.deploymentMode "oss" }} - name: gcs-log-creds-volume mountPath: /secrets/gcs-log-creds readOnly: true @@ -258,7 +258,7 @@ spec: {{ toYaml .Values.global.extraContainers | indent 8 }} {{- end }} volumes: - {{- if eq .Values.deploymentMode "oss" }} + {{- if eq .Values.global.deploymentMode "oss" }} - name: gcs-log-creds-volume secret: secretName: {{ ternary (printf "%s-gcs-log-creds" ( .Release.Name )) (.Values.global.credVolumeOverride) (eq .Values.global.deploymentMode "oss") }} From e1596bcf743815b43a8c1740cb38496e631b9e42 Mon Sep 17 00:00:00 2001 From: Vikram G <38922566+ghanvik@users.noreply.github.com> Date: Mon, 5 Sep 2022 13:46:58 -0700 Subject: [PATCH 031/200] Update README.md (#16312) Fixed link to Airbyte specification --- docs/connector-development/cdk-python/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/connector-development/cdk-python/README.md b/docs/connector-development/cdk-python/README.md index 4b06cfc91f19..878329d86b03 100644 --- a/docs/connector-development/cdk-python/README.md +++ b/docs/connector-development/cdk-python/README.md @@ -8,7 +8,7 @@ The Airbyte Python CDK is a framework for rapidly developing production-grade Ai The CDK provides an improved developer experience by providing basic implementation structure and abstracting away low-level glue boilerplate. -This document is a general introduction to the CDK. Readers should have basic familiarity with the [Airbyte Specification](https://docs.airbyte.io/architecture/airbyte-protocol) before proceeding. +This document is a general introduction to the CDK. Readers should have basic familiarity with the [Airbyte Specification](https://docs.airbyte.com/understanding-airbyte/airbyte-protocol/) before proceeding. If you have any issues with troubleshooting or want to learn more about the CDK from the Airbyte team, head to [the Connector Development section of our Discourse forum](https://discuss.airbyte.io/c/connector-development/16) to inquire further! From 498d70089f3a93dc55c57d03c4b58820b1032832 Mon Sep 17 00:00:00 2001 From: Bhupesh Varshney Date: Tue, 6 Sep 2022 02:23:41 +0530 Subject: [PATCH 032/200] Source S3: Doc fix grammar & typo describing parquet file source (#16264) --- docs/integrations/sources/s3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/s3.md b/docs/integrations/sources/s3.md index 56b1202e1111..e83f47b855ce 100644 --- a/docs/integrations/sources/s3.md +++ b/docs/integrations/sources/s3.md @@ -185,7 +185,7 @@ Since CSV files are effectively plain text, providing specific reader options is ### Parquet -Apache Parquet file is a column-oriented data storage format of the Apache Hadoop ecosystem. It provides efficient data compression and encoding schemes with enhanced performance to handle complex data in bulk. For now this solutiion are iterating through individual files at the abstract-level thus partitioned parquet datasets are unsupported. The following settings are available: +Apache Parquet file is a column-oriented data storage format of the Apache Hadoop ecosystem. It provides efficient data compression and encoding schemes with enhanced performance to handle complex data in bulk. For now, the solution involves iterating through individual files at the abstract level thus partitioned parquet datasets are unsupported. The following settings are available: * `buffer_size` : If positive, perform read buffering when deserializing individual column chunks. Otherwise IO calls are unbuffered. * `columns` : If not None, only these columns will be read from the file. From 8b5362ef114872912dd7fbd1405ae16e9f3f652d Mon Sep 17 00:00:00 2001 From: Denys Davydov Date: Tue, 6 Sep 2022 00:13:25 +0300 Subject: [PATCH 033/200] Source Greenhouse: support incremental syncs (#16338) * #1386 Source Greenhouse: support incremental syncs - first try * #1386 Source Greenhouse: implement incremental syncs * #1386 source greenhouse: upd changelog * Increased unittest to 90 * Updated link in spec * auto-bump connector version [ci skip] * Updated release stage Co-authored-by: Serhii Lazebnyi Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 4 +- .../src/main/resources/seed/source_specs.yaml | 4 +- .../connectors/source-greenhouse/Dockerfile | 6 +- .../acceptance-test-config.yml | 9 +- .../integration_tests/abnormal_state.json | 58 +++++ .../integration_tests/configured_catalog.json | 154 +++++++++--- .../configured_catalog_const_records.json | 220 ------------------ .../configured_catalog_users_only.json | 8 +- .../integration_tests/expected_records.txt | 57 +++-- .../incremental_configured_catalog.json | 186 +++++++++++++++ .../connectors/source-greenhouse/setup.py | 2 +- .../source_greenhouse/components.py | 116 +++++++++ .../source_greenhouse/greenhouse.yaml | 101 ++++---- .../source_greenhouse/spec.json | 2 +- .../unit_tests/test_components.py | 49 ++++ docs/integrations/sources/greenhouse.md | 15 +- 16 files changed, 645 insertions(+), 346 deletions(-) create mode 100644 airbyte-integrations/connectors/source-greenhouse/integration_tests/abnormal_state.json delete mode 100644 airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog_const_records.json create mode 100644 airbyte-integrations/connectors/source-greenhouse/integration_tests/incremental_configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-greenhouse/source_greenhouse/components.py create mode 100644 airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index f16263d7c9d2..ce03659c4fed 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -403,11 +403,11 @@ - name: Greenhouse sourceDefinitionId: 59f1e50a-331f-4f09-b3e8-2e8d4d355f44 dockerRepository: airbyte/source-greenhouse - dockerImageTag: 0.2.9 + dockerImageTag: 0.2.10 documentationUrl: https://docs.airbyte.io/integrations/sources/greenhouse icon: greenhouse.svg sourceType: api - releaseStage: alpha + releaseStage: beta - name: Harness sourceDefinitionId: 6fe89830-d04d-401b-aad6-6552ffa5c4af dockerRepository: farosai/airbyte-harness-source diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index b03d04def912..07f25c93109a 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3661,9 +3661,9 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-greenhouse:0.2.9" +- dockerImage: "airbyte/source-greenhouse:0.2.10" spec: - documentationUrl: "https://docs.airbyte.io/integrations/sources/greenhouse" + documentationUrl: "https://docs.airbyte.com/integrations/sources/greenhouse" connectionSpecification: $schema: "http://json-schema.org/draft-07/schema#" title: "Greenhouse Spec" diff --git a/airbyte-integrations/connectors/source-greenhouse/Dockerfile b/airbyte-integrations/connectors/source-greenhouse/Dockerfile index 8a5be1498076..66f56636848b 100644 --- a/airbyte-integrations/connectors/source-greenhouse/Dockerfile +++ b/airbyte-integrations/connectors/source-greenhouse/Dockerfile @@ -4,13 +4,13 @@ FROM python:3.9-slim RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* WORKDIR /airbyte/integration_code -COPY source_greenhouse ./source_greenhouse -COPY main.py ./ COPY setup.py ./ RUN pip install . +COPY source_greenhouse ./source_greenhouse +COPY main.py ./ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.9 +LABEL io.airbyte.version=0.2.10 LABEL io.airbyte.name=airbyte/source-greenhouse diff --git a/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml index 6c30ad302d24..bd26bb598f73 100644 --- a/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-greenhouse/acceptance-test-config.yml @@ -19,13 +19,14 @@ tests: configured_catalog_path: "integration_tests/configured_catalog.json" expect_records: path: "integration_tests/expected_records.txt" - extra_fields: yes - exact_order: yes - extra_records: no - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_users_only.json" full_refresh: - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog_const_records.json" + configured_catalog_path: "integration_tests/configured_catalog.json" - config_path: "secrets/config_users_only.json" configured_catalog_path: "integration_tests/configured_catalog_users_only.json" + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/incremental_configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-greenhouse/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..5c4e4b44e5ef --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/abnormal_state.json @@ -0,0 +1,58 @@ +{ + "candidates": { + "updated_at": "2222-01-21T00:00:00.000Z" + }, + "demographics_answers": { + "updated_at": "2222-01-21T00:00:00.000Z" + }, + "users": { + "updated_at": "2222-01-21T00:00:00.000Z" + }, + "scorecards": { + "updated_at": "2222-01-21T00:00:00.000Z" + }, + "offers": { + "updated_at": "2222-01-21T00:00:00.000Z" + }, + "job_stages": { + "updated_at": "2222-01-21T00:00:00.000Z" + }, + "job_posts": { + "updated_at": "2222-01-21T00:00:00.000Z" + }, + "interviews": { + "updated_at": "2222-01-21T00:00:00.000Z" + }, + "jobs": { + "updated_at": "2222-01-21T00:00:00.000Z" + }, + "applications": { + "applied_at": "2222-01-21T00:00:00.000Z" + }, + "jobs_stages": { + "4177046003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "4177048003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "4446240003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "4466310003": {"updated_at": "2222-01-01T00:00:00.000Z"} + }, + "applications_demographics_answers": { + "19214950003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "19214993003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "19215172003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "19215333003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "44933447003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "44937562003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "47459993003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "48693310003": {"updated_at": "2222-01-01T00:00:00.000Z"} + }, + "applications_interviews": { + "19214950003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "19214993003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "19215172003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "19215333003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "44933447003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "44937562003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "47459993003": {"updated_at": "2222-01-01T00:00:00.000Z"}, + "48693310003": {"updated_at": "2222-01-01T00:00:00.000Z"} + } +} diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog.json index c45ff14aa8f3..510dfda96156 100644 --- a/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog.json @@ -4,20 +4,28 @@ "stream": { "name": "applications", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["applied_at"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["applied_at"], "destination_sync_mode": "overwrite" }, { "stream": { "name": "candidates", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], "destination_sync_mode": "overwrite" }, { @@ -25,9 +33,11 @@ "name": "close_reasons", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "source_defined_cursor": false, + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], "destination_sync_mode": "overwrite" }, { @@ -35,9 +45,11 @@ "name": "degrees", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "source_defined_cursor": false, + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], "destination_sync_mode": "overwrite" }, { @@ -45,59 +57,81 @@ "name": "departments", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "source_defined_cursor": false, + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], "destination_sync_mode": "overwrite" }, { "stream": { "name": "job_posts", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], "destination_sync_mode": "overwrite" }, { "stream": { "name": "jobs", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], "destination_sync_mode": "overwrite" }, { "stream": { "name": "offers", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], "destination_sync_mode": "overwrite" }, { "stream": { "name": "scorecards", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], "destination_sync_mode": "overwrite" }, { "stream": { "name": "users", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], "destination_sync_mode": "overwrite" }, { @@ -105,135 +139,191 @@ "name": "custom_fields", "json_schema": {}, "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "source_defined_cursor": false, + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], "destination_sync_mode": "overwrite" }, { "stream": { "name": "demographics_question_sets", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], "destination_sync_mode": "overwrite" }, { "stream": { "name": "demographics_questions", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], "destination_sync_mode": "overwrite" }, { "stream": { "name": "demographics_answer_options", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], "destination_sync_mode": "overwrite" }, { "stream": { "name": "demographics_answers", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], "destination_sync_mode": "overwrite" }, { "stream": { "name": "applications_demographics_answers", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], "destination_sync_mode": "overwrite" }, { "stream": { "name": "demographics_question_sets_questions", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], "destination_sync_mode": "overwrite" }, { "stream": { "name": "demographics_answers_answer_options", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], "destination_sync_mode": "overwrite" }, { "stream": { "name": "interviews", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], "destination_sync_mode": "overwrite" }, { "stream": { "name": "applications_interviews", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], "destination_sync_mode": "overwrite" }, { "stream": { "name": "sources", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], "destination_sync_mode": "overwrite" }, { "stream": { "name": "rejection_reasons", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], "destination_sync_mode": "overwrite" }, { "stream": { "name": "jobs_openings", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], "destination_sync_mode": "overwrite" }, { "stream": { "name": "job_stages", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], "destination_sync_mode": "overwrite" }, { "stream": { "name": "jobs_stages", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], "destination_sync_mode": "overwrite" } ] diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog_const_records.json b/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog_const_records.json deleted file mode 100644 index ab9dfc20ec80..000000000000 --- a/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog_const_records.json +++ /dev/null @@ -1,220 +0,0 @@ -{ - "streams": [ - { - "stream": { - "name": "close_reasons", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "degrees", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "departments", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "job_posts", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "jobs", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "offers", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "scorecards", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "users", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "custom_fields", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "interviews", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "applications_interviews", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "sources", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "rejection_reasons", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "jobs_openings", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "job_stages", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "jobs_stages", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "demographics_question_sets", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "demographics_questions", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "demographics_answer_options", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "demographics_answers", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "applications_demographics_answers", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "demographics_question_sets_questions", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - }, - { - "stream": { - "name": "demographics_answers_answer_options", - "json_schema": {}, - "supported_sync_modes": ["full_refresh"] - }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" - } - ] -} diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog_users_only.json b/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog_users_only.json index 6a3c2bc4b197..2590c5f94df5 100644 --- a/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog_users_only.json +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/configured_catalog_users_only.json @@ -4,10 +4,14 @@ "stream": { "name": "users", "json_schema": {}, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_primary_key": [["id"]], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] }, "sync_mode": "full_refresh", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], "destination_sync_mode": "overwrite" } ] diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt index ba3733a1036e..c5ae337060fd 100644 --- a/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/expected_records.txt @@ -1,19 +1,17 @@ -{"stream":"applications","data":{"status":"active","source":{"public_name":"HRMARKET","id":4000067003},"rejection_reason":null,"rejection_details":null,"rejected_at":null,"prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":{"name":"John Lafleur","id":4218086003}},"prospect":true,"location":null,"last_activity_at":"2020-11-24T23:24:37.049Z","jobs":[],"job_post_id":null,"id":19214950003,"current_stage":null,"credited_to":{"name":"John Lafleur","last_name":"Lafleur","id":4218086003,"first_name":"John","employee_id":null},"candidate_id":17130511003,"applied_at":"2020-11-24T23:24:37.023Z","answers":[]},"emitted_at":1660156521774} -{"stream":"applications","data":{"status":"active","source":{"public_name":"Jobs page on your website","id":4000177003},"rejection_reason":null,"rejection_details":null,"rejected_at":null,"prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":{"name":"John Lafleur","id":4218086003}},"prospect":true,"location":null,"last_activity_at":"2020-11-24T23:25:13.804Z","jobs":[],"job_post_id":null,"id":19214993003,"current_stage":null,"credited_to":{"name":"John Lafleur","last_name":"Lafleur","id":4218086003,"first_name":"John","employee_id":null},"candidate_id":17130554003,"applied_at":"2020-11-24T23:25:13.781Z","answers":[]},"emitted_at":1660156521777} -{"stream":"applications","data":{"status":"active","source":{"public_name":"Internal Applicant","id":4000142003},"rejection_reason":null,"rejection_details":null,"rejected_at":null,"prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":null},"prospect":false,"location":null,"last_activity_at":"2020-11-24T23:28:19.779Z","jobs":[{"name":"Test job","id":4177046003}],"job_post_id":null,"id":19215172003,"current_stage":{"name":"Preliminary Phone Screen","id":5245804003},"credited_to":{"name":"John Lafleur","last_name":"Lafleur","id":4218086003,"first_name":"John","employee_id":null},"candidate_id":17130732003,"applied_at":"2020-11-24T23:28:19.712Z","answers":[]},"emitted_at":1660156521777} -{"stream":"applications","data":{"status":"active","source":{"public_name":"Referral","id":4000161003},"rejection_reason":null,"rejection_details":null,"rejected_at":null,"prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":null},"prospect":false,"location":null,"last_activity_at":"2020-12-05T02:50:25.811Z","jobs":[{"name":"Test Job 2","id":4177048003}],"job_post_id":null,"id":19215333003,"current_stage":{"name":"Offer","id":5245823003},"credited_to":{"name":"John Lafleur","last_name":"Lafleur","id":4218086003,"first_name":"John","employee_id":null},"candidate_id":17130848003,"applied_at":"2020-11-24T23:30:14.394Z","answers":[]},"emitted_at":1660156521778} -{"stream":"applications","data":{"status":"rejected","source":{"public_name":"Test agency","id":4013544003},"rejection_reason":{"type":{"name":"We rejected them","id":4000000003},"name":"Other (add notes below)","id":4000004003},"rejection_details":{},"rejected_at":"2021-09-29T16:38:03.637Z","prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":null},"prospect":false,"location":null,"last_activity_at":"2021-09-29T16:38:03.660Z","jobs":[{"name":"Test job","id":4177046003}],"job_post_id":null,"id":44933447003,"current_stage":{"name":"Phone Interview","id":5245805003},"credited_to":{"name":"John Lafleur","last_name":"Lafleur","id":4218086003,"first_name":"John","employee_id":null},"candidate_id":40513954003,"applied_at":"2021-09-29T16:37:27.589Z","answers":[]},"emitted_at":1660156521778} -{"stream":"applications","data":{"status":"active","source":{"public_name":"Test agency","id":4013544003},"rejection_reason":null,"rejection_details":null,"rejected_at":null,"prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":null},"prospect":false,"location":null,"last_activity_at":"2021-10-10T16:22:13.708Z","jobs":[{"name":"Test job","id":4177046003}],"job_post_id":null,"id":44937562003,"current_stage":{"name":"Preliminary Phone Screen","id":5245804003},"credited_to":{"name":"Greenhouse Admin","last_name":"Admin","id":4218085003,"first_name":"Greenhouse","employee_id":null},"candidate_id":40517966003,"applied_at":"2021-09-29T17:20:36.063Z","answers":[]},"emitted_at":1660156521778} -{"stream":"applications","data":{"status":"active","source":{"public_name":"Test agency","id":4013544003},"rejection_reason":null,"rejection_details":null,"rejected_at":null,"prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":null},"prospect":false,"location":null,"last_activity_at":"2021-11-03T19:56:07.402Z","jobs":[{"name":"Test job 3","id":4466310003}],"job_post_id":4797691003,"id":47459993003,"current_stage":{"name":"Application Review","id":7332462003},"credited_to":{"name":"John Lafleur","last_name":"Lafleur","id":4218086003,"first_name":"John","employee_id":null},"candidate_id":42921157003,"applied_at":"2021-11-03T19:51:14.644Z","answers":[{"question":"Website","answer":null},{"question":"LinkedIn Profile","answer":null}]},"emitted_at":1660156521779} -{"stream":"applications","data":{"status":"active","source":{"public_name":"Bubblesort","id":4000032003},"rejection_reason":null,"rejection_details":null,"rejected_at":null,"prospective_office":null,"prospective_department":null,"prospect_detail":{"prospect_stage":null,"prospect_pool":null,"prospect_owner":null},"prospect":false,"location":null,"last_activity_at":"2021-11-22T08:41:55.713Z","jobs":[{"name":"Copy of Test Job 2","id":4446240003}],"job_post_id":null,"id":48693310003,"current_stage":{"name":"Application Review","id":7179760003},"credited_to":{"name":"emily.brooks+airbyte_integration@greenhouse.io","last_name":null,"id":4218087003,"first_name":null,"employee_id":null},"candidate_id":44081361003,"applied_at":"2021-11-22T08:41:55.640Z","answers":[]},"emitted_at":1660156521779} -{ "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2020-11-24T23:24:37.050Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "Test","last_activity": "2020-11-24T23:24:37.049Z","is_private": false,"id": 17130511003,"first_name": "Test","employments": [ ],"email_addresses": [ ],"educations": [ ],"created_at": "2020-11-24T23:24:37.018Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 19214950003 ],"addresses": [ ] },"emitted_at": 1660156191149 } - { "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2020-11-24T23:25:13.806Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "Test2","last_activity": "2020-11-24T23:25:13.804Z","is_private": false,"id": 17130554003,"first_name": "Test2","employments": [ ],"email_addresses": [ ],"educations": [ ],"created_at": "2020-11-24T23:25:13.777Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 19214993003 ],"addresses": [ ] },"emitted_at": 1660156191151 } - { "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2020-11-24T23:28:19.781Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "Lastname","last_activity": "2020-11-24T23:28:19.779Z","is_private": false,"id": 17130732003,"first_name": "Name","employments": [ ],"email_addresses": [ ],"educations": [ ],"created_at": "2020-11-24T23:28:19.710Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 19215172003 ],"addresses": [ ] },"emitted_at": 1660156191152 } - { "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2020-12-05T02:50:25.823Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "D","last_activity": "2020-12-05T02:50:25.811Z","is_private": false,"id": 17130848003,"first_name": "Jack","employments": [ ],"email_addresses": [ ],"educations": [ ],"created_at": "2020-11-24T23:30:14.386Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 19215333003 ],"addresses": [ ] },"emitted_at": 1660156191152 } - { "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2021-09-29T16:38:03.672Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "User","last_activity": "2021-09-29T16:38:03.660Z","is_private": false,"id": 40513954003,"first_name": "Test","employments": [ ],"email_addresses": [ ],"educations": [ ],"created_at": "2021-09-29T16:37:27.585Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 44933447003 ],"addresses": [ ] },"emitted_at": 1660156191152 } - { "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2021-10-10T16:22:13.718Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "Scheduled Interview","last_activity": "2021-10-10T16:22:13.708Z","is_private": false,"id": 40517966003,"first_name": "Test","employments": [ ],"email_addresses": [ { "value": "vadym.hevlich@zazmic.com","type": "personal" } ],"educations": [ ],"created_at": "2021-09-29T17:20:36.038Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 44937562003 ],"addresses": [ ] },"emitted_at": 1660156191153 } - { "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2021-11-03T19:56:07.423Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "Candidate","last_activity": "2021-11-03T19:56:07.402Z","is_private": false,"id": 42921157003,"first_name": "Test","employments": [ ],"email_addresses": [ { "value": "vadym.hevlich@zazmic.com","type": "work" } ],"educations": [ ],"created_at": "2021-11-03T19:51:14.639Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 47459993003 ],"addresses": [ ] },"emitted_at": 1660156191153 } - { "stream": "candidates","data": { "website_addresses": [ ],"updated_at": "2021-11-22T08:41:55.716Z","title": null,"tags": [ ],"social_media_addresses": [ ],"recruiter": null,"photo_url": null,"phone_numbers": [ ],"last_name": "Cherniaev","last_activity": "2021-11-22T08:41:55.713Z","is_private": false,"id": 44081361003,"first_name": "Yurii","employments": [ ],"email_addresses": [ ],"educations": [ ],"created_at": "2021-11-22T08:41:55.634Z","coordinator": null,"company": null,"can_email": true,"application_ids": [ 48693310003 ],"addresses": [ ] },"emitted_at": 1660156191153 } +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "HRMARKET", "id": 4000067003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "John Lafleur", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:24:37.049Z", "jobs": [], "job_post_id": null, "id": 19214950003, "current_stage": null, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130511003, "attachments": [], "applied_at": "2020-11-24T23:24:37.023Z", "answers": []}, "emitted_at": 1662402660037} +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Jobs page on your website", "id": 4000177003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "John Lafleur", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:25:13.804Z", "jobs": [], "job_post_id": null, "id": 19214993003, "current_stage": null, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130554003, "attachments": [], "applied_at": "2020-11-24T23:25:13.781Z", "answers": []}, "emitted_at": 1662402660042} +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Internal Applicant", "id": 4000142003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2020-11-24T23:28:19.779Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 19215172003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130732003, "attachments": [], "applied_at": "2020-11-24T23:28:19.712Z", "answers": []}, "emitted_at": 1662402660043} +{"stream": "applications", "data": {"status": "rejected", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": {"type": {"name": "We rejected them", "id": 4000000003}, "name": "Other (add notes below)", "id": 4000004003}, "rejection_details": {}, "rejected_at": "2021-09-29T16:38:03.637Z", "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-09-29T16:38:03.660Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 44933447003, "current_stage": {"name": "Phone Interview", "id": 5245805003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 40513954003, "attachments": [], "applied_at": "2021-09-29T16:37:27.589Z", "answers": []}, "emitted_at": 1662402660044} +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-10-10T16:22:13.708Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 44937562003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "Greenhouse Admin", "last_name": "Admin", "id": 4218085003, "first_name": "Greenhouse", "employee_id": null}, "candidate_id": 40517966003, "attachments": [], "applied_at": "2021-09-29T17:20:36.063Z", "answers": []}, "emitted_at": 1662402660045} +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-11-03T19:56:07.402Z", "jobs": [{"name": "Test job 3", "id": 4466310003}], "job_post_id": 4797691003, "id": 47459993003, "current_stage": {"name": "Application Review", "id": 7332462003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 42921157003, "attachments": [], "applied_at": "2021-11-03T19:51:14.644Z", "answers": [{"question": "Website", "answer": null}, {"question": "LinkedIn Profile", "answer": null}]}, "emitted_at": 1662402660046} +{"stream": "applications", "data": {"status": "active", "source": {"public_name": "Bubblesort", "id": 4000032003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-11-22T08:41:55.713Z", "jobs": [{"name": "Copy of Test Job 2", "id": 4446240003}], "job_post_id": null, "id": 48693310003, "current_stage": {"name": "Application Review", "id": 7179760003}, "credited_to": {"name": "emily.brooks+airbyte_integration@greenhouse.io", "last_name": null, "id": 4218087003, "first_name": null, "employee_id": null}, "candidate_id": 44081361003, "attachments": [], "applied_at": "2021-11-22T08:41:55.640Z", "answers": []}, "emitted_at": 1662402660047} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:24:37.050Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Test", "last_activity": "2020-11-24T23:24:37.049Z", "is_private": false, "id": 17130511003, "first_name": "Test", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:24:37.018Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "HRMARKET", "id": 4000067003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "John Lafleur", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:24:37.049Z", "jobs": [], "job_post_id": null, "id": 19214950003, "current_stage": null, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130511003, "attachments": [], "applied_at": "2020-11-24T23:24:37.023Z", "answers": []}], "application_ids": [19214950003], "addresses": []}, "emitted_at": 1662403631906} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:25:13.806Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Test2", "last_activity": "2020-11-24T23:25:13.804Z", "is_private": false, "id": 17130554003, "first_name": "Test2", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:25:13.777Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Jobs page on your website", "id": 4000177003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": {"name": "John Lafleur", "id": 4218086003}}, "prospect": true, "location": null, "last_activity_at": "2020-11-24T23:25:13.804Z", "jobs": [], "job_post_id": null, "id": 19214993003, "current_stage": null, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130554003, "attachments": [], "applied_at": "2020-11-24T23:25:13.781Z", "answers": []}], "application_ids": [19214993003], "addresses": []}, "emitted_at": 1662403631910} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2020-11-24T23:28:19.781Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Lastname", "last_activity": "2020-11-24T23:28:19.779Z", "is_private": false, "id": 17130732003, "first_name": "Name", "employments": [], "email_addresses": [], "educations": [], "created_at": "2020-11-24T23:28:19.710Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Internal Applicant", "id": 4000142003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2020-11-24T23:28:19.779Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 19215172003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 17130732003, "attachments": [], "applied_at": "2020-11-24T23:28:19.712Z", "answers": []}], "application_ids": [19215172003], "addresses": []}, "emitted_at": 1662403631912} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2021-09-29T16:38:03.672Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "User", "last_activity": "2021-09-29T16:38:03.660Z", "is_private": false, "id": 40513954003, "first_name": "Test", "employments": [], "email_addresses": [], "educations": [], "created_at": "2021-09-29T16:37:27.585Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "rejected", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": {"type": {"name": "We rejected them", "id": 4000000003}, "name": "Other (add notes below)", "id": 4000004003}, "rejection_details": {}, "rejected_at": "2021-09-29T16:38:03.637Z", "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-09-29T16:38:03.660Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 44933447003, "current_stage": {"name": "Phone Interview", "id": 5245805003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 40513954003, "attachments": [], "applied_at": "2021-09-29T16:37:27.589Z", "answers": []}], "application_ids": [44933447003], "addresses": []}, "emitted_at": 1662403631914} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2021-10-10T16:22:13.718Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Scheduled Interview", "last_activity": "2021-10-10T16:22:13.708Z", "is_private": false, "id": 40517966003, "first_name": "Test", "employments": [], "email_addresses": [{"value": "vadym.hevlich@zazmic.com", "type": "personal"}], "educations": [], "created_at": "2021-09-29T17:20:36.038Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-10-10T16:22:13.708Z", "jobs": [{"name": "Test job", "id": 4177046003}], "job_post_id": null, "id": 44937562003, "current_stage": {"name": "Preliminary Phone Screen", "id": 5245804003}, "credited_to": {"name": "Greenhouse Admin", "last_name": "Admin", "id": 4218085003, "first_name": "Greenhouse", "employee_id": null}, "candidate_id": 40517966003, "attachments": [], "applied_at": "2021-09-29T17:20:36.063Z", "answers": []}], "application_ids": [44937562003], "addresses": []}, "emitted_at": 1662403631916} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2021-11-03T19:56:07.423Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Candidate", "last_activity": "2021-11-03T19:56:07.402Z", "is_private": false, "id": 42921157003, "first_name": "Test", "employments": [], "email_addresses": [{"value": "vadym.hevlich@zazmic.com", "type": "work"}], "educations": [], "created_at": "2021-11-03T19:51:14.639Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Test agency", "id": 4013544003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-11-03T19:56:07.402Z", "jobs": [{"name": "Test job 3", "id": 4466310003}], "job_post_id": 4797691003, "id": 47459993003, "current_stage": {"name": "Application Review", "id": 7332462003}, "credited_to": {"name": "John Lafleur", "last_name": "Lafleur", "id": 4218086003, "first_name": "John", "employee_id": null}, "candidate_id": 42921157003, "attachments": [], "applied_at": "2021-11-03T19:51:14.644Z", "answers": [{"question": "Website", "answer": null}, {"question": "LinkedIn Profile", "answer": null}]}], "application_ids": [47459993003], "addresses": []}, "emitted_at": 1662403631917} +{"stream": "candidates", "data": {"website_addresses": [], "updated_at": "2021-11-22T08:41:55.716Z", "title": null, "tags": [], "social_media_addresses": [], "recruiter": null, "photo_url": null, "phone_numbers": [], "last_name": "Cherniaev", "last_activity": "2021-11-22T08:41:55.713Z", "is_private": false, "id": 44081361003, "first_name": "Yurii", "employments": [], "email_addresses": [], "educations": [], "created_at": "2021-11-22T08:41:55.634Z", "coordinator": null, "company": null, "can_email": true, "attachments": [], "applications": [{"status": "active", "source": {"public_name": "Bubblesort", "id": 4000032003}, "rejection_reason": null, "rejection_details": null, "rejected_at": null, "prospective_office": null, "prospective_department": null, "prospect_detail": {"prospect_stage": null, "prospect_pool": null, "prospect_owner": null}, "prospect": false, "location": null, "last_activity_at": "2021-11-22T08:41:55.713Z", "jobs": [{"name": "Copy of Test Job 2", "id": 4446240003}], "job_post_id": null, "id": 48693310003, "current_stage": {"name": "Application Review", "id": 7179760003}, "credited_to": {"name": "emily.brooks+airbyte_integration@greenhouse.io", "last_name": null, "id": 4218087003, "first_name": null, "employee_id": null}, "candidate_id": 44081361003, "attachments": [], "applied_at": "2021-11-22T08:41:55.640Z", "answers": []}], "application_ids": [48693310003], "addresses": []}, "emitted_at": 1662403631919} {"stream":"close_reasons","data":{"id":4010635003,"name":"Not Filling"},"emitted_at":1660156523420} {"stream":"close_reasons","data":{"id":4010634003,"name":"On Hold"},"emitted_at":1660156523422} {"stream":"close_reasons","data":{"id":4010633003,"name":"Hire - New Headcount"},"emitted_at":1660156523422} @@ -46,22 +44,21 @@ {"stream":"users","data":{"id":4218087003,"name":"emily.brooks+airbyte_integration@greenhouse.io","first_name":null,"last_name":null,"primary_email_address":"emily.brooks+airbyte_integration@greenhouse.io","updated_at":"2020-11-18T14:09:08.991Z","created_at":"2020-11-18T14:09:08.809Z","disabled":false,"site_admin":true,"emails":["emily.brooks+airbyte_integration@greenhouse.io"],"employee_id":null,"linked_candidate_ids":[]},"emitted_at":1660156525826} {"stream":"users","data":{"id":4460715003,"name":"Vadym Ratniuk","first_name":"Vadym","last_name":"Ratniuk","primary_email_address":"vadym.ratniuk@globallogic.com","updated_at":"2021-09-18T10:09:16.846Z","created_at":"2021-09-14T14:03:01.050Z","disabled":false,"site_admin":false,"emails":["vadym.ratniuk@globallogic.com"],"employee_id":null,"linked_candidate_ids":[]},"emitted_at":1660156525826} {"stream":"users","data":{"id":4481107003,"name":"Vadym Hevlich","first_name":"Vadym","last_name":"Hevlich","primary_email_address":"vadym.hevlich@zazmic.com","updated_at":"2021-10-10T17:49:28.058Z","created_at":"2021-10-10T17:48:41.978Z","disabled":false,"site_admin":true,"emails":["vadym.hevlich@zazmic.com"],"employee_id":null,"linked_candidate_ids":[]},"emitted_at":1660156525827} -{"stream":"custom_fields","data":{"id":4680898003,"name":"School Name","active":true,"field_type":"candidate","priority":0,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"school_name","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10845822003,"name":"Abraham Baldwin Agricultural College","priority":0,"external_id":null},{"id":10845823003,"name":"Academy of Art University","priority":1,"external_id":null},{"id":10845824003,"name":"Acadia University","priority":2,"external_id":null},{"id":10845825003,"name":"Adams State University","priority":3,"external_id":null},{"id":10845826003,"name":"Adelphi University","priority":4,"external_id":null},{"id":10845827003,"name":"Adrian College","priority":5,"external_id":null},{"id":10845828003,"name":"Adventist University of Health Sciences","priority":6,"external_id":null},{"id":10845829003,"name":"Agnes Scott College","priority":7,"external_id":null},{"id":10845830003,"name":"AIB College of Business","priority":8,"external_id":null},{"id":10845831003,"name":"Alaska Pacific University","priority":9,"external_id":null},{"id":10845832003,"name":"Albany College of Pharmacy and Health Sciences","priority":10,"external_id":null},{"id":10845833003,"name":"Albany State University","priority":11,"external_id":null},{"id":10845834003,"name":"Albertus Magnus College","priority":12,"external_id":null},{"id":10845835003,"name":"Albion College","priority":13,"external_id":null},{"id":10845836003,"name":"Albright College","priority":14,"external_id":null},{"id":10845837003,"name":"Alderson Broaddus University","priority":15,"external_id":null},{"id":10845838003,"name":"Alfred University","priority":16,"external_id":null},{"id":10845839003,"name":"Alice Lloyd College","priority":17,"external_id":null},{"id":10845840003,"name":"Allegheny College","priority":18,"external_id":null},{"id":10845841003,"name":"Allen College","priority":19,"external_id":null},{"id":10845842003,"name":"Allen University","priority":20,"external_id":null},{"id":10845843003,"name":"Alliant International University","priority":21,"external_id":null},{"id":10845844003,"name":"Alma College","priority":22,"external_id":null},{"id":10845845003,"name":"Alvernia University","priority":23,"external_id":null},{"id":10845846003,"name":"Alverno College","priority":24,"external_id":null},{"id":10845847003,"name":"Amberton University","priority":25,"external_id":null},{"id":10845848003,"name":"American Academy of Art","priority":26,"external_id":null},{"id":10845849003,"name":"American Indian College of the Assemblies of God","priority":27,"external_id":null},{"id":10845850003,"name":"American InterContinental University","priority":28,"external_id":null},{"id":10845851003,"name":"American International College","priority":29,"external_id":null},{"id":10845852003,"name":"American Jewish University","priority":30,"external_id":null},{"id":10845853003,"name":"American Public University System","priority":31,"external_id":null},{"id":10845854003,"name":"American University","priority":32,"external_id":null},{"id":10845855003,"name":"American University in Bulgaria","priority":33,"external_id":null},{"id":10845856003,"name":"American University in Cairo","priority":34,"external_id":null},{"id":10845857003,"name":"American University of Beirut","priority":35,"external_id":null},{"id":10845858003,"name":"American University of Paris","priority":36,"external_id":null},{"id":10845859003,"name":"American University of Puerto Rico","priority":37,"external_id":null},{"id":10845860003,"name":"Amherst College","priority":38,"external_id":null},{"id":10845861003,"name":"Amridge University","priority":39,"external_id":null},{"id":10845862003,"name":"Anderson University","priority":40,"external_id":null},{"id":10845863003,"name":"Andrews University","priority":41,"external_id":null},{"id":10845864003,"name":"Angelo State University","priority":42,"external_id":null},{"id":10845865003,"name":"Anna Maria College","priority":43,"external_id":null},{"id":10845866003,"name":"Antioch University","priority":44,"external_id":null},{"id":10845867003,"name":"Appalachian Bible College","priority":45,"external_id":null},{"id":10845868003,"name":"Aquinas College","priority":46,"external_id":null},{"id":10845869003,"name":"Arcadia University","priority":47,"external_id":null},{"id":10845870003,"name":"Argosy University","priority":48,"external_id":null},{"id":10845871003,"name":"Arizona Christian University","priority":49,"external_id":null},{"id":10845872003,"name":"Arizona State University - West","priority":50,"external_id":null},{"id":10845873003,"name":"Arkansas Baptist College","priority":51,"external_id":null},{"id":10845874003,"name":"Arkansas Tech University","priority":52,"external_id":null},{"id":10845875003,"name":"Armstrong Atlantic State University","priority":53,"external_id":null},{"id":10845876003,"name":"Art Academy of Cincinnati","priority":54,"external_id":null},{"id":10845877003,"name":"Art Center College of Design","priority":55,"external_id":null},{"id":10845878003,"name":"Art Institute of Atlanta","priority":56,"external_id":null},{"id":10845879003,"name":"Art Institute of Colorado","priority":57,"external_id":null},{"id":10845880003,"name":"Art Institute of Houston","priority":58,"external_id":null},{"id":10845881003,"name":"Art Institute of Pittsburgh","priority":59,"external_id":null},{"id":10845882003,"name":"Art Institute of Portland","priority":60,"external_id":null},{"id":10845883003,"name":"Art Institute of Seattle","priority":61,"external_id":null},{"id":10845884003,"name":"Asbury University","priority":62,"external_id":null},{"id":10845885003,"name":"Ashford University","priority":63,"external_id":null},{"id":10845886003,"name":"Ashland University","priority":64,"external_id":null},{"id":10845887003,"name":"Assumption College","priority":65,"external_id":null},{"id":10845888003,"name":"Athens State University","priority":66,"external_id":null},{"id":10845889003,"name":"Auburn University - Montgomery","priority":67,"external_id":null},{"id":10845890003,"name":"Augsburg College","priority":68,"external_id":null},{"id":10845891003,"name":"Augustana College","priority":69,"external_id":null},{"id":10845892003,"name":"Aurora University","priority":70,"external_id":null},{"id":10845893003,"name":"Austin College","priority":71,"external_id":null},{"id":10845894003,"name":"Alcorn State University","priority":72,"external_id":null},{"id":10845895003,"name":"Ave Maria University","priority":73,"external_id":null},{"id":10845896003,"name":"Averett University","priority":74,"external_id":null},{"id":10845897003,"name":"Avila University","priority":75,"external_id":null},{"id":10845898003,"name":"Azusa Pacific University","priority":76,"external_id":null},{"id":10845899003,"name":"Babson College","priority":77,"external_id":null},{"id":10845900003,"name":"Bacone College","priority":78,"external_id":null},{"id":10845901003,"name":"Baker College of Flint","priority":79,"external_id":null},{"id":10845902003,"name":"Baker University","priority":80,"external_id":null},{"id":10845903003,"name":"Baldwin Wallace University","priority":81,"external_id":null},{"id":10845904003,"name":"Christian Brothers University","priority":82,"external_id":null},{"id":10845905003,"name":"Abilene Christian University","priority":83,"external_id":null},{"id":10845906003,"name":"Arizona State University","priority":84,"external_id":null},{"id":10845907003,"name":"Auburn University","priority":85,"external_id":null},{"id":10845908003,"name":"Alabama A&M University","priority":86,"external_id":null},{"id":10845909003,"name":"Alabama State University","priority":87,"external_id":null},{"id":10845910003,"name":"Arkansas State University","priority":88,"external_id":null},{"id":10845911003,"name":"Baptist Bible College","priority":89,"external_id":null},{"id":10845912003,"name":"Baptist Bible College and Seminary","priority":90,"external_id":null},{"id":10845913003,"name":"Baptist College of Florida","priority":91,"external_id":null},{"id":10845914003,"name":"Baptist Memorial College of Health Sciences","priority":92,"external_id":null},{"id":10845915003,"name":"Baptist Missionary Association Theological Seminary","priority":93,"external_id":null},{"id":10845916003,"name":"Bard College","priority":94,"external_id":null},{"id":10845917003,"name":"Bard College at Simon's Rock","priority":95,"external_id":null},{"id":10845918003,"name":"Barnard College","priority":96,"external_id":null},{"id":10845919003,"name":"Barry University","priority":97,"external_id":null},{"id":10845920003,"name":"Barton College","priority":98,"external_id":null},{"id":10845921003,"name":"Bastyr University","priority":99,"external_id":null},{"id":10845922003,"name":"Bates College","priority":100,"external_id":null},{"id":10845923003,"name":"Bauder College","priority":101,"external_id":null},{"id":10845924003,"name":"Bay Path College","priority":102,"external_id":null},{"id":10845925003,"name":"Bay State College","priority":103,"external_id":null},{"id":10845926003,"name":"Bayamon Central University","priority":104,"external_id":null},{"id":10845927003,"name":"Beacon College","priority":105,"external_id":null},{"id":10845928003,"name":"Becker College","priority":106,"external_id":null},{"id":10845929003,"name":"Belhaven University","priority":107,"external_id":null},{"id":10845930003,"name":"Bellarmine University","priority":108,"external_id":null},{"id":10845931003,"name":"Bellevue College","priority":109,"external_id":null},{"id":10845932003,"name":"Bellevue University","priority":110,"external_id":null},{"id":10845933003,"name":"Bellin College","priority":111,"external_id":null},{"id":10845934003,"name":"Belmont Abbey College","priority":112,"external_id":null},{"id":10845935003,"name":"Belmont University","priority":113,"external_id":null},{"id":10845936003,"name":"Beloit College","priority":114,"external_id":null},{"id":10845937003,"name":"Bemidji State University","priority":115,"external_id":null},{"id":10845938003,"name":"Benedict College","priority":116,"external_id":null},{"id":10845939003,"name":"Benedictine College","priority":117,"external_id":null},{"id":10845940003,"name":"Benedictine University","priority":118,"external_id":null},{"id":10845941003,"name":"Benjamin Franklin Institute of Technology","priority":119,"external_id":null},{"id":10845942003,"name":"Bennett College","priority":120,"external_id":null},{"id":10845943003,"name":"Bennington College","priority":121,"external_id":null},{"id":10845944003,"name":"Bentley University","priority":122,"external_id":null},{"id":10845945003,"name":"Berea College","priority":123,"external_id":null},{"id":10845946003,"name":"Berkeley College","priority":124,"external_id":null},{"id":10845947003,"name":"Berklee College of Music","priority":125,"external_id":null},{"id":10845948003,"name":"Berry College","priority":126,"external_id":null},{"id":10845949003,"name":"Bethany College","priority":127,"external_id":null},{"id":10845950003,"name":"Bethany Lutheran College","priority":128,"external_id":null},{"id":10845951003,"name":"Bethel College","priority":129,"external_id":null},{"id":10845952003,"name":"Bethel University","priority":130,"external_id":null},{"id":10845953003,"name":"BI Norwegian Business School","priority":131,"external_id":null},{"id":10845954003,"name":"Binghamton University - SUNY","priority":132,"external_id":null},{"id":10845955003,"name":"Biola University","priority":133,"external_id":null},{"id":10845956003,"name":"Birmingham-Southern College","priority":134,"external_id":null},{"id":10845957003,"name":"Bismarck State College","priority":135,"external_id":null},{"id":10845958003,"name":"Black Hills State University","priority":136,"external_id":null},{"id":10845959003,"name":"Blackburn College","priority":137,"external_id":null},{"id":10845960003,"name":"Blessing-Rieman College of Nursing","priority":138,"external_id":null},{"id":10845961003,"name":"Bloomfield College","priority":139,"external_id":null},{"id":10845962003,"name":"Bloomsburg University of Pennsylvania","priority":140,"external_id":null},{"id":10845963003,"name":"Blue Mountain College","priority":141,"external_id":null},{"id":10845964003,"name":"Bluefield College","priority":142,"external_id":null},{"id":10845965003,"name":"Bluefield State College","priority":143,"external_id":null},{"id":10845966003,"name":"Bluffton University","priority":144,"external_id":null},{"id":10845967003,"name":"Boricua College","priority":145,"external_id":null},{"id":10845968003,"name":"Boston Architectural College","priority":146,"external_id":null},{"id":10845969003,"name":"Boston Conservatory","priority":147,"external_id":null},{"id":10845970003,"name":"Boston University","priority":148,"external_id":null},{"id":10845971003,"name":"Bowdoin College","priority":149,"external_id":null},{"id":10845972003,"name":"Bowie State University","priority":150,"external_id":null},{"id":10845973003,"name":"Bradley University","priority":151,"external_id":null},{"id":10845974003,"name":"Brandeis University","priority":152,"external_id":null},{"id":10845975003,"name":"Brandman University","priority":153,"external_id":null},{"id":10845976003,"name":"Brazosport College","priority":154,"external_id":null},{"id":10845977003,"name":"Brenau University","priority":155,"external_id":null},{"id":10845978003,"name":"Brescia University","priority":156,"external_id":null},{"id":10845979003,"name":"Brevard College","priority":157,"external_id":null},{"id":10845980003,"name":"Brewton-Parker College","priority":158,"external_id":null},{"id":10845981003,"name":"Briar Cliff University","priority":159,"external_id":null},{"id":10845982003,"name":"Briarcliffe College","priority":160,"external_id":null},{"id":10845983003,"name":"Bridgewater College","priority":161,"external_id":null},{"id":10845984003,"name":"Bridgewater State University","priority":162,"external_id":null},{"id":10845985003,"name":"Brigham Young University - Hawaii","priority":163,"external_id":null},{"id":10845986003,"name":"Brigham Young University - Idaho","priority":164,"external_id":null},{"id":10845987003,"name":"Brock University","priority":165,"external_id":null},{"id":10845988003,"name":"Bryan College","priority":166,"external_id":null},{"id":10845989003,"name":"Bryn Athyn College of the New Church","priority":167,"external_id":null},{"id":10845990003,"name":"Bryn Mawr College","priority":168,"external_id":null},{"id":10845991003,"name":"Boston College","priority":169,"external_id":null},{"id":10845992003,"name":"Buena Vista University","priority":170,"external_id":null},{"id":10845993003,"name":"Burlington College","priority":171,"external_id":null},{"id":10845994003,"name":"Bowling Green State University","priority":172,"external_id":null},{"id":10845995003,"name":"Brown University","priority":173,"external_id":null},{"id":10845996003,"name":"Appalachian State University","priority":174,"external_id":null},{"id":10845997003,"name":"Brigham Young University - Provo","priority":175,"external_id":null},{"id":10845998003,"name":"Boise State University","priority":176,"external_id":null},{"id":10845999003,"name":"Bethune-Cookman University","priority":177,"external_id":null},{"id":10846000003,"name":"Bryant University","priority":178,"external_id":null},{"id":10846001003,"name":"Cabarrus College of Health Sciences","priority":179,"external_id":null},{"id":10846002003,"name":"Cabrini College","priority":180,"external_id":null},{"id":10846003003,"name":"Cairn University","priority":181,"external_id":null},{"id":10846004003,"name":"Caldwell College","priority":182,"external_id":null},{"id":10846005003,"name":"California Baptist University","priority":183,"external_id":null},{"id":10846006003,"name":"California College of the Arts","priority":184,"external_id":null},{"id":10846007003,"name":"California Institute of Integral Studies","priority":185,"external_id":null},{"id":10846008003,"name":"California Institute of Technology","priority":186,"external_id":null},{"id":10846009003,"name":"California Institute of the Arts","priority":187,"external_id":null},{"id":10846010003,"name":"California Lutheran University","priority":188,"external_id":null},{"id":10846011003,"name":"California Maritime Academy","priority":189,"external_id":null},{"id":10846012003,"name":"California State Polytechnic University - Pomona","priority":190,"external_id":null},{"id":10846013003,"name":"California State University - Bakersfield","priority":191,"external_id":null},{"id":10846014003,"name":"California State University - Channel Islands","priority":192,"external_id":null},{"id":10846015003,"name":"California State University - Chico","priority":193,"external_id":null},{"id":10846016003,"name":"California State University - Dominguez Hills","priority":194,"external_id":null},{"id":10846017003,"name":"California State University - East Bay","priority":195,"external_id":null},{"id":10846018003,"name":"California State University - Fullerton","priority":196,"external_id":null},{"id":10846019003,"name":"California State University - Los Angeles","priority":197,"external_id":null},{"id":10846020003,"name":"California State University - Monterey Bay","priority":198,"external_id":null},{"id":10846021003,"name":"California State University - Northridge","priority":199,"external_id":null},{"id":10846022003,"name":"California State University - San Bernardino","priority":200,"external_id":null},{"id":10846023003,"name":"California State University - San Marcos","priority":201,"external_id":null},{"id":10846024003,"name":"California State University - Stanislaus","priority":202,"external_id":null},{"id":10846025003,"name":"California University of Pennsylvania","priority":203,"external_id":null},{"id":10846026003,"name":"Calumet College of St. Joseph","priority":204,"external_id":null},{"id":10846027003,"name":"Calvary Bible College and Theological Seminary","priority":205,"external_id":null},{"id":10846028003,"name":"Calvin College","priority":206,"external_id":null},{"id":10846029003,"name":"Cambridge College","priority":207,"external_id":null},{"id":10846030003,"name":"Cameron University","priority":208,"external_id":null},{"id":10846031003,"name":"Campbellsville University","priority":209,"external_id":null},{"id":10846032003,"name":"Canisius College","priority":210,"external_id":null},{"id":10846033003,"name":"Capella University","priority":211,"external_id":null},{"id":10846034003,"name":"Capital University","priority":212,"external_id":null},{"id":10846035003,"name":"Capitol College","priority":213,"external_id":null},{"id":10846036003,"name":"Cardinal Stritch University","priority":214,"external_id":null},{"id":10846037003,"name":"Caribbean University","priority":215,"external_id":null},{"id":10846038003,"name":"Carleton College","priority":216,"external_id":null},{"id":10846039003,"name":"Carleton University","priority":217,"external_id":null},{"id":10846040003,"name":"Carlos Albizu University","priority":218,"external_id":null},{"id":10846041003,"name":"Carlow University","priority":219,"external_id":null},{"id":10846042003,"name":"Carnegie Mellon University","priority":220,"external_id":null},{"id":10846043003,"name":"Carroll College","priority":221,"external_id":null},{"id":10846044003,"name":"Carroll University","priority":222,"external_id":null},{"id":10846045003,"name":"Carson-Newman University","priority":223,"external_id":null},{"id":10846046003,"name":"Carthage College","priority":224,"external_id":null},{"id":10846047003,"name":"Case Western Reserve University","priority":225,"external_id":null},{"id":10846048003,"name":"Castleton State College","priority":226,"external_id":null},{"id":10846049003,"name":"Catawba College","priority":227,"external_id":null},{"id":10846050003,"name":"Cazenovia College","priority":228,"external_id":null},{"id":10846051003,"name":"Cedar Crest College","priority":229,"external_id":null},{"id":10846052003,"name":"Cedarville University","priority":230,"external_id":null},{"id":10846053003,"name":"Centenary College","priority":231,"external_id":null},{"id":10846054003,"name":"Centenary College of Louisiana","priority":232,"external_id":null},{"id":10846055003,"name":"Central Baptist College","priority":233,"external_id":null},{"id":10846056003,"name":"Central Bible College","priority":234,"external_id":null},{"id":10846057003,"name":"Central Christian College","priority":235,"external_id":null},{"id":10846058003,"name":"Central College","priority":236,"external_id":null},{"id":10846059003,"name":"Central Methodist University","priority":237,"external_id":null},{"id":10846060003,"name":"Central Penn College","priority":238,"external_id":null},{"id":10846061003,"name":"Central State University","priority":239,"external_id":null},{"id":10846062003,"name":"Central Washington University","priority":240,"external_id":null},{"id":10846063003,"name":"Centre College","priority":241,"external_id":null},{"id":10846064003,"name":"Chadron State College","priority":242,"external_id":null},{"id":10846065003,"name":"Chamberlain College of Nursing","priority":243,"external_id":null},{"id":10846066003,"name":"Chaminade University of Honolulu","priority":244,"external_id":null},{"id":10846067003,"name":"Champlain College","priority":245,"external_id":null},{"id":10846068003,"name":"Chancellor University","priority":246,"external_id":null},{"id":10846069003,"name":"Chapman University","priority":247,"external_id":null},{"id":10846070003,"name":"Charles R. Drew University of Medicine and Science","priority":248,"external_id":null},{"id":10846071003,"name":"Charter Oak State College","priority":249,"external_id":null},{"id":10846072003,"name":"Chatham University","priority":250,"external_id":null},{"id":10846073003,"name":"Chestnut Hill College","priority":251,"external_id":null},{"id":10846074003,"name":"Cheyney University of Pennsylvania","priority":252,"external_id":null},{"id":10846075003,"name":"Chicago State University","priority":253,"external_id":null},{"id":10846076003,"name":"Chipola College","priority":254,"external_id":null},{"id":10846077003,"name":"Chowan University","priority":255,"external_id":null},{"id":10846078003,"name":"Christendom College","priority":256,"external_id":null},{"id":10846079003,"name":"Baylor University","priority":257,"external_id":null},{"id":10846080003,"name":"Central Connecticut State University","priority":258,"external_id":null},{"id":10846081003,"name":"Central Michigan University","priority":259,"external_id":null},{"id":10846082003,"name":"Charleston Southern University","priority":260,"external_id":null},{"id":10846083003,"name":"California State University - Sacramento","priority":261,"external_id":null},{"id":10846084003,"name":"California State University - Fresno","priority":262,"external_id":null},{"id":10846085003,"name":"Campbell University","priority":263,"external_id":null},{"id":10846086003,"name":"Christopher Newport University","priority":264,"external_id":null},{"id":10846087003,"name":"Cincinnati Christian University","priority":265,"external_id":null},{"id":10846088003,"name":"Cincinnati College of Mortuary Science","priority":266,"external_id":null},{"id":10846089003,"name":"City University of Seattle","priority":267,"external_id":null},{"id":10846090003,"name":"Claflin University","priority":268,"external_id":null},{"id":10846091003,"name":"Claremont McKenna College","priority":269,"external_id":null},{"id":10846092003,"name":"Clarion University of Pennsylvania","priority":270,"external_id":null},{"id":10846093003,"name":"Clark Atlanta University","priority":271,"external_id":null},{"id":10846094003,"name":"Clark University","priority":272,"external_id":null},{"id":10846095003,"name":"Clarke University","priority":273,"external_id":null},{"id":10846096003,"name":"Clarkson College","priority":274,"external_id":null},{"id":10846097003,"name":"Clarkson University","priority":275,"external_id":null},{"id":10846098003,"name":"Clayton State University","priority":276,"external_id":null},{"id":10846099003,"name":"Clear Creek Baptist Bible College","priority":277,"external_id":null},{"id":10846100003,"name":"Clearwater Christian College","priority":278,"external_id":null},{"id":10846101003,"name":"Cleary University","priority":279,"external_id":null},{"id":10846102003,"name":"College of William and Mary","priority":280,"external_id":null},{"id":10846103003,"name":"Cleveland Chiropractic College","priority":281,"external_id":null},{"id":10846104003,"name":"Cleveland Institute of Art","priority":282,"external_id":null},{"id":10846105003,"name":"Cleveland Institute of Music","priority":283,"external_id":null},{"id":10846106003,"name":"Cleveland State University","priority":284,"external_id":null},{"id":10846107003,"name":"Coe College","priority":285,"external_id":null},{"id":10846108003,"name":"Cogswell Polytechnical College","priority":286,"external_id":null},{"id":10846109003,"name":"Coker College","priority":287,"external_id":null},{"id":10846110003,"name":"Colby College","priority":288,"external_id":null},{"id":10846111003,"name":"Colby-Sawyer College","priority":289,"external_id":null},{"id":10846112003,"name":"College at Brockport - SUNY","priority":290,"external_id":null},{"id":10846113003,"name":"College for Creative Studies","priority":291,"external_id":null},{"id":10846114003,"name":"College of Charleston","priority":292,"external_id":null},{"id":10846115003,"name":"College of Idaho","priority":293,"external_id":null},{"id":10846116003,"name":"College of Mount St. Joseph","priority":294,"external_id":null},{"id":10846117003,"name":"College of Mount St. Vincent","priority":295,"external_id":null},{"id":10846118003,"name":"College of New Jersey","priority":296,"external_id":null},{"id":10846119003,"name":"College of New Rochelle","priority":297,"external_id":null},{"id":10846120003,"name":"College of Our Lady of the Elms","priority":298,"external_id":null},{"id":10846121003,"name":"College of Saints John Fisher & Thomas More","priority":299,"external_id":null},{"id":10846122003,"name":"College of Southern Nevada","priority":300,"external_id":null},{"id":10846123003,"name":"College of St. Benedict","priority":301,"external_id":null},{"id":10846124003,"name":"College of St. Elizabeth","priority":302,"external_id":null},{"id":10846125003,"name":"College of St. Joseph","priority":303,"external_id":null},{"id":10846126003,"name":"College of St. Mary","priority":304,"external_id":null},{"id":10846127003,"name":"College of St. Rose","priority":305,"external_id":null},{"id":10846128003,"name":"College of St. Scholastica","priority":306,"external_id":null},{"id":10846129003,"name":"College of the Atlantic","priority":307,"external_id":null},{"id":10846130003,"name":"College of the Holy Cross","priority":308,"external_id":null},{"id":10846131003,"name":"College of the Ozarks","priority":309,"external_id":null},{"id":10846132003,"name":"College of Wooster","priority":310,"external_id":null},{"id":10846133003,"name":"Colorado Christian University","priority":311,"external_id":null},{"id":10846134003,"name":"Colorado College","priority":312,"external_id":null},{"id":10846135003,"name":"Colorado Mesa University","priority":313,"external_id":null},{"id":10846136003,"name":"Colorado School of Mines","priority":314,"external_id":null},{"id":10846137003,"name":"Colorado State University - Pueblo","priority":315,"external_id":null},{"id":10846138003,"name":"Colorado Technical University","priority":316,"external_id":null},{"id":10846139003,"name":"Columbia College","priority":317,"external_id":null},{"id":10846140003,"name":"Columbia College Chicago","priority":318,"external_id":null},{"id":10846141003,"name":"Columbia College of Nursing","priority":319,"external_id":null},{"id":10846142003,"name":"Columbia International University","priority":320,"external_id":null},{"id":10846143003,"name":"Columbus College of Art and Design","priority":321,"external_id":null},{"id":10846144003,"name":"Columbus State University","priority":322,"external_id":null},{"id":10846145003,"name":"Conception Seminary College","priority":323,"external_id":null},{"id":10846146003,"name":"Concord University","priority":324,"external_id":null},{"id":10846147003,"name":"Concordia College","priority":325,"external_id":null},{"id":10846148003,"name":"Concordia College - Moorhead","priority":326,"external_id":null},{"id":10846149003,"name":"Concordia University","priority":327,"external_id":null},{"id":10846150003,"name":"Concordia University Chicago","priority":328,"external_id":null},{"id":10846151003,"name":"Concordia University Texas","priority":329,"external_id":null},{"id":10846152003,"name":"Concordia University Wisconsin","priority":330,"external_id":null},{"id":10846153003,"name":"Concordia University - St. Paul","priority":331,"external_id":null},{"id":10846154003,"name":"Connecticut College","priority":332,"external_id":null},{"id":10846155003,"name":"Converse College","priority":333,"external_id":null},{"id":10846156003,"name":"Cooper Union","priority":334,"external_id":null},{"id":10846157003,"name":"Coppin State University","priority":335,"external_id":null},{"id":10846158003,"name":"Corban University","priority":336,"external_id":null},{"id":10846159003,"name":"Corcoran College of Art and Design","priority":337,"external_id":null},{"id":10846160003,"name":"Cornell College","priority":338,"external_id":null},{"id":10846161003,"name":"Cornerstone University","priority":339,"external_id":null},{"id":10846162003,"name":"Cornish College of the Arts","priority":340,"external_id":null},{"id":10846163003,"name":"Covenant College","priority":341,"external_id":null},{"id":10846164003,"name":"Cox College","priority":342,"external_id":null},{"id":10846165003,"name":"Creighton University","priority":343,"external_id":null},{"id":10846166003,"name":"Criswell College","priority":344,"external_id":null},{"id":10846167003,"name":"Crown College","priority":345,"external_id":null},{"id":10846168003,"name":"Culinary Institute of America","priority":346,"external_id":null},{"id":10846169003,"name":"Culver-Stockton College","priority":347,"external_id":null},{"id":10846170003,"name":"Cumberland University","priority":348,"external_id":null},{"id":10846171003,"name":"Columbia University","priority":349,"external_id":null},{"id":10846172003,"name":"Cornell University","priority":350,"external_id":null},{"id":10846173003,"name":"Colorado State University","priority":351,"external_id":null},{"id":10846174003,"name":"University of Virginia","priority":352,"external_id":null},{"id":10846175003,"name":"Colgate University","priority":353,"external_id":null},{"id":10846176003,"name":"CUNY - Baruch College","priority":354,"external_id":null},{"id":10846177003,"name":"CUNY - Brooklyn College","priority":355,"external_id":null},{"id":10846178003,"name":"CUNY - City College","priority":356,"external_id":null},{"id":10846179003,"name":"CUNY - College of Staten Island","priority":357,"external_id":null},{"id":10846180003,"name":"CUNY - Hunter College","priority":358,"external_id":null},{"id":10846181003,"name":"CUNY - John Jay College of Criminal Justice","priority":359,"external_id":null},{"id":10846182003,"name":"CUNY - Lehman College","priority":360,"external_id":null},{"id":10846183003,"name":"CUNY - Medgar Evers College","priority":361,"external_id":null},{"id":10846184003,"name":"CUNY - New York City College of Technology","priority":362,"external_id":null},{"id":10846185003,"name":"CUNY - Queens College","priority":363,"external_id":null},{"id":10846186003,"name":"CUNY - York College","priority":364,"external_id":null},{"id":10846187003,"name":"Curry College","priority":365,"external_id":null},{"id":10846188003,"name":"Curtis Institute of Music","priority":366,"external_id":null},{"id":10846189003,"name":"D'Youville College","priority":367,"external_id":null},{"id":10846190003,"name":"Daemen College","priority":368,"external_id":null},{"id":10846191003,"name":"Dakota State University","priority":369,"external_id":null},{"id":10846192003,"name":"Dakota Wesleyan University","priority":370,"external_id":null},{"id":10846193003,"name":"Dalhousie University","priority":371,"external_id":null},{"id":10846194003,"name":"Dallas Baptist University","priority":372,"external_id":null},{"id":10846195003,"name":"Dallas Christian College","priority":373,"external_id":null},{"id":10846196003,"name":"Dalton State College","priority":374,"external_id":null},{"id":10846197003,"name":"Daniel Webster College","priority":375,"external_id":null},{"id":10846198003,"name":"Davenport University","priority":376,"external_id":null},{"id":10846199003,"name":"Davis and Elkins College","priority":377,"external_id":null},{"id":10846200003,"name":"Davis College","priority":378,"external_id":null},{"id":10846201003,"name":"Daytona State College","priority":379,"external_id":null},{"id":10846202003,"name":"Dean College","priority":380,"external_id":null},{"id":10846203003,"name":"Defiance College","priority":381,"external_id":null},{"id":10846204003,"name":"Delaware Valley College","priority":382,"external_id":null},{"id":10846205003,"name":"Delta State University","priority":383,"external_id":null},{"id":10846206003,"name":"Denison University","priority":384,"external_id":null},{"id":10846207003,"name":"DePaul University","priority":385,"external_id":null},{"id":10846208003,"name":"DePauw University","priority":386,"external_id":null},{"id":10846209003,"name":"DEREE - The American College of Greece","priority":387,"external_id":null},{"id":10846210003,"name":"DeSales University","priority":388,"external_id":null},{"id":10846211003,"name":"DeVry University","priority":389,"external_id":null},{"id":10846212003,"name":"Dickinson College","priority":390,"external_id":null},{"id":10846213003,"name":"Dickinson State University","priority":391,"external_id":null},{"id":10846214003,"name":"Dillard University","priority":392,"external_id":null},{"id":10846215003,"name":"Divine Word College","priority":393,"external_id":null},{"id":10846216003,"name":"Dixie State College of Utah","priority":394,"external_id":null},{"id":10846217003,"name":"Doane College","priority":395,"external_id":null},{"id":10846218003,"name":"Dominican College","priority":396,"external_id":null},{"id":10846219003,"name":"Dominican University","priority":397,"external_id":null},{"id":10846220003,"name":"Dominican University of California","priority":398,"external_id":null},{"id":10846221003,"name":"Donnelly College","priority":399,"external_id":null},{"id":10846222003,"name":"Dordt College","priority":400,"external_id":null},{"id":10846223003,"name":"Dowling College","priority":401,"external_id":null},{"id":10846224003,"name":"Drew University","priority":402,"external_id":null},{"id":10846225003,"name":"Drexel University","priority":403,"external_id":null},{"id":10846226003,"name":"Drury University","priority":404,"external_id":null},{"id":10846227003,"name":"Dunwoody College of Technology","priority":405,"external_id":null},{"id":10846228003,"name":"Earlham College","priority":406,"external_id":null},{"id":10846229003,"name":"Drake University","priority":407,"external_id":null},{"id":10846230003,"name":"East Central University","priority":408,"external_id":null},{"id":10846231003,"name":"East Stroudsburg University of Pennsylvania","priority":409,"external_id":null},{"id":10846232003,"name":"East Tennessee State University","priority":410,"external_id":null},{"id":10846233003,"name":"East Texas Baptist University","priority":411,"external_id":null},{"id":10846234003,"name":"East-West University","priority":412,"external_id":null},{"id":10846235003,"name":"Eastern Connecticut State University","priority":413,"external_id":null},{"id":10846236003,"name":"Eastern Mennonite University","priority":414,"external_id":null},{"id":10846237003,"name":"Eastern Nazarene College","priority":415,"external_id":null},{"id":10846238003,"name":"Eastern New Mexico University","priority":416,"external_id":null},{"id":10846239003,"name":"Eastern Oregon University","priority":417,"external_id":null},{"id":10846240003,"name":"Eastern University","priority":418,"external_id":null},{"id":10846241003,"name":"Eckerd College","priority":419,"external_id":null},{"id":10846242003,"name":"ECPI University","priority":420,"external_id":null},{"id":10846243003,"name":"Edgewood College","priority":421,"external_id":null},{"id":10846244003,"name":"Edinboro University of Pennsylvania","priority":422,"external_id":null},{"id":10846245003,"name":"Edison State College","priority":423,"external_id":null},{"id":10846246003,"name":"Edward Waters College","priority":424,"external_id":null},{"id":10846247003,"name":"Elizabeth City State University","priority":425,"external_id":null},{"id":10846248003,"name":"Elizabethtown College","priority":426,"external_id":null},{"id":10846249003,"name":"Elmhurst College","priority":427,"external_id":null},{"id":10846250003,"name":"Elmira College","priority":428,"external_id":null},{"id":10846251003,"name":"Embry-Riddle Aeronautical University","priority":429,"external_id":null},{"id":10846252003,"name":"Embry-Riddle Aeronautical University - Prescott","priority":430,"external_id":null},{"id":10846253003,"name":"Emerson College","priority":431,"external_id":null},{"id":10846254003,"name":"Duquesne University","priority":432,"external_id":null},{"id":10846255003,"name":"Eastern Washington University","priority":433,"external_id":null},{"id":10846256003,"name":"Eastern Illinois University","priority":434,"external_id":null},{"id":10846257003,"name":"Eastern Kentucky University","priority":435,"external_id":null},{"id":10846258003,"name":"Eastern Michigan University","priority":436,"external_id":null},{"id":10846259003,"name":"Elon University","priority":437,"external_id":null},{"id":10846260003,"name":"Delaware State University","priority":438,"external_id":null},{"id":10846261003,"name":"Duke University","priority":439,"external_id":null},{"id":10846262003,"name":"California Polytechnic State University - San Luis Obispo","priority":440,"external_id":null},{"id":10846263003,"name":"Emmanuel College","priority":441,"external_id":null},{"id":10846264003,"name":"Emmaus Bible College","priority":442,"external_id":null},{"id":10846265003,"name":"Emory and Henry College","priority":443,"external_id":null},{"id":10846266003,"name":"Emory University","priority":444,"external_id":null},{"id":10846267003,"name":"Emporia State University","priority":445,"external_id":null},{"id":10846268003,"name":"Endicott College","priority":446,"external_id":null},{"id":10846269003,"name":"Erskine College","priority":447,"external_id":null},{"id":10846270003,"name":"Escuela de Artes Plasticas de Puerto Rico","priority":448,"external_id":null},{"id":10846271003,"name":"Eureka College","priority":449,"external_id":null},{"id":10846272003,"name":"Evangel University","priority":450,"external_id":null},{"id":10846273003,"name":"Everest College - Phoenix","priority":451,"external_id":null},{"id":10846274003,"name":"Everglades University","priority":452,"external_id":null},{"id":10846275003,"name":"Evergreen State College","priority":453,"external_id":null},{"id":10846276003,"name":"Excelsior College","priority":454,"external_id":null},{"id":10846277003,"name":"Fairfield University","priority":455,"external_id":null},{"id":10846278003,"name":"Fairleigh Dickinson University","priority":456,"external_id":null},{"id":10846279003,"name":"Fairmont State University","priority":457,"external_id":null},{"id":10846280003,"name":"Faith Baptist Bible College and Theological Seminary","priority":458,"external_id":null},{"id":10846281003,"name":"Farmingdale State College - SUNY","priority":459,"external_id":null},{"id":10846282003,"name":"Fashion Institute of Technology","priority":460,"external_id":null},{"id":10846283003,"name":"Faulkner University","priority":461,"external_id":null},{"id":10846284003,"name":"Fayetteville State University","priority":462,"external_id":null},{"id":10846285003,"name":"Felician College","priority":463,"external_id":null},{"id":10846286003,"name":"Ferris State University","priority":464,"external_id":null},{"id":10846287003,"name":"Ferrum College","priority":465,"external_id":null},{"id":10846288003,"name":"Finlandia University","priority":466,"external_id":null},{"id":10846289003,"name":"Fisher College","priority":467,"external_id":null},{"id":10846290003,"name":"Fisk University","priority":468,"external_id":null},{"id":10846291003,"name":"Fitchburg State University","priority":469,"external_id":null},{"id":10846292003,"name":"Five Towns College","priority":470,"external_id":null},{"id":10846293003,"name":"Flagler College","priority":471,"external_id":null},{"id":10846294003,"name":"Florida Christian College","priority":472,"external_id":null},{"id":10846295003,"name":"Florida College","priority":473,"external_id":null},{"id":10846296003,"name":"Florida Gulf Coast University","priority":474,"external_id":null},{"id":10846297003,"name":"Florida Institute of Technology","priority":475,"external_id":null},{"id":10846298003,"name":"Florida Memorial University","priority":476,"external_id":null},{"id":10846299003,"name":"Florida Southern College","priority":477,"external_id":null},{"id":10846300003,"name":"Florida State College - Jacksonville","priority":478,"external_id":null},{"id":10846301003,"name":"Fontbonne University","priority":479,"external_id":null},{"id":10846302003,"name":"Fort Hays State University","priority":480,"external_id":null},{"id":10846303003,"name":"Fort Lewis College","priority":481,"external_id":null},{"id":10846304003,"name":"Fort Valley State University","priority":482,"external_id":null},{"id":10846305003,"name":"Framingham State University","priority":483,"external_id":null},{"id":10846306003,"name":"Francis Marion University","priority":484,"external_id":null},{"id":10846307003,"name":"Franciscan University of Steubenville","priority":485,"external_id":null},{"id":10846308003,"name":"Frank Lloyd Wright School of Architecture","priority":486,"external_id":null},{"id":10846309003,"name":"Franklin and Marshall College","priority":487,"external_id":null},{"id":10846310003,"name":"Franklin College","priority":488,"external_id":null},{"id":10846311003,"name":"Franklin College Switzerland","priority":489,"external_id":null},{"id":10846312003,"name":"Franklin Pierce University","priority":490,"external_id":null},{"id":10846313003,"name":"Franklin University","priority":491,"external_id":null},{"id":10846314003,"name":"Franklin W. Olin College of Engineering","priority":492,"external_id":null},{"id":10846315003,"name":"Freed-Hardeman University","priority":493,"external_id":null},{"id":10846316003,"name":"Fresno Pacific University","priority":494,"external_id":null},{"id":10846317003,"name":"Friends University","priority":495,"external_id":null},{"id":10846318003,"name":"Frostburg State University","priority":496,"external_id":null},{"id":10846319003,"name":"Gallaudet University","priority":497,"external_id":null},{"id":10846320003,"name":"Gannon University","priority":498,"external_id":null},{"id":10846321003,"name":"Geneva College","priority":499,"external_id":null},{"id":10846322003,"name":"George Fox University","priority":500,"external_id":null},{"id":10846323003,"name":"George Mason University","priority":501,"external_id":null},{"id":10846324003,"name":"George Washington University","priority":502,"external_id":null},{"id":10846325003,"name":"Georgetown College","priority":503,"external_id":null},{"id":10846326003,"name":"Georgia College & State University","priority":504,"external_id":null},{"id":10846327003,"name":"Georgia Gwinnett College","priority":505,"external_id":null},{"id":10846328003,"name":"Georgia Regents University","priority":506,"external_id":null},{"id":10846329003,"name":"Georgia Southwestern State University","priority":507,"external_id":null},{"id":10846330003,"name":"Georgian Court University","priority":508,"external_id":null},{"id":10846331003,"name":"Gettysburg College","priority":509,"external_id":null},{"id":10846332003,"name":"Glenville State College","priority":510,"external_id":null},{"id":10846333003,"name":"God's Bible School and College","priority":511,"external_id":null},{"id":10846334003,"name":"Goddard College","priority":512,"external_id":null},{"id":10846335003,"name":"Golden Gate University","priority":513,"external_id":null},{"id":10846336003,"name":"Goldey-Beacom College","priority":514,"external_id":null},{"id":10846337003,"name":"Goldfarb School of Nursing at Barnes-Jewish College","priority":515,"external_id":null},{"id":10846338003,"name":"Gonzaga University","priority":516,"external_id":null},{"id":10846339003,"name":"Gordon College","priority":517,"external_id":null},{"id":10846340003,"name":"Fordham University","priority":518,"external_id":null},{"id":10846341003,"name":"Georgia Institute of Technology","priority":519,"external_id":null},{"id":10846342003,"name":"Gardner-Webb University","priority":520,"external_id":null},{"id":10846343003,"name":"Georgia Southern University","priority":521,"external_id":null},{"id":10846344003,"name":"Georgia State University","priority":522,"external_id":null},{"id":10846345003,"name":"Florida State University","priority":523,"external_id":null},{"id":10846346003,"name":"Dartmouth College","priority":524,"external_id":null},{"id":10846347003,"name":"Florida International University","priority":525,"external_id":null},{"id":10846348003,"name":"Georgetown University","priority":526,"external_id":null},{"id":10846349003,"name":"Furman University","priority":527,"external_id":null},{"id":10846350003,"name":"Gordon State College","priority":528,"external_id":null},{"id":10846351003,"name":"Goshen College","priority":529,"external_id":null},{"id":10846352003,"name":"Goucher College","priority":530,"external_id":null},{"id":10846353003,"name":"Governors State University","priority":531,"external_id":null},{"id":10846354003,"name":"Grace Bible College","priority":532,"external_id":null},{"id":10846355003,"name":"Grace College and Seminary","priority":533,"external_id":null},{"id":10846356003,"name":"Grace University","priority":534,"external_id":null},{"id":10846357003,"name":"Graceland University","priority":535,"external_id":null},{"id":10846358003,"name":"Grand Canyon University","priority":536,"external_id":null},{"id":10846359003,"name":"Grand Valley State University","priority":537,"external_id":null},{"id":10846360003,"name":"Grand View University","priority":538,"external_id":null},{"id":10846361003,"name":"Granite State College","priority":539,"external_id":null},{"id":10846362003,"name":"Gratz College","priority":540,"external_id":null},{"id":10846363003,"name":"Great Basin College","priority":541,"external_id":null},{"id":10846364003,"name":"Great Lakes Christian College","priority":542,"external_id":null},{"id":10846365003,"name":"Green Mountain College","priority":543,"external_id":null},{"id":10846366003,"name":"Greensboro College","priority":544,"external_id":null},{"id":10846367003,"name":"Greenville College","priority":545,"external_id":null},{"id":10846368003,"name":"Grinnell College","priority":546,"external_id":null},{"id":10846369003,"name":"Grove City College","priority":547,"external_id":null},{"id":10846370003,"name":"Guilford College","priority":548,"external_id":null},{"id":10846371003,"name":"Gustavus Adolphus College","priority":549,"external_id":null},{"id":10846372003,"name":"Gwynedd-Mercy College","priority":550,"external_id":null},{"id":10846373003,"name":"Hamilton College","priority":551,"external_id":null},{"id":10846374003,"name":"Hamline University","priority":552,"external_id":null},{"id":10846375003,"name":"Hampden-Sydney College","priority":553,"external_id":null},{"id":10846376003,"name":"Hampshire College","priority":554,"external_id":null},{"id":10846377003,"name":"Hannibal-LaGrange University","priority":555,"external_id":null},{"id":10846378003,"name":"Hanover College","priority":556,"external_id":null},{"id":10846379003,"name":"Hardin-Simmons University","priority":557,"external_id":null},{"id":10846380003,"name":"Harding University","priority":558,"external_id":null},{"id":10846381003,"name":"Harrington College of Design","priority":559,"external_id":null},{"id":10846382003,"name":"Harris-Stowe State University","priority":560,"external_id":null},{"id":10846383003,"name":"Harrisburg University of Science and Technology","priority":561,"external_id":null},{"id":10846384003,"name":"Hartwick College","priority":562,"external_id":null},{"id":10846385003,"name":"Harvey Mudd College","priority":563,"external_id":null},{"id":10846386003,"name":"Haskell Indian Nations University","priority":564,"external_id":null},{"id":10846387003,"name":"Hastings College","priority":565,"external_id":null},{"id":10846388003,"name":"Haverford College","priority":566,"external_id":null},{"id":10846389003,"name":"Hawaii Pacific University","priority":567,"external_id":null},{"id":10846390003,"name":"Hebrew Theological College","priority":568,"external_id":null},{"id":10846391003,"name":"Heidelberg University","priority":569,"external_id":null},{"id":10846392003,"name":"Hellenic College","priority":570,"external_id":null},{"id":10846393003,"name":"Henderson State University","priority":571,"external_id":null},{"id":10846394003,"name":"Hendrix College","priority":572,"external_id":null},{"id":10846395003,"name":"Heritage University","priority":573,"external_id":null},{"id":10846396003,"name":"Herzing University","priority":574,"external_id":null},{"id":10846397003,"name":"Hesser College","priority":575,"external_id":null},{"id":10846398003,"name":"High Point University","priority":576,"external_id":null},{"id":10846399003,"name":"Hilbert College","priority":577,"external_id":null},{"id":10846400003,"name":"Hillsdale College","priority":578,"external_id":null},{"id":10846401003,"name":"Hiram College","priority":579,"external_id":null},{"id":10846402003,"name":"Hobart and William Smith Colleges","priority":580,"external_id":null},{"id":10846403003,"name":"Hodges University","priority":581,"external_id":null},{"id":10846404003,"name":"Hofstra University","priority":582,"external_id":null},{"id":10846405003,"name":"Hollins University","priority":583,"external_id":null},{"id":10846406003,"name":"Holy Apostles College and Seminary","priority":584,"external_id":null},{"id":10846407003,"name":"Indiana State University","priority":585,"external_id":null},{"id":10846408003,"name":"Holy Family University","priority":586,"external_id":null},{"id":10846409003,"name":"Holy Names University","priority":587,"external_id":null},{"id":10846410003,"name":"Hood College","priority":588,"external_id":null},{"id":10846411003,"name":"Hope College","priority":589,"external_id":null},{"id":10846412003,"name":"Hope International University","priority":590,"external_id":null},{"id":10846413003,"name":"Houghton College","priority":591,"external_id":null},{"id":10846414003,"name":"Howard Payne University","priority":592,"external_id":null},{"id":10846415003,"name":"Hult International Business School","priority":593,"external_id":null},{"id":10846416003,"name":"Humboldt State University","priority":594,"external_id":null},{"id":10846417003,"name":"Humphreys College","priority":595,"external_id":null},{"id":10846418003,"name":"Huntingdon College","priority":596,"external_id":null},{"id":10846419003,"name":"Huntington University","priority":597,"external_id":null},{"id":10846420003,"name":"Husson University","priority":598,"external_id":null},{"id":10846421003,"name":"Huston-Tillotson University","priority":599,"external_id":null},{"id":10846422003,"name":"Illinois College","priority":600,"external_id":null},{"id":10846423003,"name":"Illinois Institute of Art at Chicago","priority":601,"external_id":null},{"id":10846424003,"name":"Illinois Institute of Technology","priority":602,"external_id":null},{"id":10846425003,"name":"Illinois Wesleyan University","priority":603,"external_id":null},{"id":10846426003,"name":"Immaculata University","priority":604,"external_id":null},{"id":10846427003,"name":"Indian River State College","priority":605,"external_id":null},{"id":10846428003,"name":"Indiana Institute of Technology","priority":606,"external_id":null},{"id":10846429003,"name":"Indiana University East","priority":607,"external_id":null},{"id":10846430003,"name":"Indiana University Northwest","priority":608,"external_id":null},{"id":10846431003,"name":"Indiana University of Pennsylvania","priority":609,"external_id":null},{"id":10846432003,"name":"Indiana University Southeast","priority":610,"external_id":null},{"id":10846433003,"name":"Illinois State University","priority":611,"external_id":null},{"id":10846434003,"name":"Indiana University - Bloomington","priority":612,"external_id":null},{"id":10846435003,"name":"Davidson College","priority":613,"external_id":null},{"id":10846436003,"name":"Idaho State University","priority":614,"external_id":null},{"id":10846437003,"name":"Harvard University","priority":615,"external_id":null},{"id":10846438003,"name":"Howard University","priority":616,"external_id":null},{"id":10846439003,"name":"Houston Baptist University","priority":617,"external_id":null},{"id":10846440003,"name":"Indiana University - Kokomo","priority":618,"external_id":null},{"id":10846441003,"name":"Indiana University - South Bend","priority":619,"external_id":null},{"id":10846442003,"name":"Indiana University-Purdue University - Fort Wayne","priority":620,"external_id":null},{"id":10846443003,"name":"Indiana University-Purdue University - Indianapolis","priority":621,"external_id":null},{"id":10846444003,"name":"Indiana Wesleyan University","priority":622,"external_id":null},{"id":10846445003,"name":"Institute of American Indian and Alaska Native Culture and Arts Development","priority":623,"external_id":null},{"id":10846446003,"name":"Inter American University of Puerto Rico - Aguadilla","priority":624,"external_id":null},{"id":10846447003,"name":"Inter American University of Puerto Rico - Arecibo","priority":625,"external_id":null},{"id":10846448003,"name":"Inter American University of Puerto Rico - Barranquitas","priority":626,"external_id":null},{"id":10846449003,"name":"Inter American University of Puerto Rico - Bayamon","priority":627,"external_id":null},{"id":10846450003,"name":"Inter American University of Puerto Rico - Fajardo","priority":628,"external_id":null},{"id":10846451003,"name":"Inter American University of Puerto Rico - Guayama","priority":629,"external_id":null},{"id":10846452003,"name":"Inter American University of Puerto Rico - Metropolitan Campus","priority":630,"external_id":null},{"id":10846453003,"name":"Inter American University of Puerto Rico - Ponce","priority":631,"external_id":null},{"id":10846454003,"name":"Inter American University of Puerto Rico - San German","priority":632,"external_id":null},{"id":10846455003,"name":"International College of the Cayman Islands","priority":633,"external_id":null},{"id":10846456003,"name":"Iona College","priority":634,"external_id":null},{"id":10846457003,"name":"Iowa Wesleyan College","priority":635,"external_id":null},{"id":10846458003,"name":"Ithaca College","priority":636,"external_id":null},{"id":10846459003,"name":"Jarvis Christian College","priority":637,"external_id":null},{"id":10846460003,"name":"Jewish Theological Seminary of America","priority":638,"external_id":null},{"id":10846461003,"name":"John Brown University","priority":639,"external_id":null},{"id":10846462003,"name":"John Carroll University","priority":640,"external_id":null},{"id":10846463003,"name":"John F. Kennedy University","priority":641,"external_id":null},{"id":10846464003,"name":"Johns Hopkins University","priority":642,"external_id":null},{"id":10846465003,"name":"Johnson & Wales University","priority":643,"external_id":null},{"id":10846466003,"name":"Johnson C. Smith University","priority":644,"external_id":null},{"id":10846467003,"name":"Johnson State College","priority":645,"external_id":null},{"id":10846468003,"name":"Johnson University","priority":646,"external_id":null},{"id":10846469003,"name":"Jones International University","priority":647,"external_id":null},{"id":10846470003,"name":"Judson College","priority":648,"external_id":null},{"id":10846471003,"name":"Judson University","priority":649,"external_id":null},{"id":10846472003,"name":"Juilliard School","priority":650,"external_id":null},{"id":10846473003,"name":"Juniata College","priority":651,"external_id":null},{"id":10846474003,"name":"Kalamazoo College","priority":652,"external_id":null},{"id":10846475003,"name":"Kansas City Art Institute","priority":653,"external_id":null},{"id":10846476003,"name":"Kansas Wesleyan University","priority":654,"external_id":null},{"id":10846477003,"name":"Kaplan University","priority":655,"external_id":null},{"id":10846478003,"name":"Kean University","priority":656,"external_id":null},{"id":10846479003,"name":"Keene State College","priority":657,"external_id":null},{"id":10846480003,"name":"Keiser University","priority":658,"external_id":null},{"id":10846481003,"name":"Kendall College","priority":659,"external_id":null},{"id":10846482003,"name":"Kennesaw State University","priority":660,"external_id":null},{"id":10846483003,"name":"Kentucky Christian University","priority":661,"external_id":null},{"id":10846484003,"name":"Kentucky State University","priority":662,"external_id":null},{"id":10846485003,"name":"Kentucky Wesleyan College","priority":663,"external_id":null},{"id":10846486003,"name":"Kenyon College","priority":664,"external_id":null},{"id":10846487003,"name":"Kettering College","priority":665,"external_id":null},{"id":10846488003,"name":"Kettering University","priority":666,"external_id":null},{"id":10846489003,"name":"Keuka College","priority":667,"external_id":null},{"id":10846490003,"name":"Keystone College","priority":668,"external_id":null},{"id":10846491003,"name":"King University","priority":669,"external_id":null},{"id":10846492003,"name":"King's College","priority":670,"external_id":null},{"id":10846493003,"name":"Knox College","priority":671,"external_id":null},{"id":10846494003,"name":"Kutztown University of Pennsylvania","priority":672,"external_id":null},{"id":10846495003,"name":"Kuyper College","priority":673,"external_id":null},{"id":10846496003,"name":"La Roche College","priority":674,"external_id":null},{"id":10846497003,"name":"La Salle University","priority":675,"external_id":null},{"id":10846498003,"name":"La Sierra University","priority":676,"external_id":null},{"id":10846499003,"name":"LaGrange College","priority":677,"external_id":null},{"id":10846500003,"name":"Laguna College of Art and Design","priority":678,"external_id":null},{"id":10846501003,"name":"Lake Erie College","priority":679,"external_id":null},{"id":10846502003,"name":"Lake Forest College","priority":680,"external_id":null},{"id":10846503003,"name":"Lake Superior State University","priority":681,"external_id":null},{"id":10846504003,"name":"Lakeland College","priority":682,"external_id":null},{"id":10846505003,"name":"Lakeview College of Nursing","priority":683,"external_id":null},{"id":10846506003,"name":"Lancaster Bible College","priority":684,"external_id":null},{"id":10846507003,"name":"Lander University","priority":685,"external_id":null},{"id":10846508003,"name":"Lane College","priority":686,"external_id":null},{"id":10846509003,"name":"Langston University","priority":687,"external_id":null},{"id":10846510003,"name":"Lasell College","priority":688,"external_id":null},{"id":10846511003,"name":"Lawrence Technological University","priority":689,"external_id":null},{"id":10846512003,"name":"Lawrence University","priority":690,"external_id":null},{"id":10846513003,"name":"Le Moyne College","priority":691,"external_id":null},{"id":10846514003,"name":"Lebanon Valley College","priority":692,"external_id":null},{"id":10846515003,"name":"Lee University","priority":693,"external_id":null},{"id":10846516003,"name":"Lees-McRae College","priority":694,"external_id":null},{"id":10846517003,"name":"Kansas State University","priority":695,"external_id":null},{"id":10846518003,"name":"James Madison University","priority":696,"external_id":null},{"id":10846519003,"name":"Lafayette College","priority":697,"external_id":null},{"id":10846520003,"name":"Jacksonville University","priority":698,"external_id":null},{"id":10846521003,"name":"Kent State University","priority":699,"external_id":null},{"id":10846522003,"name":"Lamar University","priority":700,"external_id":null},{"id":10846523003,"name":"Jackson State University","priority":701,"external_id":null},{"id":10846524003,"name":"Lehigh University","priority":702,"external_id":null},{"id":10846525003,"name":"Jacksonville State University","priority":703,"external_id":null},{"id":10846526003,"name":"LeMoyne-Owen College","priority":704,"external_id":null},{"id":10846527003,"name":"Lenoir-Rhyne University","priority":705,"external_id":null},{"id":10846528003,"name":"Lesley University","priority":706,"external_id":null},{"id":10846529003,"name":"LeTourneau University","priority":707,"external_id":null},{"id":10846530003,"name":"Lewis & Clark College","priority":708,"external_id":null},{"id":10846531003,"name":"Lewis University","priority":709,"external_id":null},{"id":10846532003,"name":"Lewis-Clark State College","priority":710,"external_id":null},{"id":10846533003,"name":"Lexington College","priority":711,"external_id":null},{"id":10846534003,"name":"Life Pacific College","priority":712,"external_id":null},{"id":10846535003,"name":"Life University","priority":713,"external_id":null},{"id":10846536003,"name":"LIM College","priority":714,"external_id":null},{"id":10846537003,"name":"Limestone College","priority":715,"external_id":null},{"id":10846538003,"name":"Lincoln Christian University","priority":716,"external_id":null},{"id":10846539003,"name":"Lincoln College","priority":717,"external_id":null},{"id":10846540003,"name":"Lincoln Memorial University","priority":718,"external_id":null},{"id":10846541003,"name":"Lincoln University","priority":719,"external_id":null},{"id":10846542003,"name":"Lindenwood University","priority":720,"external_id":null},{"id":10846543003,"name":"Lindsey Wilson College","priority":721,"external_id":null},{"id":10846544003,"name":"Linfield College","priority":722,"external_id":null},{"id":10846545003,"name":"Lipscomb University","priority":723,"external_id":null},{"id":10846546003,"name":"LIU Post","priority":724,"external_id":null},{"id":10846547003,"name":"Livingstone College","priority":725,"external_id":null},{"id":10846548003,"name":"Lock Haven University of Pennsylvania","priority":726,"external_id":null},{"id":10846549003,"name":"Loma Linda University","priority":727,"external_id":null},{"id":10846550003,"name":"Longwood University","priority":728,"external_id":null},{"id":10846551003,"name":"Loras College","priority":729,"external_id":null},{"id":10846552003,"name":"Louisiana College","priority":730,"external_id":null},{"id":10846553003,"name":"Louisiana State University Health Sciences Center","priority":731,"external_id":null},{"id":10846554003,"name":"Louisiana State University - Alexandria","priority":732,"external_id":null},{"id":10846555003,"name":"Louisiana State University - Shreveport","priority":733,"external_id":null},{"id":10846556003,"name":"Lourdes University","priority":734,"external_id":null},{"id":10846557003,"name":"Loyola Marymount University","priority":735,"external_id":null},{"id":10846558003,"name":"Loyola University Chicago","priority":736,"external_id":null},{"id":10846559003,"name":"Loyola University Maryland","priority":737,"external_id":null},{"id":10846560003,"name":"Loyola University New Orleans","priority":738,"external_id":null},{"id":10846561003,"name":"Lubbock Christian University","priority":739,"external_id":null},{"id":10846562003,"name":"Luther College","priority":740,"external_id":null},{"id":10846563003,"name":"Lycoming College","priority":741,"external_id":null},{"id":10846564003,"name":"Lyme Academy College of Fine Arts","priority":742,"external_id":null},{"id":10846565003,"name":"Lynchburg College","priority":743,"external_id":null},{"id":10846566003,"name":"Lyndon State College","priority":744,"external_id":null},{"id":10846567003,"name":"Lynn University","priority":745,"external_id":null},{"id":10846568003,"name":"Lyon College","priority":746,"external_id":null},{"id":10846569003,"name":"Macalester College","priority":747,"external_id":null},{"id":10846570003,"name":"MacMurray College","priority":748,"external_id":null},{"id":10846571003,"name":"Madonna University","priority":749,"external_id":null},{"id":10846572003,"name":"Maharishi University of Management","priority":750,"external_id":null},{"id":10846573003,"name":"Maine College of Art","priority":751,"external_id":null},{"id":10846574003,"name":"Maine Maritime Academy","priority":752,"external_id":null},{"id":10846575003,"name":"Malone University","priority":753,"external_id":null},{"id":10846576003,"name":"Manchester University","priority":754,"external_id":null},{"id":10846577003,"name":"Manhattan Christian College","priority":755,"external_id":null},{"id":10846578003,"name":"Manhattan College","priority":756,"external_id":null},{"id":10846579003,"name":"Manhattan School of Music","priority":757,"external_id":null},{"id":10846580003,"name":"Manhattanville College","priority":758,"external_id":null},{"id":10846581003,"name":"Mansfield University of Pennsylvania","priority":759,"external_id":null},{"id":10846582003,"name":"Maranatha Baptist Bible College","priority":760,"external_id":null},{"id":10846583003,"name":"Marian University","priority":761,"external_id":null},{"id":10846584003,"name":"Marietta College","priority":762,"external_id":null},{"id":10846585003,"name":"Marlboro College","priority":763,"external_id":null},{"id":10846586003,"name":"Marquette University","priority":764,"external_id":null},{"id":10846587003,"name":"Mars Hill University","priority":765,"external_id":null},{"id":10846588003,"name":"Martin Luther College","priority":766,"external_id":null},{"id":10846589003,"name":"Martin Methodist College","priority":767,"external_id":null},{"id":10846590003,"name":"Martin University","priority":768,"external_id":null},{"id":10846591003,"name":"Mary Baldwin College","priority":769,"external_id":null},{"id":10846592003,"name":"Marygrove College","priority":770,"external_id":null},{"id":10846593003,"name":"Maryland Institute College of Art","priority":771,"external_id":null},{"id":10846594003,"name":"Marylhurst University","priority":772,"external_id":null},{"id":10846595003,"name":"Marymount Manhattan College","priority":773,"external_id":null},{"id":10846596003,"name":"Marymount University","priority":774,"external_id":null},{"id":10846597003,"name":"Maryville College","priority":775,"external_id":null},{"id":10846598003,"name":"Maryville University of St. Louis","priority":776,"external_id":null},{"id":10846599003,"name":"Marywood University","priority":777,"external_id":null},{"id":10846600003,"name":"Massachusetts College of Art and Design","priority":778,"external_id":null},{"id":10846601003,"name":"Massachusetts College of Liberal Arts","priority":779,"external_id":null},{"id":10846602003,"name":"Massachusetts College of Pharmacy and Health Sciences","priority":780,"external_id":null},{"id":10846603003,"name":"Massachusetts Institute of Technology","priority":781,"external_id":null},{"id":10846604003,"name":"Massachusetts Maritime Academy","priority":782,"external_id":null},{"id":10846605003,"name":"Master's College and Seminary","priority":783,"external_id":null},{"id":10846606003,"name":"Mayville State University","priority":784,"external_id":null},{"id":10846607003,"name":"McDaniel College","priority":785,"external_id":null},{"id":10846608003,"name":"McGill University","priority":786,"external_id":null},{"id":10846609003,"name":"McKendree University","priority":787,"external_id":null},{"id":10846610003,"name":"McMurry University","priority":788,"external_id":null},{"id":10846611003,"name":"McPherson College","priority":789,"external_id":null},{"id":10846612003,"name":"Medaille College","priority":790,"external_id":null},{"id":10846613003,"name":"Marist College","priority":791,"external_id":null},{"id":10846614003,"name":"McNeese State University","priority":792,"external_id":null},{"id":10846615003,"name":"Louisiana Tech University","priority":793,"external_id":null},{"id":10846616003,"name":"Marshall University","priority":794,"external_id":null},{"id":10846617003,"name":"Medical University of South Carolina","priority":795,"external_id":null},{"id":10846618003,"name":"Memorial University of Newfoundland","priority":796,"external_id":null},{"id":10846619003,"name":"Memphis College of Art","priority":797,"external_id":null},{"id":10846620003,"name":"Menlo College","priority":798,"external_id":null},{"id":10846621003,"name":"Mercy College","priority":799,"external_id":null},{"id":10846622003,"name":"Mercy College of Health Sciences","priority":800,"external_id":null},{"id":10846623003,"name":"Mercy College of Ohio","priority":801,"external_id":null},{"id":10846624003,"name":"Mercyhurst University","priority":802,"external_id":null},{"id":10846625003,"name":"Meredith College","priority":803,"external_id":null},{"id":10846626003,"name":"Merrimack College","priority":804,"external_id":null},{"id":10846627003,"name":"Messiah College","priority":805,"external_id":null},{"id":10846628003,"name":"Methodist University","priority":806,"external_id":null},{"id":10846629003,"name":"Metropolitan College of New York","priority":807,"external_id":null},{"id":10846630003,"name":"Metropolitan State University","priority":808,"external_id":null},{"id":10846631003,"name":"Metropolitan State University of Denver","priority":809,"external_id":null},{"id":10846632003,"name":"Miami Dade College","priority":810,"external_id":null},{"id":10846633003,"name":"Miami International University of Art & Design","priority":811,"external_id":null},{"id":10846634003,"name":"Michigan Technological University","priority":812,"external_id":null},{"id":10846635003,"name":"Mid-America Christian University","priority":813,"external_id":null},{"id":10846636003,"name":"Mid-Atlantic Christian University","priority":814,"external_id":null},{"id":10846637003,"name":"Mid-Continent University","priority":815,"external_id":null},{"id":10846638003,"name":"MidAmerica Nazarene University","priority":816,"external_id":null},{"id":10846639003,"name":"Middle Georgia State College","priority":817,"external_id":null},{"id":10846640003,"name":"Middlebury College","priority":818,"external_id":null},{"id":10846641003,"name":"Midland College","priority":819,"external_id":null},{"id":10846642003,"name":"Midland University","priority":820,"external_id":null},{"id":10846643003,"name":"Midstate College","priority":821,"external_id":null},{"id":10846644003,"name":"Midway College","priority":822,"external_id":null},{"id":10846645003,"name":"Midwestern State University","priority":823,"external_id":null},{"id":10846646003,"name":"Miles College","priority":824,"external_id":null},{"id":10846647003,"name":"Millersville University of Pennsylvania","priority":825,"external_id":null},{"id":10846648003,"name":"Milligan College","priority":826,"external_id":null},{"id":10846649003,"name":"Millikin University","priority":827,"external_id":null},{"id":10846650003,"name":"Mills College","priority":828,"external_id":null},{"id":10846651003,"name":"Millsaps College","priority":829,"external_id":null},{"id":10846652003,"name":"Milwaukee Institute of Art and Design","priority":830,"external_id":null},{"id":10846653003,"name":"Milwaukee School of Engineering","priority":831,"external_id":null},{"id":10846654003,"name":"Minneapolis College of Art and Design","priority":832,"external_id":null},{"id":10846655003,"name":"Minnesota State University - Mankato","priority":833,"external_id":null},{"id":10846656003,"name":"Minnesota State University - Moorhead","priority":834,"external_id":null},{"id":10846657003,"name":"Minot State University","priority":835,"external_id":null},{"id":10846658003,"name":"Misericordia University","priority":836,"external_id":null},{"id":10846659003,"name":"Mississippi College","priority":837,"external_id":null},{"id":10846660003,"name":"Mississippi University for Women","priority":838,"external_id":null},{"id":10846661003,"name":"Missouri Baptist University","priority":839,"external_id":null},{"id":10846662003,"name":"Missouri Southern State University","priority":840,"external_id":null},{"id":10846663003,"name":"Missouri University of Science & Technology","priority":841,"external_id":null},{"id":10846664003,"name":"Missouri Valley College","priority":842,"external_id":null},{"id":10846665003,"name":"Missouri Western State University","priority":843,"external_id":null},{"id":10846666003,"name":"Mitchell College","priority":844,"external_id":null},{"id":10846667003,"name":"Molloy College","priority":845,"external_id":null},{"id":10846668003,"name":"Monmouth College","priority":846,"external_id":null},{"id":10846669003,"name":"Monroe College","priority":847,"external_id":null},{"id":10846670003,"name":"Montana State University - Billings","priority":848,"external_id":null},{"id":10846671003,"name":"Montana State University - Northern","priority":849,"external_id":null},{"id":10846672003,"name":"Montana Tech of the University of Montana","priority":850,"external_id":null},{"id":10846673003,"name":"Montclair State University","priority":851,"external_id":null},{"id":10846674003,"name":"Monterrey Institute of Technology and Higher Education - Monterrey","priority":852,"external_id":null},{"id":10846675003,"name":"Montreat College","priority":853,"external_id":null},{"id":10846676003,"name":"Montserrat College of Art","priority":854,"external_id":null},{"id":10846677003,"name":"Moody Bible Institute","priority":855,"external_id":null},{"id":10846678003,"name":"Moore College of Art & Design","priority":856,"external_id":null},{"id":10846679003,"name":"Moravian College","priority":857,"external_id":null},{"id":10846680003,"name":"Morehouse College","priority":858,"external_id":null},{"id":10846681003,"name":"Morningside College","priority":859,"external_id":null},{"id":10846682003,"name":"Morris College","priority":860,"external_id":null},{"id":10846683003,"name":"Morrisville State College","priority":861,"external_id":null},{"id":10846684003,"name":"Mount Aloysius College","priority":862,"external_id":null},{"id":10846685003,"name":"Mount Angel Seminary","priority":863,"external_id":null},{"id":10846686003,"name":"Mount Carmel College of Nursing","priority":864,"external_id":null},{"id":10846687003,"name":"Mount Holyoke College","priority":865,"external_id":null},{"id":10846688003,"name":"Mount Ida College","priority":866,"external_id":null},{"id":10846689003,"name":"Mount Marty College","priority":867,"external_id":null},{"id":10846690003,"name":"Mount Mary University","priority":868,"external_id":null},{"id":10846691003,"name":"Mount Mercy University","priority":869,"external_id":null},{"id":10846692003,"name":"Mount Olive College","priority":870,"external_id":null},{"id":10846693003,"name":"Mississippi State University","priority":871,"external_id":null},{"id":10846694003,"name":"Montana State University","priority":872,"external_id":null},{"id":10846695003,"name":"Mississippi Valley State University","priority":873,"external_id":null},{"id":10846696003,"name":"Monmouth University","priority":874,"external_id":null},{"id":10846697003,"name":"Morehead State University","priority":875,"external_id":null},{"id":10846698003,"name":"Miami University - Oxford","priority":876,"external_id":null},{"id":10846699003,"name":"Morgan State University","priority":877,"external_id":null},{"id":10846700003,"name":"Missouri State University","priority":878,"external_id":null},{"id":10846701003,"name":"Michigan State University","priority":879,"external_id":null},{"id":10846702003,"name":"Mount St. Mary College","priority":880,"external_id":null},{"id":10846703003,"name":"Mount St. Mary's College","priority":881,"external_id":null},{"id":10846704003,"name":"Mount St. Mary's University","priority":882,"external_id":null},{"id":10846705003,"name":"Mount Vernon Nazarene University","priority":883,"external_id":null},{"id":10846706003,"name":"Muhlenberg College","priority":884,"external_id":null},{"id":10846707003,"name":"Multnomah University","priority":885,"external_id":null},{"id":10846708003,"name":"Muskingum University","priority":886,"external_id":null},{"id":10846709003,"name":"Naropa University","priority":887,"external_id":null},{"id":10846710003,"name":"National American University","priority":888,"external_id":null},{"id":10846711003,"name":"National Graduate School of Quality Management","priority":889,"external_id":null},{"id":10846712003,"name":"National Hispanic University","priority":890,"external_id":null},{"id":10846713003,"name":"National Labor College","priority":891,"external_id":null},{"id":10846714003,"name":"National University","priority":892,"external_id":null},{"id":10846715003,"name":"National-Louis University","priority":893,"external_id":null},{"id":10846716003,"name":"Nazarene Bible College","priority":894,"external_id":null},{"id":10846717003,"name":"Nazareth College","priority":895,"external_id":null},{"id":10846718003,"name":"Nebraska Methodist College","priority":896,"external_id":null},{"id":10846719003,"name":"Nebraska Wesleyan University","priority":897,"external_id":null},{"id":10846720003,"name":"Neumann University","priority":898,"external_id":null},{"id":10846721003,"name":"Nevada State College","priority":899,"external_id":null},{"id":10846722003,"name":"New College of Florida","priority":900,"external_id":null},{"id":10846723003,"name":"New England College","priority":901,"external_id":null},{"id":10846724003,"name":"New England Conservatory of Music","priority":902,"external_id":null},{"id":10846725003,"name":"New England Institute of Art","priority":903,"external_id":null},{"id":10846726003,"name":"New England Institute of Technology","priority":904,"external_id":null},{"id":10846727003,"name":"New Jersey City University","priority":905,"external_id":null},{"id":10846728003,"name":"New Jersey Institute of Technology","priority":906,"external_id":null},{"id":10846729003,"name":"New Mexico Highlands University","priority":907,"external_id":null},{"id":10846730003,"name":"New Mexico Institute of Mining and Technology","priority":908,"external_id":null},{"id":10846731003,"name":"New Orleans Baptist Theological Seminary","priority":909,"external_id":null},{"id":10846732003,"name":"New School","priority":910,"external_id":null},{"id":10846733003,"name":"New York Institute of Technology","priority":911,"external_id":null},{"id":10846734003,"name":"New York University","priority":912,"external_id":null},{"id":10846735003,"name":"Newberry College","priority":913,"external_id":null},{"id":10846736003,"name":"Newbury College","priority":914,"external_id":null},{"id":10846737003,"name":"Newman University","priority":915,"external_id":null},{"id":10846738003,"name":"Niagara University","priority":916,"external_id":null},{"id":10846739003,"name":"Nichols College","priority":917,"external_id":null},{"id":10846740003,"name":"North Carolina Wesleyan College","priority":918,"external_id":null},{"id":10846741003,"name":"North Central College","priority":919,"external_id":null},{"id":10846742003,"name":"North Central University","priority":920,"external_id":null},{"id":10846743003,"name":"North Greenville University","priority":921,"external_id":null},{"id":10846744003,"name":"North Park University","priority":922,"external_id":null},{"id":10846745003,"name":"Northcentral University","priority":923,"external_id":null},{"id":10846746003,"name":"Northeastern Illinois University","priority":924,"external_id":null},{"id":10846747003,"name":"Northeastern State University","priority":925,"external_id":null},{"id":10846748003,"name":"Northeastern University","priority":926,"external_id":null},{"id":10846749003,"name":"Northern Kentucky University","priority":927,"external_id":null},{"id":10846750003,"name":"Northern Michigan University","priority":928,"external_id":null},{"id":10846751003,"name":"Northern New Mexico College","priority":929,"external_id":null},{"id":10846752003,"name":"Northern State University","priority":930,"external_id":null},{"id":10846753003,"name":"Northland College","priority":931,"external_id":null},{"id":10846754003,"name":"Northwest Christian University","priority":932,"external_id":null},{"id":10846755003,"name":"Northwest Florida State College","priority":933,"external_id":null},{"id":10846756003,"name":"Northwest Missouri State University","priority":934,"external_id":null},{"id":10846757003,"name":"Northwest Nazarene University","priority":935,"external_id":null},{"id":10846758003,"name":"Northwest University","priority":936,"external_id":null},{"id":10846759003,"name":"Northwestern College","priority":937,"external_id":null},{"id":10846760003,"name":"Northwestern Health Sciences University","priority":938,"external_id":null},{"id":10846761003,"name":"Northwestern Oklahoma State University","priority":939,"external_id":null},{"id":10846762003,"name":"Northwood University","priority":940,"external_id":null},{"id":10846763003,"name":"Norwich University","priority":941,"external_id":null},{"id":10846764003,"name":"Notre Dame College of Ohio","priority":942,"external_id":null},{"id":10846765003,"name":"Notre Dame de Namur University","priority":943,"external_id":null},{"id":10846766003,"name":"Notre Dame of Maryland University","priority":944,"external_id":null},{"id":10846767003,"name":"Nova Scotia College of Art and Design","priority":945,"external_id":null},{"id":10846768003,"name":"Nova Southeastern University","priority":946,"external_id":null},{"id":10846769003,"name":"Nyack College","priority":947,"external_id":null},{"id":10846770003,"name":"Oakland City University","priority":948,"external_id":null},{"id":10846771003,"name":"Oakland University","priority":949,"external_id":null},{"id":10846772003,"name":"Oakwood University","priority":950,"external_id":null},{"id":10846773003,"name":"Oberlin College","priority":951,"external_id":null},{"id":10846774003,"name":"Occidental College","priority":952,"external_id":null},{"id":10846775003,"name":"Oglala Lakota College","priority":953,"external_id":null},{"id":10846776003,"name":"North Carolina A&T State University","priority":954,"external_id":null},{"id":10846777003,"name":"Northern Illinois University","priority":955,"external_id":null},{"id":10846778003,"name":"North Dakota State University","priority":956,"external_id":null},{"id":10846779003,"name":"Nicholls State University","priority":957,"external_id":null},{"id":10846780003,"name":"North Carolina Central University","priority":958,"external_id":null},{"id":10846781003,"name":"Norfolk State University","priority":959,"external_id":null},{"id":10846782003,"name":"Northwestern State University of Louisiana","priority":960,"external_id":null},{"id":10846783003,"name":"Northern Arizona University","priority":961,"external_id":null},{"id":10846784003,"name":"North Carolina State University - Raleigh","priority":962,"external_id":null},{"id":10846785003,"name":"Northwestern University","priority":963,"external_id":null},{"id":10846786003,"name":"Oglethorpe University","priority":964,"external_id":null},{"id":10846787003,"name":"Ohio Christian University","priority":965,"external_id":null},{"id":10846788003,"name":"Ohio Dominican University","priority":966,"external_id":null},{"id":10846789003,"name":"Ohio Northern University","priority":967,"external_id":null},{"id":10846790003,"name":"Ohio Valley University","priority":968,"external_id":null},{"id":10846791003,"name":"Ohio Wesleyan University","priority":969,"external_id":null},{"id":10846792003,"name":"Oklahoma Baptist University","priority":970,"external_id":null},{"id":10846793003,"name":"Oklahoma Christian University","priority":971,"external_id":null},{"id":10846794003,"name":"Oklahoma City University","priority":972,"external_id":null},{"id":10846795003,"name":"Oklahoma Panhandle State University","priority":973,"external_id":null},{"id":10846796003,"name":"Oklahoma State University Institute of Technology - Okmulgee","priority":974,"external_id":null},{"id":10846797003,"name":"Oklahoma State University - Oklahoma City","priority":975,"external_id":null},{"id":10846798003,"name":"Oklahoma Wesleyan University","priority":976,"external_id":null},{"id":10846799003,"name":"Olivet College","priority":977,"external_id":null},{"id":10846800003,"name":"Olivet Nazarene University","priority":978,"external_id":null},{"id":10846801003,"name":"Olympic College","priority":979,"external_id":null},{"id":10846802003,"name":"Oral Roberts University","priority":980,"external_id":null},{"id":10846803003,"name":"Oregon College of Art and Craft","priority":981,"external_id":null},{"id":10846804003,"name":"Oregon Health and Science University","priority":982,"external_id":null},{"id":10846805003,"name":"Oregon Institute of Technology","priority":983,"external_id":null},{"id":10846806003,"name":"Otis College of Art and Design","priority":984,"external_id":null},{"id":10846807003,"name":"Ottawa University","priority":985,"external_id":null},{"id":10846808003,"name":"Otterbein University","priority":986,"external_id":null},{"id":10846809003,"name":"Ouachita Baptist University","priority":987,"external_id":null},{"id":10846810003,"name":"Our Lady of Holy Cross College","priority":988,"external_id":null},{"id":10846811003,"name":"Our Lady of the Lake College","priority":989,"external_id":null},{"id":10846812003,"name":"Our Lady of the Lake University","priority":990,"external_id":null},{"id":10846813003,"name":"Pace University","priority":991,"external_id":null},{"id":10846814003,"name":"Pacific Lutheran University","priority":992,"external_id":null},{"id":10846815003,"name":"Pacific Northwest College of Art","priority":993,"external_id":null},{"id":10846816003,"name":"Pacific Oaks College","priority":994,"external_id":null},{"id":10846817003,"name":"Pacific Union College","priority":995,"external_id":null},{"id":10846818003,"name":"Pacific University","priority":996,"external_id":null},{"id":10846819003,"name":"Paine College","priority":997,"external_id":null},{"id":10846820003,"name":"Palm Beach Atlantic University","priority":998,"external_id":null},{"id":10846821003,"name":"Palmer College of Chiropractic","priority":999,"external_id":null},{"id":10846822003,"name":"Park University","priority":1000,"external_id":null},{"id":10846823003,"name":"Parker University","priority":1001,"external_id":null},{"id":10846824003,"name":"Patten University","priority":1002,"external_id":null},{"id":10846825003,"name":"Paul Smith's College","priority":1003,"external_id":null},{"id":10846826003,"name":"Peirce College","priority":1004,"external_id":null},{"id":10846827003,"name":"Peninsula College","priority":1005,"external_id":null},{"id":10846828003,"name":"Pennsylvania College of Art and Design","priority":1006,"external_id":null},{"id":10846829003,"name":"Pennsylvania College of Technology","priority":1007,"external_id":null},{"id":10846830003,"name":"Pennsylvania State University - Erie, The Behrend College","priority":1008,"external_id":null},{"id":10846831003,"name":"Pennsylvania State University - Harrisburg","priority":1009,"external_id":null},{"id":10846832003,"name":"Pepperdine University","priority":1010,"external_id":null},{"id":10846833003,"name":"Peru State College","priority":1011,"external_id":null},{"id":10846834003,"name":"Pfeiffer University","priority":1012,"external_id":null},{"id":10846835003,"name":"Philadelphia University","priority":1013,"external_id":null},{"id":10846836003,"name":"Philander Smith College","priority":1014,"external_id":null},{"id":10846837003,"name":"Piedmont College","priority":1015,"external_id":null},{"id":10846838003,"name":"Pine Manor College","priority":1016,"external_id":null},{"id":10846839003,"name":"Pittsburg State University","priority":1017,"external_id":null},{"id":10846840003,"name":"Pitzer College","priority":1018,"external_id":null},{"id":10846841003,"name":"Plaza College","priority":1019,"external_id":null},{"id":10846842003,"name":"Plymouth State University","priority":1020,"external_id":null},{"id":10846843003,"name":"Point Loma Nazarene University","priority":1021,"external_id":null},{"id":10846844003,"name":"Point Park University","priority":1022,"external_id":null},{"id":10846845003,"name":"Point University","priority":1023,"external_id":null},{"id":10846846003,"name":"Polytechnic Institute of New York University","priority":1024,"external_id":null},{"id":10846847003,"name":"Pomona College","priority":1025,"external_id":null},{"id":10846848003,"name":"Pontifical Catholic University of Puerto Rico","priority":1026,"external_id":null},{"id":10846849003,"name":"Pontifical College Josephinum","priority":1027,"external_id":null},{"id":10846850003,"name":"Post University","priority":1028,"external_id":null},{"id":10846851003,"name":"Potomac College","priority":1029,"external_id":null},{"id":10846852003,"name":"Pratt Institute","priority":1030,"external_id":null},{"id":10846853003,"name":"Prescott College","priority":1031,"external_id":null},{"id":10846854003,"name":"Presentation College","priority":1032,"external_id":null},{"id":10846855003,"name":"Principia College","priority":1033,"external_id":null},{"id":10846856003,"name":"Providence College","priority":1034,"external_id":null},{"id":10846857003,"name":"Puerto Rico Conservatory of Music","priority":1035,"external_id":null},{"id":10846858003,"name":"Purchase College - SUNY","priority":1036,"external_id":null},{"id":10846859003,"name":"Purdue University - Calumet","priority":1037,"external_id":null},{"id":10846860003,"name":"Purdue University - North Central","priority":1038,"external_id":null},{"id":10846861003,"name":"Queens University of Charlotte","priority":1039,"external_id":null},{"id":10846862003,"name":"Oklahoma State University","priority":1040,"external_id":null},{"id":10846863003,"name":"Oregon State University","priority":1041,"external_id":null},{"id":10846864003,"name":"Portland State University","priority":1042,"external_id":null},{"id":10846865003,"name":"Old Dominion University","priority":1043,"external_id":null},{"id":10846866003,"name":"Prairie View A&M University","priority":1044,"external_id":null},{"id":10846867003,"name":"Presbyterian College","priority":1045,"external_id":null},{"id":10846868003,"name":"Purdue University - West Lafayette","priority":1046,"external_id":null},{"id":10846869003,"name":"Ohio University","priority":1047,"external_id":null},{"id":10846870003,"name":"Princeton University","priority":1048,"external_id":null},{"id":10846871003,"name":"Quincy University","priority":1049,"external_id":null},{"id":10846872003,"name":"Quinnipiac University","priority":1050,"external_id":null},{"id":10846873003,"name":"Radford University","priority":1051,"external_id":null},{"id":10846874003,"name":"Ramapo College of New Jersey","priority":1052,"external_id":null},{"id":10846875003,"name":"Randolph College","priority":1053,"external_id":null},{"id":10846876003,"name":"Randolph-Macon College","priority":1054,"external_id":null},{"id":10846877003,"name":"Ranken Technical College","priority":1055,"external_id":null},{"id":10846878003,"name":"Reed College","priority":1056,"external_id":null},{"id":10846879003,"name":"Regent University","priority":1057,"external_id":null},{"id":10846880003,"name":"Regent's American College London","priority":1058,"external_id":null},{"id":10846881003,"name":"Regis College","priority":1059,"external_id":null},{"id":10846882003,"name":"Regis University","priority":1060,"external_id":null},{"id":10846883003,"name":"Reinhardt University","priority":1061,"external_id":null},{"id":10846884003,"name":"Rensselaer Polytechnic Institute","priority":1062,"external_id":null},{"id":10846885003,"name":"Research College of Nursing","priority":1063,"external_id":null},{"id":10846886003,"name":"Resurrection University","priority":1064,"external_id":null},{"id":10846887003,"name":"Rhode Island College","priority":1065,"external_id":null},{"id":10846888003,"name":"Rhode Island School of Design","priority":1066,"external_id":null},{"id":10846889003,"name":"Rhodes College","priority":1067,"external_id":null},{"id":10846890003,"name":"Richard Stockton College of New Jersey","priority":1068,"external_id":null},{"id":10846891003,"name":"Richmond - The American International University in London","priority":1069,"external_id":null},{"id":10846892003,"name":"Rider University","priority":1070,"external_id":null},{"id":10846893003,"name":"Ringling College of Art and Design","priority":1071,"external_id":null},{"id":10846894003,"name":"Ripon College","priority":1072,"external_id":null},{"id":10846895003,"name":"Rivier University","priority":1073,"external_id":null},{"id":10846896003,"name":"Roanoke College","priority":1074,"external_id":null},{"id":10846897003,"name":"Robert B. Miller College","priority":1075,"external_id":null},{"id":10846898003,"name":"Roberts Wesleyan College","priority":1076,"external_id":null},{"id":10846899003,"name":"Rochester College","priority":1077,"external_id":null},{"id":10846900003,"name":"Rochester Institute of Technology","priority":1078,"external_id":null},{"id":10846901003,"name":"Rockford University","priority":1079,"external_id":null},{"id":10846902003,"name":"Rockhurst University","priority":1080,"external_id":null},{"id":10846903003,"name":"Rocky Mountain College","priority":1081,"external_id":null},{"id":10846904003,"name":"Rocky Mountain College of Art and Design","priority":1082,"external_id":null},{"id":10846905003,"name":"Roger Williams University","priority":1083,"external_id":null},{"id":10846906003,"name":"Rogers State University","priority":1084,"external_id":null},{"id":10846907003,"name":"Rollins College","priority":1085,"external_id":null},{"id":10846908003,"name":"Roosevelt University","priority":1086,"external_id":null},{"id":10846909003,"name":"Rosalind Franklin University of Medicine and Science","priority":1087,"external_id":null},{"id":10846910003,"name":"Rose-Hulman Institute of Technology","priority":1088,"external_id":null},{"id":10846911003,"name":"Rosemont College","priority":1089,"external_id":null},{"id":10846912003,"name":"Rowan University","priority":1090,"external_id":null},{"id":10846913003,"name":"Rush University","priority":1091,"external_id":null},{"id":10846914003,"name":"Rust College","priority":1092,"external_id":null},{"id":10846915003,"name":"Rutgers, the State University of New Jersey - Camden","priority":1093,"external_id":null},{"id":10846916003,"name":"Rutgers, the State University of New Jersey - Newark","priority":1094,"external_id":null},{"id":10846917003,"name":"Ryerson University","priority":1095,"external_id":null},{"id":10846918003,"name":"Sacred Heart Major Seminary","priority":1096,"external_id":null},{"id":10846919003,"name":"Saginaw Valley State University","priority":1097,"external_id":null},{"id":10846920003,"name":"Salem College","priority":1098,"external_id":null},{"id":10846921003,"name":"Salem International University","priority":1099,"external_id":null},{"id":10846922003,"name":"Salem State University","priority":1100,"external_id":null},{"id":10846923003,"name":"Salisbury University","priority":1101,"external_id":null},{"id":10846924003,"name":"Salish Kootenai College","priority":1102,"external_id":null},{"id":10846925003,"name":"Salve Regina University","priority":1103,"external_id":null},{"id":10846926003,"name":"Samuel Merritt University","priority":1104,"external_id":null},{"id":10846927003,"name":"San Diego Christian College","priority":1105,"external_id":null},{"id":10846928003,"name":"San Francisco Art Institute","priority":1106,"external_id":null},{"id":10846929003,"name":"San Francisco Conservatory of Music","priority":1107,"external_id":null},{"id":10846930003,"name":"San Francisco State University","priority":1108,"external_id":null},{"id":10846931003,"name":"Sanford College of Nursing","priority":1109,"external_id":null},{"id":10846932003,"name":"Santa Clara University","priority":1110,"external_id":null},{"id":10846933003,"name":"Santa Fe University of Art and Design","priority":1111,"external_id":null},{"id":10846934003,"name":"Sarah Lawrence College","priority":1112,"external_id":null},{"id":10846935003,"name":"Savannah College of Art and Design","priority":1113,"external_id":null},{"id":10846936003,"name":"School of the Art Institute of Chicago","priority":1114,"external_id":null},{"id":10846937003,"name":"School of Visual Arts","priority":1115,"external_id":null},{"id":10846938003,"name":"Schreiner University","priority":1116,"external_id":null},{"id":10846939003,"name":"Scripps College","priority":1117,"external_id":null},{"id":10846940003,"name":"Seattle Pacific University","priority":1118,"external_id":null},{"id":10846941003,"name":"Seattle University","priority":1119,"external_id":null},{"id":10846942003,"name":"Seton Hall University","priority":1120,"external_id":null},{"id":10846943003,"name":"Seton Hill University","priority":1121,"external_id":null},{"id":10846944003,"name":"Sewanee - University of the South","priority":1122,"external_id":null},{"id":10846945003,"name":"Shaw University","priority":1123,"external_id":null},{"id":10846946003,"name":"Shawnee State University","priority":1124,"external_id":null},{"id":10846947003,"name":"Shenandoah University","priority":1125,"external_id":null},{"id":10846948003,"name":"Shepherd University","priority":1126,"external_id":null},{"id":10846949003,"name":"Shimer College","priority":1127,"external_id":null},{"id":10846950003,"name":"Sacred Heart University","priority":1128,"external_id":null},{"id":10846951003,"name":"Robert Morris University","priority":1129,"external_id":null},{"id":10846952003,"name":"Sam Houston State University","priority":1130,"external_id":null},{"id":10846953003,"name":"Samford University","priority":1131,"external_id":null},{"id":10846954003,"name":"Savannah State University","priority":1132,"external_id":null},{"id":10846955003,"name":"San Jose State University","priority":1133,"external_id":null},{"id":10846956003,"name":"Rutgers, the State University of New Jersey - New Brunswick","priority":1134,"external_id":null},{"id":10846957003,"name":"San Diego State University","priority":1135,"external_id":null},{"id":10846958003,"name":"Shippensburg University of Pennsylvania","priority":1136,"external_id":null},{"id":10846959003,"name":"Shorter University","priority":1137,"external_id":null},{"id":10846960003,"name":"Siena College","priority":1138,"external_id":null},{"id":10846961003,"name":"Siena Heights University","priority":1139,"external_id":null},{"id":10846962003,"name":"Sierra Nevada College","priority":1140,"external_id":null},{"id":10846963003,"name":"Silver Lake College","priority":1141,"external_id":null},{"id":10846964003,"name":"Simmons College","priority":1142,"external_id":null},{"id":10846965003,"name":"Simon Fraser University","priority":1143,"external_id":null},{"id":10846966003,"name":"Simpson College","priority":1144,"external_id":null},{"id":10846967003,"name":"Simpson University","priority":1145,"external_id":null},{"id":10846968003,"name":"Sinte Gleska University","priority":1146,"external_id":null},{"id":10846969003,"name":"Sitting Bull College","priority":1147,"external_id":null},{"id":10846970003,"name":"Skidmore College","priority":1148,"external_id":null},{"id":10846971003,"name":"Slippery Rock University of Pennsylvania","priority":1149,"external_id":null},{"id":10846972003,"name":"Smith College","priority":1150,"external_id":null},{"id":10846973003,"name":"Sojourner-Douglass College","priority":1151,"external_id":null},{"id":10846974003,"name":"Soka University of America","priority":1152,"external_id":null},{"id":10846975003,"name":"Sonoma State University","priority":1153,"external_id":null},{"id":10846976003,"name":"South College","priority":1154,"external_id":null},{"id":10846977003,"name":"South Dakota School of Mines and Technology","priority":1155,"external_id":null},{"id":10846978003,"name":"South Seattle Community College","priority":1156,"external_id":null},{"id":10846979003,"name":"South Texas College","priority":1157,"external_id":null},{"id":10846980003,"name":"South University","priority":1158,"external_id":null},{"id":10846981003,"name":"Southeastern Oklahoma State University","priority":1159,"external_id":null},{"id":10846982003,"name":"Southeastern University","priority":1160,"external_id":null},{"id":10846983003,"name":"Southern Adventist University","priority":1161,"external_id":null},{"id":10846984003,"name":"Southern Arkansas University","priority":1162,"external_id":null},{"id":10846985003,"name":"Southern Baptist Theological Seminary","priority":1163,"external_id":null},{"id":10846986003,"name":"Southern California Institute of Architecture","priority":1164,"external_id":null},{"id":10846987003,"name":"Southern Connecticut State University","priority":1165,"external_id":null},{"id":10846988003,"name":"Southern Illinois University - Edwardsville","priority":1166,"external_id":null},{"id":10846989003,"name":"Southern Nazarene University","priority":1167,"external_id":null},{"id":10846990003,"name":"Southern New Hampshire University","priority":1168,"external_id":null},{"id":10846991003,"name":"Southern Oregon University","priority":1169,"external_id":null},{"id":10846992003,"name":"Southern Polytechnic State University","priority":1170,"external_id":null},{"id":10846993003,"name":"Southern University - New Orleans","priority":1171,"external_id":null},{"id":10846994003,"name":"Southern Vermont College","priority":1172,"external_id":null},{"id":10846995003,"name":"Southern Wesleyan University","priority":1173,"external_id":null},{"id":10846996003,"name":"Southwest Baptist University","priority":1174,"external_id":null},{"id":10846997003,"name":"Southwest Minnesota State University","priority":1175,"external_id":null},{"id":10846998003,"name":"Southwest University of Visual Arts","priority":1176,"external_id":null},{"id":10846999003,"name":"Southwestern Adventist University","priority":1177,"external_id":null},{"id":10847000003,"name":"Southwestern Assemblies of God University","priority":1178,"external_id":null},{"id":10847001003,"name":"Southwestern Christian College","priority":1179,"external_id":null},{"id":10847002003,"name":"Southwestern Christian University","priority":1180,"external_id":null},{"id":10847003003,"name":"Southwestern College","priority":1181,"external_id":null},{"id":10847004003,"name":"Southwestern Oklahoma State University","priority":1182,"external_id":null},{"id":10847005003,"name":"Southwestern University","priority":1183,"external_id":null},{"id":10847006003,"name":"Spalding University","priority":1184,"external_id":null},{"id":10847007003,"name":"Spelman College","priority":1185,"external_id":null},{"id":10847008003,"name":"Spring Arbor University","priority":1186,"external_id":null},{"id":10847009003,"name":"Spring Hill College","priority":1187,"external_id":null},{"id":10847010003,"name":"Springfield College","priority":1188,"external_id":null},{"id":10847011003,"name":"St. Ambrose University","priority":1189,"external_id":null},{"id":10847012003,"name":"St. Anselm College","priority":1190,"external_id":null},{"id":10847013003,"name":"St. Anthony College of Nursing","priority":1191,"external_id":null},{"id":10847014003,"name":"St. Augustine College","priority":1192,"external_id":null},{"id":10847015003,"name":"St. Augustine's University","priority":1193,"external_id":null},{"id":10847016003,"name":"St. Bonaventure University","priority":1194,"external_id":null},{"id":10847017003,"name":"St. Catharine College","priority":1195,"external_id":null},{"id":10847018003,"name":"St. Catherine University","priority":1196,"external_id":null},{"id":10847019003,"name":"St. Charles Borromeo Seminary","priority":1197,"external_id":null},{"id":10847020003,"name":"St. Cloud State University","priority":1198,"external_id":null},{"id":10847021003,"name":"St. Edward's University","priority":1199,"external_id":null},{"id":10847022003,"name":"St. Francis College","priority":1200,"external_id":null},{"id":10847023003,"name":"St. Francis Medical Center College of Nursing","priority":1201,"external_id":null},{"id":10847024003,"name":"St. Gregory's University","priority":1202,"external_id":null},{"id":10847025003,"name":"St. John Fisher College","priority":1203,"external_id":null},{"id":10847026003,"name":"St. John Vianney College Seminary","priority":1204,"external_id":null},{"id":10847027003,"name":"St. John's College","priority":1205,"external_id":null},{"id":10847028003,"name":"St. John's University","priority":1206,"external_id":null},{"id":10847029003,"name":"St. Joseph Seminary College","priority":1207,"external_id":null},{"id":10847030003,"name":"St. Joseph's College","priority":1208,"external_id":null},{"id":10847031003,"name":"St. Joseph's College New York","priority":1209,"external_id":null},{"id":10847032003,"name":"St. Joseph's University","priority":1210,"external_id":null},{"id":10847033003,"name":"St. Lawrence University","priority":1211,"external_id":null},{"id":10847034003,"name":"St. Leo University","priority":1212,"external_id":null},{"id":10847035003,"name":"Southern University and A&M College","priority":1213,"external_id":null},{"id":10847036003,"name":"Southern Methodist University","priority":1214,"external_id":null},{"id":10847037003,"name":"Southeast Missouri State University","priority":1215,"external_id":null},{"id":10847038003,"name":"Southern Utah University","priority":1216,"external_id":null},{"id":10847039003,"name":"South Dakota State University","priority":1217,"external_id":null},{"id":10847040003,"name":"St. Francis University","priority":1218,"external_id":null},{"id":10847041003,"name":"Southeastern Louisiana University","priority":1219,"external_id":null},{"id":10847042003,"name":"Southern Illinois University - Carbondale","priority":1220,"external_id":null},{"id":10847043003,"name":"St. Louis College of Pharmacy","priority":1221,"external_id":null},{"id":10847044003,"name":"St. Louis University","priority":1222,"external_id":null},{"id":10847045003,"name":"St. Luke's College of Health Sciences","priority":1223,"external_id":null},{"id":10847046003,"name":"St. Martin's University","priority":1224,"external_id":null},{"id":10847047003,"name":"St. Mary's College","priority":1225,"external_id":null},{"id":10847048003,"name":"St. Mary's College of California","priority":1226,"external_id":null},{"id":10847049003,"name":"St. Mary's College of Maryland","priority":1227,"external_id":null},{"id":10847050003,"name":"St. Mary's Seminary and University","priority":1228,"external_id":null},{"id":10847051003,"name":"St. Mary's University of Minnesota","priority":1229,"external_id":null},{"id":10847052003,"name":"St. Mary's University of San Antonio","priority":1230,"external_id":null},{"id":10847053003,"name":"St. Mary-of-the-Woods College","priority":1231,"external_id":null},{"id":10847054003,"name":"St. Michael's College","priority":1232,"external_id":null},{"id":10847055003,"name":"St. Norbert College","priority":1233,"external_id":null},{"id":10847056003,"name":"St. Olaf College","priority":1234,"external_id":null},{"id":10847057003,"name":"St. Paul's College","priority":1235,"external_id":null},{"id":10847058003,"name":"St. Peter's University","priority":1236,"external_id":null},{"id":10847059003,"name":"St. Petersburg College","priority":1237,"external_id":null},{"id":10847060003,"name":"St. Thomas Aquinas College","priority":1238,"external_id":null},{"id":10847061003,"name":"St. Thomas University","priority":1239,"external_id":null},{"id":10847062003,"name":"St. Vincent College","priority":1240,"external_id":null},{"id":10847063003,"name":"St. Xavier University","priority":1241,"external_id":null},{"id":10847064003,"name":"Stephens College","priority":1242,"external_id":null},{"id":10847065003,"name":"Sterling College","priority":1243,"external_id":null},{"id":10847066003,"name":"Stevens Institute of Technology","priority":1244,"external_id":null},{"id":10847067003,"name":"Stevenson University","priority":1245,"external_id":null},{"id":10847068003,"name":"Stillman College","priority":1246,"external_id":null},{"id":10847069003,"name":"Stonehill College","priority":1247,"external_id":null},{"id":10847070003,"name":"Strayer University","priority":1248,"external_id":null},{"id":10847071003,"name":"Suffolk University","priority":1249,"external_id":null},{"id":10847072003,"name":"Sul Ross State University","priority":1250,"external_id":null},{"id":10847073003,"name":"Sullivan University","priority":1251,"external_id":null},{"id":10847074003,"name":"SUNY Buffalo State","priority":1252,"external_id":null},{"id":10847075003,"name":"SUNY College of Agriculture and Technology - Cobleskill","priority":1253,"external_id":null},{"id":10847076003,"name":"SUNY College of Environmental Science and Forestry","priority":1254,"external_id":null},{"id":10847077003,"name":"SUNY College of Technology - Alfred","priority":1255,"external_id":null},{"id":10847078003,"name":"SUNY College of Technology - Canton","priority":1256,"external_id":null},{"id":10847079003,"name":"SUNY College of Technology - Delhi","priority":1257,"external_id":null},{"id":10847080003,"name":"SUNY College - Cortland","priority":1258,"external_id":null},{"id":10847081003,"name":"SUNY College - Old Westbury","priority":1259,"external_id":null},{"id":10847082003,"name":"SUNY College - Oneonta","priority":1260,"external_id":null},{"id":10847083003,"name":"SUNY College - Potsdam","priority":1261,"external_id":null},{"id":10847084003,"name":"SUNY Downstate Medical Center","priority":1262,"external_id":null},{"id":10847085003,"name":"SUNY Empire State College","priority":1263,"external_id":null},{"id":10847086003,"name":"SUNY Institute of Technology - Utica/Rome","priority":1264,"external_id":null},{"id":10847087003,"name":"SUNY Maritime College","priority":1265,"external_id":null},{"id":10847088003,"name":"SUNY Upstate Medical University","priority":1266,"external_id":null},{"id":10847089003,"name":"SUNY - Fredonia","priority":1267,"external_id":null},{"id":10847090003,"name":"SUNY - Geneseo","priority":1268,"external_id":null},{"id":10847091003,"name":"SUNY - New Paltz","priority":1269,"external_id":null},{"id":10847092003,"name":"SUNY - Oswego","priority":1270,"external_id":null},{"id":10847093003,"name":"SUNY - Plattsburgh","priority":1271,"external_id":null},{"id":10847094003,"name":"Swarthmore College","priority":1272,"external_id":null},{"id":10847095003,"name":"Sweet Briar College","priority":1273,"external_id":null},{"id":10847096003,"name":"Tabor College","priority":1274,"external_id":null},{"id":10847097003,"name":"Talladega College","priority":1275,"external_id":null},{"id":10847098003,"name":"Tarleton State University","priority":1276,"external_id":null},{"id":10847099003,"name":"Taylor University","priority":1277,"external_id":null},{"id":10847100003,"name":"Tennessee Wesleyan College","priority":1278,"external_id":null},{"id":10847101003,"name":"Texas A&M International University","priority":1279,"external_id":null},{"id":10847102003,"name":"Texas A&M University - Commerce","priority":1280,"external_id":null},{"id":10847103003,"name":"Texas A&M University - Corpus Christi","priority":1281,"external_id":null},{"id":10847104003,"name":"Texas A&M University - Galveston","priority":1282,"external_id":null},{"id":10847105003,"name":"Texas A&M University - Kingsville","priority":1283,"external_id":null},{"id":10847106003,"name":"Texas A&M University - Texarkana","priority":1284,"external_id":null},{"id":10847107003,"name":"Texas College","priority":1285,"external_id":null},{"id":10847108003,"name":"Texas Lutheran University","priority":1286,"external_id":null},{"id":10847109003,"name":"Bucknell University","priority":1287,"external_id":null},{"id":10847110003,"name":"Butler University","priority":1288,"external_id":null},{"id":10847111003,"name":"Stephen F. Austin State University","priority":1289,"external_id":null},{"id":10847112003,"name":"Texas A&M University - College Station","priority":1290,"external_id":null},{"id":10847113003,"name":"Stanford University","priority":1291,"external_id":null},{"id":10847114003,"name":"Stetson University","priority":1292,"external_id":null},{"id":10847115003,"name":"Stony Brook University - SUNY","priority":1293,"external_id":null},{"id":10847116003,"name":"Syracuse University","priority":1294,"external_id":null},{"id":10847117003,"name":"Texas Christian University","priority":1295,"external_id":null},{"id":10847118003,"name":"Temple University","priority":1296,"external_id":null},{"id":10847119003,"name":"Clemson University","priority":1297,"external_id":null},{"id":10847120003,"name":"Texas Southern University","priority":1298,"external_id":null},{"id":10847121003,"name":"Austin Peay State University","priority":1299,"external_id":null},{"id":10847122003,"name":"Tennessee State University","priority":1300,"external_id":null},{"id":10847123003,"name":"Ball State University","priority":1301,"external_id":null},{"id":10847124003,"name":"Texas Tech University Health Sciences Center","priority":1302,"external_id":null},{"id":10847125003,"name":"Texas Wesleyan University","priority":1303,"external_id":null},{"id":10847126003,"name":"Texas Woman's University","priority":1304,"external_id":null},{"id":10847127003,"name":"The Catholic University of America","priority":1305,"external_id":null},{"id":10847128003,"name":"The Sage Colleges","priority":1306,"external_id":null},{"id":10847129003,"name":"Thiel College","priority":1307,"external_id":null},{"id":10847130003,"name":"Thomas Aquinas College","priority":1308,"external_id":null},{"id":10847131003,"name":"Thomas College","priority":1309,"external_id":null},{"id":10847132003,"name":"Thomas Edison State College","priority":1310,"external_id":null},{"id":10847133003,"name":"Thomas Jefferson University","priority":1311,"external_id":null},{"id":10847134003,"name":"Thomas More College","priority":1312,"external_id":null},{"id":10847135003,"name":"Thomas More College of Liberal Arts","priority":1313,"external_id":null},{"id":10847136003,"name":"Thomas University","priority":1314,"external_id":null},{"id":10847137003,"name":"Tiffin University","priority":1315,"external_id":null},{"id":10847138003,"name":"Tilburg University","priority":1316,"external_id":null},{"id":10847139003,"name":"Toccoa Falls College","priority":1317,"external_id":null},{"id":10847140003,"name":"Tougaloo College","priority":1318,"external_id":null},{"id":10847141003,"name":"Touro College","priority":1319,"external_id":null},{"id":10847142003,"name":"Transylvania University","priority":1320,"external_id":null},{"id":10847143003,"name":"Trent University","priority":1321,"external_id":null},{"id":10847144003,"name":"Trevecca Nazarene University","priority":1322,"external_id":null},{"id":10847145003,"name":"Trident University International","priority":1323,"external_id":null},{"id":10847146003,"name":"Trine University","priority":1324,"external_id":null},{"id":10847147003,"name":"Trinity Christian College","priority":1325,"external_id":null},{"id":10847148003,"name":"Trinity College","priority":1326,"external_id":null},{"id":10847149003,"name":"Trinity College of Nursing & Health Sciences","priority":1327,"external_id":null},{"id":10847150003,"name":"Trinity International University","priority":1328,"external_id":null},{"id":10847151003,"name":"Trinity Lutheran College","priority":1329,"external_id":null},{"id":10847152003,"name":"Trinity University","priority":1330,"external_id":null},{"id":10847153003,"name":"Trinity Western University","priority":1331,"external_id":null},{"id":10847154003,"name":"Truett McConnell College","priority":1332,"external_id":null},{"id":10847155003,"name":"Truman State University","priority":1333,"external_id":null},{"id":10847156003,"name":"Tufts University","priority":1334,"external_id":null},{"id":10847157003,"name":"Tusculum College","priority":1335,"external_id":null},{"id":10847158003,"name":"Tuskegee University","priority":1336,"external_id":null},{"id":10847159003,"name":"Union College","priority":1337,"external_id":null},{"id":10847160003,"name":"Union Institute and University","priority":1338,"external_id":null},{"id":10847161003,"name":"Union University","priority":1339,"external_id":null},{"id":10847162003,"name":"United States Coast Guard Academy","priority":1340,"external_id":null},{"id":10847163003,"name":"United States International University - Kenya","priority":1341,"external_id":null},{"id":10847164003,"name":"United States Merchant Marine Academy","priority":1342,"external_id":null},{"id":10847165003,"name":"United States Sports Academy","priority":1343,"external_id":null},{"id":10847166003,"name":"Unity College","priority":1344,"external_id":null},{"id":10847167003,"name":"Universidad Adventista de las Antillas","priority":1345,"external_id":null},{"id":10847168003,"name":"Universidad del Este","priority":1346,"external_id":null},{"id":10847169003,"name":"Universidad del Turabo","priority":1347,"external_id":null},{"id":10847170003,"name":"Universidad Metropolitana","priority":1348,"external_id":null},{"id":10847171003,"name":"Universidad Politecnica De Puerto Rico","priority":1349,"external_id":null},{"id":10847172003,"name":"University of Advancing Technology","priority":1350,"external_id":null},{"id":10847173003,"name":"University of Alabama - Huntsville","priority":1351,"external_id":null},{"id":10847174003,"name":"University of Alaska - Anchorage","priority":1352,"external_id":null},{"id":10847175003,"name":"University of Alaska - Fairbanks","priority":1353,"external_id":null},{"id":10847176003,"name":"University of Alaska - Southeast","priority":1354,"external_id":null},{"id":10847177003,"name":"University of Alberta","priority":1355,"external_id":null},{"id":10847178003,"name":"University of Arkansas for Medical Sciences","priority":1356,"external_id":null},{"id":10847179003,"name":"University of Arkansas - Fort Smith","priority":1357,"external_id":null},{"id":10847180003,"name":"University of Arkansas - Little Rock","priority":1358,"external_id":null},{"id":10847181003,"name":"University of Arkansas - Monticello","priority":1359,"external_id":null},{"id":10847182003,"name":"University of Baltimore","priority":1360,"external_id":null},{"id":10847183003,"name":"University of Bridgeport","priority":1361,"external_id":null},{"id":10847184003,"name":"University of British Columbia","priority":1362,"external_id":null},{"id":10847185003,"name":"University of Calgary","priority":1363,"external_id":null},{"id":10847186003,"name":"University of California - Riverside","priority":1364,"external_id":null},{"id":10847187003,"name":"Holy Cross College","priority":1365,"external_id":null},{"id":10847188003,"name":"Towson University","priority":1366,"external_id":null},{"id":10847189003,"name":"United States Military Academy","priority":1367,"external_id":null},{"id":10847190003,"name":"The Citadel","priority":1368,"external_id":null},{"id":10847191003,"name":"Troy University","priority":1369,"external_id":null},{"id":10847192003,"name":"University of California - Davis","priority":1370,"external_id":null},{"id":10847193003,"name":"Grambling State University","priority":1371,"external_id":null},{"id":10847194003,"name":"University at Albany - SUNY","priority":1372,"external_id":null},{"id":10847195003,"name":"University at Buffalo - SUNY","priority":1373,"external_id":null},{"id":10847196003,"name":"United States Naval Academy","priority":1374,"external_id":null},{"id":10847197003,"name":"University of Arizona","priority":1375,"external_id":null},{"id":10847198003,"name":"University of California - Los Angeles","priority":1376,"external_id":null},{"id":10847199003,"name":"Florida A&M University","priority":1377,"external_id":null},{"id":10847200003,"name":"Texas State University","priority":1378,"external_id":null},{"id":10847201003,"name":"University of Alabama - Birmingham","priority":1379,"external_id":null},{"id":10847202003,"name":"University of California - Santa Cruz","priority":1380,"external_id":null},{"id":10847203003,"name":"University of Central Missouri","priority":1381,"external_id":null},{"id":10847204003,"name":"University of Central Oklahoma","priority":1382,"external_id":null},{"id":10847205003,"name":"University of Charleston","priority":1383,"external_id":null},{"id":10847206003,"name":"University of Chicago","priority":1384,"external_id":null},{"id":10847207003,"name":"University of Cincinnati - UC Blue Ash College","priority":1385,"external_id":null},{"id":10847208003,"name":"University of Colorado - Colorado Springs","priority":1386,"external_id":null},{"id":10847209003,"name":"University of Colorado - Denver","priority":1387,"external_id":null},{"id":10847210003,"name":"University of Dallas","priority":1388,"external_id":null},{"id":10847211003,"name":"University of Denver","priority":1389,"external_id":null},{"id":10847212003,"name":"University of Detroit Mercy","priority":1390,"external_id":null},{"id":10847213003,"name":"University of Dubuque","priority":1391,"external_id":null},{"id":10847214003,"name":"University of Evansville","priority":1392,"external_id":null},{"id":10847215003,"name":"University of Findlay","priority":1393,"external_id":null},{"id":10847216003,"name":"University of Great Falls","priority":1394,"external_id":null},{"id":10847217003,"name":"University of Guam","priority":1395,"external_id":null},{"id":10847218003,"name":"University of Guelph","priority":1396,"external_id":null},{"id":10847219003,"name":"University of Hartford","priority":1397,"external_id":null},{"id":10847220003,"name":"University of Hawaii - Hilo","priority":1398,"external_id":null},{"id":10847221003,"name":"University of Hawaii - Maui College","priority":1399,"external_id":null},{"id":10847222003,"name":"University of Hawaii - West Oahu","priority":1400,"external_id":null},{"id":10847223003,"name":"University of Houston - Clear Lake","priority":1401,"external_id":null},{"id":10847224003,"name":"University of Houston - Downtown","priority":1402,"external_id":null},{"id":10847225003,"name":"University of Houston - Victoria","priority":1403,"external_id":null},{"id":10847226003,"name":"University of Illinois - Chicago","priority":1404,"external_id":null},{"id":10847227003,"name":"University of Illinois - Springfield","priority":1405,"external_id":null},{"id":10847228003,"name":"University of Indianapolis","priority":1406,"external_id":null},{"id":10847229003,"name":"University of Jamestown","priority":1407,"external_id":null},{"id":10847230003,"name":"University of La Verne","priority":1408,"external_id":null},{"id":10847231003,"name":"University of Maine - Augusta","priority":1409,"external_id":null},{"id":10847232003,"name":"University of Maine - Farmington","priority":1410,"external_id":null},{"id":10847233003,"name":"University of Maine - Fort Kent","priority":1411,"external_id":null},{"id":10847234003,"name":"University of Maine - Machias","priority":1412,"external_id":null},{"id":10847235003,"name":"University of Maine - Presque Isle","priority":1413,"external_id":null},{"id":10847236003,"name":"University of Mary","priority":1414,"external_id":null},{"id":10847237003,"name":"University of Mary Hardin-Baylor","priority":1415,"external_id":null},{"id":10847238003,"name":"University of Mary Washington","priority":1416,"external_id":null},{"id":10847239003,"name":"University of Maryland - Baltimore","priority":1417,"external_id":null},{"id":10847240003,"name":"University of Maryland - Baltimore County","priority":1418,"external_id":null},{"id":10847241003,"name":"University of Maryland - Eastern Shore","priority":1419,"external_id":null},{"id":10847242003,"name":"University of Maryland - University College","priority":1420,"external_id":null},{"id":10847243003,"name":"University of Massachusetts - Boston","priority":1421,"external_id":null},{"id":10847244003,"name":"University of Massachusetts - Dartmouth","priority":1422,"external_id":null},{"id":10847245003,"name":"University of Massachusetts - Lowell","priority":1423,"external_id":null},{"id":10847246003,"name":"University of Medicine and Dentistry of New Jersey","priority":1424,"external_id":null},{"id":10847247003,"name":"University of Michigan - Dearborn","priority":1425,"external_id":null},{"id":10847248003,"name":"University of Michigan - Flint","priority":1426,"external_id":null},{"id":10847249003,"name":"University of Minnesota - Crookston","priority":1427,"external_id":null},{"id":10847250003,"name":"University of Minnesota - Duluth","priority":1428,"external_id":null},{"id":10847251003,"name":"University of Minnesota - Morris","priority":1429,"external_id":null},{"id":10847252003,"name":"University of Mississippi Medical Center","priority":1430,"external_id":null},{"id":10847253003,"name":"University of Missouri - Kansas City","priority":1431,"external_id":null},{"id":10847254003,"name":"University of Missouri - St. Louis","priority":1432,"external_id":null},{"id":10847255003,"name":"University of Mobile","priority":1433,"external_id":null},{"id":10847256003,"name":"University of Montana - Western","priority":1434,"external_id":null},{"id":10847257003,"name":"University of Montevallo","priority":1435,"external_id":null},{"id":10847258003,"name":"University of Mount Union","priority":1436,"external_id":null},{"id":10847259003,"name":"University of Nebraska Medical Center","priority":1437,"external_id":null},{"id":10847260003,"name":"University of Nebraska - Kearney","priority":1438,"external_id":null},{"id":10847261003,"name":"University of Dayton","priority":1439,"external_id":null},{"id":10847262003,"name":"University of Delaware","priority":1440,"external_id":null},{"id":10847263003,"name":"University of Florida","priority":1441,"external_id":null},{"id":10847264003,"name":"University of Iowa","priority":1442,"external_id":null},{"id":10847265003,"name":"University of Idaho","priority":1443,"external_id":null},{"id":10847266003,"name":"University of Kentucky","priority":1444,"external_id":null},{"id":10847267003,"name":"University of Massachusetts - Amherst","priority":1445,"external_id":null},{"id":10847268003,"name":"University of Maine","priority":1446,"external_id":null},{"id":10847269003,"name":"University of Michigan - Ann Arbor","priority":1447,"external_id":null},{"id":10847270003,"name":"University of Cincinnati","priority":1448,"external_id":null},{"id":10847271003,"name":"University of Miami","priority":1449,"external_id":null},{"id":10847272003,"name":"University of Louisiana - Monroe","priority":1450,"external_id":null},{"id":10847273003,"name":"University of Missouri","priority":1451,"external_id":null},{"id":10847274003,"name":"University of Mississippi","priority":1452,"external_id":null},{"id":10847275003,"name":"University of Memphis","priority":1453,"external_id":null},{"id":10847276003,"name":"University of Houston","priority":1454,"external_id":null},{"id":10847277003,"name":"University of Colorado - Boulder","priority":1455,"external_id":null},{"id":10847278003,"name":"University of Nebraska - Omaha","priority":1456,"external_id":null},{"id":10847279003,"name":"University of New Brunswick","priority":1457,"external_id":null},{"id":10847280003,"name":"University of New England","priority":1458,"external_id":null},{"id":10847281003,"name":"University of New Haven","priority":1459,"external_id":null},{"id":10847282003,"name":"University of New Orleans","priority":1460,"external_id":null},{"id":10847283003,"name":"University of North Alabama","priority":1461,"external_id":null},{"id":10847284003,"name":"University of North Carolina School of the Arts","priority":1462,"external_id":null},{"id":10847285003,"name":"University of North Carolina - Asheville","priority":1463,"external_id":null},{"id":10847286003,"name":"University of North Carolina - Greensboro","priority":1464,"external_id":null},{"id":10847287003,"name":"University of North Carolina - Pembroke","priority":1465,"external_id":null},{"id":10847288003,"name":"University of North Carolina - Wilmington","priority":1466,"external_id":null},{"id":10847289003,"name":"University of North Florida","priority":1467,"external_id":null},{"id":10847290003,"name":"University of North Georgia","priority":1468,"external_id":null},{"id":10847291003,"name":"University of Northwestern Ohio","priority":1469,"external_id":null},{"id":10847292003,"name":"University of Northwestern - St. Paul","priority":1470,"external_id":null},{"id":10847293003,"name":"University of Ottawa","priority":1471,"external_id":null},{"id":10847294003,"name":"University of Phoenix","priority":1472,"external_id":null},{"id":10847295003,"name":"University of Pikeville","priority":1473,"external_id":null},{"id":10847296003,"name":"University of Portland","priority":1474,"external_id":null},{"id":10847297003,"name":"University of Prince Edward Island","priority":1475,"external_id":null},{"id":10847298003,"name":"University of Puerto Rico - Aguadilla","priority":1476,"external_id":null},{"id":10847299003,"name":"University of Puerto Rico - Arecibo","priority":1477,"external_id":null},{"id":10847300003,"name":"University of Puerto Rico - Bayamon","priority":1478,"external_id":null},{"id":10847301003,"name":"University of Puerto Rico - Cayey","priority":1479,"external_id":null},{"id":10847302003,"name":"University of Puerto Rico - Humacao","priority":1480,"external_id":null},{"id":10847303003,"name":"University of Puerto Rico - Mayaguez","priority":1481,"external_id":null},{"id":10847304003,"name":"University of Puerto Rico - Medical Sciences Campus","priority":1482,"external_id":null},{"id":10847305003,"name":"University of Puerto Rico - Ponce","priority":1483,"external_id":null},{"id":10847306003,"name":"University of Puerto Rico - Rio Piedras","priority":1484,"external_id":null},{"id":10847307003,"name":"University of Puget Sound","priority":1485,"external_id":null},{"id":10847308003,"name":"University of Redlands","priority":1486,"external_id":null},{"id":10847309003,"name":"University of Regina","priority":1487,"external_id":null},{"id":10847310003,"name":"University of Rio Grande","priority":1488,"external_id":null},{"id":10847311003,"name":"University of Rochester","priority":1489,"external_id":null},{"id":10847312003,"name":"University of San Francisco","priority":1490,"external_id":null},{"id":10847313003,"name":"University of Saskatchewan","priority":1491,"external_id":null},{"id":10847314003,"name":"University of Science and Arts of Oklahoma","priority":1492,"external_id":null},{"id":10847315003,"name":"University of Scranton","priority":1493,"external_id":null},{"id":10847316003,"name":"University of Sioux Falls","priority":1494,"external_id":null},{"id":10847317003,"name":"University of South Carolina - Aiken","priority":1495,"external_id":null},{"id":10847318003,"name":"University of South Carolina - Beaufort","priority":1496,"external_id":null},{"id":10847319003,"name":"University of South Carolina - Upstate","priority":1497,"external_id":null},{"id":10847320003,"name":"University of South Florida - St. Petersburg","priority":1498,"external_id":null},{"id":10847321003,"name":"University of Southern Indiana","priority":1499,"external_id":null},{"id":10847322003,"name":"University of Southern Maine","priority":1500,"external_id":null},{"id":10847323003,"name":"University of St. Francis","priority":1501,"external_id":null},{"id":10847324003,"name":"University of St. Joseph","priority":1502,"external_id":null},{"id":10847325003,"name":"University of St. Mary","priority":1503,"external_id":null},{"id":10847326003,"name":"University of St. Thomas","priority":1504,"external_id":null},{"id":10847327003,"name":"University of Tampa","priority":1505,"external_id":null},{"id":10847328003,"name":"University of Texas Health Science Center - Houston","priority":1506,"external_id":null},{"id":10847329003,"name":"University of Texas Health Science Center - San Antonio","priority":1507,"external_id":null},{"id":10847330003,"name":"University of Texas Medical Branch - Galveston","priority":1508,"external_id":null},{"id":10847331003,"name":"University of Texas of the Permian Basin","priority":1509,"external_id":null},{"id":10847332003,"name":"University of Texas - Arlington","priority":1510,"external_id":null},{"id":10847333003,"name":"University of Texas - Brownsville","priority":1511,"external_id":null},{"id":10847334003,"name":"University of Texas - Pan American","priority":1512,"external_id":null},{"id":10847335003,"name":"University of Oregon","priority":1513,"external_id":null},{"id":10847336003,"name":"University of New Mexico","priority":1514,"external_id":null},{"id":10847337003,"name":"University of Pennsylvania","priority":1515,"external_id":null},{"id":10847338003,"name":"University of North Dakota","priority":1516,"external_id":null},{"id":10847339003,"name":"University of Nevada - Reno","priority":1517,"external_id":null},{"id":10847340003,"name":"University of New Hampshire","priority":1518,"external_id":null},{"id":10847341003,"name":"University of Texas - Austin","priority":1519,"external_id":null},{"id":10847342003,"name":"University of Southern Mississippi","priority":1520,"external_id":null},{"id":10847343003,"name":"University of Rhode Island","priority":1521,"external_id":null},{"id":10847344003,"name":"University of South Dakota","priority":1522,"external_id":null},{"id":10847345003,"name":"University of Tennessee","priority":1523,"external_id":null},{"id":10847346003,"name":"University of North Texas","priority":1524,"external_id":null},{"id":10847347003,"name":"University of North Carolina - Charlotte","priority":1525,"external_id":null},{"id":10847348003,"name":"University of Texas - San Antonio","priority":1526,"external_id":null},{"id":10847349003,"name":"University of Notre Dame","priority":1527,"external_id":null},{"id":10847350003,"name":"University of Southern California","priority":1528,"external_id":null},{"id":10847351003,"name":"University of Texas - Tyler","priority":1529,"external_id":null},{"id":10847352003,"name":"University of the Arts","priority":1530,"external_id":null},{"id":10847353003,"name":"University of the Cumberlands","priority":1531,"external_id":null},{"id":10847354003,"name":"University of the District of Columbia","priority":1532,"external_id":null},{"id":10847355003,"name":"University of the Ozarks","priority":1533,"external_id":null},{"id":10847356003,"name":"University of the Pacific","priority":1534,"external_id":null},{"id":10847357003,"name":"University of the Sacred Heart","priority":1535,"external_id":null},{"id":10847358003,"name":"University of the Sciences","priority":1536,"external_id":null},{"id":10847359003,"name":"University of the Southwest","priority":1537,"external_id":null},{"id":10847360003,"name":"University of the Virgin Islands","priority":1538,"external_id":null},{"id":10847361003,"name":"University of the West","priority":1539,"external_id":null},{"id":10847362003,"name":"University of Toronto","priority":1540,"external_id":null},{"id":10847363003,"name":"University of Vermont","priority":1541,"external_id":null},{"id":10847364003,"name":"University of Victoria","priority":1542,"external_id":null},{"id":10847365003,"name":"University of Virginia - Wise","priority":1543,"external_id":null},{"id":10847366003,"name":"University of Waterloo","priority":1544,"external_id":null},{"id":10847367003,"name":"University of West Alabama","priority":1545,"external_id":null},{"id":10847368003,"name":"University of West Florida","priority":1546,"external_id":null},{"id":10847369003,"name":"University of West Georgia","priority":1547,"external_id":null},{"id":10847370003,"name":"University of Windsor","priority":1548,"external_id":null},{"id":10847371003,"name":"University of Winnipeg","priority":1549,"external_id":null},{"id":10847372003,"name":"University of Wisconsin - Eau Claire","priority":1550,"external_id":null},{"id":10847373003,"name":"University of Wisconsin - Green Bay","priority":1551,"external_id":null},{"id":10847374003,"name":"University of Wisconsin - La Crosse","priority":1552,"external_id":null},{"id":10847375003,"name":"University of Wisconsin - Milwaukee","priority":1553,"external_id":null},{"id":10847376003,"name":"University of Wisconsin - Oshkosh","priority":1554,"external_id":null},{"id":10847377003,"name":"University of Wisconsin - Parkside","priority":1555,"external_id":null},{"id":10847378003,"name":"University of Wisconsin - Platteville","priority":1556,"external_id":null},{"id":10847379003,"name":"University of Wisconsin - River Falls","priority":1557,"external_id":null},{"id":10847380003,"name":"University of Wisconsin - Stevens Point","priority":1558,"external_id":null},{"id":10847381003,"name":"University of Wisconsin - Stout","priority":1559,"external_id":null},{"id":10847382003,"name":"University of Wisconsin - Superior","priority":1560,"external_id":null},{"id":10847383003,"name":"University of Wisconsin - Whitewater","priority":1561,"external_id":null},{"id":10847384003,"name":"Upper Iowa University","priority":1562,"external_id":null},{"id":10847385003,"name":"Urbana University","priority":1563,"external_id":null},{"id":10847386003,"name":"Ursinus College","priority":1564,"external_id":null},{"id":10847387003,"name":"Ursuline College","priority":1565,"external_id":null},{"id":10847388003,"name":"Utah Valley University","priority":1566,"external_id":null},{"id":10847389003,"name":"Utica College","priority":1567,"external_id":null},{"id":10847390003,"name":"Valdosta State University","priority":1568,"external_id":null},{"id":10847391003,"name":"Valley City State University","priority":1569,"external_id":null},{"id":10847392003,"name":"Valley Forge Christian College","priority":1570,"external_id":null},{"id":10847393003,"name":"VanderCook College of Music","priority":1571,"external_id":null},{"id":10847394003,"name":"Vanguard University of Southern California","priority":1572,"external_id":null},{"id":10847395003,"name":"Vassar College","priority":1573,"external_id":null},{"id":10847396003,"name":"Vaughn College of Aeronautics and Technology","priority":1574,"external_id":null},{"id":10847397003,"name":"Vermont Technical College","priority":1575,"external_id":null},{"id":10847398003,"name":"Victory University","priority":1576,"external_id":null},{"id":10847399003,"name":"Vincennes University","priority":1577,"external_id":null},{"id":10847400003,"name":"Virginia Commonwealth University","priority":1578,"external_id":null},{"id":10847401003,"name":"Virginia Intermont College","priority":1579,"external_id":null},{"id":10847402003,"name":"Virginia State University","priority":1580,"external_id":null},{"id":10847403003,"name":"Virginia Union University","priority":1581,"external_id":null},{"id":10847404003,"name":"Virginia Wesleyan College","priority":1582,"external_id":null},{"id":10847405003,"name":"Viterbo University","priority":1583,"external_id":null},{"id":10847406003,"name":"Voorhees College","priority":1584,"external_id":null},{"id":10847407003,"name":"Wabash College","priority":1585,"external_id":null},{"id":10847408003,"name":"Walden University","priority":1586,"external_id":null},{"id":10847409003,"name":"Waldorf College","priority":1587,"external_id":null},{"id":10847410003,"name":"Walla Walla University","priority":1588,"external_id":null},{"id":10847411003,"name":"Walsh College of Accountancy and Business Administration","priority":1589,"external_id":null},{"id":10847412003,"name":"Walsh University","priority":1590,"external_id":null},{"id":10847413003,"name":"Warner Pacific College","priority":1591,"external_id":null},{"id":10847414003,"name":"Warner University","priority":1592,"external_id":null},{"id":10847415003,"name":"Warren Wilson College","priority":1593,"external_id":null},{"id":10847416003,"name":"Wartburg College","priority":1594,"external_id":null},{"id":10847417003,"name":"Washburn University","priority":1595,"external_id":null},{"id":10847418003,"name":"Washington Adventist University","priority":1596,"external_id":null},{"id":10847419003,"name":"Washington and Jefferson College","priority":1597,"external_id":null},{"id":10847420003,"name":"Washington and Lee University","priority":1598,"external_id":null},{"id":10847421003,"name":"Washington College","priority":1599,"external_id":null},{"id":10847422003,"name":"Washington University in St. Louis","priority":1600,"external_id":null},{"id":10847423003,"name":"Watkins College of Art, Design & Film","priority":1601,"external_id":null},{"id":10847424003,"name":"Wayland Baptist University","priority":1602,"external_id":null},{"id":10847425003,"name":"Wayne State College","priority":1603,"external_id":null},{"id":10847426003,"name":"Wayne State University","priority":1604,"external_id":null},{"id":10847427003,"name":"Waynesburg University","priority":1605,"external_id":null},{"id":10847428003,"name":"Valparaiso University","priority":1606,"external_id":null},{"id":10847429003,"name":"Villanova University","priority":1607,"external_id":null},{"id":10847430003,"name":"Virginia Tech","priority":1608,"external_id":null},{"id":10847431003,"name":"Washington State University","priority":1609,"external_id":null},{"id":10847432003,"name":"University of Toledo","priority":1610,"external_id":null},{"id":10847433003,"name":"Wagner College","priority":1611,"external_id":null},{"id":10847434003,"name":"University of Wyoming","priority":1612,"external_id":null},{"id":10847435003,"name":"University of Wisconsin - Madison","priority":1613,"external_id":null},{"id":10847436003,"name":"University of Tulsa","priority":1614,"external_id":null},{"id":10847437003,"name":"Webb Institute","priority":1615,"external_id":null},{"id":10847438003,"name":"Webber International University","priority":1616,"external_id":null},{"id":10847439003,"name":"Webster University","priority":1617,"external_id":null},{"id":10847440003,"name":"Welch College","priority":1618,"external_id":null},{"id":10847441003,"name":"Wellesley College","priority":1619,"external_id":null},{"id":10847442003,"name":"Wells College","priority":1620,"external_id":null},{"id":10847443003,"name":"Wentworth Institute of Technology","priority":1621,"external_id":null},{"id":10847444003,"name":"Wesley College","priority":1622,"external_id":null},{"id":10847445003,"name":"Wesleyan College","priority":1623,"external_id":null},{"id":10847446003,"name":"Wesleyan University","priority":1624,"external_id":null},{"id":10847447003,"name":"West Chester University of Pennsylvania","priority":1625,"external_id":null},{"id":10847448003,"name":"West Liberty University","priority":1626,"external_id":null},{"id":10847449003,"name":"West Texas A&M University","priority":1627,"external_id":null},{"id":10847450003,"name":"West Virginia State University","priority":1628,"external_id":null},{"id":10847451003,"name":"West Virginia University Institute of Technology","priority":1629,"external_id":null},{"id":10847452003,"name":"West Virginia University - Parkersburg","priority":1630,"external_id":null},{"id":10847453003,"name":"West Virginia Wesleyan College","priority":1631,"external_id":null},{"id":10847454003,"name":"Western Connecticut State University","priority":1632,"external_id":null},{"id":10847455003,"name":"Western Governors University","priority":1633,"external_id":null},{"id":10847456003,"name":"Western International University","priority":1634,"external_id":null},{"id":10847457003,"name":"Western Nevada College","priority":1635,"external_id":null},{"id":10847458003,"name":"Western New England University","priority":1636,"external_id":null},{"id":10847459003,"name":"Western New Mexico University","priority":1637,"external_id":null},{"id":10847460003,"name":"Western Oregon University","priority":1638,"external_id":null},{"id":10847461003,"name":"Western State Colorado University","priority":1639,"external_id":null},{"id":10847462003,"name":"Western University","priority":1640,"external_id":null},{"id":10847463003,"name":"Western Washington University","priority":1641,"external_id":null},{"id":10847464003,"name":"Westfield State University","priority":1642,"external_id":null},{"id":10847465003,"name":"Westminster College","priority":1643,"external_id":null},{"id":10847466003,"name":"Westmont College","priority":1644,"external_id":null},{"id":10847467003,"name":"Wheaton College","priority":1645,"external_id":null},{"id":10847468003,"name":"Wheeling Jesuit University","priority":1646,"external_id":null},{"id":10847469003,"name":"Wheelock College","priority":1647,"external_id":null},{"id":10847470003,"name":"Whitman College","priority":1648,"external_id":null},{"id":10847471003,"name":"Whittier College","priority":1649,"external_id":null},{"id":10847472003,"name":"Whitworth University","priority":1650,"external_id":null},{"id":10847473003,"name":"Wichita State University","priority":1651,"external_id":null},{"id":10847474003,"name":"Widener University","priority":1652,"external_id":null},{"id":10847475003,"name":"Wilberforce University","priority":1653,"external_id":null},{"id":10847476003,"name":"Wiley College","priority":1654,"external_id":null},{"id":10847477003,"name":"Wilkes University","priority":1655,"external_id":null},{"id":10847478003,"name":"Willamette University","priority":1656,"external_id":null},{"id":10847479003,"name":"William Carey University","priority":1657,"external_id":null},{"id":10847480003,"name":"William Jessup University","priority":1658,"external_id":null},{"id":10847481003,"name":"William Jewell College","priority":1659,"external_id":null},{"id":10847482003,"name":"William Paterson University of New Jersey","priority":1660,"external_id":null},{"id":10847483003,"name":"William Peace University","priority":1661,"external_id":null},{"id":10847484003,"name":"William Penn University","priority":1662,"external_id":null},{"id":10847485003,"name":"William Woods University","priority":1663,"external_id":null},{"id":10847486003,"name":"Williams Baptist College","priority":1664,"external_id":null},{"id":10847487003,"name":"Williams College","priority":1665,"external_id":null},{"id":10847488003,"name":"Wilmington College","priority":1666,"external_id":null},{"id":10847489003,"name":"Wilmington University","priority":1667,"external_id":null},{"id":10847490003,"name":"Wilson College","priority":1668,"external_id":null},{"id":10847491003,"name":"Wingate University","priority":1669,"external_id":null},{"id":10847492003,"name":"Winona State University","priority":1670,"external_id":null},{"id":10847493003,"name":"Winston-Salem State University","priority":1671,"external_id":null},{"id":10847494003,"name":"Winthrop University","priority":1672,"external_id":null},{"id":10847495003,"name":"Wisconsin Lutheran College","priority":1673,"external_id":null},{"id":10847496003,"name":"Wittenberg University","priority":1674,"external_id":null},{"id":10847497003,"name":"Woodbury University","priority":1675,"external_id":null},{"id":10847498003,"name":"Worcester Polytechnic Institute","priority":1676,"external_id":null},{"id":10847499003,"name":"Worcester State University","priority":1677,"external_id":null},{"id":10847500003,"name":"Wright State University","priority":1678,"external_id":null},{"id":10847501003,"name":"Xavier University","priority":1679,"external_id":null},{"id":10847502003,"name":"Xavier University of Louisiana","priority":1680,"external_id":null},{"id":10847503003,"name":"Yeshiva University","priority":1681,"external_id":null},{"id":10847504003,"name":"York College","priority":1682,"external_id":null},{"id":10847505003,"name":"York College of Pennsylvania","priority":1683,"external_id":null},{"id":10847506003,"name":"York University","priority":1684,"external_id":null},{"id":10847507003,"name":"University of Cambridge","priority":1685,"external_id":null},{"id":10847508003,"name":"UCL (University College London)","priority":1686,"external_id":null},{"id":10847509003,"name":"Imperial College London","priority":1687,"external_id":null},{"id":10847510003,"name":"University of Oxford","priority":1688,"external_id":null},{"id":10847511003,"name":"ETH Zurich (Swiss Federal Institute of Technology)","priority":1689,"external_id":null},{"id":10847512003,"name":"University of Edinburgh","priority":1690,"external_id":null},{"id":10847513003,"name":"Ecole Polytechnique Fédérale de Lausanne","priority":1691,"external_id":null},{"id":10847514003,"name":"King's College London (KCL)","priority":1692,"external_id":null},{"id":10847515003,"name":"National University of Singapore (NUS)","priority":1693,"external_id":null},{"id":10847516003,"name":"University of Hong Kong","priority":1694,"external_id":null},{"id":10847517003,"name":"Australian National University","priority":1695,"external_id":null},{"id":10847518003,"name":"Ecole normale supérieure, Paris","priority":1696,"external_id":null},{"id":10847519003,"name":"University of Bristol","priority":1697,"external_id":null},{"id":10847520003,"name":"The University of Melbourne","priority":1698,"external_id":null},{"id":10847521003,"name":"The University of Tokyo","priority":1699,"external_id":null},{"id":10847522003,"name":"The University of Manchester","priority":1700,"external_id":null},{"id":10847523003,"name":"Western Illinois University","priority":1701,"external_id":null},{"id":10847524003,"name":"Wofford College","priority":1702,"external_id":null},{"id":10847525003,"name":"Western Carolina University","priority":1703,"external_id":null},{"id":10847526003,"name":"West Virginia University","priority":1704,"external_id":null},{"id":10847527003,"name":"Yale University","priority":1705,"external_id":null},{"id":10847528003,"name":"The Hong Kong University of Science and Technology","priority":1706,"external_id":null},{"id":10847529003,"name":"Kyoto University","priority":1707,"external_id":null},{"id":10847530003,"name":"Seoul National University","priority":1708,"external_id":null},{"id":10847531003,"name":"The University of Sydney","priority":1709,"external_id":null},{"id":10847532003,"name":"The Chinese University of Hong Kong","priority":1710,"external_id":null},{"id":10847533003,"name":"Ecole Polytechnique","priority":1711,"external_id":null},{"id":10847534003,"name":"Nanyang Technological University (NTU)","priority":1712,"external_id":null},{"id":10847535003,"name":"The University of Queensland","priority":1713,"external_id":null},{"id":10847536003,"name":"University of Copenhagen","priority":1714,"external_id":null},{"id":10847537003,"name":"Peking University","priority":1715,"external_id":null},{"id":10847538003,"name":"Tsinghua University","priority":1716,"external_id":null},{"id":10847539003,"name":"Ruprecht-Karls-Universität Heidelberg","priority":1717,"external_id":null},{"id":10847540003,"name":"University of Glasgow","priority":1718,"external_id":null},{"id":10847541003,"name":"The University of New South Wales","priority":1719,"external_id":null},{"id":10847542003,"name":"Technische Universität München","priority":1720,"external_id":null},{"id":10847543003,"name":"Osaka University","priority":1721,"external_id":null},{"id":10847544003,"name":"University of Amsterdam","priority":1722,"external_id":null},{"id":10847545003,"name":"KAIST - Korea Advanced Institute of Science & Technology","priority":1723,"external_id":null},{"id":10847546003,"name":"Trinity College Dublin","priority":1724,"external_id":null},{"id":10847547003,"name":"University of Birmingham","priority":1725,"external_id":null},{"id":10847548003,"name":"The University of Warwick","priority":1726,"external_id":null},{"id":10847549003,"name":"Ludwig-Maximilians-Universität München","priority":1727,"external_id":null},{"id":10847550003,"name":"Tokyo Institute of Technology","priority":1728,"external_id":null},{"id":10847551003,"name":"Lund University","priority":1729,"external_id":null},{"id":10847552003,"name":"London School of Economics and Political Science (LSE)","priority":1730,"external_id":null},{"id":10847553003,"name":"Monash University","priority":1731,"external_id":null},{"id":10847554003,"name":"University of Helsinki","priority":1732,"external_id":null},{"id":10847555003,"name":"The University of Sheffield","priority":1733,"external_id":null},{"id":10847556003,"name":"University of Geneva","priority":1734,"external_id":null},{"id":10847557003,"name":"Leiden University","priority":1735,"external_id":null},{"id":10847558003,"name":"The University of Nottingham","priority":1736,"external_id":null},{"id":10847559003,"name":"Tohoku University","priority":1737,"external_id":null},{"id":10847560003,"name":"KU Leuven","priority":1738,"external_id":null},{"id":10847561003,"name":"University of Zurich","priority":1739,"external_id":null},{"id":10847562003,"name":"Uppsala University","priority":1740,"external_id":null},{"id":10847563003,"name":"Utrecht University","priority":1741,"external_id":null},{"id":10847564003,"name":"National Taiwan University (NTU)","priority":1742,"external_id":null},{"id":10847565003,"name":"University of St Andrews","priority":1743,"external_id":null},{"id":10847566003,"name":"The University of Western Australia","priority":1744,"external_id":null},{"id":10847567003,"name":"University of Southampton","priority":1745,"external_id":null},{"id":10847568003,"name":"Fudan University","priority":1746,"external_id":null},{"id":10847569003,"name":"University of Oslo","priority":1747,"external_id":null},{"id":10847570003,"name":"Durham University","priority":1748,"external_id":null},{"id":10847571003,"name":"Aarhus University","priority":1749,"external_id":null},{"id":10847572003,"name":"Erasmus University Rotterdam","priority":1750,"external_id":null},{"id":10847573003,"name":"Université de Montréal","priority":1751,"external_id":null},{"id":10847574003,"name":"The University of Auckland","priority":1752,"external_id":null},{"id":10847575003,"name":"Delft University of Technology","priority":1753,"external_id":null},{"id":10847576003,"name":"University of Groningen","priority":1754,"external_id":null},{"id":10847577003,"name":"University of Leeds","priority":1755,"external_id":null},{"id":10847578003,"name":"Nagoya University","priority":1756,"external_id":null},{"id":10847579003,"name":"Universität Freiburg","priority":1757,"external_id":null},{"id":10847580003,"name":"City University of Hong Kong","priority":1758,"external_id":null},{"id":10847581003,"name":"The University of Adelaide","priority":1759,"external_id":null},{"id":10847582003,"name":"Pohang University of Science And Technology (POSTECH)","priority":1760,"external_id":null},{"id":10847583003,"name":"Freie Universität Berlin","priority":1761,"external_id":null},{"id":10847584003,"name":"University of Basel","priority":1762,"external_id":null},{"id":10847585003,"name":"University of Lausanne","priority":1763,"external_id":null},{"id":10847586003,"name":"Université Pierre et Marie Curie (UPMC)","priority":1764,"external_id":null},{"id":10847587003,"name":"Yonsei University","priority":1765,"external_id":null},{"id":10847588003,"name":"University of York","priority":1766,"external_id":null},{"id":10847589003,"name":"Queen Mary, University of London (QMUL)","priority":1767,"external_id":null},{"id":10847590003,"name":"Karlsruhe Institute of Technology (KIT)","priority":1768,"external_id":null},{"id":10847591003,"name":"KTH, Royal Institute of Technology","priority":1769,"external_id":null},{"id":10847592003,"name":"Lomonosov Moscow State University","priority":1770,"external_id":null},{"id":10847593003,"name":"Maastricht University","priority":1771,"external_id":null},{"id":10847594003,"name":"University of Ghent","priority":1772,"external_id":null},{"id":10847595003,"name":"Shanghai Jiao Tong University","priority":1773,"external_id":null},{"id":10847596003,"name":"Humboldt-Universität zu Berlin","priority":1774,"external_id":null},{"id":10847597003,"name":"Universidade de São Paulo (USP)","priority":1775,"external_id":null},{"id":10847598003,"name":"Georg-August-Universität Göttingen","priority":1776,"external_id":null},{"id":10847599003,"name":"Newcastle University","priority":1777,"external_id":null},{"id":10847600003,"name":"University of Liverpool","priority":1778,"external_id":null},{"id":10847601003,"name":"Kyushu University","priority":1779,"external_id":null},{"id":10847602003,"name":"Eberhard Karls Universität Tübingen","priority":1780,"external_id":null},{"id":10847603003,"name":"Technical University of Denmark","priority":1781,"external_id":null},{"id":10847604003,"name":"Cardiff University","priority":1782,"external_id":null},{"id":10847605003,"name":"Université Catholique de Louvain (UCL)","priority":1783,"external_id":null},{"id":10847606003,"name":"University College Dublin","priority":1784,"external_id":null},{"id":10847607003,"name":"McMaster University","priority":1785,"external_id":null},{"id":10847608003,"name":"Hebrew University of Jerusalem","priority":1786,"external_id":null},{"id":10847609003,"name":"Radboud University Nijmegen","priority":1787,"external_id":null},{"id":10847610003,"name":"Hokkaido University","priority":1788,"external_id":null},{"id":10847611003,"name":"Korea University","priority":1789,"external_id":null},{"id":10847612003,"name":"University of Cape Town","priority":1790,"external_id":null},{"id":10847613003,"name":"Rheinisch-Westfälische Technische Hochschule Aachen","priority":1791,"external_id":null},{"id":10847614003,"name":"University of Aberdeen","priority":1792,"external_id":null},{"id":10847615003,"name":"Wageningen University","priority":1793,"external_id":null},{"id":10847616003,"name":"University of Bergen","priority":1794,"external_id":null},{"id":10847617003,"name":"University of Bern","priority":1795,"external_id":null},{"id":10847618003,"name":"University of Otago","priority":1796,"external_id":null},{"id":10847619003,"name":"Lancaster University","priority":1797,"external_id":null},{"id":10847620003,"name":"Eindhoven University of Technology","priority":1798,"external_id":null},{"id":10847621003,"name":"Ecole Normale Supérieure de Lyon","priority":1799,"external_id":null},{"id":10847622003,"name":"University of Vienna","priority":1800,"external_id":null},{"id":10847623003,"name":"The Hong Kong Polytechnic University","priority":1801,"external_id":null},{"id":10847624003,"name":"Sungkyunkwan University","priority":1802,"external_id":null},{"id":10847625003,"name":"Rheinische Friedrich-Wilhelms-Universität Bonn","priority":1803,"external_id":null},{"id":10847626003,"name":"Universidad Nacional Autónoma de México (UNAM)","priority":1804,"external_id":null},{"id":10847627003,"name":"Zhejiang University","priority":1805,"external_id":null},{"id":10847628003,"name":"Pontificia Universidad Católica de Chile","priority":1806,"external_id":null},{"id":10847629003,"name":"Universiti Malaya (UM)","priority":1807,"external_id":null},{"id":10847630003,"name":"Université Libre de Bruxelles (ULB)","priority":1808,"external_id":null},{"id":10847631003,"name":"University of Exeter","priority":1809,"external_id":null},{"id":10847632003,"name":"Stockholm University","priority":1810,"external_id":null},{"id":10847633003,"name":"Queen's University of Belfast","priority":1811,"external_id":null},{"id":10847634003,"name":"Vrije Universiteit Brussel (VUB)","priority":1812,"external_id":null},{"id":10847635003,"name":"University of Science and Technology of China","priority":1813,"external_id":null},{"id":10847636003,"name":"Nanjing University","priority":1814,"external_id":null},{"id":10847637003,"name":"Universitat Autónoma de Barcelona","priority":1815,"external_id":null},{"id":10847638003,"name":"University of Barcelona","priority":1816,"external_id":null},{"id":10847639003,"name":"VU University Amsterdam","priority":1817,"external_id":null},{"id":10847640003,"name":"Technion - Israel Institute of Technology","priority":1818,"external_id":null},{"id":10847641003,"name":"Technische Universität Berlin","priority":1819,"external_id":null},{"id":10847642003,"name":"University of Antwerp","priority":1820,"external_id":null},{"id":10847643003,"name":"Universität Hamburg","priority":1821,"external_id":null},{"id":10847644003,"name":"University of Bath","priority":1822,"external_id":null},{"id":10847645003,"name":"University of Bologna","priority":1823,"external_id":null},{"id":10847646003,"name":"Queen's University, Ontario","priority":1824,"external_id":null},{"id":10847647003,"name":"Université Paris-Sud 11","priority":1825,"external_id":null},{"id":10847648003,"name":"Keio University","priority":1826,"external_id":null},{"id":10847649003,"name":"University of Sussex","priority":1827,"external_id":null},{"id":10847650003,"name":"Universidad Autónoma de Madrid","priority":1828,"external_id":null},{"id":10847651003,"name":"Aalto University","priority":1829,"external_id":null},{"id":10847652003,"name":"Sapienza University of Rome","priority":1830,"external_id":null},{"id":10847653003,"name":"Tel Aviv University","priority":1831,"external_id":null},{"id":10847654003,"name":"National Tsing Hua University","priority":1832,"external_id":null},{"id":10847655003,"name":"Chalmers University of Technology","priority":1833,"external_id":null},{"id":10847656003,"name":"University of Leicester","priority":1834,"external_id":null},{"id":10847657003,"name":"Université Paris Diderot - Paris 7","priority":1835,"external_id":null},{"id":10847658003,"name":"University of Gothenburg","priority":1836,"external_id":null},{"id":10847659003,"name":"University of Turku","priority":1837,"external_id":null},{"id":10847660003,"name":"Universität Frankfurt am Main","priority":1838,"external_id":null},{"id":10847661003,"name":"Universidad de Buenos Aires","priority":1839,"external_id":null},{"id":10847662003,"name":"University College Cork","priority":1840,"external_id":null},{"id":10847663003,"name":"University of Tsukuba","priority":1841,"external_id":null},{"id":10847664003,"name":"University of Reading","priority":1842,"external_id":null},{"id":10847665003,"name":"Sciences Po Paris","priority":1843,"external_id":null},{"id":10847666003,"name":"Universidade Estadual de Campinas","priority":1844,"external_id":null},{"id":10847667003,"name":"King Fahd University of Petroleum & Minerals","priority":1845,"external_id":null},{"id":10847668003,"name":"University Complutense Madrid","priority":1846,"external_id":null},{"id":10847669003,"name":"Université Paris-Sorbonne (Paris IV)","priority":1847,"external_id":null},{"id":10847670003,"name":"University of Dundee","priority":1848,"external_id":null},{"id":10847671003,"name":"Université Joseph Fourier - Grenoble 1","priority":1849,"external_id":null},{"id":10847672003,"name":"Waseda University","priority":1850,"external_id":null},{"id":10847673003,"name":"Indian Institute of Technology Delhi (IITD)","priority":1851,"external_id":null},{"id":10847674003,"name":"Universidad de Chile","priority":1852,"external_id":null},{"id":10847675003,"name":"Université Paris 1 Panthéon-Sorbonne","priority":1853,"external_id":null},{"id":10847676003,"name":"Université de Strasbourg","priority":1854,"external_id":null},{"id":10847677003,"name":"University of Twente","priority":1855,"external_id":null},{"id":10847678003,"name":"University of East Anglia (UEA)","priority":1856,"external_id":null},{"id":10847679003,"name":"National Chiao Tung University","priority":1857,"external_id":null},{"id":10847680003,"name":"Politecnico di Milano","priority":1858,"external_id":null},{"id":10847681003,"name":"Charles University","priority":1859,"external_id":null},{"id":10847682003,"name":"Indian Institute of Technology Bombay (IITB)","priority":1860,"external_id":null},{"id":10847683003,"name":"University of Milano","priority":1861,"external_id":null},{"id":10847684003,"name":"Westfälische Wilhelms-Universität Münster","priority":1862,"external_id":null},{"id":10847685003,"name":"University of Canterbury","priority":1863,"external_id":null},{"id":10847686003,"name":"Chulalongkorn University","priority":1864,"external_id":null},{"id":10847687003,"name":"Saint-Petersburg State University","priority":1865,"external_id":null},{"id":10847688003,"name":"University of Liege","priority":1866,"external_id":null},{"id":10847689003,"name":"Universität zu Köln","priority":1867,"external_id":null},{"id":10847690003,"name":"Loughborough University","priority":1868,"external_id":null},{"id":10847691003,"name":"National Cheng Kung University","priority":1869,"external_id":null},{"id":10847692003,"name":"Universität Stuttgart","priority":1870,"external_id":null},{"id":10847693003,"name":"Hanyang University","priority":1871,"external_id":null},{"id":10847694003,"name":"American University of Beirut (AUB)","priority":1872,"external_id":null},{"id":10847695003,"name":"Norwegian University of Science And Technology","priority":1873,"external_id":null},{"id":10847696003,"name":"Beijing Normal University","priority":1874,"external_id":null},{"id":10847697003,"name":"King Saud University","priority":1875,"external_id":null},{"id":10847698003,"name":"University of Oulu","priority":1876,"external_id":null},{"id":10847699003,"name":"Kyung Hee University","priority":1877,"external_id":null},{"id":10847700003,"name":"University of Strathclyde","priority":1878,"external_id":null},{"id":10847701003,"name":"Universität Ulm","priority":1879,"external_id":null},{"id":10847702003,"name":"University of Pisa","priority":1880,"external_id":null},{"id":10847703003,"name":"Technische Universität Darmstadt","priority":1881,"external_id":null},{"id":10847704003,"name":"Technische Universität Dresden","priority":1882,"external_id":null},{"id":10847705003,"name":"Macquarie University","priority":1883,"external_id":null},{"id":10847706003,"name":"Vienna University of Technology","priority":1884,"external_id":null},{"id":10847707003,"name":"Royal Holloway University of London","priority":1885,"external_id":null},{"id":10847708003,"name":"Victoria University of Wellington","priority":1886,"external_id":null},{"id":10847709003,"name":"University of Padua","priority":1887,"external_id":null},{"id":10847710003,"name":"Universiti Kebangsaan Malaysia (UKM)","priority":1888,"external_id":null},{"id":10847711003,"name":"University of Technology, Sydney","priority":1889,"external_id":null},{"id":10847712003,"name":"Universität Konstanz","priority":1890,"external_id":null},{"id":10847713003,"name":"Universidad de Los Andes Colombia","priority":1891,"external_id":null},{"id":10847714003,"name":"Université Paris Descartes","priority":1892,"external_id":null},{"id":10847715003,"name":"Tokyo Medical and Dental University","priority":1893,"external_id":null},{"id":10847716003,"name":"University of Wollongong","priority":1894,"external_id":null},{"id":10847717003,"name":"Universität Erlangen-Nürnberg","priority":1895,"external_id":null},{"id":10847718003,"name":"Queensland University of Technology","priority":1896,"external_id":null},{"id":10847719003,"name":"Tecnológico de Monterrey (ITESM)","priority":1897,"external_id":null},{"id":10847720003,"name":"Universität Mannheim","priority":1898,"external_id":null},{"id":10847721003,"name":"Universitat Pompeu Fabra","priority":1899,"external_id":null},{"id":10847722003,"name":"Mahidol University","priority":1900,"external_id":null},{"id":10847723003,"name":"Curtin University","priority":1901,"external_id":null},{"id":10847724003,"name":"National University of Ireland, Galway","priority":1902,"external_id":null},{"id":10847725003,"name":"Universidade Federal do Rio de Janeiro","priority":1903,"external_id":null},{"id":10847726003,"name":"University of Surrey","priority":1904,"external_id":null},{"id":10847727003,"name":"Hong Kong Baptist University","priority":1905,"external_id":null},{"id":10847728003,"name":"Umeå University","priority":1906,"external_id":null},{"id":10847729003,"name":"Universität Innsbruck","priority":1907,"external_id":null},{"id":10847730003,"name":"RMIT University","priority":1908,"external_id":null},{"id":10847731003,"name":"University of Eastern Finland","priority":1909,"external_id":null},{"id":10847732003,"name":"Christian-Albrechts-Universität zu Kiel","priority":1910,"external_id":null},{"id":10847733003,"name":"Indian Institute of Technology Kanpur (IITK)","priority":1911,"external_id":null},{"id":10847734003,"name":"National Yang Ming University","priority":1912,"external_id":null},{"id":10847735003,"name":"Johannes Gutenberg Universität Mainz","priority":1913,"external_id":null},{"id":10847736003,"name":"The University of Newcastle","priority":1914,"external_id":null},{"id":10847737003,"name":"Al-Farabi Kazakh National University","priority":1915,"external_id":null},{"id":10847738003,"name":"École des Ponts ParisTech","priority":1916,"external_id":null},{"id":10847739003,"name":"University of Jyväskylä","priority":1917,"external_id":null},{"id":10847740003,"name":"L.N. Gumilyov Eurasian National University","priority":1918,"external_id":null},{"id":10847741003,"name":"Kobe University","priority":1919,"external_id":null},{"id":10847742003,"name":"University of Tromso","priority":1920,"external_id":null},{"id":10847743003,"name":"Hiroshima University","priority":1921,"external_id":null},{"id":10847744003,"name":"Université Bordeaux 1, Sciences Technologies","priority":1922,"external_id":null},{"id":10847745003,"name":"University of Indonesia","priority":1923,"external_id":null},{"id":10847746003,"name":"Universität Leipzig","priority":1924,"external_id":null},{"id":10847747003,"name":"University of Southern Denmark","priority":1925,"external_id":null},{"id":10847748003,"name":"Indian Institute of Technology Madras (IITM)","priority":1926,"external_id":null},{"id":10847749003,"name":"University of The Witwatersrand","priority":1927,"external_id":null},{"id":10847750003,"name":"University of Navarra","priority":1928,"external_id":null},{"id":10847751003,"name":"Universidad Austral - Argentina","priority":1929,"external_id":null},{"id":10847752003,"name":"Universidad Carlos III de Madrid","priority":1930,"external_id":null},{"id":10847753003,"name":"Università¡ degli Studi di Roma - Tor Vergata","priority":1931,"external_id":null},{"id":10847754003,"name":"Pontificia Universidad Católica Argentina Santa María de los Buenos Aires","priority":1932,"external_id":null},{"id":10847755003,"name":"UCA","priority":1933,"external_id":null},{"id":10847756003,"name":"Julius-Maximilians-Universität Würzburg","priority":1934,"external_id":null},{"id":10847757003,"name":"Universidad Nacional de Colombia","priority":1935,"external_id":null},{"id":10847758003,"name":"Laval University","priority":1936,"external_id":null},{"id":10847759003,"name":"Ben Gurion University of The Negev","priority":1937,"external_id":null},{"id":10847760003,"name":"Linköping University","priority":1938,"external_id":null},{"id":10847761003,"name":"Aalborg University","priority":1939,"external_id":null},{"id":10847762003,"name":"Bauman Moscow State Technical University","priority":1940,"external_id":null},{"id":10847763003,"name":"Ecole Normale Supérieure de Cachan","priority":1941,"external_id":null},{"id":10847764003,"name":"SOAS - School of Oriental and African Studies, University of London","priority":1942,"external_id":null},{"id":10847765003,"name":"University of Essex","priority":1943,"external_id":null},{"id":10847766003,"name":"University of Warsaw","priority":1944,"external_id":null},{"id":10847767003,"name":"Griffith University","priority":1945,"external_id":null},{"id":10847768003,"name":"University of South Australia","priority":1946,"external_id":null},{"id":10847769003,"name":"Massey University","priority":1947,"external_id":null},{"id":10847770003,"name":"University of Porto","priority":1948,"external_id":null},{"id":10847771003,"name":"Universitat Politècnica de Catalunya","priority":1949,"external_id":null},{"id":10847772003,"name":"Indian Institute of Technology Kharagpur (IITKGP)","priority":1950,"external_id":null},{"id":10847773003,"name":"City University London","priority":1951,"external_id":null},{"id":10847774003,"name":"Dublin City University","priority":1952,"external_id":null},{"id":10847775003,"name":"Pontificia Universidad Javeriana","priority":1953,"external_id":null},{"id":10847776003,"name":"James Cook University","priority":1954,"external_id":null},{"id":10847777003,"name":"Novosibirsk State University","priority":1955,"external_id":null},{"id":10847778003,"name":"Universidade Nova de Lisboa","priority":1956,"external_id":null},{"id":10847779003,"name":"Université Aix-Marseille","priority":1957,"external_id":null},{"id":10847780003,"name":"Universiti Sains Malaysia (USM)","priority":1958,"external_id":null},{"id":10847781003,"name":"Universiti Teknologi Malaysia (UTM)","priority":1959,"external_id":null},{"id":10847782003,"name":"Université Paris Dauphine","priority":1960,"external_id":null},{"id":10847783003,"name":"University of Coimbra","priority":1961,"external_id":null},{"id":10847784003,"name":"Brunel University","priority":1962,"external_id":null},{"id":10847785003,"name":"King Abdul Aziz University (KAU)","priority":1963,"external_id":null},{"id":10847786003,"name":"Ewha Womans University","priority":1964,"external_id":null},{"id":10847787003,"name":"Nankai University","priority":1965,"external_id":null},{"id":10847788003,"name":"Taipei Medical University","priority":1966,"external_id":null},{"id":10847789003,"name":"Universität Jena","priority":1967,"external_id":null},{"id":10847790003,"name":"Ruhr-Universität Bochum","priority":1968,"external_id":null},{"id":10847791003,"name":"Heriot-Watt University","priority":1969,"external_id":null},{"id":10847792003,"name":"Politecnico di Torino","priority":1970,"external_id":null},{"id":10847793003,"name":"Universität Bremen","priority":1971,"external_id":null},{"id":10847794003,"name":"Xi'an Jiaotong University","priority":1972,"external_id":null},{"id":10847795003,"name":"Birkbeck College, University of London","priority":1973,"external_id":null},{"id":10847796003,"name":"Oxford Brookes University","priority":1974,"external_id":null},{"id":10847797003,"name":"Jagiellonian University","priority":1975,"external_id":null},{"id":10847798003,"name":"University of Tampere","priority":1976,"external_id":null},{"id":10847799003,"name":"University of Florence","priority":1977,"external_id":null},{"id":10847800003,"name":"Deakin University","priority":1978,"external_id":null},{"id":10847801003,"name":"University of the Philippines","priority":1979,"external_id":null},{"id":10847802003,"name":"Universitat Politècnica de València","priority":1980,"external_id":null},{"id":10847803003,"name":"Sun Yat-sen University","priority":1981,"external_id":null},{"id":10847804003,"name":"Université Montpellier 2, Sciences et Techniques du Languedoc","priority":1982,"external_id":null},{"id":10847805003,"name":"Moscow State Institute of International Relations (MGIMO-University)","priority":1983,"external_id":null},{"id":10847806003,"name":"Stellenbosch University","priority":1984,"external_id":null},{"id":10847807003,"name":"Politécnica de Madrid","priority":1985,"external_id":null},{"id":10847808003,"name":"Instituto Tecnológico de Buenos Aires (ITBA)","priority":1986,"external_id":null},{"id":10847809003,"name":"La Trobe University","priority":1987,"external_id":null},{"id":10847810003,"name":"Université Paul Sabatier Toulouse III","priority":1988,"external_id":null},{"id":10847811003,"name":"Karl-Franzens-Universität Graz","priority":1989,"external_id":null},{"id":10847812003,"name":"Universität Düsseldorf","priority":1990,"external_id":null},{"id":10847813003,"name":"University of Naples - Federico Ii","priority":1991,"external_id":null},{"id":10847814003,"name":"Aston University","priority":1992,"external_id":null},{"id":10847815003,"name":"University of Turin","priority":1993,"external_id":null},{"id":10847816003,"name":"Beihang University (former BUAA)","priority":1994,"external_id":null},{"id":10847817003,"name":"Indian Institute of Technology Roorkee (IITR)","priority":1995,"external_id":null},{"id":10847818003,"name":"National Central University","priority":1996,"external_id":null},{"id":10847819003,"name":"Sogang University","priority":1997,"external_id":null},{"id":10847820003,"name":"Universität Regensburg","priority":1998,"external_id":null},{"id":10847821003,"name":"Université Lille 1, Sciences et Technologie","priority":1999,"external_id":null},{"id":10847822003,"name":"University of Tasmania","priority":2000,"external_id":null},{"id":10847823003,"name":"University of Waikato","priority":2001,"external_id":null},{"id":10847824003,"name":"Wuhan University","priority":2002,"external_id":null},{"id":10847825003,"name":"National Taiwan University of Science And Technology","priority":2003,"external_id":null},{"id":10847826003,"name":"Universidade Federal de São Paulo (UNIFESP)","priority":2004,"external_id":null},{"id":10847827003,"name":"Università degli Studi di Pavia","priority":2005,"external_id":null},{"id":10847828003,"name":"Universität Bayreuth","priority":2006,"external_id":null},{"id":10847829003,"name":"Université Claude Bernard Lyon 1","priority":2007,"external_id":null},{"id":10847830003,"name":"Université du Québec","priority":2008,"external_id":null},{"id":10847831003,"name":"Universiti Putra Malaysia (UPM)","priority":2009,"external_id":null},{"id":10847832003,"name":"University of Kent","priority":2010,"external_id":null},{"id":10847833003,"name":"University of St Gallen (HSG)","priority":2011,"external_id":null},{"id":10847834003,"name":"Bond University","priority":2012,"external_id":null},{"id":10847835003,"name":"United Arab Emirates University","priority":2013,"external_id":null},{"id":10847836003,"name":"Universidad de San Andrés","priority":2014,"external_id":null},{"id":10847837003,"name":"Universidad Nacional de La Plata","priority":2015,"external_id":null},{"id":10847838003,"name":"Universität des Saarlandes","priority":2016,"external_id":null},{"id":10847839003,"name":"American University of Sharjah (AUS)","priority":2017,"external_id":null},{"id":10847840003,"name":"Bilkent University","priority":2018,"external_id":null},{"id":10847841003,"name":"Flinders University","priority":2019,"external_id":null},{"id":10847842003,"name":"Hankuk (Korea) University of Foreign Studies","priority":2020,"external_id":null},{"id":10847843003,"name":"Middle East Technical University","priority":2021,"external_id":null},{"id":10847844003,"name":"Philipps-Universität Marburg","priority":2022,"external_id":null},{"id":10847845003,"name":"Swansea University","priority":2023,"external_id":null},{"id":10847846003,"name":"Tampere University of Technology","priority":2024,"external_id":null},{"id":10847847003,"name":"Universität Bielefeld","priority":2025,"external_id":null},{"id":10847848003,"name":"University of Manitoba","priority":2026,"external_id":null},{"id":10847849003,"name":"Chiba University","priority":2027,"external_id":null},{"id":10847850003,"name":"Moscow Institute of Physics and Technology State University","priority":2028,"external_id":null},{"id":10847851003,"name":"Tallinn University of Technology","priority":2029,"external_id":null},{"id":10847852003,"name":"Taras Shevchenko National University of Kyiv","priority":2030,"external_id":null},{"id":10847853003,"name":"Tokyo University of Science","priority":2031,"external_id":null},{"id":10847854003,"name":"University of Salamanca","priority":2032,"external_id":null},{"id":10847855003,"name":"University of Trento","priority":2033,"external_id":null},{"id":10847856003,"name":"Université de Sherbrooke","priority":2034,"external_id":null},{"id":10847857003,"name":"Université Panthéon-Assas (Paris 2)","priority":2035,"external_id":null},{"id":10847858003,"name":"University of Delhi","priority":2036,"external_id":null},{"id":10847859003,"name":"Abo Akademi University","priority":2037,"external_id":null},{"id":10847860003,"name":"Czech Technical University In Prague","priority":2038,"external_id":null},{"id":10847861003,"name":"Leibniz Universität Hannover","priority":2039,"external_id":null},{"id":10847862003,"name":"Pusan National University","priority":2040,"external_id":null},{"id":10847863003,"name":"Shanghai University","priority":2041,"external_id":null},{"id":10847864003,"name":"St. Petersburg State Politechnical University","priority":2042,"external_id":null},{"id":10847865003,"name":"Università Cattolica del Sacro Cuore","priority":2043,"external_id":null},{"id":10847866003,"name":"University of Genoa","priority":2044,"external_id":null},{"id":10847867003,"name":"Bandung Institute of Technology (ITB)","priority":2045,"external_id":null},{"id":10847868003,"name":"Bogazici University","priority":2046,"external_id":null},{"id":10847869003,"name":"Goldsmiths, University of London","priority":2047,"external_id":null},{"id":10847870003,"name":"National Sun Yat-sen University","priority":2048,"external_id":null},{"id":10847871003,"name":"Renmin (People’s) University of China","priority":2049,"external_id":null},{"id":10847872003,"name":"Universidad de Costa Rica","priority":2050,"external_id":null},{"id":10847873003,"name":"Universidad de Santiago de Chile - USACH","priority":2051,"external_id":null},{"id":10847874003,"name":"University of Tartu","priority":2052,"external_id":null},{"id":10847875003,"name":"Aristotle University of Thessaloniki","priority":2053,"external_id":null},{"id":10847876003,"name":"Auckland University of Technology","priority":2054,"external_id":null},{"id":10847877003,"name":"Bangor University","priority":2055,"external_id":null},{"id":10847878003,"name":"Charles Darwin University","priority":2056,"external_id":null},{"id":10847879003,"name":"Kingston University, London","priority":2057,"external_id":null},{"id":10847880003,"name":"Universitat de Valencia","priority":2058,"external_id":null},{"id":10847881003,"name":"Université Montpellier 1","priority":2059,"external_id":null},{"id":10847882003,"name":"University of Pretoria","priority":2060,"external_id":null},{"id":10847883003,"name":"Lincoln University","priority":2061,"external_id":null},{"id":10847884003,"name":"National Taiwan Normal University","priority":2062,"external_id":null},{"id":10847885003,"name":"National University of Sciences And Technology (NUST) Islamabad","priority":2063,"external_id":null},{"id":10847886003,"name":"Swinburne University of Technology","priority":2064,"external_id":null},{"id":10847887003,"name":"Tongji University","priority":2065,"external_id":null},{"id":10847888003,"name":"Universidad de Zaragoza","priority":2066,"external_id":null},{"id":10847889003,"name":"Universidade Federal de Minas Gerais","priority":2067,"external_id":null},{"id":10847890003,"name":"Universität Duisburg-Essen","priority":2068,"external_id":null},{"id":10847891003,"name":"Al-Imam Mohamed Ibn Saud Islamic University","priority":2069,"external_id":null},{"id":10847892003,"name":"Harbin Institute of Technology","priority":2070,"external_id":null},{"id":10847893003,"name":"People's Friendship University of Russia","priority":2071,"external_id":null},{"id":10847894003,"name":"Universidade Estadual PaulistaJúlio de Mesquita Filho' (UNESP)","priority":2072,"external_id":null},{"id":10847895003,"name":"Université Nice Sophia-Antipolis","priority":2073,"external_id":null},{"id":10847896003,"name":"University of Crete","priority":2074,"external_id":null},{"id":10847897003,"name":"University of Milano-Bicocca","priority":2075,"external_id":null},{"id":10847898003,"name":"Ateneo de Manila University","priority":2076,"external_id":null},{"id":10847899003,"name":"Beijing Institute of Technology","priority":2077,"external_id":null},{"id":10847900003,"name":"Chang Gung University","priority":2078,"external_id":null},{"id":10847901003,"name":"hung-Ang University","priority":2079,"external_id":null},{"id":10847902003,"name":"Dublin Institute of Technology","priority":2080,"external_id":null},{"id":10847903003,"name":"Huazhong University of Science and Technology","priority":2081,"external_id":null},{"id":10847904003,"name":"International Islamic University Malaysia (IIUM)","priority":2082,"external_id":null},{"id":10847905003,"name":"Johannes Kepler University Linz","priority":2083,"external_id":null},{"id":10847906003,"name":"Justus-Liebig-Universität Gießen","priority":2084,"external_id":null},{"id":10847907003,"name":"Kanazawa University","priority":2085,"external_id":null},{"id":10847908003,"name":"Keele University","priority":2086,"external_id":null},{"id":10847909003,"name":"Koc University","priority":2087,"external_id":null},{"id":10847910003,"name":"National and Kapodistrian University of Athens","priority":2088,"external_id":null},{"id":10847911003,"name":"National Research University – Higher School of Economics (HSE)","priority":2089,"external_id":null},{"id":10847912003,"name":"National Technical University of Athens","priority":2090,"external_id":null},{"id":10847913003,"name":"Okayama University","priority":2091,"external_id":null},{"id":10847914003,"name":"Sabanci University","priority":2092,"external_id":null},{"id":10847915003,"name":"Southeast University","priority":2093,"external_id":null},{"id":10847916003,"name":"Sultan Qaboos University","priority":2094,"external_id":null},{"id":10847917003,"name":"Technische Universität Braunschweig","priority":2095,"external_id":null},{"id":10847918003,"name":"Technische Universität Dortmund","priority":2096,"external_id":null},{"id":10847919003,"name":"The Catholic University of Korea","priority":2097,"external_id":null},{"id":10847920003,"name":"Tianjin University","priority":2098,"external_id":null},{"id":10847921003,"name":"Tokyo Metropolitan University","priority":2099,"external_id":null},{"id":10847922003,"name":"Universidad de Antioquia","priority":2100,"external_id":null},{"id":10847923003,"name":"University of Granada","priority":2101,"external_id":null},{"id":10847924003,"name":"Universidad de Palermo","priority":2102,"external_id":null},{"id":10847925003,"name":"Universidad Nacional de Córdoba","priority":2103,"external_id":null},{"id":10847926003,"name":"Universidade de Santiago de Compostela","priority":2104,"external_id":null},{"id":10847927003,"name":"Universidade Federal do Rio Grande Do Sul","priority":2105,"external_id":null},{"id":10847928003,"name":"University of Siena","priority":2106,"external_id":null},{"id":10847929003,"name":"University of Trieste","priority":2107,"external_id":null},{"id":10847930003,"name":"Universitas Gadjah Mada","priority":2108,"external_id":null},{"id":10847931003,"name":"Université de Lorraine","priority":2109,"external_id":null},{"id":10847932003,"name":"Université de Rennes 1","priority":2110,"external_id":null},{"id":10847933003,"name":"University of Bradford","priority":2111,"external_id":null},{"id":10847934003,"name":"University of Hull","priority":2112,"external_id":null},{"id":10847935003,"name":"University of Kwazulu-Natal","priority":2113,"external_id":null},{"id":10847936003,"name":"University of Limerick","priority":2114,"external_id":null},{"id":10847937003,"name":"University of Stirling","priority":2115,"external_id":null},{"id":10847938003,"name":"University of Szeged","priority":2116,"external_id":null},{"id":10847939003,"name":"Ural Federal University","priority":2117,"external_id":null},{"id":10847940003,"name":"Xiamen University","priority":2118,"external_id":null},{"id":10847941003,"name":"Yokohama City University","priority":2119,"external_id":null},{"id":10847942003,"name":"Aberystwyth University","priority":2120,"external_id":null},{"id":10847943003,"name":"Belarus State University","priority":2121,"external_id":null},{"id":10847944003,"name":"Cairo University","priority":2122,"external_id":null},{"id":10847945003,"name":"Chiang Mai University","priority":2123,"external_id":null},{"id":10847946003,"name":"Chonbuk National University","priority":2124,"external_id":null},{"id":10847947003,"name":"Eötvös Loránd University","priority":2125,"external_id":null},{"id":10847948003,"name":"Inha University","priority":2126,"external_id":null},{"id":10847949003,"name":"Instituto Politécnico Nacional (IPN)","priority":2127,"external_id":null},{"id":10847950003,"name":"Istanbul Technical University","priority":2128,"external_id":null},{"id":10847951003,"name":"Kumamoto University","priority":2129,"external_id":null},{"id":10847952003,"name":"Kyungpook National University","priority":2130,"external_id":null},{"id":10847953003,"name":"Lingnan University (Hong Kong)","priority":2131,"external_id":null},{"id":10847954003,"name":"Masaryk University","priority":2132,"external_id":null},{"id":10847955003,"name":"Murdoch University","priority":2133,"external_id":null},{"id":10847956003,"name":"Nagasaki University","priority":2134,"external_id":null},{"id":10847957003,"name":"National Chung Hsing University","priority":2135,"external_id":null},{"id":10847958003,"name":"National Taipei University of Technology","priority":2136,"external_id":null},{"id":10847959003,"name":"National University of Ireland Maynooth","priority":2137,"external_id":null},{"id":10847960003,"name":"Osaka City University","priority":2138,"external_id":null},{"id":10847961003,"name":"Pontificia Universidad Católica del Perú","priority":2139,"external_id":null},{"id":10847962003,"name":"Pontificia Universidade Católica de São Paulo (PUC -SP)","priority":2140,"external_id":null},{"id":10847963003,"name":"Pontificia Universidade Católica do Rio de Janeiro (PUC - Rio)","priority":2141,"external_id":null},{"id":10847964003,"name":"Qatar University","priority":2142,"external_id":null},{"id":10847965003,"name":"Rhodes University","priority":2143,"external_id":null},{"id":10847966003,"name":"Tokyo University of Agriculture and Technology","priority":2144,"external_id":null},{"id":10847967003,"name":"Tomsk Polytechnic University","priority":2145,"external_id":null},{"id":10847968003,"name":"Tomsk State University","priority":2146,"external_id":null},{"id":10847969003,"name":"Umm Al-Qura University","priority":2147,"external_id":null},{"id":10847970003,"name":"Universidad Católica Andrés Bello - UCAB","priority":2148,"external_id":null},{"id":10847971003,"name":"Universidad Central de Venezuela - UCV","priority":2149,"external_id":null},{"id":10847972003,"name":"Universidad de Belgrano","priority":2150,"external_id":null},{"id":10847973003,"name":"Universidad de Concepción","priority":2151,"external_id":null},{"id":10847974003,"name":"Universidad de Sevilla","priority":2152,"external_id":null},{"id":10847975003,"name":"Universidade Catolica Portuguesa, Lisboa","priority":2153,"external_id":null},{"id":10847976003,"name":"Universidade de Brasilia (UnB)","priority":2154,"external_id":null},{"id":10847977003,"name":"University of Lisbon","priority":2155,"external_id":null},{"id":10847978003,"name":"University of Ljubljana","priority":2156,"external_id":null},{"id":10847979003,"name":"University of Seoul","priority":2157,"external_id":null},{"id":10847980003,"name":"Abu Dhabi University","priority":2158,"external_id":null},{"id":10847981003,"name":"Ain Shams University","priority":2159,"external_id":null},{"id":10847982003,"name":"Ajou University","priority":2160,"external_id":null},{"id":10847983003,"name":"De La Salle University","priority":2161,"external_id":null},{"id":10847984003,"name":"Dongguk University","priority":2162,"external_id":null},{"id":10847985003,"name":"Gifu University","priority":2163,"external_id":null},{"id":10847986003,"name":"Hacettepe University","priority":2164,"external_id":null},{"id":10847987003,"name":"Indian Institute of Technology Guwahati (IITG)","priority":2165,"external_id":null},{"id":10847988003,"name":"Jilin University","priority":2166,"external_id":null},{"id":10847989003,"name":"Kazan Federal University","priority":2167,"external_id":null},{"id":10847990003,"name":"King Khalid University","priority":2168,"external_id":null},{"id":10847991003,"name":"Martin-Luther-Universität Halle-Wittenberg","priority":2169,"external_id":null},{"id":10847992003,"name":"National Chengchi University","priority":2170,"external_id":null},{"id":10847993003,"name":"National Technical University of UkraineKyiv Polytechnic Institute'","priority":2171,"external_id":null},{"id":10847994003,"name":"Niigata University","priority":2172,"external_id":null},{"id":10847995003,"name":"Osaka Prefecture University","priority":2173,"external_id":null},{"id":10847996003,"name":"Paris Lodron University of Salzburg","priority":2174,"external_id":null},{"id":10847997003,"name":"Sharif University of Technology","priority":2175,"external_id":null},{"id":10847998003,"name":"Southern Federal University","priority":2176,"external_id":null},{"id":10847999003,"name":"Thammasat University","priority":2177,"external_id":null},{"id":10848000003,"name":"Universidad de Guadalajara (UDG)","priority":2178,"external_id":null},{"id":10848001003,"name":"Universidad de la República (UdelaR)","priority":2179,"external_id":null},{"id":10848002003,"name":"Universidad Iberoamericana (UIA)","priority":2180,"external_id":null},{"id":10848003003,"name":"Universidad Torcuato Di Tella","priority":2181,"external_id":null},{"id":10848004003,"name":"Universidade Federal da Bahia","priority":2182,"external_id":null},{"id":10848005003,"name":"Universidade Federal de São Carlos","priority":2183,"external_id":null},{"id":10848006003,"name":"Universidade Federal de Viçosa","priority":2184,"external_id":null},{"id":10848007003,"name":"Perugia University","priority":2185,"external_id":null},{"id":10848008003,"name":"Université de Nantes","priority":2186,"external_id":null},{"id":10848009003,"name":"Université Saint-Joseph de Beyrouth","priority":2187,"external_id":null},{"id":10848010003,"name":"University of Canberra","priority":2188,"external_id":null},{"id":10848011003,"name":"University of Debrecen","priority":2189,"external_id":null},{"id":10848012003,"name":"University of Johannesburg","priority":2190,"external_id":null},{"id":10848013003,"name":"University of Mumbai","priority":2191,"external_id":null},{"id":10848014003,"name":"University of Patras","priority":2192,"external_id":null},{"id":10848015003,"name":"University of Tehran","priority":2193,"external_id":null},{"id":10848016003,"name":"University of Ulsan","priority":2194,"external_id":null},{"id":10848017003,"name":"University of Ulster","priority":2195,"external_id":null},{"id":10848018003,"name":"University of Zagreb","priority":2196,"external_id":null},{"id":10848019003,"name":"Vilnius University","priority":2197,"external_id":null},{"id":10848020003,"name":"Warsaw University of Technology","priority":2198,"external_id":null},{"id":10848021003,"name":"Al Azhar University","priority":2199,"external_id":null},{"id":10848022003,"name":"Bar-Ilan University","priority":2200,"external_id":null},{"id":10848023003,"name":"Brno University of Technology","priority":2201,"external_id":null},{"id":10848024003,"name":"Chonnam National University","priority":2202,"external_id":null},{"id":10848025003,"name":"Chungnam National University","priority":2203,"external_id":null},{"id":10848026003,"name":"Corvinus University of Budapest","priority":2204,"external_id":null},{"id":10848027003,"name":"Gunma University","priority":2205,"external_id":null},{"id":10848028003,"name":"Hallym University","priority":2206,"external_id":null},{"id":10848029003,"name":"Instituto Tecnológico Autonomo de México (ITAM)","priority":2207,"external_id":null},{"id":10848030003,"name":"Istanbul University","priority":2208,"external_id":null},{"id":10848031003,"name":"Jordan University of Science & Technology","priority":2209,"external_id":null},{"id":10848032003,"name":"Kasetsart University","priority":2210,"external_id":null},{"id":10848033003,"name":"Kazakh-British Technical University","priority":2211,"external_id":null},{"id":10848034003,"name":"Khazar University","priority":2212,"external_id":null},{"id":10848035003,"name":"London Metropolitan University","priority":2213,"external_id":null},{"id":10848036003,"name":"Middlesex University","priority":2214,"external_id":null},{"id":10848037003,"name":"Universidad Industrial de Santander","priority":2215,"external_id":null},{"id":10848038003,"name":"Pontificia Universidad Católica de Valparaíso","priority":2216,"external_id":null},{"id":10848039003,"name":"Pontificia Universidade Católica do Rio Grande do Sul","priority":2217,"external_id":null},{"id":10848040003,"name":"Qafqaz University","priority":2218,"external_id":null},{"id":10848041003,"name":"Ritsumeikan University","priority":2219,"external_id":null},{"id":10848042003,"name":"Shandong University","priority":2220,"external_id":null},{"id":10848043003,"name":"University of St. Kliment Ohridski","priority":2221,"external_id":null},{"id":10848044003,"name":"South Kazakhstan State University (SKSU)","priority":2222,"external_id":null},{"id":10848045003,"name":"Universidad Adolfo Ibáñez","priority":2223,"external_id":null},{"id":10848046003,"name":"Universidad Autónoma del Estado de México","priority":2224,"external_id":null},{"id":10848047003,"name":"Universidad Autónoma Metropolitana (UAM)","priority":2225,"external_id":null},{"id":10848048003,"name":"Universidad de Alcalá","priority":2226,"external_id":null},{"id":10848049003,"name":"Universidad Nacional Costa Rica","priority":2227,"external_id":null},{"id":10848050003,"name":"Universidad Nacional de Mar del Plata","priority":2228,"external_id":null},{"id":10848051003,"name":"Universidad Peruana Cayetano Heredia","priority":2229,"external_id":null},{"id":10848052003,"name":"Universidad Simón Bolívar Venezuela","priority":2230,"external_id":null},{"id":10848053003,"name":"Universidade Federal de Santa Catarina","priority":2231,"external_id":null},{"id":10848054003,"name":"Universidade Federal do Paraná (UFPR)","priority":2232,"external_id":null},{"id":10848055003,"name":"Universidade Federal Fluminense","priority":2233,"external_id":null},{"id":10848056003,"name":"University of Modena","priority":2234,"external_id":null},{"id":10848057003,"name":"Université Lumière Lyon 2","priority":2235,"external_id":null},{"id":10848058003,"name":"Université Toulouse 1, Capitole","priority":2236,"external_id":null},{"id":10848059003,"name":"University of Economics Prague","priority":2237,"external_id":null},{"id":10848060003,"name":"University of Hertfordshire","priority":2238,"external_id":null},{"id":10848061003,"name":"University of Plymouth","priority":2239,"external_id":null},{"id":10848062003,"name":"University of Salford","priority":2240,"external_id":null},{"id":10848063003,"name":"University of Science and Technology Beijing","priority":2241,"external_id":null},{"id":10848064003,"name":"University of Western Sydney","priority":2242,"external_id":null},{"id":10848065003,"name":"Yamaguchi University","priority":2243,"external_id":null},{"id":10848066003,"name":"Yokohama National University","priority":2244,"external_id":null},{"id":10848067003,"name":"Airlangga University","priority":2245,"external_id":null},{"id":10848068003,"name":"Alexandria University","priority":2246,"external_id":null},{"id":10848069003,"name":"Alexandru Ioan Cuza University","priority":2247,"external_id":null},{"id":10848070003,"name":"Alpen-Adria-Universität Klagenfurt","priority":2248,"external_id":null},{"id":10848071003,"name":"Aoyama Gakuin University","priority":2249,"external_id":null},{"id":10848072003,"name":"Athens University of Economy And Business","priority":2250,"external_id":null},{"id":10848073003,"name":"Babes-Bolyai University","priority":2251,"external_id":null},{"id":10848074003,"name":"Baku State University","priority":2252,"external_id":null},{"id":10848075003,"name":"Belarusian National Technical University","priority":2253,"external_id":null},{"id":10848076003,"name":"Benemérita Universidad Autónoma de Puebla","priority":2254,"external_id":null},{"id":10848077003,"name":"Bogor Agricultural University","priority":2255,"external_id":null},{"id":10848078003,"name":"Coventry University","priority":2256,"external_id":null},{"id":10848079003,"name":"Cukurova University","priority":2257,"external_id":null},{"id":10848080003,"name":"Diponegoro University","priority":2258,"external_id":null},{"id":10848081003,"name":"Donetsk National University","priority":2259,"external_id":null},{"id":10848082003,"name":"Doshisha University","priority":2260,"external_id":null},{"id":10848083003,"name":"E.A.Buketov Karaganda State University","priority":2261,"external_id":null},{"id":10848084003,"name":"Far Eastern Federal University","priority":2262,"external_id":null},{"id":10848085003,"name":"Fu Jen Catholic University","priority":2263,"external_id":null},{"id":10848086003,"name":"Kagoshima University","priority":2264,"external_id":null},{"id":10848087003,"name":"Kaunas University of Technology","priority":2265,"external_id":null},{"id":10848088003,"name":"Kazakh Ablai khan University of International Relations and World Languages","priority":2266,"external_id":null},{"id":10848089003,"name":"Kazakh National Pedagogical University Abai","priority":2267,"external_id":null},{"id":10848090003,"name":"Kazakh National Technical University","priority":2268,"external_id":null},{"id":10848091003,"name":"Khon Kaen University","priority":2269,"external_id":null},{"id":10848092003,"name":"King Faisal University","priority":2270,"external_id":null},{"id":10848093003,"name":"King Mongkut''s University of Technology Thonburi","priority":2271,"external_id":null},{"id":10848094003,"name":"Kuwait University","priority":2272,"external_id":null},{"id":10848095003,"name":"Lodz University","priority":2273,"external_id":null},{"id":10848096003,"name":"Manchester Metropolitan University","priority":2274,"external_id":null},{"id":10848097003,"name":"Lobachevsky State University of Nizhni Novgorod","priority":2275,"external_id":null},{"id":10848098003,"name":"National Technical UniversityKharkiv Polytechnic Institute'","priority":2276,"external_id":null},{"id":10848099003,"name":"Nicolaus Copernicus University","priority":2277,"external_id":null},{"id":10848100003,"name":"Northumbria University at Newcastle","priority":2278,"external_id":null},{"id":10848101003,"name":"Nottingham Trent University","priority":2279,"external_id":null},{"id":10848102003,"name":"Ochanomizu University","priority":2280,"external_id":null},{"id":10848103003,"name":"Plekhanov Russian University of Economics","priority":2281,"external_id":null},{"id":10848104003,"name":"Pontificia Universidad Catolica del Ecuador","priority":2282,"external_id":null},{"id":10848105003,"name":"Prince of Songkla University","priority":2283,"external_id":null},{"id":10848106003,"name":"S.Seifullin Kazakh Agro Technical University","priority":2284,"external_id":null},{"id":10848107003,"name":"Saitama University","priority":2285,"external_id":null},{"id":10848108003,"name":"Sepuluh Nopember Institute of Technology","priority":2286,"external_id":null},{"id":10848109003,"name":"Shinshu University","priority":2287,"external_id":null},{"id":10848110003,"name":"The Robert Gordon University","priority":2288,"external_id":null},{"id":10848111003,"name":"Tokai University","priority":2289,"external_id":null},{"id":10848112003,"name":"Universidad ANAHUAC","priority":2290,"external_id":null},{"id":10848113003,"name":"Universidad Austral de Chile","priority":2291,"external_id":null},{"id":10848114003,"name":"University Autónoma de Nuevo León (UANL)","priority":2292,"external_id":null},{"id":10848115003,"name":"Universidad de la Habana","priority":2293,"external_id":null},{"id":10848116003,"name":"Universidad de La Sabana","priority":2294,"external_id":null},{"id":10848117003,"name":"Universidad de las Américas Puebla (UDLAP)","priority":2295,"external_id":null},{"id":10848118003,"name":"Universidad de los Andes Mérida","priority":2296,"external_id":null},{"id":10848119003,"name":"University of Murcia","priority":2297,"external_id":null},{"id":10848120003,"name":"Universidad de Puerto Rico","priority":2298,"external_id":null},{"id":10848121003,"name":"Universidad de San Francisco de Quito","priority":2299,"external_id":null},{"id":10848122003,"name":"Universidad de Talca","priority":2300,"external_id":null},{"id":10848123003,"name":"Universidad del Norte","priority":2301,"external_id":null},{"id":10848124003,"name":"Universidad del Rosario","priority":2302,"external_id":null},{"id":10848125003,"name":"Universidad del Valle","priority":2303,"external_id":null},{"id":10848126003,"name":"Universidad Nacional de Cuyo","priority":2304,"external_id":null},{"id":10848127003,"name":"Universidad Nacional de Rosario","priority":2305,"external_id":null},{"id":10848128003,"name":"Universidad Nacional de Tucumán","priority":2306,"external_id":null},{"id":10848129003,"name":"Universidad Nacional del Sur","priority":2307,"external_id":null},{"id":10848130003,"name":"Universidad Nacional Mayor de San Marcos","priority":2308,"external_id":null},{"id":10848131003,"name":"Universidad Técnica Federico Santa María","priority":2309,"external_id":null},{"id":10848132003,"name":"Universidad Tecnológica Nacional (UTN)","priority":2310,"external_id":null},{"id":10848133003,"name":"Universidade do Estado do Rio de Janeiro (UERJ)","priority":2311,"external_id":null},{"id":10848134003,"name":"Universidade Estadual de Londrina (UEL)","priority":2312,"external_id":null},{"id":10848135003,"name":"Universidade Federal de Santa Maria","priority":2313,"external_id":null},{"id":10848136003,"name":"Universidade Federal do Ceará (UFC)","priority":2314,"external_id":null},{"id":10848137003,"name":"Universidade Federal do Pernambuco","priority":2315,"external_id":null},{"id":10848138003,"name":"Università Ca'' Foscari Venezia","priority":2316,"external_id":null},{"id":10848139003,"name":"Catania University","priority":2317,"external_id":null},{"id":10848140003,"name":"Università degli Studi Roma Tre","priority":2318,"external_id":null},{"id":10848141003,"name":"Université Charles-de-Gaulle Lille 3","priority":2319,"external_id":null},{"id":10848142003,"name":"Université de Caen Basse-Normandie","priority":2320,"external_id":null},{"id":10848143003,"name":"Université de Cergy-Pontoise","priority":2321,"external_id":null},{"id":10848144003,"name":"Université de Poitiers","priority":2322,"external_id":null},{"id":10848145003,"name":"Université Jean Moulin Lyon 3","priority":2323,"external_id":null},{"id":10848146003,"name":"Université Lille 2 Droit et Santé","priority":2324,"external_id":null},{"id":10848147003,"name":"Université Paris Ouest Nanterre La Défense","priority":2325,"external_id":null},{"id":10848148003,"name":"Université Paul-Valéry Montpellier 3","priority":2326,"external_id":null},{"id":10848149003,"name":"Université Pierre Mendès France - Grenoble 2","priority":2327,"external_id":null},{"id":10848150003,"name":"Université Stendhal Grenoble 3","priority":2328,"external_id":null},{"id":10848151003,"name":"Université Toulouse II, Le Mirail","priority":2329,"external_id":null},{"id":10848152003,"name":"Universiti Teknologi MARA - UiTM","priority":2330,"external_id":null},{"id":10848153003,"name":"University of Baghdad","priority":2331,"external_id":null},{"id":10848154003,"name":"University of Bahrain","priority":2332,"external_id":null},{"id":10848155003,"name":"University of Bari","priority":2333,"external_id":null},{"id":10848156003,"name":"University of Belgrade","priority":2334,"external_id":null},{"id":10848157003,"name":"University of Brawijaya","priority":2335,"external_id":null},{"id":10848158003,"name":"University of Brescia","priority":2336,"external_id":null},{"id":10848159003,"name":"University of Bucharest","priority":2337,"external_id":null},{"id":10848160003,"name":"University of Calcutta","priority":2338,"external_id":null},{"id":10848161003,"name":"University of Central Lancashire","priority":2339,"external_id":null},{"id":10848162003,"name":"University of Colombo","priority":2340,"external_id":null},{"id":10848163003,"name":"University of Dhaka","priority":2341,"external_id":null},{"id":10848164003,"name":"University of East London","priority":2342,"external_id":null},{"id":10848165003,"name":"University of Engineering & Technology (UET) Lahore","priority":2343,"external_id":null},{"id":10848166003,"name":"University of Greenwich","priority":2344,"external_id":null},{"id":10848167003,"name":"University of Jordan","priority":2345,"external_id":null},{"id":10848168003,"name":"University of Karachi","priority":2346,"external_id":null},{"id":10848169003,"name":"University of Lahore","priority":2347,"external_id":null},{"id":10848170003,"name":"University of Latvia","priority":2348,"external_id":null},{"id":10848171003,"name":"University of New England","priority":2349,"external_id":null},{"id":10848172003,"name":"University of Pune","priority":2350,"external_id":null},{"id":10848173003,"name":"University of Santo Tomas","priority":2351,"external_id":null},{"id":10848174003,"name":"University of Southern Queensland","priority":2352,"external_id":null},{"id":10848175003,"name":"University of Wroclaw","priority":2353,"external_id":null},{"id":10848176003,"name":"Verona University","priority":2354,"external_id":null},{"id":10848177003,"name":"Victoria University","priority":2355,"external_id":null},{"id":10848178003,"name":"Vilnius Gediminas Technical University","priority":2356,"external_id":null},{"id":10848179003,"name":"Voronezh State University","priority":2357,"external_id":null},{"id":10848180003,"name":"Vytautas Magnus University","priority":2358,"external_id":null},{"id":10848181003,"name":"West University of Timisoara","priority":2359,"external_id":null},{"id":10848182003,"name":"University of South Alabama","priority":2360,"external_id":null},{"id":10848183003,"name":"University of Arkansas","priority":2361,"external_id":null},{"id":10848184003,"name":"University of California - Berkeley","priority":2362,"external_id":null},{"id":10848185003,"name":"University of Connecticut","priority":2363,"external_id":null},{"id":10848186003,"name":"University of South Florida","priority":2364,"external_id":null},{"id":10848187003,"name":"University of Georgia","priority":2365,"external_id":null},{"id":10848188003,"name":"University of Hawaii - Manoa","priority":2366,"external_id":null},{"id":10848189003,"name":"Iowa State University","priority":2367,"external_id":null},{"id":10848190003,"name":"Murray State University","priority":2368,"external_id":null},{"id":10848191003,"name":"University of Louisville","priority":2369,"external_id":null},{"id":10848192003,"name":"Western Kentucky University","priority":2370,"external_id":null},{"id":10848193003,"name":"Louisiana State University - Baton Rouge","priority":2371,"external_id":null},{"id":10848194003,"name":"University of Maryland - College Park","priority":2372,"external_id":null},{"id":10848195003,"name":"University of Minnesota - Twin Cities","priority":2373,"external_id":null},{"id":10848196003,"name":"University of Montana","priority":2374,"external_id":null},{"id":10848197003,"name":"East Carolina University","priority":2375,"external_id":null},{"id":10848198003,"name":"University of North Carolina - Chapel Hill","priority":2376,"external_id":null},{"id":10848199003,"name":"Wake Forest University","priority":2377,"external_id":null},{"id":10848200003,"name":"University of Nebraska - Lincoln","priority":2378,"external_id":null},{"id":10848201003,"name":"New Mexico State University","priority":2379,"external_id":null},{"id":10848202003,"name":"Ohio State University - Columbus","priority":2380,"external_id":null},{"id":10848203003,"name":"University of Oklahoma","priority":2381,"external_id":null},{"id":10848204003,"name":"Pennsylvania State University - University Park","priority":2382,"external_id":null},{"id":10848205003,"name":"University of Pittsburgh","priority":2383,"external_id":null},{"id":10848206003,"name":"University of Tennessee - Chattanooga","priority":2384,"external_id":null},{"id":10848207003,"name":"Vanderbilt University","priority":2385,"external_id":null},{"id":10848208003,"name":"Rice University","priority":2386,"external_id":null},{"id":10848209003,"name":"University of Utah","priority":2387,"external_id":null},{"id":10848210003,"name":"University of Richmond","priority":2388,"external_id":null},{"id":10848211003,"name":"University of Arkansas - Pine Bluff","priority":2389,"external_id":null},{"id":10848212003,"name":"University of Central Florida","priority":2390,"external_id":null},{"id":10848213003,"name":"Florida Atlantic University","priority":2391,"external_id":null},{"id":10848214003,"name":"Hampton University","priority":2392,"external_id":null},{"id":10848215003,"name":"Liberty University","priority":2393,"external_id":null},{"id":10848216003,"name":"Mercer University","priority":2394,"external_id":null},{"id":10848217003,"name":"Middle Tennessee State University","priority":2395,"external_id":null},{"id":10848218003,"name":"University of Nevada - Las Vegas","priority":2396,"external_id":null},{"id":10848219003,"name":"South Carolina State University","priority":2397,"external_id":null},{"id":10848220003,"name":"University of Tennessee - Martin","priority":2398,"external_id":null},{"id":10848221003,"name":"Weber State University","priority":2399,"external_id":null},{"id":10848222003,"name":"Youngstown State University","priority":2400,"external_id":null},{"id":10848223003,"name":"University of the Incarnate Word","priority":2401,"external_id":null},{"id":10848224003,"name":"University of Washington","priority":2402,"external_id":null},{"id":10848225003,"name":"University of Louisiana - Lafayette","priority":2403,"external_id":null},{"id":10848226003,"name":"Coastal Carolina University","priority":2404,"external_id":null},{"id":10848227003,"name":"Utah State University","priority":2405,"external_id":null},{"id":10848228003,"name":"University of Alabama","priority":2406,"external_id":null},{"id":10848229003,"name":"University of Illinois - Urbana-Champaign","priority":2407,"external_id":null},{"id":10848230003,"name":"United States Air Force Academy","priority":2408,"external_id":null},{"id":10848231003,"name":"University of Akron","priority":2409,"external_id":null},{"id":10848232003,"name":"University of Central Arkansas","priority":2410,"external_id":null},{"id":10848233003,"name":"University of Kansas","priority":2411,"external_id":null},{"id":10848234003,"name":"University of Northern Colorado","priority":2412,"external_id":null},{"id":10848235003,"name":"University of Northern Iowa","priority":2413,"external_id":null},{"id":10848236003,"name":"University of South Carolina","priority":2414,"external_id":null},{"id":10848237003,"name":"Tennessee Technological University","priority":2415,"external_id":null},{"id":10848238003,"name":"University of Texas - El Paso","priority":2416,"external_id":null},{"id":10848239003,"name":"Texas Tech University","priority":2417,"external_id":null},{"id":10848240003,"name":"Tulane University","priority":2418,"external_id":null},{"id":10848241003,"name":"Virginia Military Institute","priority":2419,"external_id":null},{"id":10848242003,"name":"Western Michigan University","priority":2420,"external_id":null},{"id":10848243003,"name":"Wilfrid Laurier University","priority":2421,"external_id":null},{"id":10848244003,"name":"University of San Diego","priority":2422,"external_id":null},{"id":10848245003,"name":"University of California - San Diego","priority":2423,"external_id":null},{"id":10848246003,"name":"Brooks Institute of Photography","priority":2424,"external_id":null},{"id":10848247003,"name":"Acupuncture and Integrative Medicine College - Berkeley","priority":2425,"external_id":null},{"id":10848248003,"name":"Southern Alberta Institute of Technology","priority":2426,"external_id":null},{"id":10848249003,"name":"Susquehanna University","priority":2427,"external_id":null},{"id":10848250003,"name":"University of Texas - Dallas","priority":2428,"external_id":null},{"id":10848251003,"name":"Thunderbird School of Global Management","priority":2429,"external_id":null},{"id":10848252003,"name":"Presidio Graduate School","priority":2430,"external_id":null},{"id":10848253003,"name":"École supérieure de commerce de Dijon","priority":2431,"external_id":null},{"id":10848254003,"name":"University of California - San Francisco","priority":2432,"external_id":null},{"id":10848255003,"name":"Hack Reactor","priority":2433,"external_id":null},{"id":10848256003,"name":"St. Mary''s College of California","priority":2434,"external_id":null},{"id":10848257003,"name":"New England Law","priority":2435,"external_id":null},{"id":10848258003,"name":"University of California, Merced","priority":2436,"external_id":null},{"id":10848259003,"name":"University of California, Hastings College of the Law","priority":2437,"external_id":null},{"id":10848260003,"name":"V.N. Karazin Kharkiv National University","priority":2438,"external_id":null},{"id":10848261003,"name":"SIM University (UniSIM)","priority":2439,"external_id":null},{"id":10848262003,"name":"Singapore Management University (SMU)","priority":2440,"external_id":null},{"id":10848263003,"name":"Singapore University of Technology and Design (SUTD)","priority":2441,"external_id":null},{"id":10848264003,"name":"Singapore Institute of Technology (SIT)","priority":2442,"external_id":null},{"id":10848265003,"name":"Nanyang Polytechnic (NYP)","priority":2443,"external_id":null},{"id":10848266003,"name":"Ngee Ann Polytechnic (NP)","priority":2444,"external_id":null},{"id":10848267003,"name":"Republic Polytechnic (RP)","priority":2445,"external_id":null},{"id":10848268003,"name":"Singapore Polytechnic (SP)","priority":2446,"external_id":null},{"id":10848269003,"name":"Temasek Polytechnic (TP)","priority":2447,"external_id":null},{"id":10848270003,"name":"INSEAD","priority":2448,"external_id":null},{"id":10848271003,"name":"Fundação Getúlio Vargas","priority":2449,"external_id":null},{"id":10848272003,"name":"Acharya Nagarjuna University","priority":2450,"external_id":null},{"id":10848273003,"name":"University of California - Santa Barbara","priority":2451,"external_id":null},{"id":10848274003,"name":"University of California - Irvine","priority":2452,"external_id":null},{"id":10848275003,"name":"California State University - Long Beach","priority":2453,"external_id":null},{"id":10848276003,"name":"Robert Morris University Illinois","priority":2454,"external_id":null},{"id":10848277003,"name":"Harold Washington College - City Colleges of Chicago","priority":2455,"external_id":null},{"id":10848278003,"name":"Harry S Truman College - City Colleges of Chicago","priority":2456,"external_id":null},{"id":10848279003,"name":"Kennedy-King College - City Colleges of Chicago","priority":2457,"external_id":null},{"id":10848280003,"name":"Malcolm X College - City Colleges of Chicago","priority":2458,"external_id":null},{"id":10848281003,"name":"Olive-Harvey College - City Colleges of Chicago","priority":2459,"external_id":null},{"id":10848282003,"name":"Richard J Daley College - City Colleges of Chicago","priority":2460,"external_id":null},{"id":10848283003,"name":"Wilbur Wright College - City Colleges of Chicago","priority":2461,"external_id":null},{"id":10848284003,"name":"Abertay University","priority":2462,"external_id":null},{"id":10848285003,"name":"Pontifícia Universidade Católica de Minas Gerais","priority":2463,"external_id":null},{"id":10848286003,"name":"Other","priority":2464,"external_id":null}]},"emitted_at":1660156526565} {"stream":"custom_fields","data":{"id":4680899003,"name":"Degree","active":true,"field_type":"candidate","priority":1,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"degree","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10848287003,"name":"High School","priority":0,"external_id":null},{"id":10848288003,"name":"Associate's Degree","priority":1,"external_id":null},{"id":10848289003,"name":"Bachelor's Degree","priority":2,"external_id":null},{"id":10848290003,"name":"Master's Degree","priority":3,"external_id":null},{"id":10848291003,"name":"Master of Business Administration (M.B.A.)","priority":4,"external_id":null},{"id":10848292003,"name":"Juris Doctor (J.D.)","priority":5,"external_id":null},{"id":10848293003,"name":"Doctor of Medicine (M.D.)","priority":6,"external_id":null},{"id":10848294003,"name":"Doctor of Philosophy (Ph.D.)","priority":7,"external_id":null},{"id":10848295003,"name":"Engineer's Degree","priority":8,"external_id":null},{"id":10848296003,"name":"Other","priority":9,"external_id":null}]},"emitted_at":1660156526606} {"stream":"custom_fields","data":{"id":4680900003,"name":"Discipline","active":true,"field_type":"candidate","priority":2,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"discipline","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10848297003,"name":"Accounting","priority":0,"external_id":null},{"id":10848298003,"name":"African Studies","priority":1,"external_id":null},{"id":10848299003,"name":"Agriculture","priority":2,"external_id":null},{"id":10848300003,"name":"Anthropology","priority":3,"external_id":null},{"id":10848301003,"name":"Applied Health Services","priority":4,"external_id":null},{"id":10848302003,"name":"Architecture","priority":5,"external_id":null},{"id":10848303003,"name":"Art","priority":6,"external_id":null},{"id":10848304003,"name":"Asian Studies","priority":7,"external_id":null},{"id":10848305003,"name":"Biology","priority":8,"external_id":null},{"id":10848306003,"name":"Business","priority":9,"external_id":null},{"id":10848307003,"name":"Business Administration","priority":10,"external_id":null},{"id":10848308003,"name":"Chemistry","priority":11,"external_id":null},{"id":10848309003,"name":"Classical Languages","priority":12,"external_id":null},{"id":10848310003,"name":"Communications & Film","priority":13,"external_id":null},{"id":10848311003,"name":"Computer Science","priority":14,"external_id":null},{"id":10848312003,"name":"Dentistry","priority":15,"external_id":null},{"id":10848313003,"name":"Developing Nations","priority":16,"external_id":null},{"id":10848314003,"name":"Discipline Unknown","priority":17,"external_id":null},{"id":10848315003,"name":"Earth Sciences","priority":18,"external_id":null},{"id":10848316003,"name":"Economics","priority":19,"external_id":null},{"id":10848317003,"name":"Education","priority":20,"external_id":null},{"id":10848318003,"name":"Electronics","priority":21,"external_id":null},{"id":10848319003,"name":"Engineering","priority":22,"external_id":null},{"id":10848320003,"name":"English Studies","priority":23,"external_id":null},{"id":10848321003,"name":"Environmental Studies","priority":24,"external_id":null},{"id":10848322003,"name":"European Studies","priority":25,"external_id":null},{"id":10848323003,"name":"Fashion","priority":26,"external_id":null},{"id":10848324003,"name":"Finance","priority":27,"external_id":null},{"id":10848325003,"name":"Fine Arts","priority":28,"external_id":null},{"id":10848326003,"name":"General Studies","priority":29,"external_id":null},{"id":10848327003,"name":"Health Services","priority":30,"external_id":null},{"id":10848328003,"name":"History","priority":31,"external_id":null},{"id":10848329003,"name":"Human Resources Management","priority":32,"external_id":null},{"id":10848330003,"name":"Humanities","priority":33,"external_id":null},{"id":10848331003,"name":"Industrial Arts & Carpentry","priority":34,"external_id":null},{"id":10848332003,"name":"Information Systems","priority":35,"external_id":null},{"id":10848333003,"name":"International Relations","priority":36,"external_id":null},{"id":10848334003,"name":"Journalism","priority":37,"external_id":null},{"id":10848335003,"name":"Languages","priority":38,"external_id":null},{"id":10848336003,"name":"Latin American Studies","priority":39,"external_id":null},{"id":10848337003,"name":"Law","priority":40,"external_id":null},{"id":10848338003,"name":"Linguistics","priority":41,"external_id":null},{"id":10848339003,"name":"Manufacturing & Mechanics","priority":42,"external_id":null},{"id":10848340003,"name":"Mathematics","priority":43,"external_id":null},{"id":10848341003,"name":"Medicine","priority":44,"external_id":null},{"id":10848342003,"name":"Middle Eastern Studies","priority":45,"external_id":null},{"id":10848343003,"name":"Naval Science","priority":46,"external_id":null},{"id":10848344003,"name":"North American Studies","priority":47,"external_id":null},{"id":10848345003,"name":"Nuclear Technics","priority":48,"external_id":null},{"id":10848346003,"name":"Operations Research & Strategy","priority":49,"external_id":null},{"id":10848347003,"name":"Organizational Theory","priority":50,"external_id":null},{"id":10848348003,"name":"Philosophy","priority":51,"external_id":null},{"id":10848349003,"name":"Physical Education","priority":52,"external_id":null},{"id":10848350003,"name":"Physical Sciences","priority":53,"external_id":null},{"id":10848351003,"name":"Physics","priority":54,"external_id":null},{"id":10848352003,"name":"Political Science","priority":55,"external_id":null},{"id":10848353003,"name":"Psychology","priority":56,"external_id":null},{"id":10848354003,"name":"Public Policy","priority":57,"external_id":null},{"id":10848355003,"name":"Public Service","priority":58,"external_id":null},{"id":10848356003,"name":"Religious Studies","priority":59,"external_id":null},{"id":10848357003,"name":"Russian & Soviet Studies","priority":60,"external_id":null},{"id":10848358003,"name":"Scandinavian Studies","priority":61,"external_id":null},{"id":10848359003,"name":"Science","priority":62,"external_id":null},{"id":10848360003,"name":"Slavic Studies","priority":63,"external_id":null},{"id":10848361003,"name":"Social Science","priority":64,"external_id":null},{"id":10848362003,"name":"Social Sciences","priority":65,"external_id":null},{"id":10848363003,"name":"Sociology","priority":66,"external_id":null},{"id":10848364003,"name":"Speech","priority":67,"external_id":null},{"id":10848365003,"name":"Statistics & Decision Theory","priority":68,"external_id":null},{"id":10848366003,"name":"Urban Studies","priority":69,"external_id":null},{"id":10848367003,"name":"Veterinary Medicine","priority":70,"external_id":null},{"id":10848368003,"name":"Other","priority":71,"external_id":null}]},"emitted_at":1660156526607} -{"stream":"custom_fields","data":{"id":4680901003,"name":"Employment Type","active":true,"field_type":"job","priority":0,"value_type":"single_select","private":false,"required":false,"require_approval":true,"trigger_new_version":false,"name_key":"employment_type","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10845796003,"name":"Full-time","priority":0,"external_id":null},{"id":10845797003,"name":"Part-time","priority":1,"external_id":null},{"id":10845798003,"name":"Intern","priority":2,"external_id":null},{"id":10845799003,"name":"Contract","priority":3,"external_id":null},{"id":10845800003,"name":"Temporary","priority":4,"external_id":null}]},"emitted_at":1660156526608} -{"stream":"custom_fields","data":{"id":4680902003,"name":"Start Date","active":true,"field_type":"offer","priority":0,"value_type":"date","private":true,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"start_date","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[]},"emitted_at":1660156526608} -{"stream":"custom_fields","data":{"id":4680903003,"name":"Employment Type","active":true,"field_type":"offer","priority":1,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":true,"name_key":"employment_type","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10845801003,"name":"Full-time","priority":0,"external_id":null},{"id":10845802003,"name":"Part-time","priority":1,"external_id":null},{"id":10845803003,"name":"Intern","priority":2,"external_id":null},{"id":10845804003,"name":"Contract","priority":3,"external_id":null},{"id":10845805003,"name":"Temporary","priority":4,"external_id":null}]},"emitted_at":1660156526608} -{"stream":"custom_fields","data":{"id":4680904003,"name":"Offer Documents","active":true,"field_type":"offer","priority":2,"value_type":"short_text","private":true,"required":false,"require_approval":false,"trigger_new_version":true,"name_key":"offer_documents","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[]},"emitted_at":1660156526608} -{"stream":"custom_fields","data":{"id":4680905003,"name":"Relationship","active":true,"field_type":"referral_question","priority":0,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"relationship","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10845806003,"name":"Coworker","priority":0,"external_id":null},{"id":10845807003,"name":"School","priority":1,"external_id":null},{"id":10845808003,"name":"Manager","priority":2,"external_id":null},{"id":10845809003,"name":"Reported","priority":3,"external_id":null},{"id":10845810003,"name":"Friend","priority":4,"external_id":null},{"id":10845811003,"name":"Do not know","priority":5,"external_id":null}]},"emitted_at":1660156526608} -{"stream":"custom_fields","data":{"id":4680906003,"name":"Work History","active":true,"field_type":"referral_question","priority":1,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"work_history","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10845812003,"name":"0-1","priority":0,"external_id":null},{"id":10845813003,"name":"2-5","priority":1,"external_id":null},{"id":10845814003,"name":"5+","priority":2,"external_id":null}]},"emitted_at":1660156526609} -{"stream":"custom_fields","data":{"id":4680907003,"name":"Rating","active":true,"field_type":"referral_question","priority":2,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"rating","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10845815003,"name":"Superstar","priority":0,"external_id":null},{"id":10845816003,"name":"Top 5%","priority":1,"external_id":null},{"id":10845817003,"name":"Top 10%","priority":2,"external_id":null},{"id":10845818003,"name":"Top 25%","priority":3,"external_id":null},{"id":10845819003,"name":"Top 50%","priority":4,"external_id":null}]},"emitted_at":1660156526609} -{"stream":"custom_fields","data":{"id":4680908003,"name":"When we reach out","active":true,"field_type":"referral_question","priority":3,"value_type":"single_select","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"when_we_reach_out","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[{"id":10845820003,"name":"You may mention me","priority":0,"external_id":null},{"id":10845821003,"name":"I wish to remain anonymous","priority":1,"external_id":null}]},"emitted_at":1660156526609} -{"stream":"custom_fields","data":{"id":4680909003,"name":"They know they're being referred","active":true,"field_type":"referral_question","priority":4,"value_type":"yes_no","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"they_know_they_re_being_referred","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[]},"emitted_at":1660156526609} -{"stream":"custom_fields","data":{"id":4680910003,"name":"Referral Notes","active":true,"field_type":"referral_question","priority":5,"value_type":"long_text","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"referral_notes","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[]},"emitted_at":1660156526609} -{"stream":"custom_fields","data":{"id":7431124003,"name":"Test User","active":true,"field_type":"agency_question","priority":0,"value_type":"yes_no","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"test_user","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[]},"emitted_at":1660156526610} -{"stream":"custom_fields","data":{"id":7431125003,"name":"Test User","active":true,"field_type":"agency_question","priority":1,"value_type":"short_text","private":false,"required":true,"require_approval":false,"trigger_new_version":false,"name_key":"test_user_agency_question_1633884465.559642","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[]},"emitted_at":1660156526610} -{"stream":"custom_fields","data":{"id":7431126003,"name":"Test User","active":true,"field_type":"referral_question","priority":6,"value_type":"yes_no","private":false,"required":false,"require_approval":false,"trigger_new_version":false,"name_key":"test_user","description":null,"expose_in_job_board_api":false,"api_only":false,"offices":[],"departments":[],"template_token_string":null,"custom_field_options":[]},"emitted_at":1660156526610} +{"stream": "custom_fields", "data": {"id": 4680901003, "name": "Employment Type", "active": true, "field_type": "job", "priority": 0, "value_type": "single_select", "private": false, "required": false, "require_approval": true, "trigger_new_version": false, "name_key": "employment_type", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845796003, "name": "Full-time", "priority": 0, "external_id": null}, {"id": 10845797003, "name": "Part-time", "priority": 1, "external_id": null}, {"id": 10845798003, "name": "Intern", "priority": 2, "external_id": null}, {"id": 10845799003, "name": "Contract", "priority": 3, "external_id": null}, {"id": 10845800003, "name": "Temporary", "priority": 4, "external_id": null}]}, "emitted_at": 1662401663417} +{"stream": "custom_fields", "data": {"id": 4680902003, "name": "Start Date", "active": true, "field_type": "offer", "priority": 0, "value_type": "date", "private": true, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "start_date", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1662401663418} +{"stream": "custom_fields", "data": {"id": 4680903003, "name": "Employment Type", "active": true, "field_type": "offer", "priority": 1, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": true, "name_key": "employment_type", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845801003, "name": "Full-time", "priority": 0, "external_id": null}, {"id": 10845802003, "name": "Part-time", "priority": 1, "external_id": null}, {"id": 10845803003, "name": "Intern", "priority": 2, "external_id": null}, {"id": 10845804003, "name": "Contract", "priority": 3, "external_id": null}, {"id": 10845805003, "name": "Temporary", "priority": 4, "external_id": null}]}, "emitted_at": 1662401663418} +{"stream": "custom_fields", "data": {"id": 4680904003, "name": "Offer Documents", "active": true, "field_type": "offer", "priority": 2, "value_type": "short_text", "private": true, "required": false, "require_approval": false, "trigger_new_version": true, "name_key": "offer_documents", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1662401663418} +{"stream": "custom_fields", "data": {"id": 4680905003, "name": "Relationship", "active": true, "field_type": "referral_question", "priority": 0, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "relationship", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845806003, "name": "Coworker", "priority": 0, "external_id": null}, {"id": 10845807003, "name": "School", "priority": 1, "external_id": null}, {"id": 10845808003, "name": "Manager", "priority": 2, "external_id": null}, {"id": 10845809003, "name": "Reported", "priority": 3, "external_id": null}, {"id": 10845810003, "name": "Friend", "priority": 4, "external_id": null}, {"id": 10845811003, "name": "Do not know", "priority": 5, "external_id": null}]}, "emitted_at": 1662401663418} +{"stream": "custom_fields", "data": {"id": 4680906003, "name": "Work History", "active": true, "field_type": "referral_question", "priority": 1, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "work_history", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845812003, "name": "0-1", "priority": 0, "external_id": null}, {"id": 10845813003, "name": "2-5", "priority": 1, "external_id": null}, {"id": 10845814003, "name": "5+", "priority": 2, "external_id": null}]}, "emitted_at": 1662401663418} +{"stream": "custom_fields", "data": {"id": 4680907003, "name": "Rating", "active": true, "field_type": "referral_question", "priority": 2, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "rating", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845815003, "name": "Superstar", "priority": 0, "external_id": null}, {"id": 10845816003, "name": "Top 5%", "priority": 1, "external_id": null}, {"id": 10845817003, "name": "Top 10%", "priority": 2, "external_id": null}, {"id": 10845818003, "name": "Top 25%", "priority": 3, "external_id": null}, {"id": 10845819003, "name": "Top 50%", "priority": 4, "external_id": null}]}, "emitted_at": 1662401663419} +{"stream": "custom_fields", "data": {"id": 4680908003, "name": "When we reach out", "active": true, "field_type": "referral_question", "priority": 3, "value_type": "single_select", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "when_we_reach_out", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": [{"id": 10845820003, "name": "You may mention me", "priority": 0, "external_id": null}, {"id": 10845821003, "name": "I wish to remain anonymous", "priority": 1, "external_id": null}]}, "emitted_at": 1662401663419} +{"stream": "custom_fields", "data": {"id": 4680909003, "name": "They know they're being referred", "active": true, "field_type": "referral_question", "priority": 4, "value_type": "yes_no", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "they_know_they_re_being_referred", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1662401663419} +{"stream": "custom_fields", "data": {"id": 4680910003, "name": "Referral Notes", "active": true, "field_type": "referral_question", "priority": 5, "value_type": "long_text", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "referral_notes", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1662401663419} +{"stream": "custom_fields", "data": {"id": 7431124003, "name": "Test User", "active": true, "field_type": "agency_question", "priority": 0, "value_type": "yes_no", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "test_user", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1662401663419} +{"stream": "custom_fields", "data": {"id": 7431125003, "name": "Test User", "active": true, "field_type": "agency_question", "priority": 1, "value_type": "short_text", "private": false, "required": true, "require_approval": false, "trigger_new_version": false, "name_key": "test_user_agency_question_1633884465.559642", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1662401663419} +{"stream": "custom_fields", "data": {"id": 7431126003, "name": "Test User", "active": true, "field_type": "referral_question", "priority": 6, "value_type": "yes_no", "private": false, "required": false, "require_approval": false, "trigger_new_version": false, "name_key": "test_user", "description": null, "expose_in_job_board_api": false, "api_only": false, "offices": [], "departments": [], "template_token_string": null, "custom_field_options": []}, "emitted_at": 1662401663419} {"stream":"demographics_question_sets","data":{"title":"Test Question Set 1","id":4000197003,"description":"

Test Question Set 1 description

","active":true},"emitted_at":1660156526996} {"stream":"demographics_question_sets","data":{"title":"Test Question Set 2","id":4000198003,"description":"

Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.

","active":true},"emitted_at":1660156526998} {"stream":"demographics_question_sets","data":{"title":"U.S. Standard Demographic Questions","id":4002702003,"description":"We invite applicants to share their demographic background. If you choose to complete this survey, your responses may be used to identify areas of improvement in our hiring process.","active":true},"emitted_at":1660156526998} diff --git a/airbyte-integrations/connectors/source-greenhouse/integration_tests/incremental_configured_catalog.json b/airbyte-integrations/connectors/source-greenhouse/integration_tests/incremental_configured_catalog.json new file mode 100644 index 000000000000..e9fdd8fa30da --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/integration_tests/incremental_configured_catalog.json @@ -0,0 +1,186 @@ +{ + "streams": [ + { + "stream": { + "name": "applications", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["applied_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "primary_key": [["id"]], + "cursor_field": ["applied_at"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "candidates", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "job_posts", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "jobs", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "offers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "scorecards", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "demographics_answers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "applications_demographics_answers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "interviews", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "applications_interviews", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "job_stages", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "jobs_stages", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "primary_key": [["id"]], + "cursor_field": ["updated_at"], + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-greenhouse/setup.py b/airbyte-integrations/connectors/source-greenhouse/setup.py index be75115f62e8..6707890b19ac 100644 --- a/airbyte-integrations/connectors/source-greenhouse/setup.py +++ b/airbyte-integrations/connectors/source-greenhouse/setup.py @@ -16,7 +16,7 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-cdk~=0.1.79"], + install_requires=["airbyte-cdk~=0.1.79", "dataclasses-jsonschema==2.15.1"], package_data={"": ["*.json", "*.yaml", "schemas/*.json"]}, extras_require={ "tests": TEST_REQUIREMENTS, diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/components.py b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/components.py new file mode 100644 index 000000000000..1eb4cdd82448 --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/components.py @@ -0,0 +1,116 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import datetime +from dataclasses import InitVar, dataclass +from typing import Any, ClassVar, Iterable, Mapping, MutableMapping, Optional, Union + +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.declarative.stream_slicers import StreamSlicer +from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState +from airbyte_cdk.sources.streams.core import Stream + + +@dataclass +class GreenHouseSlicer(StreamSlicer): + options: InitVar[Mapping[str, Any]] + cursor_field: str + request_cursor_field: str + + START_DATETIME: ClassVar[str] = "1970-01-01T00:00:00.000Z" + DATETIME_FORMAT: ClassVar[str] = "%Y-%m-%dT%H:%M:%S.%fZ" + + def __post_init__(self, options: Mapping[str, Any]): + self._state = {} + + def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState, *args, **kwargs) -> Iterable[StreamSlice]: + yield {self.request_cursor_field: stream_state.get(self.cursor_field, self.START_DATETIME)} + + def _max_dt_str(self, *args: str) -> Optional[str]: + new_state_candidates = list(map(lambda x: datetime.datetime.strptime(x, self.DATETIME_FORMAT), filter(None, args))) + if not new_state_candidates: + return + max_dt = max(new_state_candidates) + # `.%f` gives us microseconds, we need milliseconds + (dt, micro) = max_dt.strftime(self.DATETIME_FORMAT).split(".") + return "%s.%03dZ" % (dt, int(micro[:-1:]) / 1000) + + def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): + # stream_state can be passed in as a stream_slice parameter - it's a framework flaw, so we have to workaround it + slice_state = stream_slice.get(self.cursor_field) + current_state = self._state.get(self.cursor_field) + last_cursor = last_record and last_record[self.cursor_field] + max_dt = self._max_dt_str(slice_state, current_state, last_cursor) + if not max_dt: + return + self._state[self.cursor_field] = max_dt + + def get_stream_state(self) -> StreamState: + return self._state + + def get_request_params( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> MutableMapping[str, Any]: + return stream_slice or {} + + def get_request_headers(self, *args, **kwargs) -> Mapping[str, Any]: + return {} + + def get_request_body_data(self, *args, **kwargs) -> Optional[Union[Mapping, str]]: + return {} + + def get_request_body_json(self, *args, **kwargs) -> Optional[Mapping]: + return {} + + +@dataclass +class GreenHouseSubstreamSlicer(GreenHouseSlicer): + parent_stream: Stream + stream_slice_field: str + parent_key: str + + def stream_slices(self, sync_mode: SyncMode, stream_state: StreamState) -> Iterable[StreamSlice]: + for parent_stream_slice in self.parent_stream.stream_slices(sync_mode=sync_mode, cursor_field=None, stream_state=stream_state): + for parent_record in self.parent_stream.read_records( + sync_mode=SyncMode.full_refresh, cursor_field=None, stream_slice=parent_stream_slice, stream_state=None + ): + parent_state_value = parent_record.get(self.parent_key) + yield { + self.stream_slice_field: parent_state_value, + self.request_cursor_field: stream_state.get(str(parent_state_value), {}).get(self.cursor_field, self.START_DATETIME), + } + + def update_cursor(self, stream_slice: StreamSlice, last_record: Optional[Record] = None): + if last_record: + # stream_slice is really a stream slice + substream_id = str(stream_slice[self.stream_slice_field]) + current_state = self._state.get(substream_id, {}).get(self.cursor_field) + last_state = last_record[self.cursor_field] + max_dt = self._max_dt_str(last_state, current_state) + self._state[substream_id] = {self.cursor_field: max_dt} + return + # stream_slice here may be a stream slice or a state + if self.stream_slice_field in stream_slice: + return + substream_ids = map(lambda x: str(x), set(stream_slice.keys()) | set(self._state.keys())) + for id_ in substream_ids: + self._state[id_] = { + self.cursor_field: self._max_dt_str( + stream_slice.get(id_, {}).get(self.cursor_field), self._state.get(id_, {}).get(self.cursor_field) + ) + } + + def get_request_params( + self, + *, + stream_state: Optional[StreamState] = None, + stream_slice: Optional[StreamSlice] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> MutableMapping[str, Any]: + # ignore other fields in a slice + return {self.request_cursor_field: stream_slice[self.request_cursor_field]} diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/greenhouse.yaml b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/greenhouse.yaml index 28dc67fc49ad..f0f6e1252bee 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/greenhouse.yaml +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/greenhouse.yaml @@ -45,18 +45,34 @@ definitions: $ref: "*ref(definitions.retriever)" requester: $ref: "*ref(definitions.requester)" - applications_stream: + base_incremental_stream: $ref: "*ref(definitions.base_stream)" + stream_cursor_field: "updated_at" + retriever: + $ref: "*ref(definitions.retriever)" + requester: "*ref(definitions.requester)" + stream_slicer: + request_cursor_field: "updated_after" + cursor_field: "updated_at" + class_name: source_greenhouse.components.GreenHouseSlicer + applications_stream: + $ref: "*ref(definitions.base_incremental_stream)" $options: name: "applications" path: "applications" - primary_key: "id" + stream_cursor_field: "applied_at" + retriever: + $ref: "*ref(definitions.retriever)" + requester: "*ref(definitions.requester)" + stream_slicer: + request_cursor_field: "created_after" + cursor_field: "applied_at" + class_name: source_greenhouse.components.GreenHouseSlicer candidates_stream: - $ref: "*ref(definitions.base_stream)" + $ref: "*ref(definitions.base_incremental_stream)" $options: name: "candidates" path: "candidates" - primary_key: "id" close_reasons_stream: $ref: "*ref(definitions.base_stream)" $options: @@ -74,7 +90,7 @@ definitions: name: "departments" path: "departments" jobs_stream: - $ref: "*ref(definitions.base_stream)" + $ref: "*ref(definitions.base_incremental_stream)" $options: name: "jobs" path: "jobs" @@ -96,39 +112,39 @@ definitions: parent_key: "id" stream_slice_field: "parent_id" applications_demographics_answers_stream: + $ref: "*ref(definitions.base_stream)" $options: name: "applications_demographics_answers" - primary_key: "id" - schema_loader: - $ref: "*ref(definitions.schema_loader)" + stream_cursor_field: "updated_at" retriever: $ref: "*ref(definitions.retriever)" requester: $ref: "*ref(definitions.requester)" path: "applications/{{ stream_slice.parent_id }}/demographics/answers" stream_slicer: - type: SubstreamSlicer - parent_stream_configs: - - stream: "*ref(definitions.applications_stream)" - parent_key: "id" - stream_slice_field: "parent_id" + class_name: source_greenhouse.components.GreenHouseSubstreamSlicer + parent_stream: "*ref(definitions.applications_stream)" + request_cursor_field: "updated_after" + stream_slice_field: "parent_id" + cursor_field: "updated_at" + parent_key: "id" applications_interviews_stream: + $ref: "*ref(definitions.base_stream)" $options: name: "applications_interviews" - primary_key: "id" - schema_loader: - $ref: "*ref(definitions.schema_loader)" + stream_cursor_field: "updated_at" retriever: $ref: "*ref(definitions.retriever)" requester: $ref: "*ref(definitions.requester)" path: "applications/{{ stream_slice.parent_id }}/scheduled_interviews" stream_slicer: - type: SubstreamSlicer - parent_stream_configs: - - stream: "*ref(definitions.applications_stream)" - parent_key: "id" - stream_slice_field: "parent_id" + class_name: source_greenhouse.components.GreenHouseSubstreamSlicer + parent_stream: "*ref(definitions.applications_stream)" + request_cursor_field: "updated_after" + stream_slice_field: "parent_id" + cursor_field: "updated_at" + parent_key: "id" custom_fields_stream: $ref: "*ref(definitions.base_stream)" $options: @@ -179,39 +195,40 @@ definitions: parent_key: "id" stream_slice_field: "parent_id" interviews_stream: - $ref: "*ref(definitions.base_stream)" + $ref: "*ref(definitions.base_incremental_stream)" $options: name: "interviews" path: "scheduled_interviews" job_posts_stream: - $ref: "*ref(definitions.base_stream)" + $ref: "*ref(definitions.base_incremental_stream)" $options: name: "job_posts" path: "job_posts" job_stages_stream: - $ref: "*ref(definitions.base_stream)" + $ref: "*ref(definitions.base_incremental_stream)" $options: name: "job_stages" path: "job_stages" jobs_stages_stream: + $ref: "*ref(definitions.base_stream)" $options: name: "jobs_stages" - primary_key: "id" - schema_loader: - $ref: "*ref(definitions.schema_loader)" - retriever: - $ref: "*ref(definitions.retriever)" - requester: - $ref: "*ref(definitions.requester)" - path: "jobs/{{ stream_slice.parent_id }}/stages" - stream_slicer: - type: SubstreamSlicer - parent_stream_configs: - - stream: "*ref(definitions.jobs_stream)" - parent_key: "id" - stream_slice_field: "parent_id" + path: "jobs/{{ stream_slice.parent_id }}/stages" + stream_cursor_field: "updated_at" + retriever: + $ref: "*ref(definitions.retriever)" + requester: + $ref: "*ref(definitions.requester)" + path: "jobs/{{ stream_slice.parent_id }}/stages" + stream_slicer: + class_name: source_greenhouse.components.GreenHouseSubstreamSlicer + parent_stream: "*ref(definitions.jobs_stream)" + request_cursor_field: "updated_after" + stream_slice_field: "parent_id" + cursor_field: "updated_at" + parent_key: "id" offers_stream: - $ref: "*ref(definitions.base_stream)" + $ref: "*ref(definitions.base_incremental_stream)" $options: name: "offers" path: "offers" @@ -221,7 +238,7 @@ definitions: name: "rejection_reasons" path: "rejection_reasons" scorecards_stream: - $ref: "*ref(definitions.base_stream)" + $ref: "*ref(definitions.base_incremental_stream)" $options: name: "scorecards" path: "scorecards" @@ -231,12 +248,12 @@ definitions: name: "sources" path: "sources" users_stream: - $ref: "*ref(definitions.base_stream)" + $ref: "*ref(definitions.base_incremental_stream)" $options: name: "users" path: "users" demographics_answers_stream: - $ref: "*ref(definitions.base_stream)" + $ref: "*ref(definitions.base_incremental_stream)" $options: name: "demographics_answers" path: "demographics/answers" diff --git a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/spec.json b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/spec.json index 7d64621c2531..9ecfdedf7a65 100644 --- a/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/spec.json +++ b/airbyte-integrations/connectors/source-greenhouse/source_greenhouse/spec.json @@ -1,5 +1,5 @@ { - "documentationUrl": "https://docs.airbyte.io/integrations/sources/greenhouse", + "documentationUrl": "https://docs.airbyte.com/integrations/sources/greenhouse", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "Greenhouse Spec", diff --git a/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py new file mode 100644 index 000000000000..9c2b7f90713a --- /dev/null +++ b/airbyte-integrations/connectors/source-greenhouse/unit_tests/test_components.py @@ -0,0 +1,49 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from unittest.mock import MagicMock + +import pytest +from airbyte_cdk.models import SyncMode +from source_greenhouse.components import GreenHouseSlicer, GreenHouseSubstreamSlicer + + +def test_slicer(): + date_time = "2022-09-05T10:10:10.000000Z" + date_time_dict = {date_time: date_time} + slicer = GreenHouseSlicer(cursor_field=date_time, options=None, request_cursor_field=None) + slicer.update_cursor(stream_slice=date_time_dict, last_record=date_time_dict) + assert slicer.get_stream_state() == {date_time: "2022-09-05T10:10:10.000Z"} + assert slicer.get_request_headers() == {} + assert slicer.get_request_body_data() == {} + assert slicer.get_request_body_json() == {} + + +@pytest.mark.parametrize( + "last_record, expected, records", + [ + ( + {"2022-09-05T10:10:10.000000Z": "2022-09-05T10:10:10.000000Z"}, + {"parent_key": {"2022-09-05T10:10:10.000000Z": "2022-09-05T10:10:10.000Z"}}, + [{"parent_key": "parent_key"}], + ), + (None, {}, []), + ], +) +def test_sub_slicer(last_record, expected, records): + date_time = "2022-09-05T10:10:10.000000Z" + parent_slicer = GreenHouseSlicer(cursor_field=date_time, options=None, request_cursor_field=None) + GreenHouseSlicer.read_records = MagicMock(return_value=records) + slicer = GreenHouseSubstreamSlicer( + cursor_field=date_time, + options=None, + request_cursor_field=None, + parent_stream=parent_slicer, + stream_slice_field=date_time, + parent_key="parent_key", + ) + stream_slice = next(slicer.stream_slices(SyncMode, {})) if records else {} + slicer.update_cursor(stream_slice=stream_slice, last_record=last_record) + assert slicer.get_stream_state() == expected diff --git a/docs/integrations/sources/greenhouse.md b/docs/integrations/sources/greenhouse.md index db0ff48ca401..ffc4f5bb3afb 100644 --- a/docs/integrations/sources/greenhouse.md +++ b/docs/integrations/sources/greenhouse.md @@ -64,11 +64,12 @@ Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------| -| 0.2.9 | 2022-08-22 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml and schemas at runtime | -| 0.2.8 | 2022-08-10 | [15344](https://github.com/airbytehq/airbyte/pull/15344) | Migrate connector to config-based framework | -| 0.2.7 | 2022-04-15 | [11941](https://github.com/airbytehq/airbyte/pull/11941) | Correct Schema data type for Applications, Candidates, Scorecards and Users | -| 0.2.6 | 2021-11-08 | [7607](https://github.com/airbytehq/airbyte/pull/7607) | Implement demographics streams support. Update SAT for demographics streams | -| 0.2.5 | 2021-09-22 | [6377](https://github.com/airbytehq/airbyte/pull/6377) | Refactor the connector to use CDK. Implement additional stream support | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------| +| 0.2.10 | 2022-09-05 | [16338](https://github.com/airbytehq/airbyte/pull/16338) | Implement incremental syncs & fix SATs | +| 0.2.9 | 2022-08-22 | [15800](https://github.com/airbytehq/airbyte/pull/15800) | Bugfix to allow reading sentry.yaml and schemas at runtime | +| 0.2.8 | 2022-08-10 | [15344](https://github.com/airbytehq/airbyte/pull/15344) | Migrate connector to config-based framework | +| 0.2.7 | 2022-04-15 | [11941](https://github.com/airbytehq/airbyte/pull/11941) | Correct Schema data type for Applications, Candidates, Scorecards and Users | +| 0.2.6 | 2021-11-08 | [7607](https://github.com/airbytehq/airbyte/pull/7607) | Implement demographics streams support. Update SAT for demographics streams | +| 0.2.5 | 2021-09-22 | [6377](https://github.com/airbytehq/airbyte/pull/6377) | Refactor the connector to use CDK. Implement additional stream support | | 0.2.4 | 2021-09-15 | [6238](https://github.com/airbytehq/airbyte/pull/6238) | Add identification of accessible streams for API keys with limited permissions | From 5ccea654ad4fb53ea3c79b5d45e037439e92c202 Mon Sep 17 00:00:00 2001 From: Miles Armstrong Date: Mon, 5 Sep 2022 22:42:15 +0100 Subject: [PATCH 034/200] Add airbyte-metrics to list of charts to be packaged (#16270) --- .github/workflows/publish-helm-charts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-helm-charts.yml b/.github/workflows/publish-helm-charts.yml index f2b03415c563..732c8e4bae8f 100644 --- a/.github/workflows/publish-helm-charts.yml +++ b/.github/workflows/publish-helm-charts.yml @@ -56,7 +56,7 @@ jobs: - name: "Helm package" shell: bash run: | - declare -a StringArray=("airbyte-bootloader" "airbyte-server" "airbyte-temporal" "airbyte-webapp" "airbyte-pod-sweeper" "airbyte-worker") + declare -a StringArray=("airbyte-bootloader" "airbyte-server" "airbyte-temporal" "airbyte-webapp" "airbyte-pod-sweeper" "airbyte-worker", "airbyte-metrics") for val in ${StringArray[@]}; do cd ./airbyte/charts/${val} && helm dep update && cd $GITHUB_WORKSPACE helm package ./airbyte/charts/${val} -d airbyte-oss --version ${{ needs.generate-semantic-version.outputs.next-version }} From ec70b322f397519586eeeeb75bee40ff029b0be6 Mon Sep 17 00:00:00 2001 From: "Pedro S. Lopez" Date: Mon, 5 Sep 2022 20:46:00 -0400 Subject: [PATCH 035/200] Source: Instagram - Use latest CDK version (#16340) --- .../src/main/resources/seed/source_definitions.yaml | 2 +- .../init/src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-instagram/Dockerfile | 2 +- .../connectors/source-instagram/setup.py | 2 +- docs/integrations/sources/instagram.md | 13 +++++++------ 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index ce03659c4fed..368981b10151 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -457,7 +457,7 @@ - name: Instagram sourceDefinitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 dockerRepository: airbyte/source-instagram - dockerImageTag: 0.1.9 + dockerImageTag: 0.1.10 documentationUrl: https://docs.airbyte.com/integrations/sources/instagram icon: instagram.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 07f25c93109a..ced1ae87b172 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4104,7 +4104,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-instagram:0.1.9" +- dockerImage: "airbyte/source-instagram:0.1.10" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/instagram" changelogUrl: "https://docs.airbyte.io/integrations/sources/instagram" diff --git a/airbyte-integrations/connectors/source-instagram/Dockerfile b/airbyte-integrations/connectors/source-instagram/Dockerfile index 1684aaba026d..9e9bb4270ebb 100644 --- a/airbyte-integrations/connectors/source-instagram/Dockerfile +++ b/airbyte-integrations/connectors/source-instagram/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.9 +LABEL io.airbyte.version=0.1.10 LABEL io.airbyte.name=airbyte/source-instagram diff --git a/airbyte-integrations/connectors/source-instagram/setup.py b/airbyte-integrations/connectors/source-instagram/setup.py index 2564950e9980..47c27dce37be 100644 --- a/airbyte-integrations/connectors/source-instagram/setup.py +++ b/airbyte-integrations/connectors/source-instagram/setup.py @@ -6,7 +6,7 @@ from setuptools import find_packages, setup MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", + "airbyte-cdk~=0.1.81", "cached_property~=1.5", "facebook_business~=11.0", "pendulum>=2,<3", diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index bc6ba5786436..065daeac11cd 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -99,9 +99,10 @@ See Facebook's [documentation on rate limiting](https://developers.facebook.com/ ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.9 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | -| 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | added check for empty state and fixed tests. | -| 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous format of STATE. | -| 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling. - fix sync fail with HTTP status 400. - integrate SAT. | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:-------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| 0.1.10 | 2022-09-05 | [16340](https://github.com/airbytehq/airbyte/pull/16340) | Update to latest version of the CDK (v0.1.81) | +| 0.1.9 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | +| 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | added check for empty state and fixed tests. | +| 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous format of STATE. | +| 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling. - fix sync fail with HTTP status 400. - integrate SAT. | From 46112cfaa12a692780be42ac83cf658e4482f9f6 Mon Sep 17 00:00:00 2001 From: Arsen Losenko <20901439+arsenlosenko@users.noreply.github.com> Date: Tue, 6 Sep 2022 13:16:19 +0300 Subject: [PATCH 036/200] Source Pinterest: support OAuth (#16271) * Source Pinterest: Initial setup of OAuth flow * Remove previously added class and method for auth * Update Java part of OAuth flow, update spec * Update spec and add additional methods to Java OAuth flow * Add backwards compatibility for old config structure * Add missing imports * Revert previous changes in source_specs.yaml * Cleanup in imports and source_specs * Add missing imports * Add missing imports * Remove 'subdomain' logic from Java OAuth flow * Update docs * Update docs accordingly to comments in PR * Refactor credentials variable * Fix typo * Update acceptance-test-config.yml * Specify integer type for AD_ACCOUNT_ID value in schemas * updated SAT tests, fixed Max Rate Limit error handling * updated unit_tests * updated schemas, added caching for Boards and AdAccounts stream to reduce API Calls for child streams, commented out Incremental and Full refresh SAT tests * auto-bump connector version [ci skip] Co-authored-by: Oleksandr Bazarnov Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 113 +++++++++++---- .../connectors/source-pinterest/Dockerfile | 2 +- .../connectors/source-pinterest/README.md | 3 +- .../acceptance-test-config.yml | 47 +++++- .../schemas/ad_analytics.json | 2 +- .../schemas/ad_group_analytics.json | 2 +- .../schemas/campaign_analytics.json | 2 +- .../source_pinterest/source.py | 30 ++-- .../source_pinterest/spec.json | 123 ++++++++++++---- .../unit_tests/test_streams.py | 2 +- .../oauth/OAuthImplementationFactory.java | 1 + .../oauth/flows/PinterestOAuthFlow.java | 135 ++++++++++++++++++ docs/integrations/sources/pinterest.md | 9 +- 14 files changed, 391 insertions(+), 82 deletions(-) create mode 100644 airbyte-oauth/src/main/java/io/airbyte/oauth/flows/PinterestOAuthFlow.java diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 368981b10151..c191aa2dd546 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -750,7 +750,7 @@ - name: Pinterest sourceDefinitionId: 5cb7e5fe-38c2-11ec-8d3d-0242ac130003 dockerRepository: airbyte/source-pinterest - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/sources/pinterest icon: pinterest.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index ced1ae87b172..ae356f687f4f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -7281,7 +7281,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-pinterest:0.1.2" +- dockerImage: "airbyte/source-pinterest:0.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/pinterest" connectionSpecification: @@ -7289,35 +7289,9 @@ title: "Pinterest Spec" type: "object" required: - - "client_id" - - "client_secret" - - "refresh_token" + - "start_date" additionalProperties: true properties: - client_id: - type: "string" - title: "Client ID" - description: "Your Pinterest client ID. See the docs for instructions on how to generate it." - airbyte_secret: true - client_secret: - type: "string" - title: "Client Secret" - description: "Your Pinterest client secret. See the docs for instructions on how to generate it." - airbyte_secret: true - refresh_token: - type: "string" - title: "Refresh Token" - description: "Your Pinterest refresh token. See the docs for instructions on how to generate it." - airbyte_secret: true - access_token: - type: "string" - title: "Access Token" - description: "Your Pinterest access token. See the docs for instructions on how to generate it." - airbyte_secret: true start_date: type: "string" title: "Start Date" @@ -7325,9 +7299,92 @@ \ it would be defaulted to 2020-07-28." examples: - "2020-07-28" + credentials: + title: "Authorization Method" + type: "object" + oneOf: + - type: "object" + title: "OAuth2.0" + required: + - "auth_method" + - "refresh_token" + properties: + auth_method: + type: "string" + const: "oauth2.0" + order: 0 + client_id: + type: "string" + title: "Client ID" + description: "The Client ID of your OAuth application" + airbyte_secret: true + client_secret: + type: "string" + title: "Client Secret" + description: "The Client Secret of your OAuth application." + airbyte_secret: true + refresh_token: + type: "string" + title: "Refresh Token" + description: "Refresh Token to obtain new Access Token, when it's\ + \ expired." + airbyte_secret: true + - type: "object" + title: "Access Token" + required: + - "auth_method" + - "access_token" + properties: + auth_method: + type: "string" + const: "access_token" + order: 0 + access_token: + type: "string" + title: "Access Token" + description: "The Access Token to make authenticated requests." + airbyte_secret: true supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] + advanced_auth: + auth_flow_type: "oauth2.0" + predicate_key: + - "credentials" + - "auth_method" + predicate_value: "oauth2.0" + oauth_config_specification: + complete_oauth_output_specification: + type: "object" + additionalProperties: false + properties: + refresh_token: + type: "string" + path_in_connector_config: + - "credentials" + - "refresh_token" + complete_oauth_server_input_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + client_secret: + type: "string" + complete_oauth_server_output_specification: + type: "object" + additionalProperties: false + properties: + client_id: + type: "string" + path_in_connector_config: + - "credentials" + - "client_id" + client_secret: + type: "string" + path_in_connector_config: + - "credentials" + - "client_secret" - dockerImage: "airbyte/source-pipedrive:0.1.12" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/pipedrive" diff --git a/airbyte-integrations/connectors/source-pinterest/Dockerfile b/airbyte-integrations/connectors/source-pinterest/Dockerfile index c5bbeb3e7a18..b545e06990f8 100644 --- a/airbyte-integrations/connectors/source-pinterest/Dockerfile +++ b/airbyte-integrations/connectors/source-pinterest/Dockerfile @@ -34,5 +34,5 @@ COPY source_pinterest ./source_pinterest ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/source-pinterest diff --git a/airbyte-integrations/connectors/source-pinterest/README.md b/airbyte-integrations/connectors/source-pinterest/README.md index 51522743e4de..c930a85d87ed 100644 --- a/airbyte-integrations/connectors/source-pinterest/README.md +++ b/airbyte-integrations/connectors/source-pinterest/README.md @@ -102,7 +102,8 @@ Customize `acceptance-test-config.yml` file to configure tests. See [Source Acce If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. To run your integration tests with acceptance tests, from the connector root, run ``` -python -m pytest integration_tests -p integration_tests.acceptance +docker build . --no-cache -t airbyte/source-pinterest:dev \ +&& python -m pytest -p source_acceptance_test.plugin ``` To run your integration tests with docker diff --git a/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml index 40fb3d07026e..2fb21134f7c6 100644 --- a/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-pinterest/acceptance-test-config.yml @@ -3,21 +3,54 @@ connector_image: airbyte/source-pinterest:dev tests: spec: + # TODO: remove backward compatibility checks once updated to version `>0.1.3` + # because for OAuth2.0 implementation the specs are different - spec_path: "source_pinterest/spec.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.2" connection: - config_path: "secrets/config.json" status: "succeed" - config_path: "integration_tests/invalid_config.json" status: "failed" + - config_path: "secrets/config_oauth.json" + status: "succeed" discovery: + # TODO: remove backward compatibility checks once updated to version `>0.1.3` - config_path: "secrets/config.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.2" + - config_path: "secrets/config_oauth.json" + backward_compatibility_tests_config: + disable_for_version: "0.1.2" basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] - incremental: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" - full_refresh: - - config_path: "secrets/config.json" - configured_catalog_path: "integration_tests/configured_catalog.json" + # empty streams could be produced because of very low rate limits + empty_streams: [ + "ad_account_analytics", + "ad_accounts", + "ad_analytics", + "ad_group_analytics", + "ad_groups", + "ads", + "board_pins", + "board_section_pins", + "board_sections", + "boards", + "campaign_analytics", + "campaigns", + "user_account_analytics", + ] + + # INFO: `incremental` and `full_refresh` tests are commented out because of very small Rate Limits for Pinterest API + # They simply not going to pass with Trial Account, having 300 api calls in total. + # The basic_read test is totaly enough to verify key things of this connector. + # Once upgraded to Standard Plan - they could be uncomment back. + + # incremental: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # full_refresh: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_analytics.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_analytics.json index bcd32affea49..5a986349c6e0 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_analytics.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_analytics.json @@ -7,7 +7,7 @@ "format": "date" }, "AD_ACCOUNT_ID": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "AD_ID": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_group_analytics.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_group_analytics.json index 25cb790dfa3b..17fa3968a802 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_group_analytics.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/ad_group_analytics.json @@ -7,7 +7,7 @@ "format": "date" }, "AD_ACCOUNT_ID": { - "type": ["string"] + "type": ["integer"] }, "AD_ID": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics.json index fd4f48481921..25ee8aeea80e 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/schemas/campaign_analytics.json @@ -7,7 +7,7 @@ "format": "date" }, "AD_ACCOUNT_ID": { - "type": ["null", "string"] + "type": ["null", "integer"] }, "AD_ID": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py index 40ba2e0489b9..f0585e66ef37 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py @@ -24,6 +24,7 @@ class PinterestStream(HttpStream, ABC): primary_key = "id" data_fields = ["items"] raise_on_http_errors = True + max_rate_limit_exceeded = False def __init__(self, config: Mapping[str, Any]): super().__init__(authenticator=config["authenticator"]) @@ -60,18 +61,24 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, """ data = response.json() - exceeded_rate_limit = False if isinstance(data, dict): - exceeded_rate_limit = data.get("code") == 8 + self.max_rate_limit_exceeded = data.get("code") == 8 - if not exceeded_rate_limit: + if not self.max_rate_limit_exceeded: for data_field in self.data_fields: data = data.get(data_field, []) for record in data: yield record + def should_retry(self, response: requests.Response) -> bool: + # when max rate limit exceeded, we should skip the stream. + if response.status_code == 429 and response.json().get("code") == 8: + self.logger.error(f"For stream {self.name} max rate limit exceeded.") + setattr(self, "raise_on_http_errors", False) + return 500 <= response.status_code < 600 + class PinterestSubStream(HttpSubStream): def stream_slices( @@ -90,11 +97,15 @@ def stream_slices( class Boards(PinterestStream): + use_cache = True + def path(self, **kwargs) -> str: return "boards" class AdAccounts(PinterestStream): + use_cache = True + def path(self, **kwargs) -> str: return "ad_accounts" @@ -196,12 +207,6 @@ class PinterestAnalyticsStream(IncrementalPinterestSubStream): granularity = "DAY" analytics_target_ids = None - def should_retry(self, response: requests.Response) -> bool: - if response.status_code == 429: - self.logger.error(f"For stream {self.name} rate limit exceeded.") - setattr(self, "raise_on_http_errors", False) - return 500 <= response.status_code < 600 - def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: @@ -289,8 +294,11 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class SourcePinterest(AbstractSource): @staticmethod def get_authenticator(config): - user_pass = (config.get("client_id") + ":" + config.get("client_secret")).encode("ascii") - auth = "Basic " + standard_b64encode(user_pass).decode("ascii") + config = config.get("credentials") or config + credentials_base64_encoded = standard_b64encode( + (config.get("client_id") + ":" + config.get("client_secret")).encode("ascii") + ).decode("ascii") + auth = f"Basic {credentials_base64_encoded}" return Oauth2Authenticator( token_refresh_endpoint=f"{PinterestStream.url_base}oauth/token", diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json b/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json index e219f3e6016d..031300ddd5da 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/spec.json @@ -4,38 +4,111 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Pinterest Spec", "type": "object", - "required": ["client_id", "client_secret", "refresh_token"], + "required": ["start_date"], "additionalProperties": true, "properties": { - "client_id": { - "type": "string", - "title": "Client ID", - "description": "Your Pinterest client ID. See the docs for instructions on how to generate it.", - "airbyte_secret": true - }, - "client_secret": { - "type": "string", - "title": "Client Secret", - "description": "Your Pinterest client secret. See the docs for instructions on how to generate it.", - "airbyte_secret": true - }, - "refresh_token": { - "type": "string", - "title": "Refresh Token", - "description": "Your Pinterest refresh token. See the docs for instructions on how to generate it.", - "airbyte_secret": true - }, - "access_token": { - "type": "string", - "title": "Access Token", - "description": "Your Pinterest access token. See the docs for instructions on how to generate it.", - "airbyte_secret": true - }, "start_date": { "type": "string", "title": "Start Date", "description": "A date in the format YYYY-MM-DD. If you have not set a date, it would be defaulted to 2020-07-28.", "examples": ["2020-07-28"] + }, + "credentials": { + "title": "Authorization Method", + "type": "object", + "oneOf": [ + { + "type": "object", + "title": "OAuth2.0", + "required": ["auth_method", "refresh_token"], + "properties": { + "auth_method": { + "type": "string", + "const": "oauth2.0", + "order": 0 + }, + "client_id": { + "type": "string", + "title": "Client ID", + "description": "The Client ID of your OAuth application", + "airbyte_secret": true + }, + "client_secret": { + "type": "string", + "title": "Client Secret", + "description": "The Client Secret of your OAuth application.", + "airbyte_secret": true + }, + "refresh_token": { + "type": "string", + "title": "Refresh Token", + "description": "Refresh Token to obtain new Access Token, when it's expired.", + "airbyte_secret": true + } + } + }, + { + "type": "object", + "title": "Access Token", + "required": ["auth_method", "access_token"], + "properties": { + "auth_method": { + "type": "string", + "const": "access_token", + "order": 0 + }, + "access_token": { + "type": "string", + "title": "Access Token", + "description": "The Access Token to make authenticated requests.", + "airbyte_secret": true + } + } + } + ] + } + } + }, + "advanced_auth": { + "auth_flow_type": "oauth2.0", + "predicate_key": ["credentials", "auth_method"], + "predicate_value": "oauth2.0", + "oauth_config_specification": { + "complete_oauth_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "refresh_token": { + "type": "string", + "path_in_connector_config": ["credentials", "refresh_token"] + } + } + }, + "complete_oauth_server_input_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string" + }, + "client_secret": { + "type": "string" + } + } + }, + "complete_oauth_server_output_specification": { + "type": "object", + "additionalProperties": false, + "properties": { + "client_id": { + "type": "string", + "path_in_connector_config": ["credentials", "client_id"] + }, + "client_secret": { + "type": "string", + "path_in_connector_config": ["credentials", "client_secret"] + } + } } } } diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py index f0f84ac1219b..2b5db79355da 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py @@ -77,7 +77,7 @@ def test_http_method(patch_base_class): [ (HTTPStatus.OK, False), (HTTPStatus.BAD_REQUEST, False), - (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.TOO_MANY_REQUESTS, False), (HTTPStatus.INTERNAL_SERVER_ERROR, True), ], ) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java index 8438d59e4788..7d8808ef92cf 100644 --- a/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java @@ -39,6 +39,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository, final .put("airbyte/source-microsoft-teams", new MicrosoftTeamsOAuthFlow(configRepository, httpClient)) .put("airbyte/source-notion", new NotionOAuthFlow(configRepository, httpClient)) .put("airbyte/source-bing-ads", new MicrosoftBingAdsOAuthFlow(configRepository, httpClient)) + .put("airbyte/source-pinterest", new PinterestOAuthFlow(configRepository, httpClient)) .put("airbyte/source-pipedrive", new PipeDriveOAuthFlow(configRepository, httpClient)) .put("airbyte/source-quickbooks", new QuickbooksOAuthFlow(configRepository, httpClient)) .put("airbyte/source-retently", new RetentlyOAuthFlow(configRepository, httpClient)) diff --git a/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/PinterestOAuthFlow.java b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/PinterestOAuthFlow.java new file mode 100644 index 000000000000..5552a9565886 --- /dev/null +++ b/airbyte-oauth/src/main/java/io/airbyte/oauth/flows/PinterestOAuthFlow.java @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.oauth.flows; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import io.airbyte.commons.json.Jsons; +import io.airbyte.config.persistence.ConfigRepository; +import io.airbyte.oauth.BaseOAuth2Flow; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.function.Supplier; +import org.apache.http.client.utils.URIBuilder; + +/** + * Following docs from https://developers.pinterest.com/docs/getting-started/authentication + */ +public class PinterestOAuthFlow extends BaseOAuth2Flow { + + private static final String ACCESS_TOKEN_URL = "https://api.pinterest.com/v5/oauth/token"; + + public PinterestOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient) { + super(configRepository, httpClient); + } + + @VisibleForTesting + public PinterestOAuthFlow(final ConfigRepository configRepository, final HttpClient httpClient, final Supplier stateSupplier) { + super(configRepository, httpClient, stateSupplier); + } + + @Override + protected String formatConsentUrl(final UUID definitionId, + final String clientId, + final String redirectUrl, + final JsonNode inputOAuthConfiguration) + throws IOException { + + final URIBuilder builder = new URIBuilder() + .setScheme("https") + .setHost("pinterest.com") + .setPath("oauth") + // required + .addParameter("client_id", clientId) + .addParameter("redirect_uri", redirectUrl) + .addParameter("response_type", "code") + .addParameter("scope", "ads:read,boards:read,boards:read_secret,catalogs:read,pins:read,pins:read_secret,user_accounts:read") + .addParameter("state", getState()); + + try { + return builder.build().toString(); + } catch (final URISyntaxException e) { + throw new IOException("Failed to format Consent URL for OAuth flow", e); + } + } + + @Override + protected Map completeOAuthFlow(final String clientId, + final String clientSecret, + final String authCode, + final String redirectUrl, + final JsonNode inputOAuthConfiguration, + final JsonNode oAuthParamConfig) + throws IOException { + final var accessTokenUrl = getAccessTokenUrl(inputOAuthConfiguration); + final String authorization = Base64.getEncoder() + .encodeToString((clientId + ":" + clientSecret).getBytes(StandardCharsets.UTF_8)); + final HttpRequest request = HttpRequest.newBuilder() + .POST(HttpRequest.BodyPublishers + .ofString(tokenReqContentType.getConverter().apply( + getAccessTokenQueryParameters(clientId, clientSecret, authCode, redirectUrl)))) + .uri(URI.create(accessTokenUrl)) + .header("Content-Type", tokenReqContentType.getContentType()) + .header("Authorization", "Basic " + authorization) + .build(); + + try { + final HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + return extractOAuthOutput(Jsons.deserialize(response.body()), accessTokenUrl); + } catch (final InterruptedException e) { + throw new IOException("Failed to complete Pinterest OAuth flow", e); + } + } + + @Override + protected Map getAccessTokenQueryParameters(String clientId, + String clientSecret, + String authCode, + String redirectUrl) { + return ImmutableMap.builder() + // required + .put("grant_type", "authorization_code") + .put("code", authCode) + .put("client_id", clientId) + .put("client_secret", clientSecret) + .put("redirect_uri", redirectUrl) + .put("scope", "read") + .build(); + } + + @Override + protected String getAccessTokenUrl(final JsonNode inputOAuthConfiguration) { + return ACCESS_TOKEN_URL; + } + + @Override + protected Map extractOAuthOutput(final JsonNode data, final String accessTokenUrl) throws IOException { + final Map result = new HashMap<>(); + // getting out access_token + if (data.has("access_token")) { + result.put("access_token", data.get("access_token").asText()); + } else { + throw new IOException(String.format("Missing 'access_token' in query params from %s", accessTokenUrl)); + } + // getting out refresh_token + if (data.has("refresh_token")) { + result.put("refresh_token", data.get("refresh_token").asText()); + } else { + throw new IOException(String.format("Missing 'refresh_token' in query params from %s", accessTokenUrl)); + } + return result; + } + +} diff --git a/docs/integrations/sources/pinterest.md b/docs/integrations/sources/pinterest.md index 9c1abf8ed483..fe57c7467c91 100644 --- a/docs/integrations/sources/pinterest.md +++ b/docs/integrations/sources/pinterest.md @@ -14,10 +14,10 @@ Please read [How to get your credentials](https://developers.pinterest.com/docs/ 1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. 3. On the Set up the source page, enter the name for the Pinterest connector and select **Pinterest** from the Source type dropdown. -4. Enter your `client_id` -5. Enter your `client_secret` -6. Enter your `refresh_token` -7. Enter the `start_date` you want your sync to start from +4. Enter the `start_date` you want your sync to start from +5. Choose `OAuth2.0` in `Authorization Method` list +6. Click on `Authenticate your Pinterest account` button +7. Proceed with OAuth authentication of your account in the pop-up window that appears after previous step 8. Click **Set up source** ### For Airbyte OSS: @@ -71,6 +71,7 @@ Boards streams - 10 calls per sec / per user / per app | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------ | +| 0.1.3 | 2022-09-02 | [16271](https://github.com/airbytehq/airbyte/pull/16271) | Added support of `OAuth2.0` authentication method | 0.1.2 | 2021-12-22 | [10223](https://github.com/airbytehq/airbyte/pull/10223) | Fix naming of `AD_ID` and `AD_ACCOUNT_ID` fields | | 0.1.1 | 2021-12-22 | [9043](https://github.com/airbytehq/airbyte/pull/9043) | Update connector fields title/description | | 0.1.0 | 2021-10-29 | [7493](https://github.com/airbytehq/airbyte/pull/7493) | Release Pinterest CDK Connector | From b84cc279839c9081fdb5ab43e5496c231f107d27 Mon Sep 17 00:00:00 2001 From: Arsen Losenko <20901439+arsenlosenko@users.noreply.github.com> Date: Tue, 6 Sep 2022 13:47:53 +0300 Subject: [PATCH 037/200] :tada: Source File: change releaseStage to beta (#16347) * Source File: change releaseStage to beta * Change releaseStage to beta for Pinterest --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index c191aa2dd546..4961a721a774 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -754,7 +754,7 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/pinterest icon: pinterest.svg sourceType: api - releaseStage: alpha + releaseStage: beta - name: Pipedrive sourceDefinitionId: d8286229-c680-4063-8c59-23b9b391c700 dockerRepository: airbyte/source-pipedrive From ebd0f30141ee6b04df00f448575f0634fbac9d77 Mon Sep 17 00:00:00 2001 From: Pierre Borckmans Date: Tue, 6 Sep 2022 14:38:45 +0200 Subject: [PATCH 038/200] =?UTF-8?q?=F0=9F=90=9B=20Source=20OpenWeather:=20?= =?UTF-8?q?Update=20openweather=20onecall=20api=20to=203.0=20(#16136)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update openweather onecall api to 3.0 * Bump version and update docs * correct backwards compatibility spec * auto-bump connector version [ci skip] Co-authored-by: marcosmarxm Co-authored-by: Octavia Squidington III --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 4 ++-- airbyte-integrations/connectors/source-openweather/Dockerfile | 2 +- .../source-openweather/source_openweather/spec.json | 2 +- .../source-openweather/source_openweather/streams.py | 2 +- .../connectors/source-openweather/unit_tests/test_source.py | 2 +- docs/integrations/sources/openweather.md | 1 + 7 files changed, 8 insertions(+), 7 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 4961a721a774..f743041f8a2b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -679,7 +679,7 @@ - name: OpenWeather sourceDefinitionId: d8540a80-6120-485d-b7d6-272bca477d9b dockerRepository: airbyte/source-openweather - dockerImageTag: 0.1.5 + dockerImageTag: 0.1.6 documentationUrl: https://docs.airbyte.io/integrations/sources/openweather sourceType: api releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index ae356f687f4f..657c1ba95b52 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -6622,7 +6622,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-openweather:0.1.5" +- dockerImage: "airbyte/source-openweather:0.1.6" spec: documentationUrl: "https://docsurl.com" connectionSpecification: @@ -6633,7 +6633,7 @@ - "appid" - "lat" - "lon" - additionalProperties: false + additionalProperties: true properties: lat: title: "Latitude" diff --git a/airbyte-integrations/connectors/source-openweather/Dockerfile b/airbyte-integrations/connectors/source-openweather/Dockerfile index 264f36fd53b7..0b5dfbaa095f 100644 --- a/airbyte-integrations/connectors/source-openweather/Dockerfile +++ b/airbyte-integrations/connectors/source-openweather/Dockerfile @@ -34,5 +34,5 @@ COPY source_openweather ./source_openweather ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.5 +LABEL io.airbyte.version=0.1.6 LABEL io.airbyte.name=airbyte/source-openweather diff --git a/airbyte-integrations/connectors/source-openweather/source_openweather/spec.json b/airbyte-integrations/connectors/source-openweather/source_openweather/spec.json index 61ffbc8b6961..6efb78fef792 100644 --- a/airbyte-integrations/connectors/source-openweather/source_openweather/spec.json +++ b/airbyte-integrations/connectors/source-openweather/source_openweather/spec.json @@ -5,7 +5,7 @@ "title": "Open Weather Spec", "type": "object", "required": ["appid", "lat", "lon"], - "additionalProperties": false, + "additionalProperties": true, "properties": { "lat": { "title": "Latitude", diff --git a/airbyte-integrations/connectors/source-openweather/source_openweather/streams.py b/airbyte-integrations/connectors/source-openweather/source_openweather/streams.py index 5be335e67464..230a719a74a4 100644 --- a/airbyte-integrations/connectors/source-openweather/source_openweather/streams.py +++ b/airbyte-integrations/connectors/source-openweather/source_openweather/streams.py @@ -11,7 +11,7 @@ class OneCall(HttpStream): cursor_field = ["current", "dt"] - url_base = "https://api.openweathermap.org/data/2.5/" + url_base = "https://api.openweathermap.org/data/3.0/" primary_key = None def __init__(self, appid: str, lat: float, lon: float, lang: str = None, units: str = None): diff --git a/airbyte-integrations/connectors/source-openweather/unit_tests/test_source.py b/airbyte-integrations/connectors/source-openweather/unit_tests/test_source.py index a418cba76a35..7056455dc528 100644 --- a/airbyte-integrations/connectors/source-openweather/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-openweather/unit_tests/test_source.py @@ -28,7 +28,7 @@ def test_check_connection(mocker, response_status): assert source.check_connection(logger_mock, config_mock) == (False, requests_get_mock.return_value.json.return_value.get("message")) validate_mock.assert_called_with(config_mock) requests_get_mock.assert_called_with( - "https://api.openweathermap.org/data/2.5/onecall", params={"appid": "test_appid", "lat": 1.0, "lon": 1.0} + "https://api.openweathermap.org/data/3.0/onecall", params={"appid": "test_appid", "lat": 1.0, "lon": 1.0} ) diff --git a/docs/integrations/sources/openweather.md b/docs/integrations/sources/openweather.md index 05e4e2ae9e2a..f5ab6c875b02 100644 --- a/docs/integrations/sources/openweather.md +++ b/docs/integrations/sources/openweather.md @@ -34,6 +34,7 @@ The free plan allows 60 calls per minute and 1,000,000 calls per month, you won' | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.6 | 2022-06-21 | [16136](https://github.com/airbytehq/airbyte/pull/16136) | Update openweather onecall api to 3.0. | | 0.1.5 | 2022-06-21 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | No changes. Used connector to test publish workflow changes. | | 0.1.4 | 2022-04-27 | [12397](https://github.com/airbytehq/airbyte/pull/12397) | No changes. Used connector to test publish workflow changes. | | 0.1.0 | 2021-10-27 | [7434](https://github.com/airbytehq/airbyte/pull/7434) | Initial release | From 50014742f38a60dbce86898e270f109b0974c7fb Mon Sep 17 00:00:00 2001 From: Luis Gomez <781929+lgomezm@users.noreply.github.com> Date: Tue, 6 Sep 2022 10:06:45 -0400 Subject: [PATCH 039/200] =?UTF-8?q?=F0=9F=8E=89=20Source=20Pinterest:=20Ad?= =?UTF-8?q?ded=20backoff=20strategy=20for=20rate-limit=20errors=20(#16161)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-pinterest/Dockerfile | 2 +- .../source_pinterest/source.py | 27 ++++++------- .../unit_tests/test_streams.py | 38 +++++++++++++++++++ docs/integrations/sources/pinterest.md | 1 + 6 files changed, 56 insertions(+), 16 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index f743041f8a2b..64c3ae44816b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -750,7 +750,7 @@ - name: Pinterest sourceDefinitionId: 5cb7e5fe-38c2-11ec-8d3d-0242ac130003 dockerRepository: airbyte/source-pinterest - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/pinterest icon: pinterest.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 657c1ba95b52..ad6fadaa1865 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -7281,7 +7281,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-pinterest:0.1.3" +- dockerImage: "airbyte/source-pinterest:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/pinterest" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-pinterest/Dockerfile b/airbyte-integrations/connectors/source-pinterest/Dockerfile index b545e06990f8..2140d63593cc 100644 --- a/airbyte-integrations/connectors/source-pinterest/Dockerfile +++ b/airbyte-integrations/connectors/source-pinterest/Dockerfile @@ -34,5 +34,5 @@ COPY source_pinterest ./source_pinterest ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-pinterest diff --git a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py index f0585e66ef37..11ac4766372b 100644 --- a/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py +++ b/airbyte-integrations/connectors/source-pinterest/source_pinterest/source.py @@ -18,6 +18,10 @@ from .utils import analytics_columns, to_datetime_str +# For Pinterest analytics streams rate limit is 300 calls per day / per user. +# once hit - response would contain `code` property with int. +MAX_RATE_LIMIT_CODE = 8 + class PinterestStream(HttpStream, ABC): url_base = "https://api.pinterest.com/v5/" @@ -51,20 +55,10 @@ def request_params( def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: """ - For Pinterest analytics streams rate limit is 300 calls per day / per user. - Handling of rate limits for Pinterest analytics streams described in should_retry method of PinterestAnalyticsStream. - Response example: - { - "code": 8, - "message": "You have exceeded your rate limit. Try again later." - } + Parsing response data with respect to Rate Limits. """ - data = response.json() - if isinstance(data, dict): - self.max_rate_limit_exceeded = data.get("code") == 8 - if not self.max_rate_limit_exceeded: for data_field in self.data_fields: data = data.get(data_field, []) @@ -73,12 +67,19 @@ def parse_response(self, response: requests.Response, stream_state: Mapping[str, yield record def should_retry(self, response: requests.Response) -> bool: + if isinstance(response.json(), dict): + self.max_rate_limit_exceeded = response.json().get("code", 0) == MAX_RATE_LIMIT_CODE # when max rate limit exceeded, we should skip the stream. - if response.status_code == 429 and response.json().get("code") == 8: - self.logger.error(f"For stream {self.name} max rate limit exceeded.") + if response.status_code == requests.codes.too_many_requests and self.max_rate_limit_exceeded: + self.logger.error(f"For stream {self.name} Max Rate Limit exceeded.") setattr(self, "raise_on_http_errors", False) return 500 <= response.status_code < 600 + def backoff_time(self, response: requests.Response) -> Optional[float]: + if response.status_code == requests.codes.too_many_requests: + self.logger.error(f"For stream {self.name} rate limit exceeded.") + return float(response.headers.get("X-RateLimit-Reset", 0)) + class PinterestSubStream(HttpSubStream): def stream_slices( diff --git a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py index 2b5db79355da..5312aa9de40b 100644 --- a/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-pinterest/unit_tests/test_streams.py @@ -6,6 +6,7 @@ from unittest.mock import MagicMock import pytest +import requests from source_pinterest.source import ( AdAccountAnalytics, AdAccounts, @@ -95,6 +96,43 @@ def test_backoff_time(patch_base_class): assert stream.backoff_time(response_mock) == expected_backoff_time +@pytest.mark.parametrize( + "test_response, status_code, expected", + [ + ({"code": 8, "message": "You have exceeded your rate limit. Try again later."}, 429, False), + ({"code": 7, "message": "Some other error message"}, 429, False), + ], +) +def test_should_retry_on_max_rate_limit_error(requests_mock, test_response, status_code, expected): + stream = Boards(config=MagicMock()) + url = "https://api.pinterest.com/v5/boards" + requests_mock.get("https://api.pinterest.com/v5/boards", json=test_response, status_code=status_code) + response = requests.get(url) + result = stream.should_retry(response) + assert result == expected + + +@pytest.mark.parametrize( + "test_response, test_headers, status_code, expected", + [ + ({"code": 7, "message": "Some other error message"}, {"X-RateLimit-Reset": "2"}, 429, 2.0), + ], +) +def test_backoff_on_rate_limit_error(requests_mock, test_response, status_code, test_headers, expected): + stream = Boards(config=MagicMock()) + url = "https://api.pinterest.com/v5/boards" + requests_mock.get( + "https://api.pinterest.com/v5/boards", + json=test_response, + headers=test_headers, + status_code=status_code, + ) + + response = requests.get(url) + result = stream.backoff_time(response) + assert result == expected + + @pytest.mark.parametrize( ("stream_cls, slice, expected"), [ diff --git a/docs/integrations/sources/pinterest.md b/docs/integrations/sources/pinterest.md index fe57c7467c91..d6ccff9b6daa 100644 --- a/docs/integrations/sources/pinterest.md +++ b/docs/integrations/sources/pinterest.md @@ -71,6 +71,7 @@ Boards streams - 10 calls per sec / per user / per app | Version | Date | Pull Request | Subject | | :------ | :--------- | :------------------------------------------------------- | :------------------------------------------------ | +| 0.1.4 | 2022-09-06 | [16161](https://github.com/airbytehq/airbyte/pull/16161) | Added ability to handle `429 - Too Many Requests` error with respect to `Max Rate Limit Exceeded Error` | 0.1.3 | 2022-09-02 | [16271](https://github.com/airbytehq/airbyte/pull/16271) | Added support of `OAuth2.0` authentication method | 0.1.2 | 2021-12-22 | [10223](https://github.com/airbytehq/airbyte/pull/10223) | Fix naming of `AD_ID` and `AD_ACCOUNT_ID` fields | | 0.1.1 | 2021-12-22 | [9043](https://github.com/airbytehq/airbyte/pull/9043) | Update connector fields title/description | From e593adcdd95ee6d2e2971fc7debca9b8892df457 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Tue, 6 Sep 2022 17:47:14 +0200 Subject: [PATCH 040/200] Design fixes of the authentication page (#16328) --- .../cloud/views/auth/Auth.module.scss | 21 +++++++++++-------- .../auth/OAuthLogin/OAuthLogin.module.scss | 2 +- .../views/auth/components/FormContent.tsx | 2 +- .../components/GitBlock/GitBlock.module.scss | 3 ++- .../auth/components/Header/Header.module.scss | 2 ++ 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/airbyte-webapp/src/packages/cloud/views/auth/Auth.module.scss b/airbyte-webapp/src/packages/cloud/views/auth/Auth.module.scss index 52ccaf9efa64..6560c4b7dfc3 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/Auth.module.scss +++ b/airbyte-webapp/src/packages/cloud/views/auth/Auth.module.scss @@ -2,21 +2,22 @@ .container { display: flex; - flex-direction: column; + flex-direction: row; width: 100%; + height: 100%; background: colors.$white; } .leftSide { + display: flex; + flex-direction: column; + overflow-y: auto; width: 100%; padding: 20px 36px 39px 46px; } .rightSide { - display: flex; - flex-direction: column; - justify-content: space-between; - width: 100%; + display: none; } .rightSideFrame { @@ -26,10 +27,12 @@ overflow: hidden; } -@media (min-width: 992px) and (min-height: 720px) { - .container { - flex-direction: row; - height: 100%; +@media (min-width: 992px) { + .rightSide { + display: flex; + flex-direction: column; + justify-content: space-between; + width: 100%; } .leftSide, diff --git a/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.module.scss b/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.module.scss index 3aac5d6e0c9c..b2d2e733f276 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.module.scss +++ b/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.module.scss @@ -45,6 +45,7 @@ .github { background: #333; color: colors.$white; + border: none; } .google, @@ -59,7 +60,6 @@ padding: vars.$spacing-md; gap: vars.$spacing-md; border-radius: vars.$border-radius-sm; - border: none; transition: all vars.$transition; cursor: pointer; diff --git a/airbyte-webapp/src/packages/cloud/views/auth/components/FormContent.tsx b/airbyte-webapp/src/packages/cloud/views/auth/components/FormContent.tsx index f19552e556b6..5b217c7d9802 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/components/FormContent.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/components/FormContent.tsx @@ -8,8 +8,8 @@ import { Header } from "./Header"; const MainBlock = styled.div` width: 100%; - height: calc(100% - 100px); display: flex; + flex: 1 0 auto; align-items: center; justify-content: center; `; diff --git a/airbyte-webapp/src/packages/cloud/views/auth/components/GitBlock/GitBlock.module.scss b/airbyte-webapp/src/packages/cloud/views/auth/components/GitBlock/GitBlock.module.scss index 3b631c9d8229..1f0ed161c367 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/components/GitBlock/GitBlock.module.scss +++ b/airbyte-webapp/src/packages/cloud/views/auth/components/GitBlock/GitBlock.module.scss @@ -1,4 +1,5 @@ @use "../../../../../../scss/colors"; +@use "../../../../../../scss/variables"; .container { display: flex; @@ -7,12 +8,12 @@ .link { text-decoration: none; + margin-top: variables.$spacing-2xl; .content { display: flex; flex-direction: row; align-items: center; - margin-top: 17px; .icon { margin-right: 10px; diff --git a/airbyte-webapp/src/packages/cloud/views/auth/components/Header/Header.module.scss b/airbyte-webapp/src/packages/cloud/views/auth/components/Header/Header.module.scss index a83ef2b71b14..cf8f5c787253 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/components/Header/Header.module.scss +++ b/airbyte-webapp/src/packages/cloud/views/auth/components/Header/Header.module.scss @@ -1,10 +1,12 @@ @use "../../../../../../scss/colors"; +@use "../../../../../../scss/variables"; .links { width: 100%; display: flex; flex-direction: row; justify-content: flex-end; + margin-bottom: variables.$spacing-xl; .formLink { font-size: 11px; From 4653af27a27d797534bdc7d67348fd846bbd86a9 Mon Sep 17 00:00:00 2001 From: JJ Nilbodee Date: Tue, 6 Sep 2022 17:24:33 +0100 Subject: [PATCH 041/200] fix: Google Sheets Spec typo (#16343) Co-authored-by: Sajarin --- .../source-google-sheets/google_sheets_source/spec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.yaml b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.yaml index 923e6dc3188f..3c0fd05dff9b 100644 --- a/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.yaml +++ b/airbyte-integrations/connectors/source-google-sheets/google_sheets_source/spec.yaml @@ -1,7 +1,7 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/google-sheets connectionSpecification: $schema: http://json-schema.org/draft-07/schema# - title: Stripe Source Spec + title: Google Sheets Source Spec type: object required: - spreadsheet_id From a4621f8c9696a5bcf98c946a2d98ea5ad9074205 Mon Sep 17 00:00:00 2001 From: Tim Chan Date: Tue, 6 Sep 2022 09:30:32 -0700 Subject: [PATCH 042/200] use refactored find a pat script that writes token to env (not base64 encoded) (#16321) --- .github/workflows/slash-commands.yml | 15 ++++++++++++++- tools/bin/find_non_rate_limited_PAT | 1 + 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/slash-commands.yml b/.github/workflows/slash-commands.yml index 98f8263138ed..e87c6ed15ec7 100644 --- a/.github/workflows/slash-commands.yml +++ b/.github/workflows/slash-commands.yml @@ -8,17 +8,29 @@ jobs: if: ${{ github.event.issue.pull_request }} runs-on: ubuntu-latest steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + + - name: Check PAT rate limits + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} + - name: Get PR repo and ref id: getref run: | pr_info="$(curl ${{ github.event.issue.pull_request.url }})" echo ::set-output name=ref::"$(echo $pr_info | jq -r '.head.ref')" echo ::set-output name=repo::"$(echo $pr_info | jq -r '.head.repo.full_name')" + - name: Slash Command Dispatch id: scd uses: peter-evans/slash-command-dispatch@v2 with: - token: ${{ secrets.SUPERTOPHER_PAT }} + token: ${{ env.PAT }} commands: | test test-performance @@ -34,6 +46,7 @@ jobs: gitref=${{ steps.getref.outputs.ref }} comment-id=${{ github.event.comment.id }} dispatch-type: workflow + - name: Edit comment with error message if: steps.scd.outputs.error-message uses: peter-evans/create-or-update-comment@v1 diff --git a/tools/bin/find_non_rate_limited_PAT b/tools/bin/find_non_rate_limited_PAT index f4f79df432f3..99e0e2e92907 100755 --- a/tools/bin/find_non_rate_limited_PAT +++ b/tools/bin/find_non_rate_limited_PAT @@ -42,6 +42,7 @@ for personal_access_token in $@ echo -e "$blue_text""Found a good PAT!!""$default_text" # ::set-output name is a github action magic string for output echo "::set-output name=pat::$base64_valid_pat" + echo "PAT=$personal_access_token" >> $GITHUB_ENV exit 0 else echo -e "$red_text""Rate limit exceed for this PAT!""$default_text" From 11692b9b43a0b4d324e2c8a40fe992cc0d9ffcfd Mon Sep 17 00:00:00 2001 From: Amruta Ranade <11484018+Amruta-Ranade@users.noreply.github.com> Date: Tue, 6 Sep 2022 12:33:37 -0400 Subject: [PATCH 043/200] updated google analytics docs (#16362) --- .../sources/google-analytics-universal-analytics.md | 2 ++ docs/integrations/sources/google-analytics-v4.md | 2 +- docs/integrations/sources/zendesk-talk.md | 10 +++++----- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/integrations/sources/google-analytics-universal-analytics.md b/docs/integrations/sources/google-analytics-universal-analytics.md index 884cd01530ae..8e60ed9e1e7d 100644 --- a/docs/integrations/sources/google-analytics-universal-analytics.md +++ b/docs/integrations/sources/google-analytics-universal-analytics.md @@ -2,6 +2,8 @@ This page contains the setup guide and reference information for the Google Analytics (Universal Analytics) source connector. +This connector supports Universal Analytics properties through the [Reporting API v4](https://developers.google.com/analytics/devguides/reporting/core/v4). + ## Set up Google Sheets as a source in Airbyte ### For Airbyte Cloud diff --git a/docs/integrations/sources/google-analytics-v4.md b/docs/integrations/sources/google-analytics-v4.md index 44255df8b399..b8c731210008 100644 --- a/docs/integrations/sources/google-analytics-v4.md +++ b/docs/integrations/sources/google-analytics-v4.md @@ -2,7 +2,7 @@ This page guides you through the process of setting up the Google Analytics source connector. -This connector supports [Google Analytics v4](https://developers.google.com/analytics/devguides/collection/ga4). +This connector supports GA4 properties through the [Analytics Data API v1](https://developers.google.com/analytics/devguides/reporting/data/v1). ## Prerequisites diff --git a/docs/integrations/sources/zendesk-talk.md b/docs/integrations/sources/zendesk-talk.md index 1a634a197742..cb5de11c3c15 100644 --- a/docs/integrations/sources/zendesk-talk.md +++ b/docs/integrations/sources/zendesk-talk.md @@ -14,17 +14,17 @@ We recommend creating a restricted, read-only key specifically for Airbyte acces Another option is to use OAuth2.0 for authentication. See [Zendesk docs](https://support.zendesk.com/hc/en-us/articles/4408845965210-Using-OAuth-authentication-with-your-application) for details. -## Step 2: Set up the connector in Airbyte +## Step 2: Set up the Zendesk Talk connector in Airbyte ### For Airbyte Cloud: 1. [Log into your Airbyte Cloud](https://cloud.airbyte.io/workspaces) account. 2. In the left navigation bar, click **Sources**. In the top-right corner, click **+new source**. -3. On the Set up the source page, enter the name for the connector and select **Zendesk Talk** from the Source type dropdown. +3. On the Set up the source page, enter the name for the Zendesk Talk connector and select **Zendesk Talk** from the Source type dropdown. 4. Fill in the rest of the fields: - * *Subdomain* - * *Start Date* - * *Authentication (API Token / OAuth2.0)* + - *Subdomain* + - *Start Date* + - *Authentication (API Token / OAuth2.0)* 5. Click **Set up source** ## Supported sync modes From ce201bc4129b34e27550c6c195481349eccf0360 Mon Sep 17 00:00:00 2001 From: Charles Date: Tue, 6 Sep 2022 09:34:18 -0700 Subject: [PATCH 044/200] deprecate import / export endpoints (#16175) * deprecate import / export endpoints * remove fe deps on import / export * additional fe clean up --- airbyte-api/src/main/openapi/config.yaml | 94 ------- .../airbyte/server/apis/ConfigurationApi.java | 30 --- airbyte-webapp/package-lock.json | 71 ----- airbyte-webapp/package.json | 1 - .../components/FileDropZone/FileDropZone.tsx | 79 ------ .../src/components/FileDropZone/index.tsx | 3 - airbyte-webapp/src/core/ApiServices.tsx | 2 - .../domain/deployment/DeploymentService.ts | 13 - airbyte-webapp/src/locales/en.json | 2 - .../ConfigurationsPage/ConfigurationsPage.tsx | 119 +------- .../components/ImportConfigurationModal.tsx | 101 ------- .../api/generated-api-html/index.html | 253 ------------------ 12 files changed, 2 insertions(+), 766 deletions(-) delete mode 100644 airbyte-webapp/src/components/FileDropZone/FileDropZone.tsx delete mode 100644 airbyte-webapp/src/components/FileDropZone/index.tsx delete mode 100644 airbyte-webapp/src/core/domain/deployment/DeploymentService.ts delete mode 100644 airbyte-webapp/src/pages/SettingsPage/pages/ConfigurationsPage/components/ImportConfigurationModal.tsx diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index 84398e8b4187..e7baa7d5e14a 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -2212,100 +2212,6 @@ paths: schema: type: string format: binary - /v1/deployment/export: - post: - tags: - - deployment - summary: Export Airbyte Configuration and Data Archive - operationId: exportArchive - responses: - "200": - description: Successful operation - content: - application/x-gzip: - schema: - $ref: "#/components/schemas/AirbyteArchive" - /v1/deployment/import: - post: - tags: - - deployment - summary: Import Airbyte Configuration and Data Archive - operationId: importArchive - requestBody: - content: - application/x-gzip: - schema: - $ref: "#/components/schemas/AirbyteArchive" - required: true - responses: - "200": - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/ImportRead" - /v1/deployment/export_workspace: - post: - tags: - - deployment - summary: Export Airbyte Workspace Configuration - operationId: exportWorkspace - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/WorkspaceIdRequestBody" - required: true - responses: - "200": - description: Successful operation - content: - application/x-gzip: - schema: - $ref: "#/components/schemas/AirbyteArchive" - /v1/deployment/upload_archive_resource: - post: - tags: - - deployment - summary: Upload a GZIP archive tarball and stage it in the server's cache as a temporary resource - operationId: uploadArchiveResource - requestBody: - content: - application/x-gzip: - schema: - $ref: "#/components/schemas/AirbyteArchive" - required: true - responses: - "200": - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/UploadRead" - /v1/deployment/import_into_workspace: - post: - tags: - - deployment - summary: > - Import Airbyte Configuration into Workspace (this operation might change ids of imported - configurations). Note, in order to use this api endpoint, you might need to upload a - temporary archive resource with 'deployment/upload_archive_resource' first - operationId: importIntoWorkspace - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/ImportRequestBody" - required: true - responses: - "200": - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/ImportRead" - "404": - $ref: "#/components/responses/NotFoundResponse" /v1/attempt/set_workflow_in_attempt: post: tags: diff --git a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java index 004c461c1814..f568981233d3 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java +++ b/airbyte-server/src/main/java/io/airbyte/server/apis/ConfigurationApi.java @@ -42,8 +42,6 @@ import io.airbyte.api.model.generated.DestinationSearch; import io.airbyte.api.model.generated.DestinationUpdate; import io.airbyte.api.model.generated.HealthCheckRead; -import io.airbyte.api.model.generated.ImportRead; -import io.airbyte.api.model.generated.ImportRequestBody; import io.airbyte.api.model.generated.InternalOperationResult; import io.airbyte.api.model.generated.JobDebugInfoRead; import io.airbyte.api.model.generated.JobIdRequestBody; @@ -86,7 +84,6 @@ import io.airbyte.api.model.generated.SourceReadList; import io.airbyte.api.model.generated.SourceSearch; import io.airbyte.api.model.generated.SourceUpdate; -import io.airbyte.api.model.generated.UploadRead; import io.airbyte.api.model.generated.WebBackendConnectionCreate; import io.airbyte.api.model.generated.WebBackendConnectionRead; import io.airbyte.api.model.generated.WebBackendConnectionReadList; @@ -845,33 +842,6 @@ public WebBackendWorkspaceStateResult webBackendGetWorkspaceState(final WebBacke return execute(() -> webBackendConnectionsHandler.getWorkspaceState(webBackendWorkspaceState)); } - // ARCHIVES - - @Override - public File exportArchive() { - return execute(archiveHandler::exportData); - } - - @Override - public ImportRead importArchive(final File archiveFile) { - return execute(() -> archiveHandler.importData(archiveFile)); - } - - @Override - public File exportWorkspace(final WorkspaceIdRequestBody workspaceIdRequestBody) { - return execute(() -> archiveHandler.exportWorkspace(workspaceIdRequestBody)); - } - - @Override - public UploadRead uploadArchiveResource(final File archiveFile) { - return execute(() -> archiveHandler.uploadArchiveResource(archiveFile)); - } - - @Override - public ImportRead importIntoWorkspace(final ImportRequestBody importRequestBody) { - return execute(() -> archiveHandler.importIntoWorkspace(importRequestBody)); - } - @Override public InternalOperationResult setWorkflowInAttempt(final SetWorkflowInAttemptRequestBody requestBody) { return execute(() -> attemptHandler.setWorkflowInAttempt(requestBody)); diff --git a/airbyte-webapp/package-lock.json b/airbyte-webapp/package-lock.json index 8b788d2035ac..3d6b793e3c53 100644 --- a/airbyte-webapp/package-lock.json +++ b/airbyte-webapp/package-lock.json @@ -31,7 +31,6 @@ "query-string": "^6.13.1", "react": "^17.0.2", "react-dom": "^17.0.2", - "react-dropzone": "^11.5.3", "react-helmet": "6.1.0", "react-intl": "^5.24.8", "react-lazylog": "^4.5.3", @@ -16409,14 +16408,6 @@ "node": ">= 4.5.0" } }, - "node_modules/attr-accept": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.2.2.tgz", - "integrity": "sha512-7prDjvt9HmqiZ0cl5CRjtS84sEyhsHP2coDkaZKRKVfCDo9s7iw7ChVmar78Gu9pC4SoR/28wFu/G5JJhTnqEg==", - "engines": { - "node": ">=4" - } - }, "node_modules/autoprefixer": { "version": "9.8.8", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-9.8.8.tgz", @@ -22734,22 +22725,6 @@ "url": "https://opencollective.com/webpack" } }, - "node_modules/file-selector": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/file-selector/-/file-selector-0.2.4.tgz", - "integrity": "sha512-ZDsQNbrv6qRi1YTDOEWzf5J2KjZ9KMI1Q2SGeTkCJmNNW25Jg4TW4UMcmoqcg4WrAyKRcpBXdbWRxkfrOzVRbA==", - "dependencies": { - "tslib": "^2.0.3" - }, - "engines": { - "node": ">= 10" - } - }, - "node_modules/file-selector/node_modules/tslib": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.1.0.tgz", - "integrity": "sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A==" - }, "node_modules/file-system-cache": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/file-system-cache/-/file-system-cache-1.1.0.tgz", @@ -36263,22 +36238,6 @@ "react": "17.0.2" } }, - "node_modules/react-dropzone": { - "version": "11.5.3", - "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-11.5.3.tgz", - "integrity": "sha512-68+T6sWW5L89qJnn3SD1aRazhuRBhTT9JOI1W8vI5YWsfegM4C7tlGbPH1AgEbmZY5s8E8L0QhX0e3VdAa0KWA==", - "dependencies": { - "attr-accept": "^2.2.1", - "file-selector": "^0.2.2", - "prop-types": "^15.7.2" - }, - "engines": { - "node": ">= 10" - }, - "peerDependencies": { - "react": ">= 16.8" - } - }, "node_modules/react-element-to-jsx-string": { "version": "14.3.4", "resolved": "https://registry.npmjs.org/react-element-to-jsx-string/-/react-element-to-jsx-string-14.3.4.tgz", @@ -60213,11 +60172,6 @@ "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", "dev": true }, - "attr-accept": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/attr-accept/-/attr-accept-2.2.2.tgz", - "integrity": "sha512-7prDjvt9HmqiZ0cl5CRjtS84sEyhsHP2coDkaZKRKVfCDo9s7iw7ChVmar78Gu9pC4SoR/28wFu/G5JJhTnqEg==" - }, "autoprefixer": { "version": "9.8.8", "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-9.8.8.tgz", @@ -65219,21 +65173,6 @@ } } }, - "file-selector": { - "version": "0.2.4", - "resolved": "https://registry.npmjs.org/file-selector/-/file-selector-0.2.4.tgz", - "integrity": "sha512-ZDsQNbrv6qRi1YTDOEWzf5J2KjZ9KMI1Q2SGeTkCJmNNW25Jg4TW4UMcmoqcg4WrAyKRcpBXdbWRxkfrOzVRbA==", - "requires": { - "tslib": "^2.0.3" - }, - "dependencies": { - "tslib": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.1.0.tgz", - "integrity": "sha512-hcVC3wYEziELGGmEEXue7D75zbwIIVUMWAVbHItGPx0ziyXxrOMQx4rQEVEV45Ut/1IotuEvwqPopzIOkDMf0A==" - } - } - }, "file-system-cache": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/file-system-cache/-/file-system-cache-1.1.0.tgz", @@ -75505,16 +75444,6 @@ "scheduler": "^0.20.2" } }, - "react-dropzone": { - "version": "11.5.3", - "resolved": "https://registry.npmjs.org/react-dropzone/-/react-dropzone-11.5.3.tgz", - "integrity": "sha512-68+T6sWW5L89qJnn3SD1aRazhuRBhTT9JOI1W8vI5YWsfegM4C7tlGbPH1AgEbmZY5s8E8L0QhX0e3VdAa0KWA==", - "requires": { - "attr-accept": "^2.2.1", - "file-selector": "^0.2.2", - "prop-types": "^15.7.2" - } - }, "react-element-to-jsx-string": { "version": "14.3.4", "resolved": "https://registry.npmjs.org/react-element-to-jsx-string/-/react-element-to-jsx-string-14.3.4.tgz", diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index c1b9eaf54348..45439bd0ff0c 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -47,7 +47,6 @@ "query-string": "^6.13.1", "react": "^17.0.2", "react-dom": "^17.0.2", - "react-dropzone": "^11.5.3", "react-helmet": "6.1.0", "react-intl": "^5.24.8", "react-lazylog": "^4.5.3", diff --git a/airbyte-webapp/src/components/FileDropZone/FileDropZone.tsx b/airbyte-webapp/src/components/FileDropZone/FileDropZone.tsx deleted file mode 100644 index 4fa1296c95c5..000000000000 --- a/airbyte-webapp/src/components/FileDropZone/FileDropZone.tsx +++ /dev/null @@ -1,79 +0,0 @@ -import { faFile } from "@fortawesome/free-regular-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; -import React from "react"; -import { DropzoneOptions, useDropzone } from "react-dropzone"; -import styled from "styled-components"; - -const Content = styled.div<{ hasFiles: boolean }>` - width: 100%; - outline: none; - padding: 12px 10px; - border-radius: 4px; - font-size: 14px; - line-height: 20px; - font-weight: normal; - border: 1px solid ${({ theme, hasFiles }) => (hasFiles ? theme.primaryColor : theme.greyColor0)}; - background: ${({ theme, hasFiles }) => (hasFiles ? theme.primaryColor12 : theme.greyColor0)}; - color: ${({ theme }) => theme.greyColor40}; - cursor: pointer; - min-height: 95px; - text-align: center; - display: flex; - justify-content: center; - align-items: center; - flex-direction: column; - - &:hover { - background: ${({ theme }) => theme.greyColor20}; - border-color: ${({ theme }) => theme.greyColor20}; - } - - &:active { - border-color: ${({ theme }) => theme.primaryColor}; - } - - &:disabled { - pointer-events: none; - background: ${({ theme }) => theme.greyColor55}; - } -`; - -const FileView = styled.div` - color: ${({ theme }) => theme.textColor}; - - &:first-child { - margin-top: 7px; - } -`; - -const FileIcon = styled(FontAwesomeIcon)` - font-size: 16px; - margin-right: 8px; -`; - -interface IProps { - className?: string; - mainText?: React.ReactNode; - options?: DropzoneOptions; -} - -const FileDropZone: React.FC = ({ className, mainText, options }) => { - const { acceptedFiles, getRootProps, getInputProps } = useDropzone(options); - - return ( - - - {mainText} -
- {acceptedFiles.map((file, index) => ( - - - {file.name} - - ))} -
- - ); -}; - -export default FileDropZone; diff --git a/airbyte-webapp/src/components/FileDropZone/index.tsx b/airbyte-webapp/src/components/FileDropZone/index.tsx deleted file mode 100644 index cd65b979540a..000000000000 --- a/airbyte-webapp/src/components/FileDropZone/index.tsx +++ /dev/null @@ -1,3 +0,0 @@ -import FileDropZone from "./FileDropZone"; - -export default FileDropZone; diff --git a/airbyte-webapp/src/core/ApiServices.tsx b/airbyte-webapp/src/core/ApiServices.tsx index 6f52c327ea99..5845ff9e3b5e 100644 --- a/airbyte-webapp/src/core/ApiServices.tsx +++ b/airbyte-webapp/src/core/ApiServices.tsx @@ -5,7 +5,6 @@ import { useConfig } from "config"; import { OperationService } from "./domain/connection"; import { DestinationDefinitionService } from "./domain/connector/DestinationDefinitionService"; import { SourceDefinitionService } from "./domain/connector/SourceDefinitionService"; -import { DeploymentService } from "./domain/deployment/DeploymentService"; import { HealthService } from "./health/HealthService"; import { RequestMiddleware } from "./request/RequestMiddleware"; import { useGetService, useInjectServices } from "./servicesProvider"; @@ -18,7 +17,6 @@ export const ApiServices: React.FC = React.memo(({ children }) => { () => ({ SourceDefinitionService: new SourceDefinitionService(config.apiUrl, middlewares), DestinationDefinitionService: new DestinationDefinitionService(config.apiUrl, middlewares), - DeploymentService: new DeploymentService(config.apiUrl, middlewares), OperationService: new OperationService(config.apiUrl, middlewares), HealthService: new HealthService(config.apiUrl, middlewares), }), diff --git a/airbyte-webapp/src/core/domain/deployment/DeploymentService.ts b/airbyte-webapp/src/core/domain/deployment/DeploymentService.ts deleted file mode 100644 index 612092fd3a57..000000000000 --- a/airbyte-webapp/src/core/domain/deployment/DeploymentService.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { AirbyteArchive, exportArchive, importArchive } from "../../request/AirbyteClient"; -import { AirbyteRequestService } from "../../request/AirbyteRequestService"; - -export class DeploymentService extends AirbyteRequestService { - public async exportDeployment() { - const blob = await exportArchive(this.requestOptions); - return window.URL.createObjectURL(blob); - } - - public async importDeployment(file: AirbyteArchive) { - await importArchive(file, this.requestOptions); - } -} diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 3afe8a7e0ecb..ab84608c894e 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -441,8 +441,6 @@ "admin.learnMore": "Learn more from our documentation to understand how to fill these field.", "admin.exportConfigurationText": "Download an archive of all configuration and state data. Exported data can be upgraded to another Airbyte version or can be exported to a different Airbyte deployment. For more information visit the configuration archive page in our docs.", "admin.importConfigurationText": "Upload an archive of all configuration and state data. Warning: This will overwrite all existing configuration!", - "admin.dropZoneTitle": "Drag 'n' drop file here, or click to select file", - "admin.dropZoneSubtitle": "Only *.tar and *.gz files will be accepted", "admin.logs": "Logs", "admin.logs.error": "Unable to download logs at this time.", "admin.downloadServerLogs": "Download Server Logs", diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConfigurationsPage/ConfigurationsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConfigurationsPage/ConfigurationsPage.tsx index 31b192b0002d..2e245dbef951 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConfigurationsPage/ConfigurationsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConfigurationsPage/ConfigurationsPage.tsx @@ -1,16 +1,10 @@ -import React, { useState } from "react"; +import React from "react"; import { FormattedMessage } from "react-intl"; -import { useAsyncFn } from "react-use"; import styled from "styled-components"; -import { Button, ContentCard, Link, LoadingButton } from "components"; +import { ContentCard } from "components"; import HeadTitle from "components/HeadTitle"; -import { useConfig } from "config"; -import { DeploymentService } from "core/domain/deployment/DeploymentService"; -import { useServicesProvider } from "core/servicesProvider"; - -import ImportConfigurationModal from "./components/ImportConfigurationModal"; import LogsContent from "./components/LogsContent"; const Content = styled.div` @@ -21,119 +15,10 @@ const ControlContent = styled(ContentCard)` margin-top: 12px; `; -const ButtonContent = styled.div` - padding: 29px 28px 27px; - display: flex; - align-items: center; -`; - -const Text = styled.div` - margin-left: 20px; - font-size: 11px; - line-height: 13px; - color: ${({ theme }) => theme.greyColor40}; - white-space: pre-line; - flex: 1 0 0; -`; - -const DocLink = styled(Link).attrs({ as: "a" })` - text-decoration: none; - display: inline-block; -`; - -const Warning = styled.div` - font-weight: bold; -`; - const ConfigurationsPage: React.FC = () => { - const config = useConfig(); - const { getService } = useServicesProvider(); - const [isModalOpen, setIsModalOpen] = useState(false); - const [error, setError] = useState(null); - - const [{ loading }, onImport] = useAsyncFn( - async (fileBlob: Blob) => { - try { - const reader = new FileReader(); - reader.readAsArrayBuffer(fileBlob); - - return new Promise((resolve, reject) => { - reader.onloadend = async (e) => { - const file = e?.target?.result; - if (!file) { - throw new Error("No file"); - } - - try { - const deploymentService = getService("DeploymentService"); - await deploymentService.importDeployment(new Blob([file])); - window.location.reload(); - resolve(true); - } catch (e) { - reject(e); - } - }; - }); - } catch (e) { - setError(e); - } - }, - [getService] - ); - - const [{ loading: loadingExport }, onExport] = useAsyncFn(async () => { - const deploymentService = getService("DeploymentService"); - const file = await deploymentService.exportDeployment(); - window.location.assign(file); - }, []); - return ( - }> - - - - - - ( - - {lnk} - - ), - }} - /> - - - - - }> - - - - {warn}, - }} - /> - - {isModalOpen && ( - setIsModalOpen(false)} - onSubmit={onImport} - isLoading={loading} - error={error} - cleanError={() => setError(null)} - /> - )} - - }> diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConfigurationsPage/components/ImportConfigurationModal.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConfigurationsPage/components/ImportConfigurationModal.tsx deleted file mode 100644 index 2b25004ce84e..000000000000 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConfigurationsPage/components/ImportConfigurationModal.tsx +++ /dev/null @@ -1,101 +0,0 @@ -import React, { useState } from "react"; -import { FormattedMessage } from "react-intl"; -import styled from "styled-components"; - -import { Button } from "components"; -import FileDropZone from "components/FileDropZone"; -import Modal from "components/Modal"; - -export interface IProps { - onClose: () => void; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - onSubmit: (data: any) => void; - message?: React.ReactNode; - isLoading?: boolean; - error?: Error | null | boolean; - cleanError?: () => void; -} - -const Content = styled.div` - padding: 18px 37px 28px; - font-size: 14px; - line-height: 28px; - width: 485px; -`; - -const ButtonWithMargin = styled(Button)` - margin-right: 9px; -`; - -const DropZoneSubtitle = styled.div` - font-size: 11px; - font-weight: bold; -`; - -const Bottom = styled.div` - display: flex; - justify-content: space-between; - align-items: center; - flex-direction: row; - padding-top: 27px; -`; - -const Error = styled.div` - color: ${({ theme }) => theme.dangerColor}; - font-size: 14px; - line-height: 17px; - margin-right: 10px; -`; - -const Note = styled.div` - padding-top: 8px; - text-align: center; -`; - -const DropZoneMainText = () => ( -
- - - - -
-); - -const ImportConfigurationModal: React.FC = ({ onClose, onSubmit, isLoading, error, cleanError }) => { - const [usersFile, setUsersFile] = useState(null); - - return ( - }> - - } - options={{ - onDrop: (files) => { - setUsersFile(files[0]); - cleanError?.(); - }, - maxFiles: 1, - accept: - "application/x-zip-compressed, application/zip, application/x-gzip, application/x-gtar, application/x-tgz", - }} - /> - - - - - {error ? : null} -
- - - - -
- - - - ); -}; - -export default ImportConfigurationModal; diff --git a/docs/reference/api/generated-api-html/index.html b/docs/reference/api/generated-api-html/index.html index 0de7ded6d3e0..5b98235929f7 100644 --- a/docs/reference/api/generated-api-html/index.html +++ b/docs/reference/api/generated-api-html/index.html @@ -243,14 +243,6 @@

DbMigration

  • post /v1/db_migrations/migrate
  • post /v1/db_migrations/list
    • -

    Deployment

    -

    Destination

    -

    Deployment

    -
    -
    - Up - 
    post /v1/deployment/export
    -
    Export Airbyte Configuration and Data Archive (exportArchive)
    -
    - - - - - - - -

    Return type

    -
    - - File -
    - - - - -

    Produces

    - This API call produces the following media types according to the Accept request header; - the media type will be conveyed by the Content-Type response header. -
      -
    • application/x-gzip
      • -
    - -

    Responses

    -

    200

    - Successful operation - File -
    -
    -
    -
    - Up - 
    post /v1/deployment/export_workspace
    -
    Export Airbyte Workspace Configuration (exportWorkspace)
    -
    - - -

    Consumes

    - This API call consumes the following media types via the Content-Type request header: -
      -
    • application/json
      • -
    - -

    Request body

    -
    -
    WorkspaceIdRequestBody WorkspaceIdRequestBody (required)
    - -
    Body Parameter —
    - -
    - - - - -

    Return type

    -
    - - File -
    - - - - -

    Produces

    - This API call produces the following media types according to the Accept request header; - the media type will be conveyed by the Content-Type response header. -
      -
    • application/x-gzip
      • -
    - -

    Responses

    -

    200

    - Successful operation - File -
    -
    -
    -
    - Up - 
    post /v1/deployment/import
    -
    Import Airbyte Configuration and Data Archive (importArchive)
    -
    - - -

    Consumes

    - This API call consumes the following media types via the Content-Type request header: -
      -
    • application/x-gzip
      • -
    - -

    Request body

    -
    -
    body file (required)
    - -
    Body Parameter —
    - -
    - - - - -

    Return type

    -
    - ImportRead - -
    - - - -

    Example data

    -
    Content-Type: application/json
    - 
    {
-  "reason" : "reason",
-  "status" : "succeeded"
-}
    - -

    Produces

    - This API call produces the following media types according to the Accept request header; - the media type will be conveyed by the Content-Type response header. -
      -
    • application/json
      • -
    - -

    Responses

    -

    200

    - Successful operation - ImportRead -
    -
    -
    -
    - Up - 
    post /v1/deployment/import_into_workspace
    -
    Import Airbyte Configuration into Workspace (this operation might change ids of imported configurations). Note, in order to use this api endpoint, you might need to upload a temporary archive resource with 'deployment/upload_archive_resource' first (importIntoWorkspace)
    -
    - - -

    Consumes

    - This API call consumes the following media types via the Content-Type request header: -
      -
    • application/json
      • -
    - -

    Request body

    -
    -
    ImportRequestBody ImportRequestBody (required)
    - -
    Body Parameter —
    - -
    - - - - -

    Return type

    -
    - ImportRead - -
    - - - -

    Example data

    -
    Content-Type: application/json
    - 
    {
-  "reason" : "reason",
-  "status" : "succeeded"
-}
    - -

    Produces

    - This API call produces the following media types according to the Accept request header; - the media type will be conveyed by the Content-Type response header. -
      -
    • application/json
      • -
    - -

    Responses

    -

    200

    - Successful operation - ImportRead -

    404

    - Object with given id was not found. - NotFoundKnownExceptionInfo -
    -
    -
    -
    - Up - 
    post /v1/deployment/upload_archive_resource
    -
    Upload a GZIP archive tarball and stage it in the server's cache as a temporary resource (uploadArchiveResource)
    -
    - - -

    Consumes

    - This API call consumes the following media types via the Content-Type request header: -
      -
    • application/x-gzip
      • -
    - -

    Request body

    -
    -
    body file (required)
    - -
    Body Parameter —
    - -
    - - - - -

    Return type

    -
    - UploadRead - -
    - - - -

    Example data

    -
    Content-Type: application/json
    - 
    {
-  "resourceId" : "046b6c7f-0b8a-43b9-b35d-6489e6daee91",
-  "status" : "succeeded"
-}
    - -

    Produces

    - This API call produces the following media types according to the Accept request header; - the media type will be conveyed by the Content-Type response header. -
      -
    • application/json
      • -
    - -

    Responses

    -

    200

    - Successful operation - UploadRead -
    -

    Destination

    From dd6baf905c04038d9de5bff4cf09d752afb7f9aa Mon Sep 17 00:00:00 2001 From: Yurii Bidiuk <35812734+yurii-bidiuk@users.noreply.github.com> Date: Tue, 6 Sep 2022 21:44:24 +0300 Subject: [PATCH 045/200] Source DB2: Add custom jdbc params (#16354) * DB2 add custom jdbc params * bump version * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 11 ++- .../source-db2-strict-encrypt/Dockerfile | 2 +- .../src/test/resources/expected_spec.json | 8 +- .../connectors/source-db2/Dockerfile | 2 +- .../connectors/source-db2/build.gradle | 1 + .../Db2Source.java | 5 + .../source-db2/src/main/resources/spec.json | 8 +- .../Db2SpecTest.java | 91 +++++++++++++++++++ docs/integrations/sources/db2.md | 3 +- 10 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2SpecTest.java diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 64c3ae44816b..2281cc52391b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -449,7 +449,7 @@ - name: IBM Db2 sourceDefinitionId: 447e0381-3780-4b46-bb62-00a4e3c8b8e2 dockerRepository: airbyte/source-db2 - dockerImageTag: 0.1.15 + dockerImageTag: 0.1.16 documentationUrl: https://docs.airbyte.io/integrations/sources/db2 icon: db2.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index ad6fadaa1865..8d1f68e261ed 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4016,7 +4016,7 @@ - - "client_secret" oauthFlowOutputParameters: - - "refresh_token" -- dockerImage: "airbyte/source-db2:0.1.15" +- dockerImage: "airbyte/source-db2:0.1.16" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/db2" connectionSpecification: @@ -4059,11 +4059,18 @@ type: "string" airbyte_secret: true order: 4 + jdbc_url_params: + description: "Additional properties to pass to the JDBC URL string when\ + \ connecting to the database formatted as 'key=value' pairs separated\ + \ by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." + title: "JDBC URL Params" + type: "string" + order: 5 encryption: title: "Encryption" type: "object" description: "Encryption method to use when communicating with the database" - order: 5 + order: 6 oneOf: - title: "Unencrypted" description: "Data transfer will not be encrypted." diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile index 18a2bcd8fdfa..ab7aecf54702 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-db2-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.15 +LABEL io.airbyte.version=0.1.16 LABEL io.airbyte.name=airbyte/source-db2-strict-encrypt diff --git a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/resources/expected_spec.json b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/resources/expected_spec.json index eb30afcfe92c..0e2de7cf80c0 100644 --- a/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-db2-strict-encrypt/src/test/resources/expected_spec.json @@ -37,11 +37,17 @@ "airbyte_secret": true, "order": 4 }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "title": "JDBC URL Params", + "type": "string", + "order": 5 + }, "encryption": { "title": "Encryption", "type": "object", "description": "Encryption method to use when communicating with the database", - "order": 5, + "order": 6, "oneOf": [ { "title": "TLS Encrypted (verify certificate)", diff --git a/airbyte-integrations/connectors/source-db2/Dockerfile b/airbyte-integrations/connectors/source-db2/Dockerfile index 41d96d988a14..0bb990e58705 100644 --- a/airbyte-integrations/connectors/source-db2/Dockerfile +++ b/airbyte-integrations/connectors/source-db2/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-db2 COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.15 +LABEL io.airbyte.version=0.1.16 LABEL io.airbyte.name=airbyte/source-db2 diff --git a/airbyte-integrations/connectors/source-db2/build.gradle b/airbyte-integrations/connectors/source-db2/build.gradle index fb6fda9c43d9..34d5d340fdc2 100644 --- a/airbyte-integrations/connectors/source-db2/build.gradle +++ b/airbyte-integrations/connectors/source-db2/build.gradle @@ -22,6 +22,7 @@ dependencies { testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) testImplementation project(':airbyte-test-utils') testImplementation libs.connectors.testcontainers.db2 + testImplementation project(":airbyte-json-validation") integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') integrationTestJavaImplementation project(':airbyte-integrations:connectors:source-db2') diff --git a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java index 49bbc1feca3e..de55925b1bc9 100644 --- a/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java +++ b/airbyte-integrations/connectors/source-db2/src/main/java/io.airbyte.integrations.source.db2/Db2Source.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.source.db2; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.functional.CheckedFunction; import io.airbyte.commons.json.Jsons; @@ -79,6 +80,10 @@ public JsonNode toDatabaseConfig(final JsonNode config) { .build()); } + if (config.get(JdbcUtils.JDBC_URL_PARAMS_KEY) != null && !config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText().isEmpty()) { + ((ObjectNode) result).put(JdbcUtils.JDBC_URL_PARAMS_KEY, config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText()); + } + return result; } diff --git a/airbyte-integrations/connectors/source-db2/src/main/resources/spec.json b/airbyte-integrations/connectors/source-db2/src/main/resources/spec.json index 31af53be8891..f96481498c01 100644 --- a/airbyte-integrations/connectors/source-db2/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-db2/src/main/resources/spec.json @@ -37,11 +37,17 @@ "airbyte_secret": true, "order": 4 }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3).", + "title": "JDBC URL Params", + "type": "string", + "order": 5 + }, "encryption": { "title": "Encryption", "type": "object", "description": "Encryption method to use when communicating with the database", - "order": 5, + "order": 6, "oneOf": [ { "title": "Unencrypted", diff --git a/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2SpecTest.java b/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2SpecTest.java new file mode 100644 index 000000000000..84cf9e347489 --- /dev/null +++ b/airbyte-integrations/connectors/source-db2/src/test/java/io.airbyte.integrations.source.db2/Db2SpecTest.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.db2; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.airbyte.commons.io.IOs; +import io.airbyte.commons.json.Jsons; +import io.airbyte.commons.resources.MoreResources; +import io.airbyte.protocol.models.ConnectorSpecification; +import io.airbyte.validation.json.JsonSchemaValidator; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class Db2SpecTest { + + private static JsonNode schema; + private static JsonNode config; + private static String configText; + private static JsonSchemaValidator validator; + + @BeforeAll + static void init() throws IOException { + configText = """ + { + "host": "localhost", + "port": 1521, + "db": "db", + "username": "test", + "password": "password", + "jdbc_url_params": "property1=pValue1&property2=pValue2" + } + """; + final String spec = MoreResources.readResource("spec.json"); + final File schemaFile = IOs.writeFile(Files.createTempDirectory(Path.of("/tmp"), "pg-spec-test"), "schema.json", spec).toFile(); + schema = JsonSchemaValidator.getSchema(schemaFile).get("connectionSpecification"); + validator = new JsonSchemaValidator(); + } + + @BeforeEach + void beforeEach() { + config = Jsons.deserialize(configText); + } + + @Test + void testHostMissing() { + ((ObjectNode) config).remove("host"); + assertFalse(validator.test(schema, config)); + } + + @Test + void testPortMissing() { + ((ObjectNode) config).remove("port"); + assertFalse(validator.test(schema, config)); + } + + @Test + void testDatabaseMissing() { + ((ObjectNode) config).remove("db"); + assertFalse(validator.test(schema, config)); + } + + @Test + void testUsernameMissing() { + ((ObjectNode) config).remove("username"); + assertFalse(validator.test(schema, config)); + } + + @Test + void testPasswordMissing() { + ((ObjectNode) config).remove("password"); + assertFalse(validator.test(schema, config)); + } + + @Test + void testJdbcAdditionalProperty() throws Exception { + final ConnectorSpecification spec = new Db2Source().spec(); + assertNotNull(spec.getConnectionSpecification().get("properties").get("jdbc_url_params")); + } + +} diff --git a/docs/integrations/sources/db2.md b/docs/integrations/sources/db2.md index b03abb3b0c83..d84dfb86cee5 100644 --- a/docs/integrations/sources/db2.md +++ b/docs/integrations/sources/db2.md @@ -62,7 +62,8 @@ You can also enter your own password for the keystore, but if you don't, the pas | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.15 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | +| 0.1.16 | 2022-09-06 | [16354](https://github.com/airbytehq/airbyte/pull/16354) | Add custom JDBC params | +| 0.1.15 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | | 0.1.14 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | | 0.1.13 | 2022-07-22 | [14714](https://github.com/airbytehq/airbyte/pull/14714) | Clarified error message when invalid cursor column selected | | 0.1.12 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | From 1e5e37e051c44c6abaf56070003c4dd42a46f0fb Mon Sep 17 00:00:00 2001 From: Serhii Chvaliuk Date: Wed, 7 Sep 2022 00:12:02 +0300 Subject: [PATCH 046/200] =?UTF-8?q?=F0=9F=90=9B=20Source=20google=20ads:?= =?UTF-8?q?=20remove=20"end=5Fdate"=20from=20config=20if=20empty=20value?= =?UTF-8?q?=20(#16344)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sergey Chvalyuk --- .../main/resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 4 ++-- .../connectors/source-google-ads/Dockerfile | 2 +- .../acceptance-test-config.yml | 1 + .../integration_tests/test_incremental.py | 2 +- .../source_google_ads/source.py | 17 ++++++++++++----- .../source_google_ads/spec.json | 2 +- 7 files changed, 19 insertions(+), 11 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 2281cc52391b..0d22fd9c652c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -347,7 +347,7 @@ - name: Google Ads sourceDefinitionId: 253487c0-2246-43ba-a21f-5116b20a2c50 dockerRepository: airbyte/source-google-ads - dockerImageTag: 0.1.44 + dockerImageTag: 0.1.45 documentationUrl: https://docs.airbyte.io/integrations/sources/google-ads icon: google-adwords.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 8d1f68e261ed..ca95dc1d3eb1 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2904,7 +2904,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-google-ads:0.1.44" +- dockerImage: "airbyte/source-google-ads:0.1.45" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/google-ads" connectionSpecification: @@ -2993,7 +2993,7 @@ title: "End Date (Optional)" description: "UTC date and time in the format 2017-01-25. Any data after\ \ this date will not be replicated." - pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$" + pattern: "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}$" examples: - "2017-01-30" order: 6 diff --git a/airbyte-integrations/connectors/source-google-ads/Dockerfile b/airbyte-integrations/connectors/source-google-ads/Dockerfile index 34b46891c1ee..351d0fc6d683 100644 --- a/airbyte-integrations/connectors/source-google-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-google-ads/Dockerfile @@ -13,5 +13,5 @@ COPY main.py ./ ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.44 +LABEL io.airbyte.version=0.1.45 LABEL io.airbyte.name=airbyte/source-google-ads diff --git a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml index 21cf249576d6..48f893f021f6 100644 --- a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml @@ -22,6 +22,7 @@ tests: "display_topics_performance_report", "shopping_performance_report", "unhappytable", + "click_view", ] timeout_seconds: 600 - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py b/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py index 3299e60478fd..85b85fab79ec 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/test_incremental.py @@ -35,7 +35,7 @@ def configured_catalog(): def test_incremental_sync(config, configured_catalog): today = pendulum.now().date() - start_date = today.subtract(months=2) + start_date = today.subtract(months=3) config["start_date"] = start_date.to_date_string() google_ads_client = SourceGoogleAds() diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py index 403522fa1a24..5a830a9a655b 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py @@ -38,6 +38,12 @@ class SourceGoogleAds(AbstractSource): + @staticmethod + def _validate_and_transform(config: Mapping[str, Any]): + if config.get("end_date") == "": + config.pop("end_date") + return config + @staticmethod def get_credentials(config: Mapping[str, Any]) -> MutableMapping[str, Any]: credentials = config["credentials"] @@ -52,16 +58,15 @@ def get_credentials(config: Mapping[str, Any]) -> MutableMapping[str, Any]: @staticmethod def get_incremental_stream_config(google_api: GoogleAds, config: Mapping[str, Any], customers: List[Customer]): - true_end_date = None - configured_end_date = config.get("end_date") - if configured_end_date is not None: - true_end_date = min(today(), parse(configured_end_date)).to_date_string() + end_date = config.get("end_date") + if end_date: + end_date = min(today(), parse(end_date)).to_date_string() incremental_stream_config = dict( api=google_api, customers=customers, conversion_window_days=config["conversion_window_days"], start_date=config["start_date"], - end_date=true_end_date, + end_date=end_date, ) return incremental_stream_config @@ -80,6 +85,7 @@ def is_metrics_in_custom_query(query: str) -> bool: return False def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, any]: + config = self._validate_and_transform(config) try: logger.info("Checking the config") google_api = GoogleAds(credentials=self.get_credentials(config)) @@ -109,6 +115,7 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> return False, f"Unable to connect to Google Ads API with the provided configuration - {error_messages}" def streams(self, config: Mapping[str, Any]) -> List[Stream]: + config = self._validate_and_transform(config) google_api = GoogleAds(credentials=self.get_credentials(config)) accounts = self.get_account_info(google_api, config) customers = Customer.from_accounts(accounts) diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json index 128a77125100..b929331c6080 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json @@ -75,7 +75,7 @@ "type": "string", "title": "End Date (Optional)", "description": "UTC date and time in the format 2017-01-25. Any data after this date will not be replicated.", - "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$", + "pattern": "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}$", "examples": ["2017-01-30"], "order": 6 }, From 8cd38809f9b06169786f442cd358e779d06c99d9 Mon Sep 17 00:00:00 2001 From: mlavoie-sm360 Date: Tue, 6 Sep 2022 22:16:13 +0100 Subject: [PATCH 047/200] =?UTF-8?q?=F0=9F=8E=89=20Source=20Facebook=20Mark?= =?UTF-8?q?eting:=20Added=20custom=20conversions=20stream=20(#15724)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🎉 Source Facebook Marketing: Added support for custom conversions stream * solve md conflict and update dockerfile version * Updated test_streams test * auto-bump connector version [ci skip] Co-authored-by: marcosmarxm Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../source-facebook-marketing/Dockerfile | 2 +- .../schemas/custom_conversions.json | 81 +++++++++++++++++++ .../source_facebook_marketing/source.py | 7 ++ .../streams/__init__.py | 2 + .../streams/streams.py | 9 +++ .../unit_tests/test_source.py | 2 +- .../sources/facebook-marketing.md | 2 + 9 files changed, 105 insertions(+), 4 deletions(-) create mode 100644 airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/custom_conversions.json diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0d22fd9c652c..5d37584ff1d0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -256,7 +256,7 @@ - name: Facebook Marketing sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c dockerRepository: airbyte/source-facebook-marketing - dockerImageTag: 0.2.62 + dockerImageTag: 0.2.63 documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing icon: facebook.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index ca95dc1d3eb1..2e4a586f201a 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1857,7 +1857,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-facebook-marketing:0.2.62" +- dockerImage: "airbyte/source-facebook-marketing:0.2.63" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile index 985a728288be..c2687a1c8416 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile +++ b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile @@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.62 +LABEL io.airbyte.version=0.2.63 LABEL io.airbyte.name=airbyte/source-facebook-marketing diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/custom_conversions.json b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/custom_conversions.json new file mode 100644 index 000000000000..8f699108a5d7 --- /dev/null +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/schemas/custom_conversions.json @@ -0,0 +1,81 @@ +{ + "properties": { + "id": { + "type": ["null", "string"] + }, + "account_id": { + "type": ["null", "string"] + }, + "business": { + "type": ["null", "string"] + }, + "creation_time": { + "type": "string", + "format": "date-time" + }, + "custom_event_type": { + "type": ["null", "string"] + }, + "data_sources": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "source_type": { + "type": "string" + }, + "name": { + "type": "string" + } + } + } + }, + "default_conversion_value": { + "type": ["null", "number"] + }, + "description": { + "type": ["null", "string"] + }, + "event_source_type": { + "type": ["null", "string"] + }, + "first_fired_time": { + "type": "string", + "format": "date-time" + }, + "is_archived": { + "type": ["null", "boolean"] + }, + "is_unavailable": { + "type": ["null", "boolean"] + }, + "last_fired_time": { + "type": "string", + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + }, + "offline_conversion_data_set": { + "type": ["null", "string"] + }, + "pixel": { + "type": "object", + "properties": { + "id": { + "type": "string" + } + } + }, + "retention_days": { + "type": ["null", "number"] + }, + "rule": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] +} diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py index 9a53bc42ae76..adeb1296c7e7 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py @@ -26,6 +26,7 @@ AdsInsightsPlatformAndDevice, AdsInsightsRegion, Campaigns, + CustomConversions, Images, Videos, ) @@ -116,6 +117,12 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]: page_size=config.page_size, max_batch_size=config.max_batch_size, ), + CustomConversions( + api=api, + include_deleted=config.include_deleted, + page_size=config.page_size, + max_batch_size=config.max_batch_size, + ), Images( api=api, start_date=config.start_date, diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/__init__.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/__init__.py index 83bd21d17c36..7a7822235695 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/__init__.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/__init__.py @@ -16,6 +16,7 @@ AdsInsightsPlatformAndDevice, AdsInsightsRegion, Campaigns, + CustomConversions, Images, Videos, ) @@ -33,6 +34,7 @@ "AdsInsightsPlatformAndDevice", "AdsInsightsRegion", "Campaigns", + "CustomConversions", "Images", "Videos", "Activities", diff --git a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py index 6bf55c29dc72..6787de8203eb 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/streams.py @@ -69,6 +69,15 @@ def list_objects(self, params: Mapping[str, Any]) -> Iterable: return self._api.account.get_ad_creatives(params=params) +class CustomConversions(FBMarketingStream): + """doc: https://developers.facebook.com/docs/marketing-api/reference/custom-conversion""" + + entity_prefix = "customconversion" + + def list_objects(self, params: Mapping[str, Any]) -> Iterable: + return self._api.account.get_custom_conversions(params=params) + + class Ads(FBMarketingIncrementalStream): """doc: https://developers.facebook.com/docs/marketing-api/reference/adgroup""" diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py index ed8ba0068dfa..3b0a7e3e8bf1 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_source.py @@ -92,7 +92,7 @@ def test_check_connection_exception(self, api, config, logger_mock): def test_streams(self, config, api): streams = SourceFacebookMarketing().streams(config) - assert len(streams) == 15 + assert len(streams) == 16 def test_spec(self): spec = SourceFacebookMarketing().spec() diff --git a/docs/integrations/sources/facebook-marketing.md b/docs/integrations/sources/facebook-marketing.md index 093ec63e142b..c41e00493131 100644 --- a/docs/integrations/sources/facebook-marketing.md +++ b/docs/integrations/sources/facebook-marketing.md @@ -91,6 +91,7 @@ You can replicate the following tables using the Facebook Marketing connector: * [Ads](https://developers.facebook.com/docs/marketing-api/reference/adgroup#fields) * [AdInsights](https://developers.facebook.com/docs/marketing-api/reference/adgroup/insights/) * [Campaigns](https://developers.facebook.com/docs/marketing-api/reference/ad-campaign-group#fields) +* [CustomConversions](https://developers.facebook.com/docs/marketing-api/reference/custom-conversion) * [Images](https://developers.facebook.com/docs/marketing-api/reference/ad-image) * [Videos](https://developers.facebook.com/docs/marketing-api/reference/video) @@ -120,6 +121,7 @@ Please be informed that the connector uses the `lookback_window` parameter to pe | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.2.63 | 2022-09-06 | [15724](https://github.com/airbytehq/airbyte/pull/15724) | Add the Custom Conversion stream | | 0.2.62 | 2022-09-01 | [16222](https://github.com/airbytehq/airbyte/pull/16222) | Remove `end_date` from config if empty value (re-implement #16096) | | 0.2.61 | 2022-08-29 | [16096](https://github.com/airbytehq/airbyte/pull/16096) | Remove `end_date` from config if empty value | | 0.2.60 | 2022-08-19 | [15788](https://github.com/airbytehq/airbyte/pull/15788) | Retry FacebookBadObjectError | From e8bc3d7e038193a367a7f461f3ae77a3f9b3316c Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Tue, 6 Sep 2022 23:21:00 +0200 Subject: [PATCH 048/200] Improve body of release PR (#16330) --- tools/bin/pr_body.sh | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/tools/bin/pr_body.sh b/tools/bin/pr_body.sh index f18c5549604b..5168f54fa1b6 100755 --- a/tools/bin/pr_body.sh +++ b/tools/bin/pr_body.sh @@ -14,12 +14,17 @@ echo PAGER=cat git log v${PREV_VERSION}..${GIT_REVISION} --oneline --decorate=no # The following empty 'echo' is also important for marking the end of the changelog for the Create Release Github action echo -echo "Instructions:" -echo "- *SQUASH MERGE* this PR - this is necessary to ensure the automated Create Release action is triggered." -echo "- Double check that the Create Release action was triggered and ran successfully on the commit to master \ +echo "### Instructions" +echo "1) *SQUASH MERGE* this PR - this is necessary to ensure the automated Create Release action is triggered." +echo "2) Double check that the [Create Release](https://github.com/airbytehq/airbyte/actions/workflows/create-release.yml) action was triggered and ran successfully on the commit to master \ (this should only take a few seconds)." -echo "- If the Create Release action failed due to a transient issue, retry the action. If it failed due to \ -a non-transient issue, you will need to manually create a release by following these steps:" -echo " 1. Pull most recent version of master" -echo " 2. Run ./tools/bin/tag_version.sh" -echo " 3. Create a GitHub release with the changelog" +echo "3) If the Create Release action failed due to a transient issue, retry the action. If it failed due to \ +a non-transient issue, create a release manually by following the below instructions." +echo +echo "
    " +echo "Create the GitHub release manually" +echo +echo "1. Pull most recent version of master" +echo "2. Run ./tools/bin/tag_version.sh" +echo "3. Create a GitHub release with the changelog" +echo "
    " From 68ab5233d7d14f0d9eb13427fe49ad9c96f2954a Mon Sep 17 00:00:00 2001 From: Alex Birdsall Date: Tue, 6 Sep 2022 14:43:46 -0700 Subject: [PATCH 049/200] Add advanced mode toggle (#15912) * Add useAdvancedModeSetting hook * Add advanced mode toggle to workspace settings As of this commit, it's a bit ugly since the toggle switch has no margin between it and the text input directly above; going to migrate the page to use scss modules instead of styling it by the current page conventions. * Query the current workspace inside useAdvancedMode This setting is only relevant to single-workspace views; this simplifies client code. * Style workspace settings form with scss module The switch needed a top-margin, too, so I created an scss module to hold that style. Also migrated the component-local `Header` and `Buttons` styled-components. * Only show connection state if advanced mode is enabled * Don't call setAdvancedMode until form is submitted * Add external label, extract text to i18n keys Keeping the className-passing behavior in LabeledSwitch, though, it seems to me to be a strictly better API for that component. * Add tooltip to advanced mode switch * Add unit test for useAdvancedModeSetting hook * Add unit test for the SettingsView component * Address PR comments - clean up object destructuring/composition - expand inter-item spacing from 10px to 21px to match related form * Add advanced mode toggle to OSS account settings * Remove commented-out button container component * mock useTrackPage since it was added to SettingsView Co-authored-by: lmossman --- .../LabeledSwitch/LabeledSwitch.tsx | 2 +- .../services/useAdvancedModeSetting.test.ts | 53 ++++++++++++++ .../hooks/services/useAdvancedModeSetting.ts | 19 +++++ airbyte-webapp/src/locales/en.json | 3 + .../src/packages/cloud/locales/en.json | 3 + .../WorkspaceSettingsView.module.scss | 20 +++++ .../WorkspaceSettingsView.tsx | 73 +++++++++++-------- .../components/SettingsView.test.tsx | 48 ++++++++++++ .../components/SettingsView.tsx | 4 +- .../components/AccountForm.module.scss | 7 ++ .../AccountPage/components/AccountForm.tsx | 62 +++++++++++----- airbyte-webapp/src/utils/testutils.tsx | 46 ++++++++++++ 12 files changed, 288 insertions(+), 52 deletions(-) create mode 100644 airbyte-webapp/src/hooks/services/useAdvancedModeSetting.test.ts create mode 100644 airbyte-webapp/src/hooks/services/useAdvancedModeSetting.ts create mode 100644 airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.module.scss create mode 100644 airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.test.tsx create mode 100644 airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.module.scss diff --git a/airbyte-webapp/src/components/LabeledSwitch/LabeledSwitch.tsx b/airbyte-webapp/src/components/LabeledSwitch/LabeledSwitch.tsx index 51ef4ff443c1..c62538d5be4f 100644 --- a/airbyte-webapp/src/components/LabeledSwitch/LabeledSwitch.tsx +++ b/airbyte-webapp/src/components/LabeledSwitch/LabeledSwitch.tsx @@ -13,7 +13,7 @@ interface LabeledSwitchProps extends React.InputHTMLAttributes } export const LabeledSwitch: React.FC = (props) => ( -
    +
    {props.checkbox ? ( diff --git a/airbyte-webapp/src/hooks/services/useAdvancedModeSetting.test.ts b/airbyte-webapp/src/hooks/services/useAdvancedModeSetting.test.ts new file mode 100644 index 000000000000..a91094329247 --- /dev/null +++ b/airbyte-webapp/src/hooks/services/useAdvancedModeSetting.test.ts @@ -0,0 +1,53 @@ +import { act, renderHook } from "@testing-library/react-hooks"; + +import { useAdvancedModeSetting } from "./useAdvancedModeSetting"; + +// mock `useCurrentWorkspace` with a closure so we can simulate changing +// workspaces by mutating the top-level variable it references +let mockWorkspaceId = "fakeWorkspaceId"; +const changeToWorkspace = (newWorkspaceId: string) => { + mockWorkspaceId = newWorkspaceId; +}; + +jest.mock("hooks/services/useWorkspace", () => ({ + useCurrentWorkspace() { + return { workspaceId: mockWorkspaceId }; + }, +})); + +test("it defaults to false before advanced mode is explicitly set", () => { + const { result } = renderHook(() => useAdvancedModeSetting()); + // eslint-disable-next-line prefer-const + let [isAdvancedMode, setAdvancedMode] = result.current; + + expect(isAdvancedMode).toBe(false); + + act(() => setAdvancedMode(true)); + [isAdvancedMode] = result.current; + + expect(isAdvancedMode).toBe(true); +}); + +test("it stores workspace-specific advanced mode settings", () => { + changeToWorkspace("workspaceA"); + + const { result, rerender } = renderHook(() => useAdvancedModeSetting()); + // Avoiding destructuring in this spec to avoid capturing stale values when + // rerendering in different workspaces + const setAdvancedModeA = result.current[1]; + + expect(result.current[0]).toBe(false); + act(() => setAdvancedModeA(true)); + + expect(result.current[0]).toBe(true); + + // in workspaceB, it returns the default setting of `false` + changeToWorkspace("workspaceB"); + rerender(); + expect(result.current[0]).toBe(false); + + // ...but workspaceA's manual setting is persisted + changeToWorkspace("workspaceA"); + rerender(); + expect(result.current[0]).toBe(true); +}); diff --git a/airbyte-webapp/src/hooks/services/useAdvancedModeSetting.ts b/airbyte-webapp/src/hooks/services/useAdvancedModeSetting.ts new file mode 100644 index 000000000000..f79bdd63c401 --- /dev/null +++ b/airbyte-webapp/src/hooks/services/useAdvancedModeSetting.ts @@ -0,0 +1,19 @@ +import { useLocalStorage } from "react-use"; + +import { useCurrentWorkspace } from "hooks/services/useWorkspace"; + +type SettingsByWorkspace = Record; + +export const useAdvancedModeSetting = (): [boolean, (newSetting: boolean) => void] => { + const { workspaceId } = useCurrentWorkspace(); + const [advancedModeSettingsByWorkspace, setAdvancedModeSettingsByWorkspace] = useLocalStorage( + "advancedMode", + {} + ); + + const isAdvancedMode = (advancedModeSettingsByWorkspace || {})[workspaceId] ?? false; + const setAdvancedMode = (newSetting: boolean) => + setAdvancedModeSettingsByWorkspace({ ...advancedModeSettingsByWorkspace, [workspaceId]: newSetting }); + + return [isAdvancedMode, setAdvancedMode]; +}; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index ab84608c894e..43415b301efb 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -108,6 +108,9 @@ "form.url.error": "field must be a valid URL", "form.setupGuide": "Setup Guide", "form.wait": "Please wait a little bit more…", + "form.advancedMode.label": "Advanced mode", + "form.advancedMode.switchLabel": "Enable advanced mode", + "form.advancedMode.tooltip": "When Advanced Mode is enabled, certain views will display additional technical information.", "connectionForm.validation.error": "The form is invalid. Please make sure that all fields are correct.", "connectionForm.normalization.title": "Normalization", diff --git a/airbyte-webapp/src/packages/cloud/locales/en.json b/airbyte-webapp/src/packages/cloud/locales/en.json index 83398b372238..e2703274dcde 100644 --- a/airbyte-webapp/src/packages/cloud/locales/en.json +++ b/airbyte-webapp/src/packages/cloud/locales/en.json @@ -90,6 +90,9 @@ "settings.generalSettings.changeWorkspace": "Change Workspace", "settings.generalSettings.form.name.label": "Workspace name", "settings.generalSettings.form.name.placeholder": "Workspace name", + "settings.generalSettings.form.advancedMode.label": "Advanced mode", + "settings.generalSettings.form.advancedMode.switchLabel": "Enable advanced mode", + "settings.generalSettings.form.advancedMode.tooltip": "When Advanced Mode is enabled, certain views will display additional technical information.", "settings.generalSettings.deleteLabel": "Delete your workspace", "settings.generalSettings.deleteText": "Delete", "settings.accessManagementSettings": "Access Management", diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.module.scss b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.module.scss new file mode 100644 index 000000000000..bb016d31df64 --- /dev/null +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.module.scss @@ -0,0 +1,20 @@ +.formItem { + margin-top: 21px; + width: 100%; +} + +.buttonGroup { + display: flex; + flex-direction: row; + justify-content: flex-end; + align-items: center; + + & > button { + margin-left: 5px; + } +} + +.header { + display: flex; + justify-content: space-between; +} diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx index 855d9e7a6149..7f804a66e322 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/WorkspaceSettingsView.tsx @@ -1,11 +1,13 @@ +import classNames from "classnames"; import { Field, FieldProps, Form, Formik } from "formik"; import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import styled from "styled-components"; -import { Button, LabeledInput, LoadingButton } from "components"; +import { Button, Label, LabeledInput, LabeledSwitch, LoadingButton } from "components"; +import { InfoTooltip } from "components/base/Tooltip"; import { useTrackPage, PageTrackingCodes } from "hooks/services/Analytics"; +import { useAdvancedModeSetting } from "hooks/services/useAdvancedModeSetting"; import { useCurrentWorkspace } from "hooks/services/useWorkspace"; import { useRemoveWorkspace, @@ -14,23 +16,16 @@ import { } from "packages/cloud/services/workspaces/WorkspacesService"; import { Content, SettingsCard } from "pages/SettingsPage/pages/SettingsComponents"; -const Header = styled.div` - display: flex; - justify-content: space-between; -`; +import styles from "./WorkspaceSettingsView.module.scss"; -const Buttons = styled.div` - margin-top: 10px; - width: 100%; - display: flex; - flex-direction: row; - justify-content: flex-end; - align-items: center; - - & > button { - margin-left: 5px; - } -`; +const AdvancedModeSwitchLabel = () => ( + <> + + + + + +); export const WorkspaceSettingsView: React.FC = () => { const { formatMessage } = useIntl(); @@ -39,29 +34,34 @@ export const WorkspaceSettingsView: React.FC = () => { const workspace = useCurrentWorkspace(); const removeWorkspace = useRemoveWorkspace(); const updateWorkspace = useUpdateWorkspace(); + const [isAdvancedMode, setAdvancedMode] = useAdvancedModeSetting(); return ( <> +
    - +
    } > - updateWorkspace.mutateAsync({ + initialValues={{ + name: workspace.name, + advancedMode: isAdvancedMode, + }} + onSubmit={async (payload) => { + setAdvancedMode(payload.advancedMode); + return updateWorkspace.mutateAsync({ workspaceId: workspace.workspaceId, name: payload.name, - }) - } + }); + }} > - {({ dirty, isSubmitting, resetForm, isValid }) => ( + {({ dirty, isSubmitting, resetForm, isValid, setFieldValue }) => (
    @@ -78,14 +78,27 @@ export const WorkspaceSettingsView: React.FC = () => { /> )} - + + + {({ field }: FieldProps) => ( + } + checked={field.value} + onChange={() => setFieldValue(field.name, !field.value)} + /> + )} + + +
    save changes - +
    )} @@ -93,7 +106,7 @@ export const WorkspaceSettingsView: React.FC = () => {     +
    { > - +
    } /> diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.test.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.test.tsx new file mode 100644 index 000000000000..5f9acfc73bd5 --- /dev/null +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.test.tsx @@ -0,0 +1,48 @@ +import { render, mockConnection } from "utils/testutils"; + +import SettingsView from "./SettingsView"; + +let mockIsAdvancedMode = false; +const setMockIsAdvancedMode = (newSetting: boolean) => { + mockIsAdvancedMode = newSetting; +}; +jest.mock("hooks/services/useAdvancedModeSetting", () => ({ + useAdvancedModeSetting() { + return [mockIsAdvancedMode, setMockIsAdvancedMode]; + }, +})); + +jest.mock("hooks/services/useConnectionHook", () => ({ + useDeleteConnection: () => ({ mutateAsync: () => null }), + useGetConnectionState: () => ({ state: null, globalState: null, streamState: null }), +})); + +jest.mock("hooks/services/Analytics/useAnalyticsService", () => ({ + useTrackPage: () => null, +})); + +// Mocking the DeleteBlock component is a bit ugly, but it's simpler and less +// brittle than mocking the providers it depends on; at least it's a direct, +// visible dependency of the component under test here. +// +// This mock is intentionally trivial; if anything to do with this component is +// to be tested, we'll have to bite the bullet and render it properly, within +// the necessary providers. +jest.mock("components/DeleteBlock", () => () => { + const MockDeleteBlock = () =>
    Does not actually delete anything
    ; + return ; +}); + +describe("", () => { + test("it only renders connection state when advanced mode is enabled", async () => { + let container: HTMLElement; + + setMockIsAdvancedMode(false); + ({ container } = await render()); + expect(container.textContent).not.toContain("Connection State"); + + setMockIsAdvancedMode(true); + ({ container } = await render()); + expect(container.textContent).toContain("Connection State"); + }); +}); diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx index 866b0d65cc41..8b71da6e1fa4 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.tsx @@ -3,6 +3,7 @@ import React from "react"; import DeleteBlock from "components/DeleteBlock"; import { PageTrackingCodes, useTrackPage } from "hooks/services/Analytics"; +import { useAdvancedModeSetting } from "hooks/services/useAdvancedModeSetting"; import { useDeleteConnection } from "hooks/services/useConnectionHook"; import { WebBackendConnectionRead } from "../../../../../core/request/AirbyteClient"; @@ -16,12 +17,13 @@ interface SettingsViewProps { const SettingsView: React.FC = ({ connection }) => { const { mutateAsync: deleteConnection } = useDeleteConnection(); + const [isAdvancedMode] = useAdvancedModeSetting(); useTrackPage(PageTrackingCodes.CONNECTIONS_ITEM_SETTINGS); const onDelete = () => deleteConnection(connection); return (
    - + {isAdvancedMode && }
    ); diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.module.scss b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.module.scss new file mode 100644 index 000000000000..858dca46d0f4 --- /dev/null +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.module.scss @@ -0,0 +1,7 @@ +.formItem { + margin-bottom: 21px; +} + +.submit { + margin-bottom: 10px; +} diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx index 0aed616a86ac..07d4dd29bf9d 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx @@ -4,21 +4,16 @@ import { FormattedMessage, useIntl } from "react-intl"; import styled from "styled-components"; import * as yup from "yup"; -import { LoadingButton } from "components"; -import LabeledInput from "components/LabeledInput"; +import { Label, LabeledInput, LabeledSwitch, LoadingButton } from "components"; +import { InfoTooltip } from "components/base/Tooltip"; import { Row, Cell } from "components/SimpleTableComponents"; +import { useAdvancedModeSetting } from "hooks/services/useAdvancedModeSetting"; + +import styles from "./AccountForm.module.scss"; + const InputRow = styled(Row)` height: auto; - margin-bottom: 40px; -`; - -const ButtonCell = styled(Cell)` - &:last-child { - text-align: left; - } - padding-left: 11px; - height: 9px; `; const EmailForm = styled(Form)` @@ -40,6 +35,15 @@ const Success = styled.div` color: ${({ theme }) => theme.successColor}; `; +const AdvancedModeSwitchLabel = () => ( + <> + + + + + +); + const accountValidationSchema = yup.object().shape({ email: yup.string().email("form.email.error").required("form.empty.error"), }); @@ -53,19 +57,23 @@ interface AccountFormProps { const AccountForm: React.FC = ({ email, onSubmit, successMessage, errorMessage }) => { const { formatMessage } = useIntl(); + const [isAdvancedMode, setAdvancedMode] = useAdvancedModeSetting(); return ( { + onSubmit(data); + setAdvancedMode(data.advancedMode); + }} > - {({ isSubmitting, dirty, values }) => ( + {({ isSubmitting, dirty, values, setFieldValue }) => ( - + {({ field, meta }: FieldProps) => ( @@ -81,12 +89,26 @@ const AccountForm: React.FC = ({ email, onSubmit, successMessa )} - - - - - +
    + + + {({ field }: FieldProps) => ( + } + checked={field.value} + onChange={() => setFieldValue(field.name, !field.value)} + /> + )} + +
    +
    + + + +
    {!dirty && (successMessage ? ( {successMessage} diff --git a/airbyte-webapp/src/utils/testutils.tsx b/airbyte-webapp/src/utils/testutils.tsx index f817fc673cb5..90dd4589e811 100644 --- a/airbyte-webapp/src/utils/testutils.tsx +++ b/airbyte-webapp/src/utils/testutils.tsx @@ -6,6 +6,13 @@ import { MemoryRouter } from "react-router-dom"; import { ThemeProvider } from "styled-components"; import { ConfigContext, defaultConfig } from "config"; +import { + ConnectionStatus, + DestinationRead, + NamespaceDefinitionType, + SourceRead, + WebBackendConnectionRead, +} from "core/request/AirbyteClient"; import { ServicesProvider } from "core/servicesProvider"; import { defaultFeatures, FeatureService } from "hooks/services/Feature"; import en from "locales/en.json"; @@ -53,3 +60,42 @@ export const TestWrapper: React.FC = ({ children }) => ( ); + +export const mockSource: SourceRead = { + sourceId: "test-source", + name: "test source", + sourceName: "test-source-name", + workspaceId: "test-workspace-id", + sourceDefinitionId: "test-source-definition-id", + connectionConfiguration: undefined, +}; + +export const mockDestination: DestinationRead = { + destinationId: "test-destination", + name: "test destination", + destinationName: "test destination name", + workspaceId: "test-workspace-id", + destinationDefinitionId: "test-destination-definition-id", + connectionConfiguration: undefined, +}; + +export const mockConnection: WebBackendConnectionRead = { + connectionId: "test-connection", + name: "test connection", + prefix: "test", + sourceId: "test-source", + destinationId: "test-destination", + status: ConnectionStatus.active, + schedule: undefined, + syncCatalog: { + streams: [], + }, + namespaceDefinition: NamespaceDefinitionType.source, + namespaceFormat: "", + operationIds: [], + source: mockSource, + destination: mockDestination, + operations: [], + catalogId: "", + isSyncing: false, +}; From f15234b2340d4bd9d90bd8a84c9540994944eba5 Mon Sep 17 00:00:00 2001 From: Evan Tahler Date: Tue, 6 Sep 2022 15:36:18 -0700 Subject: [PATCH 050/200] Log sync summary and sync activity seperators (#16314) * Log activity start and end * pretty-print JSON replication and failure summaries * simplify --- .../io/airbyte/commons/io/LineGobbler.java | 31 +++++++++++++++++++ .../general/DbtTransformationWorker.java | 3 ++ .../general/DefaultCheckConnectionWorker.java | 2 ++ .../general/DefaultNormalizationWorker.java | 4 ++- .../general/DefaultReplicationWorker.java | 9 +++++- 5 files changed, 47 insertions(+), 2 deletions(-) diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java b/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java index 4f02a654b24e..ccacefdf581e 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/io/LineGobbler.java @@ -7,8 +7,10 @@ import io.airbyte.commons.concurrency.VoidCallable; import io.airbyte.commons.logging.MdcScope; import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -26,6 +28,35 @@ public static void gobble(final InputStream is, final Consumer consumer) gobble(is, consumer, GENERIC, MdcScope.DEFAULT_BUILDER); } + public static void gobble(final String message, final Consumer consumer) { + final InputStream stringAsSteam = new ByteArrayInputStream(message.getBytes(StandardCharsets.UTF_8)); + gobble(stringAsSteam, consumer); + } + + public static void gobble(final String message) { + gobble(message, LOGGER::info); + } + + /** + * Used to emit a visual separator in the user-facing logs indicating a start of a meaningful + * temporal activity + * + * @param message + */ + public static void startSection(final String message) { + gobble("\r\n----- START " + message + " -----\r\n\r\n"); + } + + /** + * Used to emit a visual separator in the user-facing logs indicating a end of a meaningful temporal + * activity + * + * @param message + */ + public static void endSection(final String message) { + gobble("\r\n----- END " + message + " -----\r\n\r\n"); + } + public static void gobble(final InputStream is, final Consumer consumer, final MdcScope.Builder mdcScopeBuilder) { gobble(is, consumer, GENERIC, mdcScopeBuilder); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/general/DbtTransformationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/general/DbtTransformationWorker.java index 656c539e45a7..abc454b28787 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/general/DbtTransformationWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/general/DbtTransformationWorker.java @@ -4,6 +4,7 @@ package io.airbyte.workers.general; +import io.airbyte.commons.io.LineGobbler; import io.airbyte.config.OperatorDbtInput; import io.airbyte.config.ResourceRequirements; import io.airbyte.workers.Worker; @@ -42,6 +43,7 @@ public DbtTransformationWorker(final String jobId, @Override public Void run(final OperatorDbtInput operatorDbtInput, final Path jobRoot) throws WorkerException { final long startTime = System.currentTimeMillis(); + LineGobbler.startSection("DBT TRANSFORMATION"); try (dbtTransformationRunner) { LOGGER.info("Running dbt transformation."); @@ -65,6 +67,7 @@ public Void run(final OperatorDbtInput operatorDbtInput, final Path jobRoot) thr final Duration duration = Duration.ofMillis(System.currentTimeMillis() - startTime); LOGGER.info("Dbt Transformation executed in {}.", duration.toMinutesPart()); + LineGobbler.endSection("DBT TRANSFORMATION"); return null; } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionWorker.java index 08bc8e8513e9..ee188f4051e7 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultCheckConnectionWorker.java @@ -58,6 +58,7 @@ public DefaultCheckConnectionWorker(final WorkerConfigs workerConfigs, final Int @Override public ConnectorJobOutput run(final StandardCheckConnectionInput input, final Path jobRoot) throws WorkerException { + LineGobbler.startSection("CHECK"); try { process = integrationLauncher.check( @@ -88,6 +89,7 @@ public ConnectorJobOutput run(final StandardCheckConnectionInput input, final Pa LOGGER.debug("Check connection job subprocess finished with exit code {}", exitCode); LOGGER.debug("Check connection job received output: {}", output); + LineGobbler.endSection("CHECK"); return new ConnectorJobOutput().withOutputType(OutputType.CHECK_CONNECTION).withCheckConnection(output); } else { final String message = String.format("Error checking connection, status: %s, exit code: %d", status, exitCode); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultNormalizationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultNormalizationWorker.java index cf9b6735f69c..d48c196b9ff3 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultNormalizationWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultNormalizationWorker.java @@ -4,6 +4,7 @@ package io.airbyte.workers.general; +import io.airbyte.commons.io.LineGobbler; import io.airbyte.config.Configs.WorkerEnvironment; import io.airbyte.config.FailureReason; import io.airbyte.config.NormalizationInput; @@ -54,7 +55,7 @@ public NormalizationSummary run(final NormalizationInput input, final Path jobRo final long startTime = System.currentTimeMillis(); try (normalizationRunner) { - LOGGER.info("Running normalization."); + LineGobbler.startSection("DEFAULT NORMALIZATION"); normalizationRunner.start(); Path normalizationRoot = null; @@ -92,6 +93,7 @@ public NormalizationSummary run(final NormalizationInput input, final Path jobRo } LOGGER.info("Normalization summary: {}", summary); + LineGobbler.endSection("DEFAULT NORMALIZATION"); return summary; } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java index 9002badfbc19..b91cde3c4c64 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java @@ -4,6 +4,8 @@ package io.airbyte.workers.general; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.airbyte.commons.io.LineGobbler; import io.airbyte.config.FailureReason; import io.airbyte.config.ReplicationAttemptSummary; import io.airbyte.config.ReplicationOutput; @@ -117,6 +119,7 @@ public DefaultReplicationWorker(final String jobId, @Override public final ReplicationOutput run(final StandardSyncInput syncInput, final Path jobRoot) throws WorkerException { LOGGER.info("start sync worker. job id: {} attempt id: {}", jobId, attempt); + LineGobbler.startSection("REPLICATION"); // todo (cgardens) - this should not be happening in the worker. this is configuration information // that is independent of workflow executions. @@ -246,7 +249,6 @@ else if (hasFailed.get()) { .withStartTime(startTime) .withEndTime(System.currentTimeMillis()); - LOGGER.info("sync summary: {}", summary); final ReplicationOutput output = new ReplicationOutput() .withReplicationAttemptSummary(summary) .withOutputCatalog(destinationConfig.getCatalog()); @@ -293,6 +295,11 @@ else if (hasFailed.get()) { metricReporter.trackStateMetricTrackerError(); } + final ObjectMapper mapper = new ObjectMapper(); + LOGGER.info("sync summary: {}", mapper.writerWithDefaultPrettyPrinter().writeValueAsString(summary)); + LOGGER.info("failures: {}", mapper.writerWithDefaultPrettyPrinter().writeValueAsString(failures)); + + LineGobbler.endSection("REPLICATION"); return output; } catch (final Exception e) { throw new WorkerException("Sync failed", e); From 790343e93fe1b5bc01cb1faace80bd3a63b2eab0 Mon Sep 17 00:00:00 2001 From: Parker Mossman Date: Tue, 6 Sep 2022 20:03:24 -0700 Subject: [PATCH 051/200] Remove US/Pacific-New because it doesn't work with micronaut and it isn't recognized by the US Government (#16385) --- .../airbyte/server/handlers/ConnectionSchedulerHelperTest.java | 1 - airbyte-webapp/src/config/availableCronTimeZones.json | 1 - 2 files changed, 2 deletions(-) diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionSchedulerHelperTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionSchedulerHelperTest.java index e761660f29d3..49a780908576 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionSchedulerHelperTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/ConnectionSchedulerHelperTest.java @@ -646,7 +646,6 @@ void testAvailableCronTimeZonesStayTheSame() { "US/Michigan", "US/Mountain", "US/Pacific", - "US/Pacific-New", "US/Samoa", "UTC", "Universal", diff --git a/airbyte-webapp/src/config/availableCronTimeZones.json b/airbyte-webapp/src/config/availableCronTimeZones.json index 0db41456d13f..94d1b2b92b78 100644 --- a/airbyte-webapp/src/config/availableCronTimeZones.json +++ b/airbyte-webapp/src/config/availableCronTimeZones.json @@ -550,7 +550,6 @@ "US/Michigan", "US/Mountain", "US/Pacific", - "US/Pacific-New", "US/Samoa", "UTC", "Universal", From e253b982609e8b3fd503c22a3e1a1c662d1c2c6e Mon Sep 17 00:00:00 2001 From: Augustin Date: Wed, 7 Sep 2022 09:29:52 +0200 Subject: [PATCH 052/200] doc: add airbyte_secret field in example (#16374) --- .../tutorials/cdk-tutorial-python-http/define-inputs.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/connector-development/tutorials/cdk-tutorial-python-http/define-inputs.md b/docs/connector-development/tutorials/cdk-tutorial-python-http/define-inputs.md index 1e6e01f2b945..71251b44efde 100644 --- a/docs/connector-development/tutorials/cdk-tutorial-python-http/define-inputs.md +++ b/docs/connector-development/tutorials/cdk-tutorial-python-http/define-inputs.md @@ -24,6 +24,7 @@ connectionSpecification: access_key: type: string description: API access key used to retrieve data from the Exchange Rates API. + airbyte_secret: true start_date: type: string description: Start getting data from that date. From 63deca2e0a5a7acf956b96d6a2c2c9ab2d58ccee Mon Sep 17 00:00:00 2001 From: Eugene Date: Wed, 7 Sep 2022 11:41:37 +0300 Subject: [PATCH 053/200] =?UTF-8?q?=F0=9F=8E=89Source-AlloyDB=20for=20Post?= =?UTF-8?q?gres:=20added=20new=20connector=20(#16323)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [16174] Source-AlloyDB for Postgres: added new connector --- .github/CODEOWNERS | 1 + .../init/src/main/resources/icons/alloydb.svg | 1 + .../resources/seed/source_definitions.yaml | 8 + .../src/main/resources/seed/source_specs.yaml | 405 ++++++++++++++++++ airbyte-integrations/builds.md | 1 + .../connectors/source-alloydb/.dockerignore | 3 + .../connectors/source-alloydb/Dockerfile | 20 + .../source-alloydb/acceptance-test-config.yml | 6 + .../connectors/source-alloydb/build.gradle | 33 ++ .../source/alloydb/AlloyDbSource.java | 29 ++ docs/integrations/README.md | 1 + docs/integrations/sources/alloydb.md | 332 ++++++++++++++ 12 files changed, 840 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/icons/alloydb.svg create mode 100644 airbyte-integrations/connectors/source-alloydb/.dockerignore create mode 100644 airbyte-integrations/connectors/source-alloydb/Dockerfile create mode 100644 airbyte-integrations/connectors/source-alloydb/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-alloydb/build.gradle create mode 100644 airbyte-integrations/connectors/source-alloydb/src/main/java/io/airbyte/integrations/source/alloydb/AlloyDbSource.java create mode 100644 docs/integrations/sources/alloydb.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8a84703badef..d063f4e23030 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -20,6 +20,7 @@ # JDBC-based connectors /airbyte-integrations/bases/base-java/ @airbytehq/jdbc-connectors /airbyte-integrations/connectors/source-jdbc/ @airbytehq/jdbc-connectors +/airbyte-integrations/connectors/source-alloydb/ @airbytehq/jdbc-connectors /airbyte-integrations/connectors/source-bigquery/ @airbytehq/jdbc-connectors /airbyte-integrations/connectors/source-clickhouse/ @airbytehq/jdbc-connectors /airbyte-integrations/connectors/source-cockroachdb/ @airbytehq/jdbc-connectors diff --git a/airbyte-config/init/src/main/resources/icons/alloydb.svg b/airbyte-config/init/src/main/resources/icons/alloydb.svg new file mode 100644 index 000000000000..12034d4b89fa --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/alloydb.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 5d37584ff1d0..92ae63bfe6e6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -6,6 +6,14 @@ icon: airtable.svg sourceType: api releaseStage: alpha +- name: AlloyDB for PostgreSQL + sourceDefinitionId: 1fa90628-2b9e-11ed-a261-0242ac120002 + dockerRepository: airbyte/source-alloydb + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/alloydb + icon: alloydb.svg + sourceType: database + releaseStage: alpha - name: AWS CloudTrail sourceDefinitionId: 6ff047c0-f5d5-4ce5-8c81-204a830fa7e1 dockerRepository: airbyte/source-aws-cloudtrail diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 2e4a586f201a..9be52aaedd38 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -43,6 +43,411 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-alloydb:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/postgres" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Postgres Source Spec" + type: "object" + required: + - "host" + - "port" + - "database" + - "username" + properties: + host: + title: "Host" + description: "Hostname of the database." + type: "string" + order: 0 + port: + title: "Port" + description: "Port of the database." + type: "integer" + minimum: 0 + maximum: 65536 + default: 5432 + examples: + - "5432" + order: 1 + database: + title: "Database Name" + description: "Name of the database." + type: "string" + order: 2 + schemas: + title: "Schemas" + description: "The list of schemas (case sensitive) to sync from. Defaults\ + \ to public." + type: "array" + items: + type: "string" + minItems: 0 + uniqueItems: true + default: + - "public" + order: 3 + username: + title: "Username" + description: "Username to access the database." + type: "string" + order: 4 + password: + title: "Password" + description: "Password associated with the username." + type: "string" + airbyte_secret: true + order: 5 + jdbc_url_params: + description: "Additional properties to pass to the JDBC URL string when\ + \ connecting to the database formatted as 'key=value' pairs separated\ + \ by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more\ + \ information read about     JDBC URL parameters." + title: "JDBC URL Parameters (Advanced)" + type: "string" + order: 6 + ssl: + title: "Connect using SSL" + description: "Encrypt data using SSL. When activating SSL, please select\ + \ one of the connection modes." + type: "boolean" + default: false + order: 7 + ssl_mode: + title: "SSL Modes" + description: "SSL connection modes. \n
    • disable - Disables\ + \ encryption of communication between Airbyte and source database
      • \n\ + \
    • allow - Enables encryption only when required by the source\ + \ database
      • \n
    • prefer - allows unencrypted connection only\ + \ if the source database does not support encryption
      • \n
    • require\ + \ - Always require encryption. If the source database server does not\ + \ support encryption, connection will fail
      • \n
    • verify-ca\ + \ - Always require encryption and verifies that the source database server\ + \ has a valid SSL certificate
      • \n
    • verify-full - This is\ + \ the most secure mode. Always require encryption and verifies the identity\ + \ of the source database server
    \n Read more in the docs." + type: "object" + order: 7 + oneOf: + - title: "disable" + additionalProperties: false + description: "Disable SSL." + required: + - "mode" + properties: + mode: + type: "string" + const: "disable" + enum: + - "disable" + default: "disable" + order: 0 + - title: "allow" + additionalProperties: false + description: "Allow SSL mode." + required: + - "mode" + properties: + mode: + type: "string" + const: "allow" + enum: + - "allow" + default: "allow" + order: 0 + - title: "prefer" + additionalProperties: false + description: "Prefer SSL mode." + required: + - "mode" + properties: + mode: + type: "string" + const: "prefer" + enum: + - "prefer" + default: "prefer" + order: 0 + - title: "require" + additionalProperties: false + description: "Require SSL mode." + required: + - "mode" + properties: + mode: + type: "string" + const: "require" + enum: + - "require" + default: "require" + order: 0 + - title: "verify-ca" + additionalProperties: false + description: "Verify-ca SSL mode." + required: + - "mode" + - "ca_certificate" + properties: + mode: + type: "string" + const: "verify-ca" + enum: + - "verify-ca" + default: "verify-ca" + order: 0 + ca_certificate: + type: "string" + title: "CA certificate" + description: "CA certificate" + airbyte_secret: true + multiline: true + order: 1 + client_certificate: + type: "string" + title: "Client Certificate (Optional)" + description: "Client certificate" + airbyte_secret: true + multiline: true + order: 2 + client_key: + type: "string" + title: "Client Key (Optional)" + description: "Client key" + airbyte_secret: true + multiline: true + order: 3 + client_key_password: + type: "string" + title: "Client key password (Optional)" + description: "Password for keystorage. If you do not add it - the\ + \ password will be generated automatically." + airbyte_secret: true + order: 4 + - title: "verify-full" + additionalProperties: false + description: "Verify-full SSL mode." + required: + - "mode" + - "ca_certificate" + properties: + mode: + type: "string" + const: "verify-full" + enum: + - "verify-full" + default: "verify-full" + order: 0 + ca_certificate: + type: "string" + title: "CA Certificate" + description: "CA certificate" + airbyte_secret: true + multiline: true + order: 1 + client_certificate: + type: "string" + title: "Client Certificate (Optional)" + description: "Client certificate" + airbyte_secret: true + multiline: true + order: 2 + client_key: + type: "string" + title: "Client Key (Optional)" + description: "Client key" + airbyte_secret: true + multiline: true + order: 3 + client_key_password: + type: "string" + title: "Client key password (Optional)" + description: "Password for keystorage. If you do not add it - the\ + \ password will be generated automatically." + airbyte_secret: true + order: 4 + replication_method: + type: "object" + title: "Replication Method" + description: "Replication method for extracting data from the database." + order: 8 + oneOf: + - title: "Standard" + description: "Standard replication requires no setup on the DB side but\ + \ will not be able to represent deletions incrementally." + required: + - "method" + properties: + method: + type: "string" + const: "Standard" + enum: + - "Standard" + default: "Standard" + order: 0 + - title: "Logical Replication (CDC)" + description: "Logical replication uses the Postgres write-ahead log (WAL)\ + \ to detect inserts, updates, and deletes. This needs to be configured\ + \ on the source database itself. Only available on Postgres 10 and above.\ + \ Read the docs." + required: + - "method" + - "replication_slot" + - "publication" + properties: + method: + type: "string" + const: "CDC" + enum: + - "CDC" + default: "CDC" + order: 0 + plugin: + type: "string" + title: "Plugin" + description: "A logical decoding plugin installed on the PostgreSQL\ + \ server. The `pgoutput` plugin is used by default. If the replication\ + \ table contains a lot of big jsonb values it is recommended to\ + \ use `wal2json` plugin. Read more about selecting replication plugins." + enum: + - "pgoutput" + - "wal2json" + default: "pgoutput" + order: 1 + replication_slot: + type: "string" + title: "Replication Slot" + description: "A plugin logical replication slot. Read about replication slots." + order: 2 + publication: + type: "string" + title: "Publication" + description: "A Postgres publication used for consuming changes. Read\ + \ about publications and replication identities." + order: 3 + initial_waiting_seconds: + type: "integer" + title: "Initial Waiting Time in Seconds (Advanced)" + description: "The amount of time the connector will wait when it launches\ + \ to determine if there is new data to sync or not. Defaults to\ + \ 300 seconds. Valid range: 120 seconds to 1200 seconds. Read about\ + \ initial waiting time." + default: 300 + order: 4 + min: 120 + max: 1200 + tunnel_method: + type: "object" + title: "SSH Tunnel Method" + description: "Whether to initiate an SSH tunnel before connecting to the\ + \ database, and if so, which kind of authentication to use." + oneOf: + - title: "No Tunnel" + required: + - "tunnel_method" + properties: + tunnel_method: + description: "No ssh tunnel needed to connect to database" + type: "string" + const: "NO_TUNNEL" + order: 0 + - title: "SSH Key Authentication" + required: + - "tunnel_method" + - "tunnel_host" + - "tunnel_port" + - "tunnel_user" + - "ssh_key" + properties: + tunnel_method: + description: "Connect through a jump server tunnel host using username\ + \ and ssh key" + type: "string" + const: "SSH_KEY_AUTH" + order: 0 + tunnel_host: + title: "SSH Tunnel Jump Server Host" + description: "Hostname of the jump server host that allows inbound\ + \ ssh tunnel." + type: "string" + order: 1 + tunnel_port: + title: "SSH Connection Port" + description: "Port on the proxy/jump server that accepts inbound ssh\ + \ connections." + type: "integer" + minimum: 0 + maximum: 65536 + default: 22 + examples: + - "22" + order: 2 + tunnel_user: + title: "SSH Login Username" + description: "OS-level username for logging into the jump server host." + type: "string" + order: 3 + ssh_key: + title: "SSH Private Key" + description: "OS-level user account ssh key credentials in RSA PEM\ + \ format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )" + type: "string" + airbyte_secret: true + multiline: true + order: 4 + - title: "Password Authentication" + required: + - "tunnel_method" + - "tunnel_host" + - "tunnel_port" + - "tunnel_user" + - "tunnel_user_password" + properties: + tunnel_method: + description: "Connect through a jump server tunnel host using username\ + \ and password authentication" + type: "string" + const: "SSH_PASSWORD_AUTH" + order: 0 + tunnel_host: + title: "SSH Tunnel Jump Server Host" + description: "Hostname of the jump server host that allows inbound\ + \ ssh tunnel." + type: "string" + order: 1 + tunnel_port: + title: "SSH Connection Port" + description: "Port on the proxy/jump server that accepts inbound ssh\ + \ connections." + type: "integer" + minimum: 0 + maximum: 65536 + default: 22 + examples: + - "22" + order: 2 + tunnel_user: + title: "SSH Login Username" + description: "OS-level username for logging into the jump server host" + type: "string" + order: 3 + tunnel_user_password: + title: "Password" + description: "OS-level password for logging into the jump server host" + type: "string" + airbyte_secret: true + order: 4 + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-aws-cloudtrail:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/aws-cloudtrail" diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 85ff8df3058c..7bf5b9737db7 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -6,6 +6,7 @@ | :--- | :--- | | 3PL Central | [![source-amazon-seller-partner](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-tplcentral%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-tplcentral) | | Airtable | [![source-airtable](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-airtable%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-airtable) | +| AlloyDB | [![source-alloydb](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-alloydb%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-alloydb) | | Amazon Seller Partner | [![source-amazon-seller-partner](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-amazon-seller-partner%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-amazon-seller-partner) | | Amplitude | [![source-amplitude](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-amplitude%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-amplitude) | | Apify Dataset | [![source-amplitude](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-apify-dataset%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-apify-dataset) | diff --git a/airbyte-integrations/connectors/source-alloydb/.dockerignore b/airbyte-integrations/connectors/source-alloydb/.dockerignore new file mode 100644 index 000000000000..65c7d0ad3e73 --- /dev/null +++ b/airbyte-integrations/connectors/source-alloydb/.dockerignore @@ -0,0 +1,3 @@ +* +!Dockerfile +!build diff --git a/airbyte-integrations/connectors/source-alloydb/Dockerfile b/airbyte-integrations/connectors/source-alloydb/Dockerfile new file mode 100644 index 000000000000..699100679d40 --- /dev/null +++ b/airbyte-integrations/connectors/source-alloydb/Dockerfile @@ -0,0 +1,20 @@ +FROM airbyte/integration-base-java:dev AS build + +WORKDIR /airbyte + +ENV APPLICATION source-alloydb + +COPY build/distributions/${APPLICATION}*.tar ${APPLICATION}.tar + +RUN tar xf ${APPLICATION}.tar --strip-components=1 && rm -rf ${APPLICATION}.tar + +FROM airbyte/integration-base-java:dev + +WORKDIR /airbyte + +ENV APPLICATION source-alloydb + +COPY --from=build /airbyte /airbyte + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-alloydb diff --git a/airbyte-integrations/connectors/source-alloydb/acceptance-test-config.yml b/airbyte-integrations/connectors/source-alloydb/acceptance-test-config.yml new file mode 100644 index 000000000000..94c25ae1348d --- /dev/null +++ b/airbyte-integrations/connectors/source-alloydb/acceptance-test-config.yml @@ -0,0 +1,6 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-alloydb:dev +tests: + spec: + - spec_path: "src/test/resources/expected_spec.json" diff --git a/airbyte-integrations/connectors/source-alloydb/build.gradle b/airbyte-integrations/connectors/source-alloydb/build.gradle new file mode 100644 index 000000000000..4e4bd028d8c0 --- /dev/null +++ b/airbyte-integrations/connectors/source-alloydb/build.gradle @@ -0,0 +1,33 @@ +plugins { + id 'application' + id 'airbyte-docker' + id 'airbyte-integration-test-java' +} + +application { + mainClass = 'io.airbyte.integrations.source.alloydb.AlloyDbSource' + applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0'] +} + +dependencies { + implementation project(':airbyte-db:db-lib') + + implementation project(':airbyte-integrations:bases:base-java') + implementation project(':airbyte-integrations:connectors:source-postgres') + implementation project(':airbyte-protocol:protocol-models') + implementation project(':airbyte-integrations:connectors:source-jdbc') + implementation project(':airbyte-integrations:connectors:source-relational-db') + + + testImplementation testFixtures(project(':airbyte-integrations:connectors:source-jdbc')) + testImplementation project(':airbyte-test-utils') + + testImplementation libs.connectors.testcontainers.postgresql + + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') + + implementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + integrationTestJavaImplementation files(project(':airbyte-integrations:bases:base-java').airbyteDocker.outputs) + integrationTestJavaImplementation project(':airbyte-integrations:bases:standard-source-test') + +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-alloydb/src/main/java/io/airbyte/integrations/source/alloydb/AlloyDbSource.java b/airbyte-integrations/connectors/source-alloydb/src/main/java/io/airbyte/integrations/source/alloydb/AlloyDbSource.java new file mode 100644 index 000000000000..efa806eb94ab --- /dev/null +++ b/airbyte-integrations/connectors/source-alloydb/src/main/java/io/airbyte/integrations/source/alloydb/AlloyDbSource.java @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.alloydb; + +import io.airbyte.integrations.base.IntegrationRunner; +import io.airbyte.integrations.base.Source; +import io.airbyte.integrations.source.postgres.PostgresSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class AlloyDbSource { + + private static final Logger LOGGER = LoggerFactory.getLogger(AlloyDbSource.class); + + /** + * AlloyDB for PostgreSQL is a fully managed PostgreSQL-compatible database service. So the + * source-postgres connector is used under the hood. For more details please check the + * https://cloud.google.com/alloydb + */ + public static void main(final String[] args) throws Exception { + final Source source = PostgresSource.sshWrappedSource(); + LOGGER.info("starting source: AlloyDB for {}", PostgresSource.class); + new IntegrationRunner(source).run(args); + LOGGER.info("completed source: AlloyDB for {}", PostgresSource.class); + } + +} diff --git a/docs/integrations/README.md b/docs/integrations/README.md index ec909c0cfa3c..1ca760b47a1a 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -18,6 +18,7 @@ For more information about the grading system, see [Product Release Stages](http |:--------------------------------------------------------------------------------------------| :------------------- | :------------------ | | [3PL Central](sources/tplcentral.md) | Alpha | No | | [Airtable](sources/airtable.md) | Alpha | Yes | +| [AlloyDb](sources/alloydb.md) | Alpha | Yes | | [Amazon Ads](sources/amazon-ads.md) | Beta | Yes | | [Amazon Seller Partner](sources/amazon-seller-partner.md) | Alpha | Yes | | [Amazon SQS](sources/amazon-sqs.md) | Alpha | Yes | diff --git a/docs/integrations/sources/alloydb.md b/docs/integrations/sources/alloydb.md new file mode 100644 index 000000000000..ee27832f7fec --- /dev/null +++ b/docs/integrations/sources/alloydb.md @@ -0,0 +1,332 @@ +# AlloyDB for PostgreSQL + +This page contains the setup guide and reference information for the AlloyDB for PostgreSQL. + +## Prerequisites + +- For Airbyte Open Source users, [upgrade](https://docs.airbyte.com/operator-guides/upgrading-airbyte/) your Airbyte platform to version `v0.40.0-alpha` or newer +- Allowlist the IP address `34.106.109.131` to enable access to Airbyte +- For Airbyte Cloud (and optionally for Airbyte Open Source), ensure SSL is enabled in your environment + +## Setup guide + +## When to use AlloyDB with CDC + +Configure AlloyDB with CDC if: + +- You need a record of deletions +- Your table has a primary key but doesn't have a reasonable cursor field for incremental syncing (`updated_at`). CDC allows you to sync your table incrementally + +If your goal is to maintain a snapshot of your table in the destination but the limitations prevent you from using CDC, consider using [non-CDC incremental sync](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) and occasionally reset the data and re-sync. + +If your dataset is small and you just want a snapshot of your table in the destination, consider using [Full Refresh replication](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite) for your table instead of CDC. + + +### Step 1: (Optional) Create a dedicated read-only user + +We recommend creating a dedicated read-only user for better permission control and auditing. Alternatively, you can use an existing AlloyDB user in your database. + +To create a dedicated user, run the following command: + +``` +CREATE USER PASSWORD 'your_password_here'; +``` + +Grant access to the relevant schema: + +``` +GRANT USAGE ON SCHEMA TO +``` + +:::note +To replicate data from multiple AlloyDB schemas, re-run the command to grant access to all the relevant schemas. Note that you'll need to set up multiple Airbyte sources connecting to the same AlloyDB database on multiple schemas. +::: + +Grant the user read-only access to the relevant tables: + +``` +GRANT SELECT ON ALL TABLES IN SCHEMA TO airbyte; +``` + +Allow user to see tables created in the future: + +``` +ALTER DEFAULT PRIVILEGES IN SCHEMA GRANT SELECT ON TABLES TO ; +``` + +Additionally, if you plan to configure CDC for the AlloyDB source connector, grant `REPLICATION` and `LOGIN` permissions to the user: + +``` +CREATE ROLE REPLICATION LOGIN; +``` + +and grant that role to the user: + +``` +GRANT to ; +``` + +**Syncing a subset of columns​** + +Currently, there is no way to sync a subset of columns using the AlloyDB source connector: + +- When setting up a connection, you can only choose which tables to sync, but not columns. +- If the user can only access a subset of columns, the connection check will pass. However, the data sync will fail with a permission denied exception. + +The workaround for partial table syncing is to create a view on the specific columns, and grant the user read access to that view: + +``` +CREATE VIEW as SELECT FROM ; +``` +``` +GRANT SELECT ON TABLE IN SCHEMA to ; +``` + +**Note:** The workaround works only for non-CDC setups since CDC requires data to be in tables and not views. +This issue is tracked in [#9771](https://github.com/airbytehq/airbyte/issues/9771). + +### Step 2: Set up the AlloyDB connector in Airbyte + +1. Log into your [Airbyte Cloud](https://cloud.airbyte.io/workspaces) or Airbyte Open Source account. +2. Click **Sources** and then click **+ New source**. +3. On the Set up the source page, select **AlloyDB** from the Source type dropdown. +4. Enter a name for your source. +5. For the **Host**, **Port**, and **DB Name**, enter the hostname, port number, and name for your AlloyDB database. +6. List the **Schemas** you want to sync. + :::note + The schema names are case sensitive. The 'public' schema is set by default. Multiple schemas may be used at one time. No schemas set explicitly - will sync all of existing. + ::: +7. For **User** and **Password**, enter the username and password you created in [Step 1](#step-1-optional-create-a-dedicated-read-only-user). +8. To customize the JDBC connection beyond common options, specify additional supported [JDBC URL parameters](https://jdbc.postgresql.org/documentation/head/connect.html) as key-value pairs separated by the symbol & in the **JDBC URL Parameters (Advanced)** field. + + Example: key1=value1&key2=value2&key3=value3 + + These parameters will be added at the end of the JDBC URL that the AirByte will use to connect to your AlloyDB database. + + The connector now supports `connectTimeout` and defaults to 60 seconds. Setting connectTimeout to 0 seconds will set the timeout to the longest time available. + + **Note:** Do not use the following keys in JDBC URL Params field as they will be overwritten by Airbyte: + `currentSchema`, `user`, `password`, `ssl`, and `sslmode`. + + :::warning + This is an advanced configuration option. Users are advised to use it with caution. + ::: + +9. For Airbyte Open Source, toggle the switch to connect using SSL. Airbyte Cloud uses SSL by default. +10. For Replication Method, select Standard or [Logical CDC](https://www.postgresql.org/docs/10/logical-replication.html) from the dropdown. Refer to [Configuring AlloyDB connector with Change Data Capture (CDC)](#configuring-alloydb-connector-with-change-data-capture-cdc) for more information. +11. For SSH Tunnel Method, select: + - No Tunnel for a direct connection to the database + - SSH Key Authentication to use an RSA Private as your secret for establishing the SSH tunnel + - Password Authentication to use a password as your secret for establishing the SSH tunnel + Refer to [Connect via SSH Tunnel](#connect-via-ssh-tunnel​) for more information. +12. Click **Set up source**. + +### Connect via SSH Tunnel​ + +You can connect to a AlloyDB instance via an SSH tunnel. + +When using an SSH tunnel, you are configuring Airbyte to connect to an intermediate server (also called a bastion server) that has direct access to the database. Airbyte connects to the bastion and then asks the bastion to connect directly to the server. + +To connect to a AlloyDB instance via an SSH tunnel: + +1. While [setting up](#setup-guide) the AlloyDB source connector, from the SSH tunnel dropdown, select: + - SSH Key Authentication to use an RSA Private as your secret for establishing the SSH tunnel + - Password Authentication to use a password as your secret for establishing the SSH Tunnel +2. For **SSH Tunnel Jump Server Host**, enter the hostname or IP address for the intermediate (bastion) server that Airbyte will connect to. +3. For **SSH Connection Port**, enter the port on the bastion server. The default port for SSH connections is 22. +4. For **SSH Login Username**, enter the username to use when connecting to the bastion server. **Note:** This is the operating system username and not the AlloyDB username. +5. For authentication: + - If you selected **SSH Key Authentication**, set the **SSH Private Key** to the [RSA Private Key](#generating-an-rsa-private-key​) that you are using to create the SSH connection. + - If you selected **Password Authentication**, enter the password for the operating system user to connect to the bastion server. **Note:** This is the operating system password and not the AlloyDB password. + +#### Generating an RSA Private Key​ +The connector expects an RSA key in PEM format. To generate this key, run: + +``` +ssh-keygen -t rsa -m PEM -f myuser_rsa +``` + +The command produces the private key in PEM format and the public key remains in the standard format used by the `authorized_keys` file on your bastion server. Add the public key to your bastion host to the user you want to use with Airbyte. The private key is provided via copy-and-paste to the Airbyte connector configuration screen to allow it to log into the bastion server. + +## Configuring AlloyDB connector with Change Data Capture (CDC) + +Airbyte uses [logical replication](https://www.postgresql.org/docs/10/logical-replication.html) of the Postgres write-ahead log (WAL) to incrementally capture deletes using a replication plugin. To learn more how Airbyte implements CDC, refer to [Change Data Capture (CDC)](https://docs.airbyte.com/understanding-airbyte/cdc/) + +### CDC Considerations + +- Incremental sync is only supported for tables with primary keys. For tables without primary keys, use [Full Refresh sync](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-overwrite). +- Data must be in tables and not views. +- The modifications you want to capture must be made using `DELETE`/`INSERT`/`UPDATE`. For example, changes made using `TRUNCATE`/`ALTER` will not appear in logs and therefore in your destination. +- Schema changes are not supported automatically for CDC sources. Reset and resync data if you make a schema change. +- The records produced by `DELETE` statements only contain primary keys. All other data fields are unset. +- Log-based replication only works for master instances of AlloyDB. +- Using logical replication increases disk space used on the database server. The additional data is stored until it is consumed. + - Set frequent syncs for CDC to ensure that the data doesn't fill up your disk space. + - If you stop syncing a CDC-configured AlloyDB instance with Airbyte, delete the replication slot. Otherwise, it may fill up your disk space. + +### Setting up CDC for AlloyDB + +Airbyte requires a replication slot configured only for its use. Only one source should be configured that uses this replication slot. See Setting up CDC for AlloyDB for instructions. + +#### Step 2: Select a replication plugin​ + +We recommend using a [pgoutput](https://www.postgresql.org/docs/9.6/logicaldecoding-output-plugin.html) plugin (the standard logical decoding plugin in AlloyDB). If the replication table contains multiple JSON blobs and the table size exceeds 1 GB, we recommend using a [wal2json](https://github.com/eulerto/wal2json) instead. Note that wal2json may require additional installation for Bare Metal, VMs (EC2/GCE/etc), Docker, etc. For more information read the [wal2json documentation](https://github.com/eulerto/wal2json). + +#### Step 3: Create replication slot​ + +To create a replication slot called `airbyte_slot` using pgoutput, run: + +``` +SELECT pg_create_logical_replication_slot('airbyte_slot', 'pgoutput'); +``` + +To create a replication slot called `airbyte_slot` using wal2json, run: + +``` +SELECT pg_create_logical_replication_slot('airbyte_slot', 'wal2json'); +``` + +#### Step 4: Create publications and replication identities for tables​ + +For each table you want to replicate with CDC, add the replication identity (the method of distinguishing between rows) first: + +To use primary keys to distinguish between rows, run: + +``` +ALTER TABLE tbl1 REPLICA IDENTITY DEFAULT; +``` + +After setting the replication identity, run: + +``` +CREATE PUBLICATION airbyte_publication FOR TABLE ;` +``` + +The publication name is customizable. Refer to the [Postgres docs](https://www.postgresql.org/docs/10/sql-alterpublication.html) if you need to add or remove tables from your publication in the future. + +:::note +You must add the replication identity before creating the publication. Otherwise, `ALTER`/`UPDATE`/`DELETE` statements may fail if AlloyDB cannot determine how to uniquely identify rows. +Also, the publication should include all the tables and only the tables that need to be synced. Otherwise, data from these tables may not be replicated correctly. +::: + +:::warning +The Airbyte UI currently allows selecting any tables for CDC. If a table is selected that is not part of the publication, it will not be replicated even though it is selected. If a table is part of the publication but does not have a replication identity, that replication identity will be created automatically on the first run if the Airbyte user has the necessary permissions. +::: + +#### Step 5: [Optional] Set up initial waiting time + +:::warning +This is an advanced feature. Use it if absolutely necessary. +::: + +The AlloyDB connector may need some time to start processing the data in the CDC mode in the following scenarios: + +- When the connection is set up for the first time and a snapshot is needed +- When the connector has a lot of change logs to process + +The connector waits for the default initial wait time of 5 minutes (300 seconds). Setting the parameter to a longer duration will result in slower syncs, while setting it to a shorter duration may cause the connector to not have enough time to create the initial snapshot or read through the change logs. The valid range is 120 seconds to 1200 seconds. + +If you know there are database changes to be synced, but the connector cannot read those changes, the root cause may be insufficient waiting time. In that case, you can increase the waiting time (example: set to 600 seconds) to test if it is indeed the root cause. On the other hand, if you know there are no database changes, you can decrease the wait time to speed up the zero record syncs. + +#### Step 6: Set up the AlloyDB source connector + +In [Step 2](#step-2-set-up-the-alloydb-connector-in-airbyte) of the connector setup guide, enter the replication slot and publication you just created. + +## Supported sync modes + +The AlloyDB source connector supports the following [sync modes](https://docs.airbyte.com/cloud/core-concepts#connection-sync-modes): + +- [Full Refresh - Overwrite](https://docs.airbyte.com/understanding-airbyte/glossary#full-refresh-sync) +- [Full Refresh - Append](https://docs.airbyte.com/understanding-airbyte/connections/full-refresh-append) +- [Incremental Sync - Append](https://docs.airbyte.com/understanding-airbyte/connections/incremental-append) +- [Incremental Sync - Deduped History](https://docs.airbyte.com/understanding-airbyte/connections/incremental-deduped-history) + +## Supported cursors + +- `TIMESTAMP` +- `TIMESTAMP_WITH_TIMEZONE` +- `TIME` +- `TIME_WITH_TIMEZONE` +- `DATE` +- `BIT` +- `BOOLEAN` +- `TINYINT/SMALLINT` +- `INTEGER` +- `BIGINT` +- `FLOAT/DOUBLE` +- `REAL` +- `NUMERIC/DECIMAL` +- `CHAR/NCHAR/NVARCHAR/VARCHAR/LONGVARCHAR` +- `BINARY/BLOB` + +## Data type mapping +The AlloyDb is a fully managed PostgreSQL-compatible database service. + +According to Postgres [documentation](https://www.postgresql.org/docs/14/datatype.html), Postgres data types are mapped to the following data types when synchronizing data. You can check the test values examples [here](https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceDatatypeTest.java). If you can't find the data type you are looking for or have any problems feel free to add a new test! + +| Postgres Type | Resulting Type | Notes | +|:--------------------------------------|:---------------|:------------------------------------------------------------------------------------------------------------------------------------------------| +| `bigint` | number | | +| `bigserial`, `serial8` | number | | +| `bit` | string | Fixed-length bit string (e.g. "0100"). | +| `bit varying`, `varbit` | string | Variable-length bit string (e.g. "0100"). | +| `boolean`, `bool` | boolean | | +| `box` | string | | +| `bytea` | string | Variable length binary string with hex output format prefixed with "\x" (e.g. "\x6b707a"). | +| `character`, `char` | string | | +| `character varying`, `varchar` | string | | +| `cidr` | string | | +| `circle` | string | | +| `date` | string | Parsed as ISO8601 date time at midnight. CDC mode doesn't support era indicators. Issue: [#14590](https://github.com/airbytehq/airbyte/issues/14590) | +| `double precision`, `float`, `float8` | number | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902). | +| `hstore` | string | | +| `inet` | string | | +| `integer`, `int`, `int4` | number | | +| `interval` | string | | +| `json` | string | | +| `jsonb` | string | | +| `line` | string | | +| `lseg` | string | | +| `macaddr` | string | | +| `macaddr8` | string | | +| `money` | number | | +| `numeric`, `decimal` | number | `Infinity`, `-Infinity`, and `NaN` are not supported and converted to `null`. Issue: [#8902](https://github.com/airbytehq/airbyte/issues/8902). | +| `path` | string | | +| `pg_lsn` | string | | +| `point` | string | | +| `polygon` | string | | +| `real`, `float4` | number | | +| `smallint`, `int2` | number | | +| `smallserial`, `serial2` | number | | +| `serial`, `serial4` | number | | +| `text` | string | | +| `time` | string | Parsed as a time string without a time-zone in the ISO-8601 calendar system. | +| `timetz` | string | Parsed as a time string with time-zone in the ISO-8601 calendar system. | +| `timestamp` | string | Parsed as a date-time string without a time-zone in the ISO-8601 calendar system. | +| `timestamptz` | string | Parsed as a date-time string with time-zone in the ISO-8601 calendar system. | +| `tsquery` | string | | +| `tsvector` | string | | +| `uuid` | string | | +| `xml` | string | | +| `enum` | string | | +| `tsrange` | string | | +| `array` | array | E.g. "[\"10001\",\"10002\",\"10003\",\"10004\"]". | +| composite type | string | | + +## Limitations + +- The AlloyDB source connector currently does not handle schemas larger than 4MB. +- The AlloyDB source connector does not alter the schema present in your database. Depending on the destination connected to this source, however, the schema may be altered. See the destination's documentation for more details. +- The following two schema evolution actions are currently supported: + - Adding/removing tables without resetting the entire connection at the destination + Caveat: In the CDC mode, adding a new table to a connection may become a temporary bottleneck. When a new table is added, the next sync job takes a full snapshot of the new table before it proceeds to handle any changes. + - Resetting a single table within the connection without resetting the rest of the destination tables in that connection +- Changing a column data type or removing a column might break connections. + + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.1.0 | 2022-09-05 | [16323](https://github.com/airbytehq/airbyte/pull/16323) | Initial commit. Based on source-postgres v.1.0.7 + From d105177e76c72ef3dc77c4c08aad0ae079b0dcd7 Mon Sep 17 00:00:00 2001 From: Oleksandr Sheheda Date: Wed, 7 Sep 2022 13:36:27 +0300 Subject: [PATCH 054/200] MQTT Destination: Password is not marked as a secret field in spec (#16263) * [16219] MQTT Destination: Password is not marked as a secret field in spec * [16219] MQTT Destination: Password is not marked as a secret field in spec * [16219] MQTT Destination: Password is not marked as a secret field in spec added logs for troubleshooting * [16219] MQTT Destination: Password is not marked as a secret field in spec updated maxinflight and connect_timeout for test only * [16219] MQTT Destination: Password is not marked as a secret field in spec updated maxinflight for test only * [16219] MQTT Destination: Password is not marked as a secret field in spec added config param max_in_flight for test * [16219] MQTT Destination: Password is not marked as a secret field in spec made refactoring * auto-bump connector version [ci skip] Co-authored-by: Octavia Squidington III --- .../init/src/main/resources/seed/destination_definitions.yaml | 2 +- .../init/src/main/resources/seed/destination_specs.yaml | 3 ++- airbyte-integrations/connectors/destination-mqtt/Dockerfile | 2 +- .../integrations/destination/mqtt/MqttDestinationConfig.java | 4 ++++ .../connectors/destination-mqtt/src/main/resources/spec.json | 3 ++- .../destination/mqtt/MqttDestinationAcceptanceTest.java | 1 + docs/integrations/destinations/mqtt.md | 1 + 7 files changed, 12 insertions(+), 4 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 844d41197b40..fa573110f5a7 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -161,7 +161,7 @@ - name: MQTT destinationDefinitionId: f3802bc4-5406-4752-9e8d-01e504ca8194 dockerRepository: airbyte/destination-mqtt - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/destinations/mqtt icon: mqtt.svg releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index d84246244c3a..ad5987d3e1ff 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -2406,7 +2406,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-mqtt:0.1.2" +- dockerImage: "airbyte/destination-mqtt:0.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mqtt" connectionSpecification: @@ -2447,6 +2447,7 @@ title: "Password" description: "Password to use for the connection." type: "string" + airbyte_secret: true topic_pattern: title: "Topic pattern" description: "Topic pattern in which the records will be sent. You can use\ diff --git a/airbyte-integrations/connectors/destination-mqtt/Dockerfile b/airbyte-integrations/connectors/destination-mqtt/Dockerfile index 792c70786846..4240cbc8afb1 100644 --- a/airbyte-integrations/connectors/destination-mqtt/Dockerfile +++ b/airbyte-integrations/connectors/destination-mqtt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mqtt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/destination-mqtt diff --git a/airbyte-integrations/connectors/destination-mqtt/src/main/java/io/airbyte/integrations/destination/mqtt/MqttDestinationConfig.java b/airbyte-integrations/connectors/destination-mqtt/src/main/java/io/airbyte/integrations/destination/mqtt/MqttDestinationConfig.java index 3f2b63ee7530..73fc3a569f0e 100644 --- a/airbyte-integrations/connectors/destination-mqtt/src/main/java/io/airbyte/integrations/destination/mqtt/MqttDestinationConfig.java +++ b/airbyte-integrations/connectors/destination-mqtt/src/main/java/io/airbyte/integrations/destination/mqtt/MqttDestinationConfig.java @@ -85,6 +85,10 @@ private MqttConnectOptions buildMqttConnectOptions(final JsonNode config) { options.setPassword(config.get("password").asText().toCharArray()); } + if (config.has("max_in_flight") && !config.get("max_in_flight").asText().isBlank()) { + options.setMaxInflight(config.get("max_in_flight").asInt()); + } + return options; } diff --git a/airbyte-integrations/connectors/destination-mqtt/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-mqtt/src/main/resources/spec.json index c88cf72803bb..759bc66ef0bb 100644 --- a/airbyte-integrations/connectors/destination-mqtt/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-mqtt/src/main/resources/spec.json @@ -46,7 +46,8 @@ "password": { "title": "Password", "description": "Password to use for the connection.", - "type": "string" + "type": "string", + "airbyte_secret": true }, "topic_pattern": { "title": "Topic pattern", diff --git a/airbyte-integrations/connectors/destination-mqtt/src/test-integration/java/io/airbyte/integrations/destination/mqtt/MqttDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mqtt/src/test-integration/java/io/airbyte/integrations/destination/mqtt/MqttDestinationAcceptanceTest.java index c6bbf9d810a3..b0dea439b6a0 100644 --- a/airbyte-integrations/connectors/destination-mqtt/src/test-integration/java/io/airbyte/integrations/destination/mqtt/MqttDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mqtt/src/test-integration/java/io/airbyte/integrations/destination/mqtt/MqttDestinationAcceptanceTest.java @@ -63,6 +63,7 @@ protected JsonNode getConfig() throws UnknownHostException { .put("clean_session", true) .put("message_retained", false) .put("message_qos", "EXACTLY_ONCE") + .put("max_in_flight", 1000) .build()); } diff --git a/docs/integrations/destinations/mqtt.md b/docs/integrations/destinations/mqtt.md index b9611b016741..e465ac5fdda5 100644 --- a/docs/integrations/destinations/mqtt.md +++ b/docs/integrations/destinations/mqtt.md @@ -85,5 +85,6 @@ _NOTE_: MQTT version 5 is not supported yet. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.3 | 2022-09-02 | [16263](https://github.com/airbytehq/airbyte/pull/16263) | Marked password field in spec as airbyte_secret | | 0.1.2 | 2022-07-12 | [14648](https://github.com/airbytehq/airbyte/pull/14648) | Include lifecycle management | | 0.1.1 | 2022-05-24 | [13099](https://github.com/airbytehq/airbyte/pull/13099) | Fixed build's tests | From 086d33ef715f63dc2be7b8598fb7234c2d1df38a Mon Sep 17 00:00:00 2001 From: Augustin Date: Wed, 7 Sep 2022 14:24:14 +0200 Subject: [PATCH 055/200] Env-configs: make REMOTE_CONNECTOR_CATALOG_URL optional (#16346) --- .../src/main/java/io/airbyte/config/Configs.java | 5 ++++- .../main/java/io/airbyte/config/EnvConfigs.java | 10 +++++++--- .../java/io/airbyte/config/EnvConfigsTest.java | 14 ++++++++++++++ ...finitionProviderToConfigPersistenceAdapter.java | 3 +++ 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java index dc94b824b9bf..5dac93a102c5 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java @@ -11,6 +11,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -74,8 +75,10 @@ public interface Configs { /** * Defines the URL to pull the remote connector catalog from. + * + * @return */ - URI getRemoteConnectorCatalogUrl(); + Optional getRemoteConnectorCatalogUrl(); // Docker Only diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java index c2cf65bd1b8a..5d10b1f14ef5 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -327,9 +327,13 @@ public Path getWorkspaceRoot() { } @Override - public URI getRemoteConnectorCatalogUrl() { - // Default to reuse the job database - return URI.create(getEnsureEnv(REMOTE_CONNECTOR_CATALOG_URL)); + public Optional getRemoteConnectorCatalogUrl() { + final String remoteConnectorCatalogUrl = getEnvOrDefault(REMOTE_CONNECTOR_CATALOG_URL, null); + if (remoteConnectorCatalogUrl != null) { + return Optional.of(URI.create(remoteConnectorCatalogUrl)); + } else { + return Optional.empty(); + } } // Docker Only diff --git a/airbyte-config/config-models/src/test/java/io/airbyte/config/EnvConfigsTest.java b/airbyte-config/config-models/src/test/java/io/airbyte/config/EnvConfigsTest.java index 550674ca9cf5..fef1a269f40a 100644 --- a/airbyte-config/config-models/src/test/java/io/airbyte/config/EnvConfigsTest.java +++ b/airbyte-config/config-models/src/test/java/io/airbyte/config/EnvConfigsTest.java @@ -10,10 +10,12 @@ import io.airbyte.config.Configs.DeploymentMode; import io.airbyte.config.Configs.JobErrorReportingStrategy; import io.airbyte.config.Configs.WorkerEnvironment; +import java.net.URI; import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; @@ -469,4 +471,16 @@ void testAllJobEnvMapRetrieval() { assertEquals(expected, config.getJobDefaultEnvMap()); } + @Test + void testRemoteConnectorCatalogUrl() { + envMap.put(EnvConfigs.REMOTE_CONNECTOR_CATALOG_URL, null); + assertEquals(Optional.empty(), config.getRemoteConnectorCatalogUrl()); + + envMap.put(EnvConfigs.REMOTE_CONNECTOR_CATALOG_URL, ""); + assertEquals(Optional.empty(), config.getRemoteConnectorCatalogUrl()); + + envMap.put(EnvConfigs.REMOTE_CONNECTOR_CATALOG_URL, "https://airbyte.com"); + assertEquals(Optional.of(URI.create("https://airbyte.com")), config.getRemoteConnectorCatalogUrl()); + } + } diff --git a/airbyte-config/init/src/main/java/io/airbyte/config/init/DefinitionProviderToConfigPersistenceAdapter.java b/airbyte-config/init/src/main/java/io/airbyte/config/init/DefinitionProviderToConfigPersistenceAdapter.java index a55e64d76c0a..0763c4730d0b 100644 --- a/airbyte-config/init/src/main/java/io/airbyte/config/init/DefinitionProviderToConfigPersistenceAdapter.java +++ b/airbyte-config/init/src/main/java/io/airbyte/config/init/DefinitionProviderToConfigPersistenceAdapter.java @@ -98,7 +98,10 @@ public Map> dumpConfigs() throws IOException { @Override public void loadData(ConfigPersistence seedPersistence) throws IOException { throw new UnsupportedOperationException(PERSISTENCE_READ_ONLY_ERROR_MSG); + } + public DefinitionsProvider getDefinitionsProvider() { + return definitionsProvider; } } From 479c2322d0ce408c1fdb841c39ec4109270054c0 Mon Sep 17 00:00:00 2001 From: Eugene Date: Wed, 7 Sep 2022 18:20:49 +0300 Subject: [PATCH 056/200] =?UTF-8?q?=F0=9F=8E=89Source-cockroachdb:=20added?= =?UTF-8?q?=20custom=20JDBC=20parameters=20field=20(#16394)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [10723] Source-cockroachdb: added custom JDBC parameters field --- .../main/resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 13 +++++++++++-- .../source-cockroachdb-strict-encrypt/Dockerfile | 2 +- .../src/main/resources/expected_spec.json | 6 ++++++ .../connectors/source-cockroachdb/Dockerfile | 2 +- .../source/cockroachdb/CockroachDbSource.java | 6 ++++++ .../src/main/resources/spec.json | 8 +++++++- .../source/cockroachdb/CockroachDbSpecTest.java | 15 +++++++++++++++ docs/integrations/sources/cockroachdb.md | 3 ++- 9 files changed, 50 insertions(+), 7 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 92ae63bfe6e6..e5e34117215b 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -184,7 +184,7 @@ - name: Cockroachdb sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003 dockerRepository: airbyte/source-cockroachdb - dockerImageTag: 0.1.17 + dockerImageTag: 0.1.18 documentationUrl: https://docs.airbyte.io/integrations/sources/cockroachdb icon: cockroachdb.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 9be52aaedd38..251388fedf5e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -1795,7 +1795,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-cockroachdb:0.1.17" +- dockerImage: "airbyte/source-cockroachdb:0.1.18" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/cockroachdb" connectionSpecification: @@ -1839,12 +1839,21 @@ type: "string" airbyte_secret: true order: 4 + jdbc_url_params: + description: "Additional properties to pass to the JDBC URL string when\ + \ connecting to the database formatted as 'key=value' pairs separated\ + \ by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more\ + \ information read about JDBC URL parameters." + title: "JDBC URL Parameters (Advanced)" + type: "string" + order: 5 ssl: title: "Connect using SSL" description: "Encrypt client/server communications for increased security." type: "boolean" default: false - order: 5 + order: 6 supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile index c22730f43175..86fe67b15e31 100644 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile +++ b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb-strict-encrypt COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.17 +LABEL io.airbyte.version=0.1.18 LABEL io.airbyte.name=airbyte/source-cockroachdb-strict-encrypt diff --git a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/expected_spec.json b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/expected_spec.json index 37ce4ffec1b8..964654e6dfeb 100644 --- a/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/expected_spec.json +++ b/airbyte-integrations/connectors/source-cockroachdb-strict-encrypt/src/main/resources/expected_spec.json @@ -40,6 +40,12 @@ "type": "string", "airbyte_secret": true, "order": 4 + }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", + "title": "JDBC URL Parameters (Advanced)", + "type": "string", + "order": 5 } } } diff --git a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile index 3fe2f76f9b50..1541c933439b 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/Dockerfile +++ b/airbyte-integrations/connectors/source-cockroachdb/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION source-cockroachdb COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.17 +LABEL io.airbyte.version=0.1.18 LABEL io.airbyte.name=airbyte/source-cockroachdb diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java index 2e34b38a8928..6777215e5611 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/main/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSource.java @@ -4,6 +4,8 @@ package io.airbyte.integrations.source.cockroachdb; +import static io.airbyte.db.jdbc.JdbcUtils.AMPERSAND; + import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; import io.airbyte.commons.functional.CheckedFunction; @@ -61,6 +63,10 @@ public JsonNode toDatabaseConfig(final JsonNode config) { config.get(JdbcUtils.PORT_KEY).asText(), config.get(JdbcUtils.DATABASE_KEY).asText())); + if (config.get(JdbcUtils.JDBC_URL_PARAMS_KEY) != null && !config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText().isEmpty()) { + jdbcUrl.append(config.get(JdbcUtils.JDBC_URL_PARAMS_KEY).asText()).append(AMPERSAND); + } + if (config.has(JdbcUtils.SSL_KEY) && config.get(JdbcUtils.SSL_KEY).asBoolean() || !config.has(JdbcUtils.SSL_KEY)) { additionalParameters.add("ssl=true"); additionalParameters.add("sslmode=require"); diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/main/resources/spec.json b/airbyte-integrations/connectors/source-cockroachdb/src/main/resources/spec.json index 8b0bff813171..68e798ffa05a 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-cockroachdb/src/main/resources/spec.json @@ -41,12 +41,18 @@ "airbyte_secret": true, "order": 4 }, + "jdbc_url_params": { + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (Eg. key1=value1&key2=value2&key3=value3). For more information read about JDBC URL parameters.", + "title": "JDBC URL Parameters (Advanced)", + "type": "string", + "order": 5 + }, "ssl": { "title": "Connect using SSL", "description": "Encrypt client/server communications for increased security.", "type": "boolean", "default": false, - "order": 5 + "order": 6 } } } diff --git a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSpecTest.java b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSpecTest.java index cf684f99fc8c..880101660d13 100644 --- a/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSpecTest.java +++ b/airbyte-integrations/connectors/source-cockroachdb/src/test/java/io/airbyte/integrations/source/cockroachdb/CockroachDbSpecTest.java @@ -5,6 +5,7 @@ package io.airbyte.integrations.source.cockroachdb; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import com.fasterxml.jackson.databind.JsonNode; @@ -13,6 +14,7 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; import io.airbyte.db.jdbc.JdbcUtils; +import io.airbyte.protocol.models.ConnectorSpecification; import io.airbyte.validation.json.JsonSchemaValidator; import java.io.File; import java.io.IOException; @@ -33,6 +35,7 @@ public class CockroachDbSpecTest { + "\"database\" : \"postgres_db\", " + "\"port\" : 5432, " + "\"host\" : \"localhost\", " + + "\"jdbc_url_params\" : \"property1=pValue1&property2=pValue2\", " + "\"ssl\" : true }"; private static JsonNode schema; @@ -69,4 +72,16 @@ void testWithReplicationMethodWithReplicationSlot() { assertTrue(validator.test(schema, config)); } + @Test + void testWithJdbcAdditionalProperty() { + final JsonNode config = Jsons.deserialize(CONFIGURATION); + assertTrue(validator.test(schema, config)); + } + + @Test + void testJdbcAdditionalProperty() throws Exception { + final ConnectorSpecification spec = new CockroachDbSource().spec(); + assertNotNull(spec.getConnectionSpecification().get("properties").get("jdbc_url_params")); + } + } diff --git a/docs/integrations/sources/cockroachdb.md b/docs/integrations/sources/cockroachdb.md index ce53cbd9fbf5..1538c0221a04 100644 --- a/docs/integrations/sources/cockroachdb.md +++ b/docs/integrations/sources/cockroachdb.md @@ -95,9 +95,10 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- | :--- | +| 0.1.18 | 2022-09-01 | [16394](https://github.com/airbytehq/airbyte/pull/16394) | Added custom jdbc properties field | | 0.1.17 | 2022-09-01 | [16238](https://github.com/airbytehq/airbyte/pull/16238) | Emit state messages more frequently | | 0.1.16 | 2022-08-18 | [14356](https://github.com/airbytehq/airbyte/pull/14356) | DB Sources: only show a table can sync incrementally if at least one column can be used as a cursor field | -| 0.1.13 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | +| 0.1.13 | 2022-07-14 | [14574](https://github.com/airbytehq/airbyte/pull/14574) | Removed additionalProperties:false from JDBC source connectors | | 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.11 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.1.10 | 2022-02-24 | [10235](https://github.com/airbytehq/airbyte/pull/10235) | Fix Replication Failure due Multiple portal opens | From fbeb6a4dc1b65dd6e28f47bce060b3a264862be7 Mon Sep 17 00:00:00 2001 From: Edmundo Ruiz Ghanem <168664+edmundito@users.noreply.github.com> Date: Wed, 7 Sep 2022 12:17:24 -0400 Subject: [PATCH 057/200] Add storybook (#16398) --- .../SingletonCard/SingletonCard.tsx | 2 +- .../SingletonCard/index.stories.tsx | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 airbyte-webapp/src/components/SingletonCard/index.stories.tsx diff --git a/airbyte-webapp/src/components/SingletonCard/SingletonCard.tsx b/airbyte-webapp/src/components/SingletonCard/SingletonCard.tsx index 822f98d69c32..e310f0ea3e96 100644 --- a/airbyte-webapp/src/components/SingletonCard/SingletonCard.tsx +++ b/airbyte-webapp/src/components/SingletonCard/SingletonCard.tsx @@ -69,7 +69,7 @@ const CloseButton = styled(Button)` margin-left: 10px; `; -const SingletonCard: React.FC = (props) => ( +export const SingletonCard: React.FC = (props) => ( {props.hasError && }
    diff --git a/airbyte-webapp/src/components/SingletonCard/index.stories.tsx b/airbyte-webapp/src/components/SingletonCard/index.stories.tsx new file mode 100644 index 000000000000..8321f7ccd197 --- /dev/null +++ b/airbyte-webapp/src/components/SingletonCard/index.stories.tsx @@ -0,0 +1,35 @@ +import { ComponentStory, ComponentMeta } from "@storybook/react"; + +import { SingletonCard } from "./SingletonCard"; + +export default { + title: "Ui/SingletonCard", + component: SingletonCard, + argTypes: { + title: { type: { name: "string", required: false } }, + text: { type: { name: "string", required: false } }, + onClose: { table: { disable: true } }, + }, +} as ComponentMeta; + +const Template: ComponentStory = (args) => ; + +export const Basic = Template.bind({}); +Basic.args = { + text: "This is a basic card", +}; + +export const WithTitle = Template.bind({}); +WithTitle.args = { + title: "With Title", + text: "This is a card with a title", +}; + +export const WithCloseButton = Template.bind({}); +WithCloseButton.args = { + title: "With Close button", + text: "This is a card with a close button", + onClose: () => { + console.log("Closed!"); + }, +}; From 42c5265d0d24b4d53aee403c4f0ff1a81688cd40 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 7 Sep 2022 12:28:56 -0400 Subject: [PATCH 058/200] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=94=A7=20Add=20acc?= =?UTF-8?q?essibility=20linting=20(#15859)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * turn on recommended rules * remove autofocus prop (form already autofocuses there without it anyways) * remove autofocus prop from connector request form (field autofocuses without it anyways) * remove autofocus from popout menu (doesn't autofocus even with it on) * Sidebar popout menu accessibility (role, tabindex, onfocus) * switch to use onFocus vs onClick * edit connection name button should be a real button * maybe create element instead for connectionname? * fix weird ts button things * modal and switch * add label control rule * disable for tooltip temporarily * fix rebase issues * make navbar popout buttons actual buttons * fix button appearance in sidebar * remove border from path popout button * review cleanup * Update airbyte-webapp/src/components/Modal/Modal.tsx Co-authored-by: Krishna Glick * leave modal default for now * missing comment space * make eslint jsx a11y plugin a dev dependency * fix stylelint problem * add mock intersection observer * review changes * switch component * move the event bubbling to the StatusCell * Remove autoFocus * order z-indices by value * missing bracket Co-authored-by: Krishna Glick Co-authored-by: Tim Roes --- airbyte-webapp/.eslintrc | 6 +- airbyte-webapp/package-lock.json | 432 +++++++++++------- airbyte-webapp/package.json | 1 + .../EntityTable/components/StatusCell.tsx | 11 +- .../src/components/Modal/Modal.module.scss | 32 +- airbyte-webapp/src/components/Modal/Modal.tsx | 64 +-- .../src/components/base/Popout/Popout.tsx | 1 - .../src/components/base/Switch/Switch.tsx | 2 +- .../src/components/base/Tooltip/Tooltip.tsx | 1 + .../services/Modal/ModalService.test.tsx | 6 + .../cloud/views/layout/SideBar/SideBar.tsx | 8 +- .../components/CreateWorkspaceForm.tsx | 2 +- .../components/ConnectionName.tsx | 5 +- .../pages/OnboardingPage/OnboardingPage.tsx | 1 - airbyte-webapp/src/scss/_z-indices.scss | 4 +- airbyte-webapp/src/utils/testutils.tsx | 11 + .../components/PathPopoutButton.module.scss | 1 + .../components/PathPopoutButton.tsx | 4 +- .../components/ConnectorForm.tsx | 2 +- .../ServiceForm/ServiceForm.test.tsx | 6 +- .../views/layout/SideBar/SideBar.module.scss | 3 + .../src/views/layout/SideBar/SideBar.tsx | 4 +- 22 files changed, 365 insertions(+), 242 deletions(-) diff --git a/airbyte-webapp/.eslintrc b/airbyte-webapp/.eslintrc index da88f4262ce5..4ae49f4ecee0 100644 --- a/airbyte-webapp/.eslintrc +++ b/airbyte-webapp/.eslintrc @@ -5,9 +5,10 @@ "plugin:jest/recommended", "prettier", "plugin:prettier/recommended", - "plugin:css-modules/recommended" + "plugin:css-modules/recommended", + "plugin:jsx-a11y/recommended" ], - "plugins": ["react", "@typescript-eslint", "prettier", "unused-imports", "css-modules"], + "plugins": ["react", "@typescript-eslint", "prettier", "unused-imports", "css-modules", "jsx-a11y"], "parserOptions": { "ecmaVersion": 2020, "sourceType": "module", @@ -16,6 +17,7 @@ } }, "rules": { + "jsx-a11y/label-has-associated-control": "error", "curly": "warn", "css-modules/no-undef-class": ["error", { "camelCase": true }], "css-modules/no-unused-class": ["error", { "camelCase": true }], diff --git a/airbyte-webapp/package-lock.json b/airbyte-webapp/package-lock.json index 3d6b793e3c53..3511f56d0a07 100644 --- a/airbyte-webapp/package-lock.json +++ b/airbyte-webapp/package-lock.json @@ -91,6 +91,7 @@ "eslint-config-react-app": "^7.0.1", "eslint-plugin-css-modules": "^2.11.0", "eslint-plugin-jest": "^26.5.3", + "eslint-plugin-jsx-a11y": "^6.6.1", "eslint-plugin-prettier": "^4.0.0", "eslint-plugin-unused-imports": "^2.0.0", "express": "^4.18.1", @@ -2269,9 +2270,9 @@ } }, "node_modules/@babel/runtime": { - "version": "7.18.3", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.18.3.tgz", - "integrity": "sha512-38Y8f7YUhce/K7RMwTp7m0uCumpv9hZkitCbBClqQIow1qSbCvGkcegKOXpEWCQLfWmevgRiWokZ1GkpfhbZug==", + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.18.9.tgz", + "integrity": "sha512-lkqXDcvlFT5rvEjiu6+QYO+1GXrEHRo2LOtS7E4GtX5ESIZOgepqsZBVIj6Pv+a6zqsya9VCgiK1KAK4BvJDAw==", "dependencies": { "regenerator-runtime": "^0.13.4" }, @@ -16158,14 +16159,14 @@ "dev": true }, "node_modules/array-includes": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.4.tgz", - "integrity": "sha512-ZTNSQkmWumEbiHO2GF4GmWxYVTiQyJy2XOTa15sdQSrvKn7l+180egQMqlrMOUMCyLMD7pmyQe4mMDUT6Behrw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.5.tgz", + "integrity": "sha512-iSDYZMMyTPkiFasVqfuAQnWAYcvO/SeBSCGKePoEthjp4LEMTe4uLc7b025o4jAZpHhihh8xPo99TNWUWWkGDQ==", "dev": true, "dependencies": { "call-bind": "^1.0.2", - "define-properties": "^1.1.3", - "es-abstract": "^1.19.1", + "define-properties": "^1.1.4", + "es-abstract": "^1.19.5", "get-intrinsic": "^1.1.1", "is-string": "^1.0.7" }, @@ -16437,9 +16438,9 @@ "dev": true }, "node_modules/axe-core": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.3.5.tgz", - "integrity": "sha512-WKTW1+xAzhMS5dJsxWkliixlO/PqC4VhmO9T4juNYcaTg9jzWiJsou6m5pxWYGfigWbwzJWeFY6z47a+4neRXA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.4.3.tgz", + "integrity": "sha512-32+ub6kkdhhWick/UjvEwRchgoetXqTK14INLqbGm5U2TzBkBNF3nQtLYm8ovxSkQWArjEQvftCKryjZaATu3w==", "dev": true, "engines": { "node": ">=4" @@ -19970,15 +19971,19 @@ } }, "node_modules/define-properties": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", - "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", "dev": true, "dependencies": { - "object-keys": "^1.0.12" + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" }, "engines": { "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/define-property": { @@ -20858,31 +20863,34 @@ } }, "node_modules/es-abstract": { - "version": "1.19.1", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.19.1.tgz", - "integrity": "sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==", + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.20.1.tgz", + "integrity": "sha512-WEm2oBhfoI2sImeM4OF2zE2V3BYdSF+KnSi9Sidz51fQHd7+JuF8Xgcj9/0o+OWeIeIS/MiuNnlruQrJf16GQA==", "dev": true, "dependencies": { "call-bind": "^1.0.2", "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", + "function.prototype.name": "^1.1.5", "get-intrinsic": "^1.1.1", "get-symbol-description": "^1.0.0", "has": "^1.0.3", - "has-symbols": "^1.0.2", + "has-property-descriptors": "^1.0.0", + "has-symbols": "^1.0.3", "internal-slot": "^1.0.3", "is-callable": "^1.2.4", - "is-negative-zero": "^2.0.1", + "is-negative-zero": "^2.0.2", "is-regex": "^1.1.4", - "is-shared-array-buffer": "^1.0.1", + "is-shared-array-buffer": "^1.0.2", "is-string": "^1.0.7", - "is-weakref": "^1.0.1", - "object-inspect": "^1.11.0", + "is-weakref": "^1.0.2", + "object-inspect": "^1.12.0", "object-keys": "^1.1.1", "object.assign": "^4.1.2", - "string.prototype.trimend": "^1.0.4", - "string.prototype.trimstart": "^1.0.4", - "unbox-primitive": "^1.0.1" + "regexp.prototype.flags": "^1.4.3", + "string.prototype.trimend": "^1.0.5", + "string.prototype.trimstart": "^1.0.5", + "unbox-primitive": "^1.0.2" }, "engines": { "node": ">= 0.4" @@ -21495,23 +21503,24 @@ } }, "node_modules/eslint-plugin-jsx-a11y": { - "version": "6.5.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.5.1.tgz", - "integrity": "sha512-sVCFKX9fllURnXT2JwLN5Qgo24Ug5NF6dxhkmxsMEUZhXRcGg+X3e1JbJ84YePQKBl5E0ZjAH5Q4rkdcGY99+g==", + "version": "6.6.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.6.1.tgz", + "integrity": "sha512-sXgFVNHiWffBq23uiS/JaP6eVR622DqwB4yTzKvGZGcPq6/yZ3WmOZfuBks/vHWo9GaFOqC2ZK4i6+C35knx7Q==", "dev": true, "dependencies": { - "@babel/runtime": "^7.16.3", + "@babel/runtime": "^7.18.9", "aria-query": "^4.2.2", - "array-includes": "^3.1.4", + "array-includes": "^3.1.5", "ast-types-flow": "^0.0.7", - "axe-core": "^4.3.5", + "axe-core": "^4.4.3", "axobject-query": "^2.2.0", - "damerau-levenshtein": "^1.0.7", + "damerau-levenshtein": "^1.0.8", "emoji-regex": "^9.2.2", "has": "^1.0.3", - "jsx-ast-utils": "^3.2.1", + "jsx-ast-utils": "^3.3.2", "language-tags": "^1.0.5", - "minimatch": "^3.0.4" + "minimatch": "^3.1.2", + "semver": "^6.3.0" }, "engines": { "node": ">=4.0" @@ -21526,6 +21535,18 @@ "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", "dev": true }, + "node_modules/eslint-plugin-jsx-a11y/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/eslint-plugin-prettier": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-4.0.0.tgz", @@ -24016,9 +24037,9 @@ } }, "node_modules/has-bigints": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz", - "integrity": "sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", + "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -24056,10 +24077,22 @@ "node": ">=0.10.0" } }, + "node_modules/has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "dev": true, + "dependencies": { + "get-intrinsic": "^1.1.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/has-symbols": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", - "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", "dev": true, "engines": { "node": ">= 0.4" @@ -25522,10 +25555,13 @@ "dev": true }, "node_modules/is-bigint": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.2.tgz", - "integrity": "sha512-0JV5+SOCQkIdzjBK9buARcV804Ddu7A0Qet6sHi3FimE9ne6m4BGQZfRn+NZiXbBk4F4XmHfDZIipLj9pX8dSA==", + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", + "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", "dev": true, + "dependencies": { + "has-bigints": "^1.0.1" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -25542,12 +25578,13 @@ } }, "node_modules/is-boolean-object": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.1.tgz", - "integrity": "sha512-bXdQWkECBUIAcCkeH1unwJLIpZYaa5VvuygSyS/c2lf719mTKZDU5UdDRlpd01UjADgmW8RfqaP+mRaVPdr/Ng==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", + "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", "dev": true, "dependencies": { - "call-bind": "^1.0.2" + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" }, "engines": { "node": ">= 0.4" @@ -25816,9 +25853,9 @@ "dev": true }, "node_modules/is-negative-zero": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", - "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz", + "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==", "dev": true, "engines": { "node": ">= 0.4" @@ -25845,10 +25882,13 @@ } }, "node_modules/is-number-object": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.5.tgz", - "integrity": "sha512-RU0lI/n95pMoUKu9v1BZP5MBcZuNSVJkMkAG2dJqC4z2GlkGUNeH68SuHuBKBD/XFe+LHZ+f9BKkLET60Niedw==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", + "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", "dev": true, + "dependencies": { + "has-tostringtag": "^1.0.0" + }, "engines": { "node": ">= 0.4" }, @@ -25966,10 +26006,13 @@ } }, "node_modules/is-shared-array-buffer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.1.tgz", - "integrity": "sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", + "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", "dev": true, + "dependencies": { + "call-bind": "^1.0.2" + }, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -29924,13 +29967,13 @@ } }, "node_modules/jsx-ast-utils": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.2.1.tgz", - "integrity": "sha512-uP5vu8xfy2F9A6LGC22KO7e2/vGTS1MhP+18f++ZNlf0Ohaxbc9nIEwHAsejlJKyzfZzU5UIhe5ItYkitcZnZA==", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.3.tgz", + "integrity": "sha512-fYQHZTZ8jSfmWZ0iyzfwiU4WDX4HpHbMCZ3gPlWYiCl3BoeOTsqKBqnTVfH2rYT7eP5c3sVbeSPHnnJOaTrWiw==", "dev": true, "dependencies": { - "array-includes": "^3.1.3", - "object.assign": "^4.1.2" + "array-includes": "^3.1.5", + "object.assign": "^4.1.3" }, "engines": { "node": ">=4.0" @@ -33755,14 +33798,14 @@ } }, "node_modules/object.assign": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", - "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz", + "integrity": "sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ==", "dev": true, "dependencies": { - "call-bind": "^1.0.0", - "define-properties": "^1.1.3", - "has-symbols": "^1.0.1", + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "has-symbols": "^1.0.3", "object-keys": "^1.1.1" }, "engines": { @@ -39923,13 +39966,14 @@ "dev": true }, "node_modules/regexp.prototype.flags": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.3.1.tgz", - "integrity": "sha512-JiBdRBq91WlY7uRJ0ds7R+dU02i6LKi8r3BuQhNXn+kmeLN+EfHhfjqMRis1zJxnlu88hq/4dx0P2OP3APRTOA==", + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz", + "integrity": "sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA==", "dev": true, "dependencies": { "call-bind": "^1.0.2", - "define-properties": "^1.1.3" + "define-properties": "^1.1.3", + "functions-have-names": "^1.2.2" }, "engines": { "node": ">= 0.4" @@ -43323,26 +43367,28 @@ } }, "node_modules/string.prototype.trimend": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz", - "integrity": "sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.5.tgz", + "integrity": "sha512-I7RGvmjV4pJ7O3kdf+LXFpVfdNOxtCW/2C8f6jNiW4+PQchwxkCDzlk1/7p+Wl4bqFIZeF47qAHXLuHHWKAxog==", "dev": true, "dependencies": { "call-bind": "^1.0.2", - "define-properties": "^1.1.3" + "define-properties": "^1.1.4", + "es-abstract": "^1.19.5" }, "funding": { "url": "https://github.com/sponsors/ljharb" } }, "node_modules/string.prototype.trimstart": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz", - "integrity": "sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.5.tgz", + "integrity": "sha512-THx16TJCGlsN0o6dl2o6ncWUsdgnLRSA23rRE5pyGBw/mLr3Ej/R2LaqCtgP8VNMGZsvMWnf9ooZPyY2bHvUFg==", "dev": true, "dependencies": { "call-bind": "^1.0.2", - "define-properties": "^1.1.3" + "define-properties": "^1.1.4", + "es-abstract": "^1.19.5" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -45171,14 +45217,14 @@ } }, "node_modules/unbox-primitive": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.1.tgz", - "integrity": "sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", + "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==", "dev": true, "dependencies": { - "function-bind": "^1.1.1", - "has-bigints": "^1.0.1", - "has-symbols": "^1.0.2", + "call-bind": "^1.0.2", + "has-bigints": "^1.0.2", + "has-symbols": "^1.0.3", "which-boxed-primitive": "^1.0.2" }, "funding": { @@ -49219,9 +49265,9 @@ } }, "@babel/runtime": { - "version": "7.18.3", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.18.3.tgz", - "integrity": "sha512-38Y8f7YUhce/K7RMwTp7m0uCumpv9hZkitCbBClqQIow1qSbCvGkcegKOXpEWCQLfWmevgRiWokZ1GkpfhbZug==", + "version": "7.18.9", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.18.9.tgz", + "integrity": "sha512-lkqXDcvlFT5rvEjiu6+QYO+1GXrEHRo2LOtS7E4GtX5ESIZOgepqsZBVIj6Pv+a6zqsya9VCgiK1KAK4BvJDAw==", "requires": { "regenerator-runtime": "^0.13.4" } @@ -59973,14 +60019,14 @@ "dev": true }, "array-includes": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.4.tgz", - "integrity": "sha512-ZTNSQkmWumEbiHO2GF4GmWxYVTiQyJy2XOTa15sdQSrvKn7l+180egQMqlrMOUMCyLMD7pmyQe4mMDUT6Behrw==", + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.5.tgz", + "integrity": "sha512-iSDYZMMyTPkiFasVqfuAQnWAYcvO/SeBSCGKePoEthjp4LEMTe4uLc7b025o4jAZpHhihh8xPo99TNWUWWkGDQ==", "dev": true, "requires": { "call-bind": "^1.0.2", - "define-properties": "^1.1.3", - "es-abstract": "^1.19.1", + "define-properties": "^1.1.4", + "es-abstract": "^1.19.5", "get-intrinsic": "^1.1.1", "is-string": "^1.0.7" } @@ -60196,9 +60242,9 @@ } }, "axe-core": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.3.5.tgz", - "integrity": "sha512-WKTW1+xAzhMS5dJsxWkliixlO/PqC4VhmO9T4juNYcaTg9jzWiJsou6m5pxWYGfigWbwzJWeFY6z47a+4neRXA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.4.3.tgz", + "integrity": "sha512-32+ub6kkdhhWick/UjvEwRchgoetXqTK14INLqbGm5U2TzBkBNF3nQtLYm8ovxSkQWArjEQvftCKryjZaATu3w==", "dev": true }, "axobject-query": { @@ -63013,12 +63059,13 @@ "dev": true }, "define-properties": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", - "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.4.tgz", + "integrity": "sha512-uckOqKcfaVvtBdsVkdPv3XjveQJsNQqmhXgRi8uhvWWuPYZCNlzT8qAyblUgNoXdHdjMTzAqeGjAoli8f+bzPA==", "dev": true, "requires": { - "object-keys": "^1.0.12" + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" } }, "define-property": { @@ -63759,31 +63806,34 @@ } }, "es-abstract": { - "version": "1.19.1", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.19.1.tgz", - "integrity": "sha512-2vJ6tjA/UfqLm2MPs7jxVybLoB8i1t1Jd9R3kISld20sIxPcTbLuggQOUxeWeAvIUkduv/CfMjuh4WmiXr2v9w==", + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.20.1.tgz", + "integrity": "sha512-WEm2oBhfoI2sImeM4OF2zE2V3BYdSF+KnSi9Sidz51fQHd7+JuF8Xgcj9/0o+OWeIeIS/MiuNnlruQrJf16GQA==", "dev": true, "requires": { "call-bind": "^1.0.2", "es-to-primitive": "^1.2.1", "function-bind": "^1.1.1", + "function.prototype.name": "^1.1.5", "get-intrinsic": "^1.1.1", "get-symbol-description": "^1.0.0", "has": "^1.0.3", - "has-symbols": "^1.0.2", + "has-property-descriptors": "^1.0.0", + "has-symbols": "^1.0.3", "internal-slot": "^1.0.3", "is-callable": "^1.2.4", - "is-negative-zero": "^2.0.1", + "is-negative-zero": "^2.0.2", "is-regex": "^1.1.4", - "is-shared-array-buffer": "^1.0.1", + "is-shared-array-buffer": "^1.0.2", "is-string": "^1.0.7", - "is-weakref": "^1.0.1", - "object-inspect": "^1.11.0", + "is-weakref": "^1.0.2", + "object-inspect": "^1.12.0", "object-keys": "^1.1.1", "object.assign": "^4.1.2", - "string.prototype.trimend": "^1.0.4", - "string.prototype.trimstart": "^1.0.4", - "unbox-primitive": "^1.0.1" + "regexp.prototype.flags": "^1.4.3", + "string.prototype.trimend": "^1.0.5", + "string.prototype.trimstart": "^1.0.5", + "unbox-primitive": "^1.0.2" } }, "es-array-method-boxes-properly": { @@ -64359,23 +64409,24 @@ } }, "eslint-plugin-jsx-a11y": { - "version": "6.5.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.5.1.tgz", - "integrity": "sha512-sVCFKX9fllURnXT2JwLN5Qgo24Ug5NF6dxhkmxsMEUZhXRcGg+X3e1JbJ84YePQKBl5E0ZjAH5Q4rkdcGY99+g==", + "version": "6.6.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.6.1.tgz", + "integrity": "sha512-sXgFVNHiWffBq23uiS/JaP6eVR622DqwB4yTzKvGZGcPq6/yZ3WmOZfuBks/vHWo9GaFOqC2ZK4i6+C35knx7Q==", "dev": true, "requires": { - "@babel/runtime": "^7.16.3", + "@babel/runtime": "^7.18.9", "aria-query": "^4.2.2", - "array-includes": "^3.1.4", + "array-includes": "^3.1.5", "ast-types-flow": "^0.0.7", - "axe-core": "^4.3.5", + "axe-core": "^4.4.3", "axobject-query": "^2.2.0", - "damerau-levenshtein": "^1.0.7", + "damerau-levenshtein": "^1.0.8", "emoji-regex": "^9.2.2", "has": "^1.0.3", - "jsx-ast-utils": "^3.2.1", + "jsx-ast-utils": "^3.3.2", "language-tags": "^1.0.5", - "minimatch": "^3.0.4" + "minimatch": "^3.1.2", + "semver": "^6.3.0" }, "dependencies": { "emoji-regex": { @@ -64383,6 +64434,15 @@ "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", "dev": true + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } } } }, @@ -66186,9 +66246,9 @@ } }, "has-bigints": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz", - "integrity": "sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.2.tgz", + "integrity": "sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==", "dev": true }, "has-flag": { @@ -66216,10 +66276,19 @@ } } }, + "has-property-descriptors": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", + "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "dev": true, + "requires": { + "get-intrinsic": "^1.1.1" + } + }, "has-symbols": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", - "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==", + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", "dev": true }, "has-tostringtag": { @@ -67331,10 +67400,13 @@ "dev": true }, "is-bigint": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.2.tgz", - "integrity": "sha512-0JV5+SOCQkIdzjBK9buARcV804Ddu7A0Qet6sHi3FimE9ne6m4BGQZfRn+NZiXbBk4F4XmHfDZIipLj9pX8dSA==", - "dev": true + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", + "integrity": "sha512-zB9CruMamjym81i2JZ3UMn54PKGsQzsJeo6xvN3HJJ4CAsQNB6iRutp2To77OfCNuoxspsIhzaPoO1zyCEhFOg==", + "dev": true, + "requires": { + "has-bigints": "^1.0.1" + } }, "is-binary-path": { "version": "2.1.0", @@ -67345,12 +67417,13 @@ } }, "is-boolean-object": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.1.tgz", - "integrity": "sha512-bXdQWkECBUIAcCkeH1unwJLIpZYaa5VvuygSyS/c2lf719mTKZDU5UdDRlpd01UjADgmW8RfqaP+mRaVPdr/Ng==", + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.2.tgz", + "integrity": "sha512-gDYaKHJmnj4aWxyj6YHyXVpdQawtVLHU5cb+eztPGczf6cjuTdwve5ZIEfgXqH4e57An1D1AKf8CZ3kYrQRqYA==", "dev": true, "requires": { - "call-bind": "^1.0.2" + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" } }, "is-buffer": { @@ -67536,9 +67609,9 @@ "dev": true }, "is-negative-zero": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", - "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz", + "integrity": "sha512-dqJvarLawXsFbNDeJW7zAz8ItJ9cd28YufuuFzh0G8pNHjJMnY08Dv7sYX2uF5UpQOwieAeOExEYAWWfu7ZZUA==", "dev": true }, "is-npm": { @@ -67553,10 +67626,13 @@ "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==" }, "is-number-object": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.5.tgz", - "integrity": "sha512-RU0lI/n95pMoUKu9v1BZP5MBcZuNSVJkMkAG2dJqC4z2GlkGUNeH68SuHuBKBD/XFe+LHZ+f9BKkLET60Niedw==", - "dev": true + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.7.tgz", + "integrity": "sha512-k1U0IRzLMo7ZlYIfzRu23Oh6MiIFasgpb9X76eqfFZAqwH44UI4KTBvBYIZ1dSL9ZzChTB9ShHfLkR4pdW5krQ==", + "dev": true, + "requires": { + "has-tostringtag": "^1.0.0" + } }, "is-obj": { "version": "1.0.1", @@ -67635,10 +67711,13 @@ "dev": true }, "is-shared-array-buffer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.1.tgz", - "integrity": "sha512-IU0NmyknYZN0rChcKhRO1X8LYz5Isj/Fsqh8NJOSf+N/hCOTwy29F32Ik7a+QszE63IdvmwdTPDd6cZ5pg4cwA==", - "dev": true + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", + "integrity": "sha512-sqN2UDu1/0y6uvXyStCOzyhAjCSlHceFoMKJW8W9EU9cvic/QdsZ0kEU93HEy3IUEFZIiH/3w+AH/UQbPHNdhA==", + "dev": true, + "requires": { + "call-bind": "^1.0.2" + } }, "is-stream": { "version": "2.0.0", @@ -70695,13 +70774,13 @@ "dev": true }, "jsx-ast-utils": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.2.1.tgz", - "integrity": "sha512-uP5vu8xfy2F9A6LGC22KO7e2/vGTS1MhP+18f++ZNlf0Ohaxbc9nIEwHAsejlJKyzfZzU5UIhe5ItYkitcZnZA==", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.3.tgz", + "integrity": "sha512-fYQHZTZ8jSfmWZ0iyzfwiU4WDX4HpHbMCZ3gPlWYiCl3BoeOTsqKBqnTVfH2rYT7eP5c3sVbeSPHnnJOaTrWiw==", "dev": true, "requires": { - "array-includes": "^3.1.3", - "object.assign": "^4.1.2" + "array-includes": "^3.1.5", + "object.assign": "^4.1.3" } }, "jszip": { @@ -73550,14 +73629,14 @@ } }, "object.assign": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", - "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==", + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.4.tgz", + "integrity": "sha512-1mxKf0e58bvyjSCtKYY4sRe9itRk3PJpquJOjeIkz885CczcI4IvJJDLPS72oowuSh+pBxUFROpX+TU++hxhZQ==", "dev": true, "requires": { - "call-bind": "^1.0.0", - "define-properties": "^1.1.3", - "has-symbols": "^1.0.1", + "call-bind": "^1.0.2", + "define-properties": "^1.1.4", + "has-symbols": "^1.0.3", "object-keys": "^1.1.1" } }, @@ -77964,13 +78043,14 @@ "dev": true }, "regexp.prototype.flags": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.3.1.tgz", - "integrity": "sha512-JiBdRBq91WlY7uRJ0ds7R+dU02i6LKi8r3BuQhNXn+kmeLN+EfHhfjqMRis1zJxnlu88hq/4dx0P2OP3APRTOA==", + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz", + "integrity": "sha512-fjggEOO3slI6Wvgjwflkc4NFRCTZAu5CnNfBd5qOMYhWdn67nJBBu34/TkD++eeFmd8C9r9jfXJ27+nSiRkSUA==", "dev": true, "requires": { "call-bind": "^1.0.2", - "define-properties": "^1.1.3" + "define-properties": "^1.1.3", + "functions-have-names": "^1.2.2" } }, "regexpp": { @@ -80566,23 +80646,25 @@ } }, "string.prototype.trimend": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz", - "integrity": "sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.5.tgz", + "integrity": "sha512-I7RGvmjV4pJ7O3kdf+LXFpVfdNOxtCW/2C8f6jNiW4+PQchwxkCDzlk1/7p+Wl4bqFIZeF47qAHXLuHHWKAxog==", "dev": true, "requires": { "call-bind": "^1.0.2", - "define-properties": "^1.1.3" + "define-properties": "^1.1.4", + "es-abstract": "^1.19.5" } }, "string.prototype.trimstart": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz", - "integrity": "sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.5.tgz", + "integrity": "sha512-THx16TJCGlsN0o6dl2o6ncWUsdgnLRSA23rRE5pyGBw/mLr3Ej/R2LaqCtgP8VNMGZsvMWnf9ooZPyY2bHvUFg==", "dev": true, "requires": { "call-bind": "^1.0.2", - "define-properties": "^1.1.3" + "define-properties": "^1.1.4", + "es-abstract": "^1.19.5" } }, "stringify-entities": { @@ -81898,14 +81980,14 @@ "optional": true }, "unbox-primitive": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.1.tgz", - "integrity": "sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.2.tgz", + "integrity": "sha512-61pPlCD9h51VoreyJ0BReideM3MDKMKnh6+V9L08331ipq6Q8OFXZYiqP6n/tbHx4s5I9uRhcye6BrbkizkBDw==", "dev": true, "requires": { - "function-bind": "^1.1.1", - "has-bigints": "^1.0.1", - "has-symbols": "^1.0.2", + "call-bind": "^1.0.2", + "has-bigints": "^1.0.2", + "has-symbols": "^1.0.3", "which-boxed-primitive": "^1.0.2" } }, diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 45439bd0ff0c..b5888ae6ddc9 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -107,6 +107,7 @@ "eslint-config-react-app": "^7.0.1", "eslint-plugin-css-modules": "^2.11.0", "eslint-plugin-jest": "^26.5.3", + "eslint-plugin-jsx-a11y": "^6.6.1", "eslint-plugin-prettier": "^4.0.0", "eslint-plugin-unused-imports": "^2.0.0", "express": "^4.18.1", diff --git a/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx b/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx index a3c8a1be59cc..929b6f3cc13b 100644 --- a/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx @@ -35,7 +35,16 @@ const StatusCell: React.FC = ({ enabled, isManual, id, onChangeStatus, i onChangeStatus(id); }; - return ; + return ( + // this is so we can stop event propagation so the row doesn't receive the click and redirect + // eslint-disable-next-line jsx-a11y/no-static-element-interactions +
    event.stopPropagation()} + onKeyPress={(event: React.SyntheticEvent) => event.stopPropagation()} + > + +
    + ); } if (isSyncing) { diff --git a/airbyte-webapp/src/components/Modal/Modal.module.scss b/airbyte-webapp/src/components/Modal/Modal.module.scss index 1374c393a0b9..6946eafbef14 100644 --- a/airbyte-webapp/src/components/Modal/Modal.module.scss +++ b/airbyte-webapp/src/components/Modal/Modal.module.scss @@ -1,5 +1,6 @@ @use "../../scss/colors"; @use "../../scss/variables"; +@use "../../scss/z-indices"; @keyframes fade-in { from { @@ -7,23 +8,40 @@ } } -.modal { - animation: fade-in 0.2s ease-out; - position: absolute; +.backdrop { + position: fixed; top: 0; left: 0; - width: 100vw; - height: 100vh; + right: 0; + bottom: 0; background: rgba(colors.$black, 0.5); +} + +.modalPageContainer { + position: absolute; + z-index: z-indices.$modal; +} + +.modalContainer { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + animation: fadeIn 0.2s ease-out; display: flex; justify-content: center; align-items: center; - z-index: 100; +} + +.modalPanel { + margin-left: auto; + margin-right: auto; } .card { margin-left: variables.$width-size-menu; - max-width: calc(100% - (#{variables.$width-size-menu} - #{variables.$spacing-lg}) * 2); + max-width: calc(100vw - variables.$width-size-menu - variables.$spacing-lg * 2); &.sm { width: variables.$width-modal-sm; diff --git a/airbyte-webapp/src/components/Modal/Modal.tsx b/airbyte-webapp/src/components/Modal/Modal.tsx index 1ff2bc3dd87a..299043a1a69c 100644 --- a/airbyte-webapp/src/components/Modal/Modal.tsx +++ b/airbyte-webapp/src/components/Modal/Modal.tsx @@ -1,6 +1,6 @@ +import { Dialog } from "@headlessui/react"; import classNames from "classnames"; -import React, { useEffect, useCallback } from "react"; -import { createPortal } from "react-dom"; +import React, { useState } from "react"; import ContentCard from "components/ContentCard"; @@ -22,43 +22,29 @@ const cardStyleBySize = { xl: styles.xl, }; -const Modal: React.FC = ({ children, title, onClose, clear, closeOnBackground, size, testId }) => { - const handleUserKeyPress = useCallback((event: KeyboardEvent, closeModal: () => void) => { - const { key } = event; - // Escape key - if (key === "Escape") { - closeModal(); - } - }, []); - - useEffect(() => { - if (!onClose) { - return; - } - - const onKeyDown = (event: KeyboardEvent) => handleUserKeyPress(event, onClose); - window.addEventListener("keydown", onKeyDown); - - return () => { - window.removeEventListener("keydown", onKeyDown); - }; - }, [handleUserKeyPress, onClose]); - - return createPortal( -
    (closeOnBackground && onClose ? onClose() : null)} - data-testid={testId} - > - {clear ? ( - children - ) : ( - - {children} - - )} -
    , - document.body +const Modal: React.FC = ({ children, title, size, onClose, clear, testId }) => { + const [isOpen, setIsOpen] = useState(true); + + const onModalClose = () => { + setIsOpen(false); + onClose?.(); + }; + + return ( + +
    +
    + + {clear ? ( + children + ) : ( + + {children} + + )} + +
    +
    ); }; diff --git a/airbyte-webapp/src/components/base/Popout/Popout.tsx b/airbyte-webapp/src/components/base/Popout/Popout.tsx index 28053686cab3..b42f2cdee240 100644 --- a/airbyte-webapp/src/components/base/Popout/Popout.tsx +++ b/airbyte-webapp/src/components/base/Popout/Popout.tsx @@ -66,7 +66,6 @@ const Popout: React.FC = ({ onChange, targetComponent, ...props }) targetComponent, onOpen: toggleOpen, }} - autoFocus backspaceRemovesValue={false} controlShouldRenderValue={false} hideSelectedOptions={false} diff --git a/airbyte-webapp/src/components/base/Switch/Switch.tsx b/airbyte-webapp/src/components/base/Switch/Switch.tsx index 6674b122cf6d..3c27f54c5db9 100644 --- a/airbyte-webapp/src/components/base/Switch/Switch.tsx +++ b/airbyte-webapp/src/components/base/Switch/Switch.tsx @@ -19,7 +19,7 @@ export const Switch: React.FC = ({ loading, small, checked, value, }); return ( -
    ) : ( -
    setEditingState(true)}> +
    + )} ); diff --git a/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx b/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx index 86147dd59c91..e213d145f23a 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/OnboardingPage.tsx @@ -37,7 +37,6 @@ const Content = styled.div<{ big?: boolean; medium?: boolean }>` align-items: center; min-height: 100%; position: relative; - z-index: 2; `; const Footer = styled.div` diff --git a/airbyte-webapp/src/scss/_z-indices.scss b/airbyte-webapp/src/scss/_z-indices.scss index dfbe539c2518..8b81f8320a3f 100644 --- a/airbyte-webapp/src/scss/_z-indices.scss +++ b/airbyte-webapp/src/scss/_z-indices.scss @@ -1,2 +1,4 @@ +$tooltip: 9999 + 2; +$modal: 9999 + 1; $sidebar: 9999; -$tooltip: 9999 + 1; +$panelSplitter: 0; diff --git a/airbyte-webapp/src/utils/testutils.tsx b/airbyte-webapp/src/utils/testutils.tsx index 90dd4589e811..58980169b25b 100644 --- a/airbyte-webapp/src/utils/testutils.tsx +++ b/airbyte-webapp/src/utils/testutils.tsx @@ -61,6 +61,17 @@ export const TestWrapper: React.FC = ({ children }) => ( ); +export const useMockIntersectionObserver = () => { + // IntersectionObserver isn't available in test environment but is used by the dialog component + const mockIntersectionObserver = jest.fn(); + mockIntersectionObserver.mockReturnValue({ + observe: jest.fn().mockReturnValue(null), + unobserve: jest.fn().mockReturnValue(null), + disconnect: jest.fn().mockReturnValue(null), + }); + window.IntersectionObserver = mockIntersectionObserver; +}; + export const mockSource: SourceRead = { sourceId: "test-source", name: "test source", diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/components/PathPopoutButton.module.scss b/airbyte-webapp/src/views/Connection/CatalogTree/components/PathPopoutButton.module.scss index 7d915a94dbf5..2d27778881ed 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/components/PathPopoutButton.module.scss +++ b/airbyte-webapp/src/views/Connection/CatalogTree/components/PathPopoutButton.module.scss @@ -15,6 +15,7 @@ padding: 8px; border-radius: variables.$border-radius-xs; background-color: colors.$grey-100; + border: none; cursor: pointer; &:hover { diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/components/PathPopoutButton.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/components/PathPopoutButton.tsx index 29d0cc33ed3b..0b773a483784 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/components/PathPopoutButton.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/components/PathPopoutButton.tsx @@ -15,12 +15,12 @@ interface PathPopoutButtonProps { export const PathPopoutButton: React.FC = ({ items = [], onClick, children }) => ( + } placement="bottom-start" disabled={items.length === 0} diff --git a/airbyte-webapp/src/views/Connector/RequestConnectorModal/components/ConnectorForm.tsx b/airbyte-webapp/src/views/Connector/RequestConnectorModal/components/ConnectorForm.tsx index 8f497b2790ea..a8741f5e14bb 100644 --- a/airbyte-webapp/src/views/Connector/RequestConnectorModal/components/ConnectorForm.tsx +++ b/airbyte-webapp/src/views/Connector/RequestConnectorModal/components/ConnectorForm.tsx @@ -100,7 +100,7 @@ const ConnectorForm: React.FC = ({ onSubmit, onCancel, curre ) } > - + )} diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/ServiceForm.test.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/ServiceForm.test.tsx index 732cf326c746..3449734f5729 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/ServiceForm.test.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/ServiceForm.test.tsx @@ -5,7 +5,7 @@ import React from "react"; import selectEvent from "react-select-event"; import { AirbyteJSONSchema } from "core/jsonSchema"; -import { render } from "utils/testutils"; +import { render, useMockIntersectionObserver } from "utils/testutils"; import { ServiceForm } from "views/Connector/ServiceForm"; import { DestinationDefinitionSpecificationRead } from "../../../core/request/AirbyteClient"; @@ -353,6 +353,10 @@ describe("Service Form", () => { }); test("should fill right values in array of objects field", async () => { + // IntersectionObserver isn't available in test environment but is used by headless-ui dialog + // used for this component + useMockIntersectionObserver(); + const addPriceListItem = useAddPriceListItem(container); await addPriceListItem("test-1", "1"); await addPriceListItem("test-2", "2"); diff --git a/airbyte-webapp/src/views/layout/SideBar/SideBar.module.scss b/airbyte-webapp/src/views/layout/SideBar/SideBar.module.scss index ce6754aa1d73..4321f1f01f01 100644 --- a/airbyte-webapp/src/views/layout/SideBar/SideBar.module.scss +++ b/airbyte-webapp/src/views/layout/SideBar/SideBar.module.scss @@ -2,6 +2,9 @@ @use "../../../scss/variables"; .menuItem { + background-color: transparent; + appearance: none; + border: none; color: colors.$grey-30; width: 100%; cursor: pointer; diff --git a/airbyte-webapp/src/views/layout/SideBar/SideBar.tsx b/airbyte-webapp/src/views/layout/SideBar/SideBar.tsx index 893a6c65fbe6..e3705c227c33 100644 --- a/airbyte-webapp/src/views/layout/SideBar/SideBar.tsx +++ b/airbyte-webapp/src/views/layout/SideBar/SideBar.tsx @@ -129,12 +129,12 @@ const SideBar: React.FC = () => {
  • {({ onOpen, isOpen }) => ( -
    +
    + )}
    • From a68650822c7ee7679a3a3228dcc66e839d64b514 Mon Sep 17 00:00:00 2001 From: "Krishna (kc) Glick" Date: Wed, 7 Sep 2022 12:32:01 -0400 Subject: [PATCH 059/200] Misc Improvements to assist Connection Form Refactor (#16303) * Move exports to declaration for formConfig * Add ConnectionOrPartialConnection, move ConnectionForm exports * Removing additionBottomControls as it was unused * Create and use FormError, test createFormErrorMessage * Remove unnecessary eslint ignore, remove unnecessary async/await * rename createFormErrorMessage --- .../CreateConnectionContent.tsx | 21 +------- .../src/hooks/services/useSourceHook.tsx | 11 ++-- .../components/ReplicationView.tsx | 3 +- .../components/DestinationForm.tsx | 6 +-- .../components/DestinationStep.tsx | 11 ++-- .../OnboardingPage/components/SourceStep.tsx | 11 ++-- .../components/SourceForm.tsx | 6 +-- .../src/utils/errorStatusMessage.test.tsx | 35 ++++++++++++ .../src/utils/errorStatusMessage.tsx | 6 ++- .../ConnectionForm/ConnectionForm.test.tsx | 2 +- .../ConnectionForm/ConnectionForm.tsx | 22 ++++---- .../components/CreateControls.tsx | 9 +--- .../Connection/ConnectionForm/formConfig.tsx | 53 ++++++++++--------- .../views/Connection/ConnectionForm/index.tsx | 4 +- .../src/views/Connection/FormCard.tsx | 4 +- .../Connector/ConnectorCard/ConnectorCard.tsx | 4 +- 16 files changed, 102 insertions(+), 106 deletions(-) create mode 100644 airbyte-webapp/src/utils/errorStatusMessage.test.tsx diff --git a/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx b/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx index 9cdef1509991..12cf19de6c97 100644 --- a/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx +++ b/airbyte-webapp/src/components/CreateConnectionContent/CreateConnectionContent.tsx @@ -2,7 +2,6 @@ import { faRedoAlt } from "@fortawesome/free-solid-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import React, { Suspense, useMemo } from "react"; import { FormattedMessage } from "react-intl"; -import styled from "styled-components"; import { Button, ContentCard } from "components"; import { IDataItem } from "components/base/DropDown/components/Option"; @@ -13,25 +12,14 @@ import { Action, Namespace } from "core/analytics"; import { LogsRequestError } from "core/request/LogsRequestError"; import { useAnalyticsService } from "hooks/services/Analytics"; import { useCreateConnection, ValuesProps } from "hooks/services/useConnectionHook"; -import ConnectionForm from "views/Connection/ConnectionForm"; -import { ConnectionFormProps } from "views/Connection/ConnectionForm/ConnectionForm"; +import { ConnectionForm, ConnectionFormProps } from "views/Connection/ConnectionForm"; import { DestinationRead, SourceRead, WebBackendConnectionRead } from "../../core/request/AirbyteClient"; import { useDiscoverSchema } from "../../hooks/services/useSourceHook"; import TryAfterErrorBlock from "./components/TryAfterErrorBlock"; import styles from "./CreateConnectionContent.module.scss"; -const SkipButton = styled.div` - margin-top: 6px; - - & > button { - min-width: 239px; - margin-left: 9px; - } -`; - interface CreateConnectionContentProps { - additionBottomControls?: React.ReactNode; source: SourceRead; destination: DestinationRead; afterSubmitConnection?: (connection: WebBackendConnectionRead) => void; @@ -41,7 +29,6 @@ const CreateConnectionContent: React.FC = ({ source, destination, afterSubmitConnection, - additionBottomControls, }) => { const { mutateAsync: createConnection } = useCreateConnection(); const analyticsService = useAnalyticsService(); @@ -104,10 +91,7 @@ const CreateConnectionContent: React.FC = ({ const job = LogsRequestError.extractJobInfo(schemaErrorStatus); return ( - {additionBottomControls}} - /> + {job && } ); @@ -120,7 +104,6 @@ const CreateConnectionContent: React.FC = ({ { - (async () => { - if (sourceId) { - await onDiscoverSchema(); - } - })(); + if (sourceId) { + onDiscoverSchema(); + } }, [onDiscoverSchema, sourceId]); return { schemaErrorStatus, isLoading, schema, catalogId, onDiscoverSchema }; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx index 665fa532c968..21cfbe7ea479 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx @@ -21,8 +21,7 @@ import { } from "hooks/services/useConnectionHook"; import { equal } from "utils/objects"; import { CatalogDiffModal } from "views/Connection/CatalogDiffModal/CatalogDiffModal"; -import ConnectionForm from "views/Connection/ConnectionForm"; -import { ConnectionFormSubmitResult } from "views/Connection/ConnectionForm/ConnectionForm"; +import { ConnectionForm, ConnectionFormSubmitResult } from "views/Connection/ConnectionForm"; interface ReplicationViewProps { onAfterSaveSchema: () => void; diff --git a/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/components/DestinationForm.tsx b/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/components/DestinationForm.tsx index a31baae84278..1b38901f4c46 100644 --- a/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/components/DestinationForm.tsx +++ b/airbyte-webapp/src/pages/DestinationPage/pages/CreateDestinationPage/components/DestinationForm.tsx @@ -8,7 +8,7 @@ import { LogsRequestError } from "core/request/LogsRequestError"; import { useAnalyticsService } from "hooks/services/Analytics"; import useRouter from "hooks/useRouter"; import { useGetDestinationDefinitionSpecificationAsync } from "services/connector/DestinationDefinitionSpecificationService"; -import { createFormErrorMessage } from "utils/errorStatusMessage"; +import { generateMessageFromError, FormError } from "utils/errorStatusMessage"; import { ConnectorCard } from "views/Connector/ConnectorCard"; interface DestinationFormProps { @@ -21,7 +21,7 @@ interface DestinationFormProps { afterSelectConnector?: () => void; destinationDefinitions: DestinationDefinitionRead[]; hasSuccess?: boolean; - error?: { message?: string; status?: number } | null; + error?: FormError | null; } const hasDestinationDefinitionId = (state: unknown): state is { destinationDefinitionId: string } => { @@ -75,7 +75,7 @@ export const DestinationForm: React.FC = ({ }); }; - const errorMessage = error ? createFormErrorMessage(error) : null; + const errorMessage = error ? generateMessageFromError(error) : null; return ( = ({ onNextStep, onSuccess }) => { useGetDestinationDefinitionSpecificationAsync(destinationDefinitionId); const { destinationDefinitions } = useDestinationDefinitionList(); const [successRequest, setSuccessRequest] = useState(false); - const [error, setError] = useState<{ - status: number; - response: JobInfo; - message: string; - } | null>(null); + const [error, setError] = useState(null); const { mutateAsync: createDestination } = useCreateDestination(); @@ -89,7 +84,7 @@ const DestinationStep: React.FC = ({ onNextStep, onSuccess }) => { }); }; - const errorMessage = error ? createFormErrorMessage(error) : null; + const errorMessage = error ? generateMessageFromError(error) : null; return ( = ({ onNextStep, onSuccess }) => { const { sourceDefinitions } = useSourceDefinitionList(); const [sourceDefinitionId, setSourceDefinitionId] = useState(null); const [successRequest, setSuccessRequest] = useState(false); - const [error, setError] = useState<{ - status: number; - response: JobInfo; - message: string; - } | null>(null); + const [error, setError] = useState(null); const { setDocumentationUrl, setDocumentationPanelOpen } = useDocumentationPanelContext(); const { mutateAsync: createSource } = useCreateSource(); @@ -91,7 +86,7 @@ const SourceStep: React.FC = ({ onNextStep, onSuccess }) => { ...values, }); - const errorMessage = error ? createFormErrorMessage(error) : ""; + const errorMessage = error ? generateMessageFromError(error) : ""; return ( void; sourceDefinitions: SourceDefinitionReadWithLatestTag[]; hasSuccess?: boolean; - error?: { message?: string; status?: number } | null; + error?: FormError | null; } const hasSourceDefinitionId = (state: unknown): state is { sourceDefinitionId: string } => { @@ -76,7 +76,7 @@ export const SourceForm: React.FC = ({ }); }; - const errorMessage = error ? createFormErrorMessage(error) : null; + const errorMessage = error ? generateMessageFromError(error) : null; return ( { + it("should return a provided error message", () => { + const errMsg = "test"; + expect(generateMessageFromError(new Error(errMsg))).toBe(errMsg); + }); + + it("should return null if no error message and no status, or status is 0", () => { + expect(generateMessageFromError(new Error())).toBe(null); + const fakeStatusError = new FormError(); + fakeStatusError.status = 0; + expect(generateMessageFromError(fakeStatusError)).toBe(null); + }); + + it("should return a validation error message if status is 400", () => { + const fakeStatusError = new FormError(); + fakeStatusError.status = 400; + expect(generateMessageFromError(fakeStatusError)).toMatchInlineSnapshot(` + + `); + }); + + it("should return a 'some error' message if status is > 0 and not 400", () => { + const fakeStatusError = new FormError(); + fakeStatusError.status = 401; + expect(generateMessageFromError(fakeStatusError)).toMatchInlineSnapshot(` + + `); + }); +}); diff --git a/airbyte-webapp/src/utils/errorStatusMessage.tsx b/airbyte-webapp/src/utils/errorStatusMessage.tsx index 7afcf81cb3c7..bb6cc8682bed 100644 --- a/airbyte-webapp/src/utils/errorStatusMessage.tsx +++ b/airbyte-webapp/src/utils/errorStatusMessage.tsx @@ -1,6 +1,10 @@ import { FormattedMessage } from "react-intl"; -export const createFormErrorMessage = (error: { status?: number; message?: string }): JSX.Element | string | null => { +export class FormError extends Error { + status?: number; +} + +export const generateMessageFromError = (error: FormError): JSX.Element | string | null => { if (error.message) { return error.message; } diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.test.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.test.tsx index be101452c6f1..59ef676f0a1f 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.test.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.test.tsx @@ -11,7 +11,7 @@ import { import { ConfirmationModalService } from "hooks/services/ConfirmationModal/ConfirmationModalService"; import { render } from "utils/testutils"; -import ConnectionForm, { ConnectionFormProps } from "./ConnectionForm"; +import { ConnectionForm, ConnectionFormProps } from "./ConnectionForm"; const mockSource: SourceRead = { sourceId: "test-source", diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.tsx index de8bdcc12cbe..77e502efa497 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.tsx @@ -16,7 +16,7 @@ import { import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; import { useGetDestinationDefinitionSpecification } from "services/connector/DestinationDefinitionSpecificationService"; import { useCurrentWorkspace } from "services/workspaces/WorkspacesService"; -import { createFormErrorMessage } from "utils/errorStatusMessage"; +import { generateMessageFromError } from "utils/errorStatusMessage"; import CreateControls from "./components/CreateControls"; import EditControls from "./components/EditControls"; @@ -118,10 +118,13 @@ const DirtyChangeTracker: React.FC = ({ dirty, onChange return null; }; -interface ConnectionFormProps { +export type ConnectionOrPartialConnection = + | WebBackendConnectionRead + | (Partial & Pick); + +export interface ConnectionFormProps { onSubmit: (values: ConnectionFormValues) => Promise; className?: string; - additionBottomControls?: React.ReactNode; successMessage?: React.ReactNode; onDropDownSelect?: (item: DropDownRow.IDataItem) => void; onCancel?: () => void; @@ -132,19 +135,16 @@ interface ConnectionFormProps { mode: ConnectionFormMode; additionalSchemaControl?: React.ReactNode; - connection: - | WebBackendConnectionRead - | (Partial & Pick); + connection: ConnectionOrPartialConnection; } -const ConnectionForm: React.FC = ({ +export const ConnectionForm: React.FC = ({ onSubmit, onCancel, className, onDropDownSelect, mode, successMessage, - additionBottomControls, canSubmitUntouchedForm, additionalSchemaControl, connection, @@ -194,7 +194,7 @@ const ConnectionForm: React.FC = ({ [connection.operations, workspace.workspaceId, onSubmit, clearFormChange, formId] ); - const errorMessage = submitError ? createFormErrorMessage(submitError) : null; + const errorMessage = submitError ? generateMessageFromError(submitError) : null; const frequencies = useFrequencyDropdownData(connection.scheduleData); return ( @@ -373,7 +373,6 @@ const ConnectionForm: React.FC = ({ onEndEditTransformation={toggleEditingTransformation} /> = ({ ); }; - -export type { ConnectionFormProps }; -export default ConnectionForm; diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/components/CreateControls.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/components/CreateControls.tsx index a8eaea1c0c83..6835fcc5568e 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/components/CreateControls.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/components/CreateControls.tsx @@ -8,7 +8,6 @@ interface CreateControlsProps { isSubmitting: boolean; isValid: boolean; errorMessage?: React.ReactNode; - additionBottomControls?: React.ReactNode; } const ButtonContainer = styled.div` @@ -59,12 +58,7 @@ const ErrorText = styled.div` max-width: 400px; `; -const CreateControls: React.FC = ({ - isSubmitting, - errorMessage, - additionBottomControls, - isValid, -}) => { +const CreateControls: React.FC = ({ isSubmitting, errorMessage, isValid }) => { if (isSubmitting) { return ( @@ -90,7 +84,6 @@ const CreateControls: React.FC = ({
    )}
    - {additionBottomControls || null} diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx index 0a3bd2840663..53e82610e06f 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx @@ -28,8 +28,9 @@ import { ValuesProps } from "hooks/services/useConnectionHook"; import { useCurrentWorkspace } from "services/workspaces/WorkspacesService"; import calculateInitialCatalog from "./calculateInitialCatalog"; +import { ConnectionOrPartialConnection } from "./ConnectionForm"; -interface FormikConnectionFormValues { +export interface FormikConnectionFormValues { name?: string; scheduleType?: ConnectionScheduleType | null; scheduleData?: ConnectionScheduleData | null; @@ -41,9 +42,9 @@ interface FormikConnectionFormValues { normalization?: NormalizationType; } -type ConnectionFormValues = ValuesProps; +export type ConnectionFormValues = ValuesProps; -const SUPPORTED_MODES: Array<[SyncMode, DestinationSyncMode]> = [ +export const SUPPORTED_MODES: Array<[SyncMode, DestinationSyncMode]> = [ [SyncMode.incremental, DestinationSyncMode.append_dedup], [SyncMode.full_refresh, DestinationSyncMode.overwrite], [SyncMode.incremental, DestinationSyncMode.append], @@ -57,7 +58,7 @@ const DEFAULT_SCHEDULE: ConnectionScheduleData = { }, }; -function useDefaultTransformation(): OperationCreate { +export function useDefaultTransformation(): OperationCreate { const workspace = useCurrentWorkspace(); return { name: "My dbt transformations", @@ -73,7 +74,7 @@ function useDefaultTransformation(): OperationCreate { }; } -const connectionValidationSchema = yup +export const connectionValidationSchema = yup .object({ name: yup.string().required("form.empty.error"), scheduleType: yup.string().oneOf([ConnectionScheduleType.manual, ConnectionScheduleType.basic]), @@ -172,7 +173,7 @@ const connectionValidationSchema = yup * @param initialOperations * @param workspaceId */ -function mapFormPropsToOperation( +export function mapFormPropsToOperation( values: { transformations?: OperationRead[]; normalization?: NormalizationType; @@ -210,10 +211,10 @@ function mapFormPropsToOperation( return newOperations; } -const getInitialTransformations = (operations: OperationCreate[]): OperationRead[] => +export const getInitialTransformations = (operations: OperationCreate[]): OperationRead[] => operations?.filter(isDbtTransformation) ?? []; -const getInitialNormalization = ( +export const getInitialNormalization = ( operations?: Array, isEditMode?: boolean ): NormalizationType => { @@ -227,10 +228,8 @@ const getInitialNormalization = ( : NormalizationType.basic; }; -const useInitialValues = ( - connection: - | WebBackendConnectionRead - | (Partial & Pick), +export const useInitialValues = ( + connection: ConnectionOrPartialConnection, destDefinition: DestinationDefinitionSpecificationRead, isEditMode?: boolean ): FormikConnectionFormValues => { @@ -261,10 +260,24 @@ const useInitialValues = ( } return initialValues; - }, [initialSchema, connection, isEditMode, destDefinition]); + }, [ + connection.connectionId, + connection.destination.name, + connection.name, + connection.namespaceDefinition, + connection.namespaceFormat, + connection.operations, + connection.prefix, + connection.scheduleData, + connection.source.name, + destDefinition.supportsDbt, + destDefinition.supportsNormalization, + initialSchema, + isEditMode, + ]); }; -const useFrequencyDropdownData = ( +export const useFrequencyDropdownData = ( additionalFrequency: WebBackendConnectionRead["scheduleData"] ): DropDownRow.IDataItem[] => { const { formatMessage } = useIntl(); @@ -295,15 +308,3 @@ const useFrequencyDropdownData = ( })); }, [formatMessage, additionalFrequency]); }; - -export type { ConnectionFormValues, FormikConnectionFormValues }; -export { - connectionValidationSchema, - useInitialValues, - useFrequencyDropdownData, - mapFormPropsToOperation, - SUPPORTED_MODES, - useDefaultTransformation, - getInitialNormalization, - getInitialTransformations, -}; diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/index.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/index.tsx index a4298101b981..518cc0429e56 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/index.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/index.tsx @@ -1,3 +1 @@ -import ConnectionForm from "./ConnectionForm"; - -export default ConnectionForm; +export * from "./ConnectionForm"; diff --git a/airbyte-webapp/src/views/Connection/FormCard.tsx b/airbyte-webapp/src/views/Connection/FormCard.tsx index 51946825b882..215ab247609f 100644 --- a/airbyte-webapp/src/views/Connection/FormCard.tsx +++ b/airbyte-webapp/src/views/Connection/FormCard.tsx @@ -6,7 +6,7 @@ import styled from "styled-components"; import { FormChangeTracker } from "components/FormChangeTracker"; -import { createFormErrorMessage } from "utils/errorStatusMessage"; +import { generateMessageFromError } from "utils/errorStatusMessage"; import { CollapsibleCardProps, CollapsibleCard } from "views/Connection/CollapsibleCard"; import EditControls from "views/Connection/ConnectionForm/components/EditControls"; @@ -41,7 +41,7 @@ export const FormCard = ({ form.onSubmit(values, formikHelpers); }); - const errorMessage = error ? createFormErrorMessage(error) : null; + const errorMessage = error ? generateMessageFromError(error) : null; return ( mutateAsync({ values, formikHelpers })}> diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx index 8f10b2a14bc2..3ef7ec4abb5d 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx @@ -9,7 +9,7 @@ import { Connector, ConnectorT } from "core/domain/connector"; import { SynchronousJobRead } from "core/request/AirbyteClient"; import { LogsRequestError } from "core/request/LogsRequestError"; import { useAnalyticsService } from "hooks/services/Analytics"; -import { createFormErrorMessage } from "utils/errorStatusMessage"; +import { generateMessageFromError } from "utils/errorStatusMessage"; import { ServiceForm, ServiceFormProps, ServiceFormValues } from "views/Connector/ServiceForm"; import { useTestConnector } from "./useTestConnector"; @@ -99,7 +99,7 @@ export const ConnectorCard: React.VFC Date: Wed, 7 Sep 2022 09:36:13 -0700 Subject: [PATCH 060/200] Realign dockerfile that diverged (#16290) * Realign dockerfile that diverged * Fix image label Co-authored-by: Davin Chia --- airbyte-container-orchestrator/Dockerfile | 30 +++++++++++------------ airbyte-workers/Dockerfile | 15 ++++++------ 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index 0794fd545e33..cbbbc69f6d8a 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -1,31 +1,31 @@ ARG JDK_VERSION=19-slim-bullseye ARG JDK_IMAGE=openjdk:${JDK_VERSION} -FROM ${JDK_IMAGE} AS sync-attempt +FROM ${JDK_IMAGE} AS orchestrator ARG DOCKER_BUILD_ARCH=amd64 -# Install Docker to launch orchestrator images. Eventually should be replaced with Docker-java. +# Install Docker to launch worker images. Eventually should be replaced with Docker-java. # See https://gitter.im/docker-java/docker-java?at=5f3eb87ba8c1780176603f4e for more information on why we are not currently using Docker-java +# See https://docs.docker.com/engine/install/debian/ to understand what the following commands are +# doing. RUN apt-get update && apt-get install -y \ - apt-transport-https \ ca-certificates \ - curl \ - gnupg-agent \ - software-properties-common -RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - -# arch var used to detect architecture of container. Architecture should be spcified to get proper binaries from repo. -RUN arch=$(dpkg --print-architecture) && \ - add-apt-repository \ - "deb [arch=${arch}] https://download.docker.com/linux/debian \ - $(lsb_release -cs) stable" + wget \ + gnupg \ + lsb-release +RUN mkdir -p /etc/apt/keyrings && \ + wget -O - https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \ + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null RUN apt-get update && apt-get install -y docker-ce-cli jq # Install kubectl for copying files to kube pods. Eventually should be replaced with a kube java client. # See https://github.com/airbytehq/airbyte/issues/8643 for more information on why we are using kubectl for copying. # The following commands were taken from https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/#install-using-native-package-management -RUN curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg -RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y kubectl +RUN wget -O /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list +RUN apt-get update && apt-get install -y kubectl # Don't change this manually. Bump version expects to make moves based on this string ARG VERSION=0.40.4 diff --git a/airbyte-workers/Dockerfile b/airbyte-workers/Dockerfile index 93433d9653b2..c06c8833d816 100644 --- a/airbyte-workers/Dockerfile +++ b/airbyte-workers/Dockerfile @@ -13,20 +13,21 @@ RUN apt-get update && apt-get install -y \ wget \ gnupg \ lsb-release -RUN mkdir -p /etc/apt/keyrings -RUN wget -O - https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg -RUN echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ - $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null +RUN mkdir -p /etc/apt/keyrings && \ + wget -O - https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg && \ + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null RUN apt-get update && apt-get install -y docker-ce-cli jq # Install kubectl for copying files to kube pods. Eventually should be replaced with a kube java client. # See https://github.com/airbytehq/airbyte/issues/8643 for more information on why we are using kubectl for copying. # The following commands were taken from https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/#install-using-native-package-management -RUN wget -O /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg -RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list +RUN wget -O /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list RUN apt-get update && apt-get install -y kubectl +# Don't change this manually. Bump version expects to make moves based on this string ARG VERSION=0.40.4 ENV APPLICATION airbyte-workers From 1d9608cbbeb0d61a0451b2ff5e90aa61827a14ed Mon Sep 17 00:00:00 2001 From: Brian Lai <51336873+brianjlai@users.noreply.github.com> Date: Wed, 7 Sep 2022 13:20:14 -0400 Subject: [PATCH 061/200] [per-stream cdk] Support deserialization of legacy and per-stream state (#16205) * interpret legacy and new per-stream format into AirbyteStateMessages * add ConnectorStateManager stubs for future work * remove frozen for the time being until we need to hash descriptors * add validation that AirbyteStateMessage has at least one of stream, global, or data fields * pr feedback and clean up of the code * remove changes to airbyte_protocol and perform validation in read_state() * fix import formatting --- .../airbyte_cdk/sources/abstract_source.py | 15 +- .../sources/connector_state_manager.py | 54 +++++ .../python/airbyte_cdk/sources/source.py | 39 +++- .../sources/test_connector_state_manager.py | 39 ++++ .../python/unit_tests/sources/test_source.py | 198 +++++++++++++++++- 5 files changed, 322 insertions(+), 23 deletions(-) create mode 100644 airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py create mode 100644 airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py diff --git a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py index 5e853a8be3d3..fec2d8cab972 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/abstract_source.py @@ -2,13 +2,11 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # - -import copy import logging from abc import ABC, abstractmethod from datetime import datetime from functools import lru_cache -from typing import Any, Dict, Iterator, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, Dict, Iterator, List, Mapping, MutableMapping, Optional, Tuple, Union from airbyte_cdk.models import ( AirbyteCatalog, @@ -22,6 +20,7 @@ SyncMode, ) from airbyte_cdk.models import Type as MessageType +from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager from airbyte_cdk.sources.source import Source from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.http import HttpStream @@ -91,10 +90,12 @@ def read( logger: logging.Logger, config: Mapping[str, Any], catalog: ConfiguredAirbyteCatalog, - state: MutableMapping[str, Any] = None, + state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None, ) -> Iterator[AirbyteMessage]: """Implements the Read operation from the Airbyte Specification. See https://docs.airbyte.io/architecture/airbyte-protocol.""" - connector_state = copy.deepcopy(state or {}) + state_manager = ConnectorStateManager(state=state) + connector_state = state_manager.get_legacy_state() + logger.info(f"Starting syncing {self.name}") config, internal_config = split_config(config) # TODO assert all streams exist in the connector @@ -133,6 +134,10 @@ def read( logger.info(f"Finished syncing {self.name}") + @property + def per_stream_state_enabled(self): + return False # While CDK per-stream is in active development we should keep this off + def _read_stream( self, logger: logging.Logger, diff --git a/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py b/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py new file mode 100644 index 000000000000..cbc936599e42 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py @@ -0,0 +1,54 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import copy +from typing import Any, List, Mapping, MutableMapping, Union + +from airbyte_cdk.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType + + +class ConnectorStateManager: + """ + ConnectorStateManager consolidates the various forms of a stream's incoming state message (STREAM / GLOBAL / LEGACY) under a common + interface. It also provides methods to extract and update state + """ + + # In the immediate, we only persist legacy which will be used during abstract_source.read(). In the subsequent PRs we will + # initialize the ConnectorStateManager according to the new per-stream interface received from the platform + def __init__(self, state: Union[List[AirbyteStateMessage], MutableMapping[str, Any]] = None): + if not state: + self.legacy = {} + elif self.is_migrated_legacy_state(state): + # The legacy state format received from the platform is parsed and stored as a single AirbyteStateMessage when reading + # the file. This is used for input backwards compatibility. + self.legacy = state[0].data + elif isinstance(state, MutableMapping): + # In the event that legacy state comes in as its original JSON object format, no changes to the input need to be made + self.legacy = state + else: + raise ValueError("Input state should come in the form of list of Airbyte state messages or a mapping of states") + + def get_stream_state(self, namespace: str, stream_name: str) -> AirbyteStateBlob: + # todo implement in upcoming PRs + pass + + def get_legacy_state(self) -> MutableMapping[str, Any]: + """ + Returns a deep copy of the current legacy state dictionary made up of the state of all streams for a connector + :return: A copy of the legacy state + """ + return copy.deepcopy(self.legacy, {}) + + def update_state_for_stream(self, namespace: str, stream_name: str, value: Mapping[str, Any]): + # todo implement in upcoming PRs + pass + + @staticmethod + def is_migrated_legacy_state(state: List[AirbyteStateMessage]) -> bool: + return ( + isinstance(state, List) + and len(state) == 1 + and isinstance(state[0], AirbyteStateMessage) + and state[0].type == AirbyteStateType.LEGACY + ) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/source.py b/airbyte-cdk/python/airbyte_cdk/sources/source.py index de0c1be2cceb..d16e42329ab7 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/source.py @@ -6,11 +6,10 @@ import json import logging from abc import ABC, abstractmethod -from collections import defaultdict -from typing import Any, Dict, Generic, Iterable, Mapping, MutableMapping, TypeVar +from typing import Any, Generic, Iterable, List, Mapping, MutableMapping, TypeVar, Union from airbyte_cdk.connector import BaseConnector, DefaultConnectorMixin, TConfig -from airbyte_cdk.models import AirbyteCatalog, AirbyteMessage, ConfiguredAirbyteCatalog +from airbyte_cdk.models import AirbyteCatalog, AirbyteMessage, AirbyteStateMessage, AirbyteStateType, ConfiguredAirbyteCatalog TState = TypeVar("TState") TCatalog = TypeVar("TCatalog") @@ -39,15 +38,37 @@ def discover(self, logger: logging.Logger, config: TConfig) -> AirbyteCatalog: """ -class Source(DefaultConnectorMixin, BaseSource[Mapping[str, Any], MutableMapping[str, Any], ConfiguredAirbyteCatalog], ABC): +class Source( + DefaultConnectorMixin, + BaseSource[Mapping[str, Any], Union[List[AirbyteStateMessage], MutableMapping[str, Any]], ConfiguredAirbyteCatalog], + ABC, +): # can be overridden to change an input state - def read_state(self, state_path: str) -> Dict[str, Any]: + def read_state(self, state_path: str) -> List[AirbyteStateMessage]: + """ + Retrieves the input state of a sync by reading from the specified JSON file. Incoming state can be deserialized into either + a JSON object for legacy state input or as a list of AirbyteStateMessages for the per-stream state format. Regardless of the + incoming input type, it will always be transformed and output as a list of AirbyteStateMessage(s). + :param state_path: The filepath to where the stream states are located + :return: The complete stream state based on the connector's previous sync + """ if state_path: state_obj = json.loads(open(state_path, "r").read()) - else: - state_obj = {} - state = defaultdict(dict, state_obj) - return state + if not state_obj: + return [] + is_per_stream_state = isinstance(state_obj, List) + if is_per_stream_state: + parsed_state_messages = [] + for state in state_obj: + parsed_message = AirbyteStateMessage.parse_obj(state) + if not parsed_message.stream and not parsed_message.data and not parsed_message.global_: + raise ValueError("AirbyteStateMessage should contain either a stream, global, or state field") + parsed_state_messages.append(parsed_message) + return parsed_state_messages + else: + # When the legacy JSON object format is received, always outputting an AirbyteStateMessage simplifies processing downstream + return [AirbyteStateMessage(type=AirbyteStateType.LEGACY, data=state_obj)] + return [] # can be overridden to change an input catalog def read_catalog(self, catalog_path: str) -> ConfiguredAirbyteCatalog: diff --git a/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py b/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py new file mode 100644 index 000000000000..ea5e2d219714 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from contextlib import nullcontext as does_not_raise + +import pytest +from airbyte_cdk.models import AirbyteStateMessage, AirbyteStateType +from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager + + +@pytest.mark.parametrize( + "input_state, expected_legacy_state, expected_error", + [ + pytest.param( + [AirbyteStateMessage(type=AirbyteStateType.LEGACY, data={"actresses": {"id": "seehorn_rhea"}})], + {"actresses": {"id": "seehorn_rhea"}}, + does_not_raise(), + id="test_legacy_input_state", + ), + pytest.param( + { + "actors": {"created_at": "1962-10-22"}, + "actresses": {"id": "seehorn_rhea"}, + }, + {"actors": {"created_at": "1962-10-22"}, "actresses": {"id": "seehorn_rhea"}}, + does_not_raise(), + id="test_supports_legacy_json_blob", + ), + pytest.param({}, {}, does_not_raise(), id="test_initialize_empty_mapping_by_default"), + pytest.param([], {}, does_not_raise(), id="test_initialize_empty_state"), + pytest.param("strings_are_not_allowed", None, pytest.raises(ValueError), id="test_value_error_is_raised_on_invalid_state_input"), + ], +) +def test_get_legacy_state(input_state, expected_legacy_state, expected_error): + with expected_error: + state_manager = ConnectorStateManager(input_state) + actual_legacy_state = state_manager.get_legacy_state() + assert actual_legacy_state == expected_legacy_state diff --git a/airbyte-cdk/python/unit_tests/sources/test_source.py b/airbyte-cdk/python/unit_tests/sources/test_source.py index de2b282012ee..1ebfc910ec37 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_source.py @@ -2,19 +2,30 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # - import json import logging import tempfile +from contextlib import nullcontext as does_not_raise from typing import Any, Mapping, MutableMapping from unittest.mock import MagicMock import pytest -from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode, Type +from airbyte_cdk.models import ( + AirbyteGlobalState, + AirbyteStateBlob, + AirbyteStateMessage, + AirbyteStateType, + AirbyteStreamState, + ConfiguredAirbyteCatalog, + StreamDescriptor, + SyncMode, + Type, +) from airbyte_cdk.sources import AbstractSource, Source from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.streams.http.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer +from pydantic import ValidationError class MockSource(Source): @@ -93,18 +104,187 @@ def streams(self, config): return MockAbstractSource() -def test_read_state(source): - state = {"updated_at": "yesterday"} - +@pytest.mark.parametrize( + "incoming_state, expected_state, expected_error", + [ + pytest.param( + [ + { + "type": "STREAM", + "stream": { + "stream_state": {"created_at": "2009-07-19"}, + "stream_descriptor": {"name": "movies", "namespace": "public"}, + }, + } + ], + [ + AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="movies", namespace="public"), + stream_state=AirbyteStateBlob.parse_obj({"created_at": "2009-07-19"}), + ), + ) + ], + does_not_raise(), + id="test_incoming_stream_state", + ), + pytest.param( + [ + { + "type": "STREAM", + "stream": { + "stream_state": {"created_at": "2009-07-19"}, + "stream_descriptor": {"name": "movies", "namespace": "public"}, + }, + }, + { + "type": "STREAM", + "stream": { + "stream_state": {"id": "villeneuve_denis"}, + "stream_descriptor": {"name": "directors", "namespace": "public"}, + }, + }, + { + "type": "STREAM", + "stream": { + "stream_state": {"created_at": "1995-12-27"}, + "stream_descriptor": {"name": "actors", "namespace": "public"}, + }, + }, + ], + [ + AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="movies", namespace="public"), + stream_state=AirbyteStateBlob.parse_obj({"created_at": "2009-07-19"}), + ), + ), + AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="directors", namespace="public"), + stream_state=AirbyteStateBlob.parse_obj({"id": "villeneuve_denis"}), + ), + ), + AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="actors", namespace="public"), + stream_state=AirbyteStateBlob.parse_obj({"created_at": "1995-12-27"}), + ), + ), + ], + does_not_raise(), + id="test_incoming_multiple_stream_states", + ), + pytest.param( + [ + { + "type": "GLOBAL", + "global": { + "shared_state": {"shared_key": "shared_val"}, + "stream_states": [ + {"stream_state": {"created_at": "2009-07-19"}, "stream_descriptor": {"name": "movies", "namespace": "public"}} + ], + }, + } + ], + [ + AirbyteStateMessage.parse_obj( + { + "type": AirbyteStateType.GLOBAL, + "global": AirbyteGlobalState( + shared_state=AirbyteStateBlob.parse_obj({"shared_key": "shared_val"}), + stream_states=[ + AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="movies", namespace="public"), + stream_state=AirbyteStateBlob.parse_obj({"created_at": "2009-07-19"}), + ) + ], + ), + } + ), + ], + does_not_raise(), + id="test_incoming_global_state", + ), + pytest.param( + {"movies": {"created_at": "2009-07-19"}, "directors": {"id": "villeneuve_denis"}}, + [ + AirbyteStateMessage( + type=AirbyteStateType.LEGACY, data={"movies": {"created_at": "2009-07-19"}, "directors": {"id": "villeneuve_denis"}} + ) + ], + does_not_raise(), + id="test_incoming_legacy_state", + ), + pytest.param([], [], does_not_raise(), id="test_empty_incoming_stream_state"), + pytest.param(None, [], does_not_raise(), id="test_none_incoming_state"), + pytest.param({}, [], does_not_raise(), id="test_empty_incoming_legacy_state"), + pytest.param( + [ + { + "type": "NOT_REAL", + "stream": { + "stream_state": {"created_at": "2009-07-19"}, + "stream_descriptor": {"name": "movies", "namespace": "public"}, + }, + } + ], + None, + pytest.raises(ValidationError), + id="test_invalid_stream_state_invalid_type", + ), + pytest.param( + [{"type": "STREAM", "stream": {"stream_state": {"created_at": "2009-07-19"}}}], + None, + pytest.raises(ValidationError), + id="test_invalid_stream_state_missing_descriptor", + ), + pytest.param( + [{"type": "GLOBAL", "global": {"shared_state": {"shared_key": "shared_val"}}}], + None, + pytest.raises(ValidationError), + id="test_invalid_global_state_missing_streams", + ), + pytest.param( + [ + { + "type": "GLOBAL", + "global": { + "shared_state": {"shared_key": "shared_val"}, + "stream_states": { + "stream_state": {"created_at": "2009-07-19"}, + "stream_descriptor": {"name": "movies", "namespace": "public"}, + }, + }, + } + ], + None, + pytest.raises(ValidationError), + id="test_invalid_global_state_streams_not_list", + ), + pytest.param( + [{"type": "LEGACY", "not": "something"}], + None, + pytest.raises(ValueError), + id="test_invalid_state_message_has_no_stream_global_or_data", + ), + ], +) +def test_read_state(source, incoming_state, expected_state, expected_error): with tempfile.NamedTemporaryFile("w") as state_file: - state_file.write(json.dumps(state)) + state_file.write(json.dumps(incoming_state)) state_file.flush() - actual = source.read_state(state_file.name) - assert state == actual + with expected_error: + actual = source.read_state(state_file.name) + assert actual == expected_state def test_read_state_nonexistent(source): - assert {} == source.read_state("") + assert [] == source.read_state("") def test_read_catalog(source): From 4e7ba062824a34769eec665bb42778316e836325 Mon Sep 17 00:00:00 2001 From: Alexandre Girard Date: Wed, 7 Sep 2022 10:24:55 -0700 Subject: [PATCH 062/200] Document guidelines on low-code support (#16387) * Guide for using lowcode or not * Use headers * Update docs/connector-development/config-based/overview.md Co-authored-by: Andy * Update docs/connector-development/config-based/overview.md Co-authored-by: Andy * Update docs/connector-development/config-based/overview.md Co-authored-by: Brian Lai <51336873+brianjlai@users.noreply.github.com> * Update docs/connector-development/config-based/overview.md Co-authored-by: Brian Lai <51336873+brianjlai@users.noreply.github.com> * an -> a Co-authored-by: Andy Co-authored-by: Brian Lai <51336873+brianjlai@users.noreply.github.com> --- .../config-based/overview.md | 84 +++++++++++++++---- 1 file changed, 70 insertions(+), 14 deletions(-) diff --git a/docs/connector-development/config-based/overview.md b/docs/connector-development/config-based/overview.md index 56e1f78f73ed..5f0361df64d9 100644 --- a/docs/connector-development/config-based/overview.md +++ b/docs/connector-development/config-based/overview.md @@ -15,22 +15,78 @@ The process then submits HTTP requests to the API endpoint, and extracts records See the [connector definition section](yaml-structure.md) for more information on the YAML file describing the connector. +## Does this framework support the connector I want to build? + +Not all APIs are can be built using this framework because its featureset is still limited. +This section describes guidelines for determining whether a connector for a given API can be built using the config-based framework. Please let us know through the #lowcode-earlyaccess Slack channel if you'd like to build something that falls outside what we currently support and we'd be happy to discuss and prioritize in the coming months! + +Refer to the API's documentation to answer the following questions: + +### Is this a HTTP REST API returning data as JSON? + +The API documentation should show which HTTP method must be used to retrieve data from the API. +For example, the [documentation for the Exchange Rates Data API](https://apilayer.com/marketplace/exchangerates_data-api#documentation-tab) says the GET method should be used, and that the response is a JSON object. + +Other API types such as SOAP or GraphQL are not supported. + +Other encoding schemes such as CSV or Protobuf are not supported. + +Integrations that require the use of an SDK are not supported. + +### Do queries return the data synchronously or do they trigger a bulk workflow? + +Some APIs return the data of interest as part of the response. This is the case for the [Exchange Rates Data API](https://apilayer.com/marketplace/exchangerates_data-api#documentation-tab) - each request results in a response containing the data we're interested in. + +Other APIs use bulk workflows, which means a query will trigger an asynchronous process on the integration's side. [Zendesk bulk queries](https://developer.zendesk.com/api-reference/ticketing/tickets/tickets/#bulk-mark-tickets-as-spam) are an example of such integrations. + +An initial request will trigger the workflow and return an ID and a job status. The actual data then needs to be fetched when the asynchronous job is completed. + +Asynchronous bulk workflows are not supported. + +### What is the pagination mechanism? + +The only pagination mechanisms supported are + +* Offset count passed either by query params or request header such as [Sendgrid](https://docs.sendgrid.com/api-reference/bounces-api/retrieve-all-bounces) +* Page count passed either by query params or request header such as [Greenhouse](https://developers.greenhouse.io/harvest.html#get-list-applications) +* Cursor field pointing to the URL of the next page of records such as [Sentry](https://docs.sentry.io/api/pagination/) + +### What is the authorization mechanism? + +Endpoints that require authenticating using a query param or a HTTP header, as is the case for the [Exchange Rates Data API](https://apilayer.com/marketplace/exchangerates_data-api#authentication), are supported. + +Endpoints that require authenticating using Basic Auth over HTTPS, as is the case for [Greenhouse](https://developers.greenhouse.io/harvest.html#authentication), are supported. + +Endpoints that require authenticating using OAuth 2.0, as is the case for [Strava](https://developers.strava.com/docs/authentication/#introduction), are supported. + +Other authentication schemes such as GWT are not supported. + +### Is the schema static or dynamic? + +Only static schemas are supported. + +Dynamically deriving the schema from querying an endpoint is not supported. + +### Does the endpoint have a strict rate limit + +Throttling is not supported, but the connector can use exponential backoff to avoid API bans in case it gets rate limited. This can work for APIs with high rate limits, but not for those that have strict limits on a small time-window, such as the [Reddit Ads API](https://ads-api.reddit.com/docs/#section/Rate-Limits), which limits to 1 request per second. + ## Supported features -| Feature | Support | -|--------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Transport protocol | HTTP | -| HTTP methods | GET, POST | -| Data format | Json | -| Resource type | Collections
    Sub-collection | -| [Pagination](./pagination.md) | [Page limit](./pagination.md#page-increment)
    [Offset](./pagination.md#offset-increment)
    [Cursor](./pagination.md#cursor) | -| [Authentication](./authentication.md) | [Header based](./authentication.md#ApiKeyAuthenticator)
    [Bearer](./authentication.md#BearerAuthenticator)
    [Basic](./authentication.md#BasicHttpAuthenticator)
    [OAuth](./authentication.md#OAuth) | -| Sync mode | Full refresh
    Incremental | -| Schema discovery | Only static schemas | -| [Stream slicing](./stream-slicers.md) | [Datetime](./stream-slicers.md#Datetime), [lists](./stream-slicers.md#list-stream-slicer), [parent-resource id](./stream-slicers.md#Substream-slicer) | +| Feature | Support | +|--------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Transport protocol | HTTP | +| HTTP methods | GET, POST | +| Data format | JSON | +| Resource type | Collections
    Sub-collection | +| [Pagination](./pagination.md) | [Page limit](./pagination.md#page-increment)
    [Offset](./pagination.md#offset-increment)
    [Cursor](./pagination.md#cursor) | +| [Authentication](./authentication.md) | [Header based](./authentication.md#ApiKeyAuthenticator)
    [Bearer](./authentication.md#BearerAuthenticator)
    [Basic](./authentication.md#BasicHttpAuthenticator)
    [OAuth](./authentication.md#OAuth) | +| Sync mode | Full refresh
    Incremental | +| Schema discovery | Only static schemas | +| [Stream slicing](./stream-slicers.md) | [Datetime](./stream-slicers.md#Datetime), [lists](./stream-slicers.md#list-stream-slicer), [parent-resource id](./stream-slicers.md#Substream-slicer) | | [Record transformation](./record-selector.md) | [Field selection](./record-selector.md#selecting-a-field)
    [Adding fields](./record-selector.md#adding-fields)
    [Removing fields](./record-selector.md#removing-fields)
    [Filtering records](./record-selector.md#filtering-records) | -| [Error detection](./error-handling.md) | [From HTTP status code](./error-handling.md#from-status-code)
    [From error message](./error-handling.md#from-error-message) | -| [Backoff strategies](./error-handling.md#Backoff-Strategies) | [Exponential](./error-handling.md#Exponential-backoff)
    [Constant](./error-handling.md#Constant-Backoff)
    [Derived from headers](./error-handling.md#Wait-time-defined-in-header) | +| [Error detection](./error-handling.md) | [From HTTP status code](./error-handling.md#from-status-code)
    [From error message](./error-handling.md#from-error-message) | +| [Backoff strategies](./error-handling.md#Backoff-Strategies) | [Exponential](./error-handling.md#Exponential-backoff)
    [Constant](./error-handling.md#Constant-Backoff)
    [Derived from headers](./error-handling.md#Wait-time-defined-in-header) | If a feature you require is not supported, you can [request the feature](../../contributing-to-airbyte/README.md#requesting-new-features) and use the [Python CDK](../cdk-python/README.md). @@ -67,7 +123,7 @@ There is currently only one implementation, the `SimpleRetriever`, which is defi 1. Requester: Describes how to submit requests to the API source 2. Paginator: Describes how to navigate through the API's pages -3. Record selector: Describes how to extract records from an HTTP response +3. Record selector: Describes how to extract records from a HTTP response 4. Stream Slicer: Describes how to partition the stream, enabling incremental syncs and checkpointing Each of those components (and their subcomponents) are defined by an explicit interface and one or many implementations. From bc0d7cc1c58c641e0653d19cb43d7c9c4cceab60 Mon Sep 17 00:00:00 2001 From: "Krishna (kc) Glick" Date: Wed, 7 Sep 2022 13:57:40 -0400 Subject: [PATCH 063/200] Add tests to various hooks (#16293) * Move testutils to test-utils folder * Add mock data * Add analytics provider to test renderer * Add hooks.test.ts, test useUniqueFormId * Add tests for mapFormPropsToOperation and useInitialValues --- .../components/EditorHeader.test.tsx | 2 +- .../src/components/base/Input/Input.test.tsx | 3 +- .../services/FormChangeTracker/hooks.test.ts | 19 + .../services/Modal/ModalService.test.tsx | 3 +- .../views/auth/OAuthLogin/OAuthLogin.test.tsx | 2 +- .../components/SettingsView.test.tsx | 2 +- .../test-utils/mock-data/mockConnection.json | 349 ++ .../mock-data/mockDestinationDefinition.json | 329 ++ .../src/{utils => test-utils}/testutils.tsx | 0 .../ConnectionForm/ConnectionForm.test.tsx | 2 +- .../__snapshots__/formConfig.test.ts.snap | 3193 +++++++++++++++++ .../ConnectionForm/formConfig.test.ts | 178 +- .../ServiceForm/ServiceForm.test.tsx | 2 +- .../Sections/auth/AuthButton.test.tsx | 2 +- 14 files changed, 4072 insertions(+), 14 deletions(-) create mode 100644 airbyte-webapp/src/hooks/services/FormChangeTracker/hooks.test.ts create mode 100644 airbyte-webapp/src/test-utils/mock-data/mockConnection.json create mode 100644 airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinition.json rename airbyte-webapp/src/{utils => test-utils}/testutils.tsx (100%) create mode 100644 airbyte-webapp/src/views/Connection/ConnectionForm/__snapshots__/formConfig.test.ts.snap diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.test.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.test.tsx index d803bef725ad..ff97664317e6 100644 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.test.tsx +++ b/airbyte-webapp/src/components/ArrayOfObjectsEditor/components/EditorHeader.test.tsx @@ -1,4 +1,4 @@ -import { render } from "utils/testutils"; +import { render } from "test-utils/testutils"; import { EditorHeader } from "./EditorHeader"; diff --git a/airbyte-webapp/src/components/base/Input/Input.test.tsx b/airbyte-webapp/src/components/base/Input/Input.test.tsx index 13e4ed92f398..fcd3c8d7e6de 100644 --- a/airbyte-webapp/src/components/base/Input/Input.test.tsx +++ b/airbyte-webapp/src/components/base/Input/Input.test.tsx @@ -1,7 +1,6 @@ import { fireEvent, waitFor } from "@testing-library/react"; import { act } from "react-dom/test-utils"; - -import { render } from "utils/testutils"; +import { render } from "test-utils/testutils"; import { Input } from "./Input"; diff --git a/airbyte-webapp/src/hooks/services/FormChangeTracker/hooks.test.ts b/airbyte-webapp/src/hooks/services/FormChangeTracker/hooks.test.ts new file mode 100644 index 000000000000..2de194a114c7 --- /dev/null +++ b/airbyte-webapp/src/hooks/services/FormChangeTracker/hooks.test.ts @@ -0,0 +1,19 @@ +import { renderHook } from "@testing-library/react-hooks"; + +import { useUniqueFormId } from "./hooks"; + +describe("#useUniqueFormId", () => { + it("should use what is passed into it", () => { + const { + result: { current }, + } = renderHook(() => useUniqueFormId("asdf")); + expect(current).toBe("asdf"); + }); + + it("should generate an id like /form_/", () => { + const { + result: { current }, + } = renderHook(useUniqueFormId); + expect(current).toMatch(/form_/); + }); +}); diff --git a/airbyte-webapp/src/hooks/services/Modal/ModalService.test.tsx b/airbyte-webapp/src/hooks/services/Modal/ModalService.test.tsx index 50ddc07cf8a2..46405a3e7aaa 100644 --- a/airbyte-webapp/src/hooks/services/Modal/ModalService.test.tsx +++ b/airbyte-webapp/src/hooks/services/Modal/ModalService.test.tsx @@ -1,8 +1,7 @@ import { render, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { useEffectOnce } from "react-use"; - -import { useMockIntersectionObserver } from "utils/testutils"; +import { useMockIntersectionObserver } from "test-utils/testutils"; import { ModalServiceProvider, useModalService } from "./ModalService"; import { ModalResult } from "./types"; diff --git a/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.test.tsx b/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.test.tsx index 798aded0d89b..c27ce81cce37 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.test.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/OAuthLogin/OAuthLogin.test.tsx @@ -1,10 +1,10 @@ import { render } from "@testing-library/react"; import userEvents from "@testing-library/user-event"; import { EMPTY } from "rxjs"; +import { TestWrapper } from "test-utils/testutils"; import type { useExperiment } from "hooks/services/Experiment"; import type { Experiments } from "hooks/services/Experiment/experiments"; -import { TestWrapper } from "utils/testutils"; const mockUseExperiment = jest.fn, Parameters>(); jest.doMock("hooks/services/Experiment", () => ({ diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.test.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.test.tsx index 5f9acfc73bd5..8cf4bf0478fa 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.test.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/SettingsView.test.tsx @@ -1,4 +1,4 @@ -import { render, mockConnection } from "utils/testutils"; +import { render, mockConnection } from "test-utils/testutils"; import SettingsView from "./SettingsView"; diff --git a/airbyte-webapp/src/test-utils/mock-data/mockConnection.json b/airbyte-webapp/src/test-utils/mock-data/mockConnection.json new file mode 100644 index 000000000000..bf44cc7251fb --- /dev/null +++ b/airbyte-webapp/src/test-utils/mock-data/mockConnection.json @@ -0,0 +1,349 @@ +{ + "connectionId": "a9c8e4b5-349d-4a17-bdff-5ad2f6fbd611", + "name": "Scrafty <> Heroku Postgres", + "namespaceDefinition": "source", + "namespaceFormat": "${SOURCE_NAMESPACE}", + "prefix": "", + "sourceId": "a3295ed7-4acf-4c0b-b16b-07a00e624a52", + "destinationId": "083a53bc-8bc2-4dc0-b05a-4273a96f3b93", + "syncCatalog": { + "streams": [ + { + "stream": { + "name": "pokemon", + "jsonSchema": { + "type": "object", + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "forms": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + } + }, + "moves": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "move": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "version_group_details": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "version_group": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "level_learned_at": { + "type": ["null", "integer"] + }, + "move_learn_method": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + } + } + } + } + } + } + }, + "order": { + "type": ["null", "integer"] + }, + "stats": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "stat": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "effort": { + "type": ["null", "integer"] + }, + "base_stat": { + "type": ["null", "integer"] + } + } + } + }, + "types": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "slot": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + } + } + } + }, + "height": { + "type": ["null", "integer"] + }, + "weight": { + "type": ["null", "integer"] + }, + "species": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "sprites": { + "type": ["null", "object"], + "properties": { + "back_shiny": { + "type": ["null", "string"] + }, + "back_female": { + "type": ["null", "string"] + }, + "front_shiny": { + "type": ["null", "string"] + }, + "back_default": { + "type": ["null", "string"] + }, + "front_female": { + "type": ["null", "string"] + }, + "front_default": { + "type": ["null", "string"] + }, + "back_shiny_female": { + "type": ["null", "string"] + }, + "front_shiny_female": { + "type": ["null", "string"] + } + } + }, + "abilities": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "slot": { + "type": ["null", "integer"] + }, + "ability": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "is_hidden": { + "type": ["null", "boolean"] + } + } + } + }, + "held_items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "item": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "version_details": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "rarity": { + "type": ["null", "integer"] + }, + "version": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + } + } + } + } + } + } + }, + "is_default ": { + "type": ["null", "boolean"] + }, + "game_indices": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "version": { + "type": ["null", "object"], + "properties": { + "url": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + } + } + }, + "game_index": { + "type": ["null", "integer"] + } + } + } + }, + "base_experience": { + "type": ["null", "integer"] + }, + "location_area_encounters": { + "type": ["null", "string"] + } + } + }, + "supportedSyncModes": ["full_refresh"], + "defaultCursorField": [], + "sourceDefinedPrimaryKey": [] + }, + "config": { + "syncMode": "full_refresh", + "cursorField": [], + "destinationSyncMode": "append", + "primaryKey": [], + "aliasName": "pokemon", + "selected": true + } + } + ] + }, + "scheduleType": "manual", + "status": "active", + "operationIds": ["8af8ef4d-01b1-49c8-b145-23775f34a74b"], + "source": { + "sourceDefinitionId": "6371b14b-bc68-4236-bfbd-468e8df8e968", + "sourceId": "a3295ed7-4acf-4c0b-b16b-07a00e624a52", + "workspaceId": "47c74b9b-9b89-4af1-8331-4865af6c4e4d", + "connectionConfiguration": { + "pokemon_name": "scrafty" + }, + "name": "Scrafty", + "sourceName": "PokeAPI" + }, + "destination": { + "destinationDefinitionId": "25c5221d-dce2-4163-ade9-739ef790f503", + "destinationId": "083a53bc-8bc2-4dc0-b05a-4273a96f3b93", + "workspaceId": "47c74b9b-9b89-4af1-8331-4865af6c4e4d", + "connectionConfiguration": { + "ssl": false, + "host": "asdf", + "port": 5432, + "schema": "public", + "database": "asdf", + "password": "**********", + "username": "asdf", + "tunnel_method": { + "tunnel_method": "NO_TUNNEL" + } + }, + "name": "Heroku Postgres", + "destinationName": "Postgres" + }, + "operations": [ + { + "workspaceId": "47c74b9b-9b89-4af1-8331-4865af6c4e4d", + "operationId": "8af8ef4d-01b1-49c8-b145-23775f34a74b", + "name": "Normalization", + "operatorConfiguration": { + "operatorType": "normalization", + "normalization": { + "option": "basic" + } + } + } + ], + "latestSyncJobCreatedAt": 1660227512, + "latestSyncJobStatus": "succeeded", + "isSyncing": false, + "catalogId": "bf31d1df-d7ba-4bae-b1ec-dac617b4f70c" +} diff --git a/airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinition.json b/airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinition.json new file mode 100644 index 000000000000..40a038b7bc6a --- /dev/null +++ b/airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinition.json @@ -0,0 +1,329 @@ +{ + "destinationDefinitionId": "25c5221d-dce2-4163-ade9-739ef790f503", + "documentationUrl": "https://docs.airbyte.io/integrations/destinations/postgres", + "connectionSpecification": { + "type": "object", + "title": "Postgres Destination Spec", + "$schema": "http://json-schema.org/draft-07/schema#", + "required": ["host", "port", "username", "database", "schema"], + "properties": { + "ssl": { + "type": "boolean", + "order": 6, + "title": "SSL Connection", + "default": false, + "description": "Encrypt data using SSL. When activating SSL, please select one of the connection modes." + }, + "host": { + "type": "string", + "order": 0, + "title": "Host", + "description": "Hostname of the database." + }, + "port": { + "type": "integer", + "order": 1, + "title": "Port", + "default": 5432, + "maximum": 65536, + "minimum": 0, + "examples": ["5432"], + "description": "Port of the database." + }, + "schema": { + "type": "string", + "order": 3, + "title": "Default Schema", + "default": "public", + "examples": ["public"], + "description": "The default schema tables are written to if the source does not specify a namespace. The usual value for this field is \"public\"." + }, + "database": { + "type": "string", + "order": 2, + "title": "DB Name", + "description": "Name of the database." + }, + "password": { + "type": "string", + "order": 5, + "title": "Password", + "description": "Password associated with the username.", + "airbyte_secret": true + }, + "ssl_mode": { + "type": "object", + "oneOf": [ + { + "title": "disable", + "required": ["mode"], + "properties": { + "mode": { + "enum": ["disable"], + "type": "string", + "const": "disable", + "order": 0, + "default": "disable" + } + }, + "description": "Disable SSL.", + "additionalProperties": false + }, + { + "title": "allow", + "required": ["mode"], + "properties": { + "mode": { + "enum": ["allow"], + "type": "string", + "const": "allow", + "order": 0, + "default": "allow" + } + }, + "description": "Allow SSL mode.", + "additionalProperties": false + }, + { + "title": "prefer", + "required": ["mode"], + "properties": { + "mode": { + "enum": ["prefer"], + "type": "string", + "const": "prefer", + "order": 0, + "default": "prefer" + } + }, + "description": "Prefer SSL mode.", + "additionalProperties": false + }, + { + "title": "require", + "required": ["mode"], + "properties": { + "mode": { + "enum": ["require"], + "type": "string", + "const": "require", + "order": 0, + "default": "require" + } + }, + "description": "Require SSL mode.", + "additionalProperties": false + }, + { + "title": "verify-ca", + "required": ["mode", "ca_certificate"], + "properties": { + "mode": { + "enum": ["verify-ca"], + "type": "string", + "const": "verify-ca", + "order": 0, + "default": "verify-ca" + }, + "ca_certificate": { + "type": "string", + "order": 1, + "title": "CA certificate", + "multiline": true, + "description": "CA certificate", + "airbyte_secret": true + }, + "client_key_password": { + "type": "string", + "order": 4, + "title": "Client key password (Optional)", + "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true + } + }, + "description": "Verify-ca SSL mode.", + "additionalProperties": false + }, + { + "title": "verify-full", + "required": ["mode", "ca_certificate", "client_certificate", "client_key"], + "properties": { + "mode": { + "enum": ["verify-full"], + "type": "string", + "const": "verify-full", + "order": 0, + "default": "verify-full" + }, + "client_key": { + "type": "string", + "order": 3, + "title": "Client key", + "multiline": true, + "description": "Client key", + "airbyte_secret": true + }, + "ca_certificate": { + "type": "string", + "order": 1, + "title": "CA certificate", + "multiline": true, + "description": "CA certificate", + "airbyte_secret": true + }, + "client_certificate": { + "type": "string", + "order": 2, + "title": "Client certificate", + "multiline": true, + "description": "Client certificate", + "airbyte_secret": true + }, + "client_key_password": { + "type": "string", + "order": 4, + "title": "Client key password (Optional)", + "description": "Password for keystorage. This field is optional. If you do not add it - the password will be generated automatically.", + "airbyte_secret": true + } + }, + "description": "Verify-full SSL mode.", + "additionalProperties": false + } + ], + "order": 7, + "title": "SSL modes", + "description": "SSL connection modes. \n disable - Chose this mode to disable encryption of communication between Airbyte and destination database\n allow - Chose this mode to enable encryption only when required by the source database\n prefer - Chose this mode to allow unencrypted connection only if the source database does not support encryption\n require - Chose this mode to always require encryption. If the source database server does not support encryption, connection will fail\n verify-ca - Chose this mode to always require encryption and to verify that the source database server has a valid SSL certificate\n verify-full - This is the most secure mode. Chose this mode to always require encryption and to verify the identity of the source database server\n See more information - in the docs." + }, + "username": { + "type": "string", + "order": 4, + "title": "User", + "description": "Username to use to access the database." + }, + "tunnel_method": { + "type": "object", + "oneOf": [ + { + "title": "No Tunnel", + "required": ["tunnel_method"], + "properties": { + "tunnel_method": { + "type": "string", + "const": "NO_TUNNEL", + "order": 0, + "description": "No ssh tunnel needed to connect to database" + } + } + }, + { + "title": "SSH Key Authentication", + "required": ["tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "ssh_key"], + "properties": { + "ssh_key": { + "type": "string", + "order": 4, + "title": "SSH Private Key", + "multiline": true, + "description": "OS-level user account ssh key credentials in RSA PEM format ( created with ssh-keygen -t rsa -m PEM -f myuser_rsa )", + "airbyte_secret": true + }, + "tunnel_host": { + "type": "string", + "order": 1, + "title": "SSH Tunnel Jump Server Host", + "description": "Hostname of the jump server host that allows inbound ssh tunnel." + }, + "tunnel_port": { + "type": "integer", + "order": 2, + "title": "SSH Connection Port", + "default": 22, + "maximum": 65536, + "minimum": 0, + "examples": ["22"], + "description": "Port on the proxy/jump server that accepts inbound ssh connections." + }, + "tunnel_user": { + "type": "string", + "order": 3, + "title": "SSH Login Username", + "description": "OS-level username for logging into the jump server host." + }, + "tunnel_method": { + "type": "string", + "const": "SSH_KEY_AUTH", + "order": 0, + "description": "Connect through a jump server tunnel host using username and ssh key" + } + } + }, + { + "title": "Password Authentication", + "required": ["tunnel_method", "tunnel_host", "tunnel_port", "tunnel_user", "tunnel_user_password"], + "properties": { + "tunnel_host": { + "type": "string", + "order": 1, + "title": "SSH Tunnel Jump Server Host", + "description": "Hostname of the jump server host that allows inbound ssh tunnel." + }, + "tunnel_port": { + "type": "integer", + "order": 2, + "title": "SSH Connection Port", + "default": 22, + "maximum": 65536, + "minimum": 0, + "examples": ["22"], + "description": "Port on the proxy/jump server that accepts inbound ssh connections." + }, + "tunnel_user": { + "type": "string", + "order": 3, + "title": "SSH Login Username", + "description": "OS-level username for logging into the jump server host" + }, + "tunnel_method": { + "type": "string", + "const": "SSH_PASSWORD_AUTH", + "order": 0, + "description": "Connect through a jump server tunnel host using username and password authentication" + }, + "tunnel_user_password": { + "type": "string", + "order": 4, + "title": "Password", + "description": "OS-level password for logging into the jump server host", + "airbyte_secret": true + } + } + } + ], + "title": "SSH Tunnel Method", + "description": "Whether to initiate an SSH tunnel before connecting to the database, and if so, which kind of authentication to use." + }, + "jdbc_url_params": { + "type": "string", + "order": 8, + "title": "JDBC URL Params", + "description": "Additional properties to pass to the JDBC URL string when connecting to the database formatted as 'key=value' pairs separated by the symbol '&'. (example: key1=value1&key2=value2&key3=value3)." + } + }, + "additionalProperties": true + }, + "jobInfo": { + "id": "7c3ae799-cb25-4c05-9685-f7bb1885d662", + "configType": "get_spec", + "configId": "Optional.empty", + "createdAt": 1661365436880, + "endedAt": 1661365436880, + "succeeded": true, + "logs": { + "logLines": [] + } + }, + "supportedDestinationSyncModes": ["overwrite", "append", "append_dedup"], + "supportsDbt": true, + "supportsNormalization": true +} diff --git a/airbyte-webapp/src/utils/testutils.tsx b/airbyte-webapp/src/test-utils/testutils.tsx similarity index 100% rename from airbyte-webapp/src/utils/testutils.tsx rename to airbyte-webapp/src/test-utils/testutils.tsx diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.test.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.test.tsx index 59ef676f0a1f..07d15456f909 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.test.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/ConnectionForm.test.tsx @@ -1,5 +1,6 @@ import { waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; +import { render } from "test-utils/testutils"; import { ConnectionStatus, @@ -9,7 +10,6 @@ import { WebBackendConnectionRead, } from "core/request/AirbyteClient"; import { ConfirmationModalService } from "hooks/services/ConfirmationModal/ConfirmationModalService"; -import { render } from "utils/testutils"; import { ConnectionForm, ConnectionFormProps } from "./ConnectionForm"; diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/__snapshots__/formConfig.test.ts.snap b/airbyte-webapp/src/views/Connection/ConnectionForm/__snapshots__/formConfig.test.ts.snap new file mode 100644 index 000000000000..1904aa6d7fee --- /dev/null +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/__snapshots__/formConfig.test.ts.snap @@ -0,0 +1,3193 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`#useInitialValues should generate initial values w/ edit mode: false 1`] = ` +Object { + "all": Array [ + Object { + "name": "Scrafty <> Heroku Postgres", + "namespaceDefinition": "source", + "namespaceFormat": "\${SOURCE_NAMESPACE}", + "normalization": "basic", + "prefix": "", + "scheduleData": null, + "syncCatalog": Object { + "streams": Array [ + Object { + "config": Object { + "aliasName": "pokemon", + "cursorField": Array [], + "destinationSyncMode": "overwrite", + "primaryKey": Array [], + "selected": true, + "syncMode": "full_refresh", + }, + "id": "0", + "stream": Object { + "defaultCursorField": Array [], + "jsonSchema": Object { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": Object { + "abilities": Object { + "items": Object { + "properties": Object { + "ability": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "is_hidden": Object { + "type": Array [ + "null", + "boolean", + ], + }, + "slot": Object { + "type": Array [ + "null", + "integer", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "base_experience": Object { + "type": Array [ + "null", + "integer", + ], + }, + "forms": Object { + "items": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "game_indices": Object { + "items": Object { + "properties": Object { + "game_index": Object { + "type": Array [ + "null", + "integer", + ], + }, + "version": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "height": Object { + "type": Array [ + "null", + "integer", + ], + }, + "held_items": Object { + "items": Object { + "properties": Object { + "item": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_details": Object { + "items": Object { + "properties": Object { + "rarity": Object { + "type": Array [ + "null", + "integer", + ], + }, + "version": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "id": Object { + "type": Array [ + "null", + "integer", + ], + }, + "is_default ": Object { + "type": Array [ + "null", + "boolean", + ], + }, + "location_area_encounters": Object { + "type": Array [ + "null", + "string", + ], + }, + "moves": Object { + "items": Object { + "properties": Object { + "move": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_group_details": Object { + "items": Object { + "properties": Object { + "level_learned_at": Object { + "type": Array [ + "null", + "integer", + ], + }, + "move_learn_method": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_group": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "order": Object { + "type": Array [ + "null", + "integer", + ], + }, + "species": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "sprites": Object { + "properties": Object { + "back_default": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_shiny": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_shiny_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_default": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_shiny": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_shiny_female": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "stats": Object { + "items": Object { + "properties": Object { + "base_stat": Object { + "type": Array [ + "null", + "integer", + ], + }, + "effort": Object { + "type": Array [ + "null", + "integer", + ], + }, + "stat": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "types": Object { + "items": Object { + "properties": Object { + "slot": Object { + "type": Array [ + "null", + "integer", + ], + }, + "type": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "weight": Object { + "type": Array [ + "null", + "integer", + ], + }, + }, + "type": "object", + }, + "name": "pokemon", + "sourceDefinedPrimaryKey": Array [], + "supportedSyncModes": Array [ + "full_refresh", + ], + }, + }, + ], + }, + "transformations": Array [], + }, + ], + "current": Object { + "name": "Scrafty <> Heroku Postgres", + "namespaceDefinition": "source", + "namespaceFormat": "\${SOURCE_NAMESPACE}", + "normalization": "basic", + "prefix": "", + "scheduleData": null, + "syncCatalog": Object { + "streams": Array [ + Object { + "config": Object { + "aliasName": "pokemon", + "cursorField": Array [], + "destinationSyncMode": "overwrite", + "primaryKey": Array [], + "selected": true, + "syncMode": "full_refresh", + }, + "id": "0", + "stream": Object { + "defaultCursorField": Array [], + "jsonSchema": Object { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": Object { + "abilities": Object { + "items": Object { + "properties": Object { + "ability": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "is_hidden": Object { + "type": Array [ + "null", + "boolean", + ], + }, + "slot": Object { + "type": Array [ + "null", + "integer", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "base_experience": Object { + "type": Array [ + "null", + "integer", + ], + }, + "forms": Object { + "items": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "game_indices": Object { + "items": Object { + "properties": Object { + "game_index": Object { + "type": Array [ + "null", + "integer", + ], + }, + "version": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "height": Object { + "type": Array [ + "null", + "integer", + ], + }, + "held_items": Object { + "items": Object { + "properties": Object { + "item": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_details": Object { + "items": Object { + "properties": Object { + "rarity": Object { + "type": Array [ + "null", + "integer", + ], + }, + "version": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "id": Object { + "type": Array [ + "null", + "integer", + ], + }, + "is_default ": Object { + "type": Array [ + "null", + "boolean", + ], + }, + "location_area_encounters": Object { + "type": Array [ + "null", + "string", + ], + }, + "moves": Object { + "items": Object { + "properties": Object { + "move": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_group_details": Object { + "items": Object { + "properties": Object { + "level_learned_at": Object { + "type": Array [ + "null", + "integer", + ], + }, + "move_learn_method": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_group": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "order": Object { + "type": Array [ + "null", + "integer", + ], + }, + "species": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "sprites": Object { + "properties": Object { + "back_default": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_shiny": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_shiny_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_default": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_shiny": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_shiny_female": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "stats": Object { + "items": Object { + "properties": Object { + "base_stat": Object { + "type": Array [ + "null", + "integer", + ], + }, + "effort": Object { + "type": Array [ + "null", + "integer", + ], + }, + "stat": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "types": Object { + "items": Object { + "properties": Object { + "slot": Object { + "type": Array [ + "null", + "integer", + ], + }, + "type": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "weight": Object { + "type": Array [ + "null", + "integer", + ], + }, + }, + "type": "object", + }, + "name": "pokemon", + "sourceDefinedPrimaryKey": Array [], + "supportedSyncModes": Array [ + "full_refresh", + ], + }, + }, + ], + }, + "transformations": Array [], + }, + "error": undefined, +} +`; + +exports[`#useInitialValues should generate initial values w/ edit mode: true 1`] = ` +Object { + "all": Array [ + Object { + "name": "Scrafty <> Heroku Postgres", + "namespaceDefinition": "source", + "namespaceFormat": "\${SOURCE_NAMESPACE}", + "normalization": "basic", + "prefix": "", + "scheduleData": null, + "syncCatalog": Object { + "streams": Array [ + Object { + "config": Object { + "aliasName": "pokemon", + "cursorField": Array [], + "destinationSyncMode": "append", + "primaryKey": Array [], + "selected": true, + "syncMode": "full_refresh", + }, + "id": "0", + "stream": Object { + "defaultCursorField": Array [], + "jsonSchema": Object { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": Object { + "abilities": Object { + "items": Object { + "properties": Object { + "ability": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "is_hidden": Object { + "type": Array [ + "null", + "boolean", + ], + }, + "slot": Object { + "type": Array [ + "null", + "integer", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "base_experience": Object { + "type": Array [ + "null", + "integer", + ], + }, + "forms": Object { + "items": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "game_indices": Object { + "items": Object { + "properties": Object { + "game_index": Object { + "type": Array [ + "null", + "integer", + ], + }, + "version": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "height": Object { + "type": Array [ + "null", + "integer", + ], + }, + "held_items": Object { + "items": Object { + "properties": Object { + "item": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_details": Object { + "items": Object { + "properties": Object { + "rarity": Object { + "type": Array [ + "null", + "integer", + ], + }, + "version": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "id": Object { + "type": Array [ + "null", + "integer", + ], + }, + "is_default ": Object { + "type": Array [ + "null", + "boolean", + ], + }, + "location_area_encounters": Object { + "type": Array [ + "null", + "string", + ], + }, + "moves": Object { + "items": Object { + "properties": Object { + "move": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_group_details": Object { + "items": Object { + "properties": Object { + "level_learned_at": Object { + "type": Array [ + "null", + "integer", + ], + }, + "move_learn_method": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_group": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "order": Object { + "type": Array [ + "null", + "integer", + ], + }, + "species": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "sprites": Object { + "properties": Object { + "back_default": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_shiny": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_shiny_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_default": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_shiny": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_shiny_female": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "stats": Object { + "items": Object { + "properties": Object { + "base_stat": Object { + "type": Array [ + "null", + "integer", + ], + }, + "effort": Object { + "type": Array [ + "null", + "integer", + ], + }, + "stat": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "types": Object { + "items": Object { + "properties": Object { + "slot": Object { + "type": Array [ + "null", + "integer", + ], + }, + "type": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "weight": Object { + "type": Array [ + "null", + "integer", + ], + }, + }, + "type": "object", + }, + "name": "pokemon", + "sourceDefinedPrimaryKey": Array [], + "supportedSyncModes": Array [ + "full_refresh", + ], + }, + }, + ], + }, + "transformations": Array [], + }, + ], + "current": Object { + "name": "Scrafty <> Heroku Postgres", + "namespaceDefinition": "source", + "namespaceFormat": "\${SOURCE_NAMESPACE}", + "normalization": "basic", + "prefix": "", + "scheduleData": null, + "syncCatalog": Object { + "streams": Array [ + Object { + "config": Object { + "aliasName": "pokemon", + "cursorField": Array [], + "destinationSyncMode": "append", + "primaryKey": Array [], + "selected": true, + "syncMode": "full_refresh", + }, + "id": "0", + "stream": Object { + "defaultCursorField": Array [], + "jsonSchema": Object { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": Object { + "abilities": Object { + "items": Object { + "properties": Object { + "ability": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "is_hidden": Object { + "type": Array [ + "null", + "boolean", + ], + }, + "slot": Object { + "type": Array [ + "null", + "integer", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "base_experience": Object { + "type": Array [ + "null", + "integer", + ], + }, + "forms": Object { + "items": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "game_indices": Object { + "items": Object { + "properties": Object { + "game_index": Object { + "type": Array [ + "null", + "integer", + ], + }, + "version": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "height": Object { + "type": Array [ + "null", + "integer", + ], + }, + "held_items": Object { + "items": Object { + "properties": Object { + "item": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_details": Object { + "items": Object { + "properties": Object { + "rarity": Object { + "type": Array [ + "null", + "integer", + ], + }, + "version": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "id": Object { + "type": Array [ + "null", + "integer", + ], + }, + "is_default ": Object { + "type": Array [ + "null", + "boolean", + ], + }, + "location_area_encounters": Object { + "type": Array [ + "null", + "string", + ], + }, + "moves": Object { + "items": Object { + "properties": Object { + "move": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_group_details": Object { + "items": Object { + "properties": Object { + "level_learned_at": Object { + "type": Array [ + "null", + "integer", + ], + }, + "move_learn_method": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_group": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "order": Object { + "type": Array [ + "null", + "integer", + ], + }, + "species": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "sprites": Object { + "properties": Object { + "back_default": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_shiny": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_shiny_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_default": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_shiny": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_shiny_female": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "stats": Object { + "items": Object { + "properties": Object { + "base_stat": Object { + "type": Array [ + "null", + "integer", + ], + }, + "effort": Object { + "type": Array [ + "null", + "integer", + ], + }, + "stat": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "types": Object { + "items": Object { + "properties": Object { + "slot": Object { + "type": Array [ + "null", + "integer", + ], + }, + "type": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "weight": Object { + "type": Array [ + "null", + "integer", + ], + }, + }, + "type": "object", + }, + "name": "pokemon", + "sourceDefinedPrimaryKey": Array [], + "supportedSyncModes": Array [ + "full_refresh", + ], + }, + }, + ], + }, + "transformations": Array [], + }, + "error": undefined, +} +`; + +exports[`#useInitialValues should generate initial values w/ no edit mode 1`] = ` +Object { + "all": Array [ + Object { + "name": "Scrafty <> Heroku Postgres", + "namespaceDefinition": "source", + "namespaceFormat": "\${SOURCE_NAMESPACE}", + "normalization": "basic", + "prefix": "", + "scheduleData": null, + "syncCatalog": Object { + "streams": Array [ + Object { + "config": Object { + "aliasName": "pokemon", + "cursorField": Array [], + "destinationSyncMode": "overwrite", + "primaryKey": Array [], + "selected": true, + "syncMode": "full_refresh", + }, + "id": "0", + "stream": Object { + "defaultCursorField": Array [], + "jsonSchema": Object { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": Object { + "abilities": Object { + "items": Object { + "properties": Object { + "ability": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "is_hidden": Object { + "type": Array [ + "null", + "boolean", + ], + }, + "slot": Object { + "type": Array [ + "null", + "integer", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "base_experience": Object { + "type": Array [ + "null", + "integer", + ], + }, + "forms": Object { + "items": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "game_indices": Object { + "items": Object { + "properties": Object { + "game_index": Object { + "type": Array [ + "null", + "integer", + ], + }, + "version": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "height": Object { + "type": Array [ + "null", + "integer", + ], + }, + "held_items": Object { + "items": Object { + "properties": Object { + "item": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_details": Object { + "items": Object { + "properties": Object { + "rarity": Object { + "type": Array [ + "null", + "integer", + ], + }, + "version": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "id": Object { + "type": Array [ + "null", + "integer", + ], + }, + "is_default ": Object { + "type": Array [ + "null", + "boolean", + ], + }, + "location_area_encounters": Object { + "type": Array [ + "null", + "string", + ], + }, + "moves": Object { + "items": Object { + "properties": Object { + "move": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_group_details": Object { + "items": Object { + "properties": Object { + "level_learned_at": Object { + "type": Array [ + "null", + "integer", + ], + }, + "move_learn_method": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_group": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "order": Object { + "type": Array [ + "null", + "integer", + ], + }, + "species": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "sprites": Object { + "properties": Object { + "back_default": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_shiny": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_shiny_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_default": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_shiny": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_shiny_female": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "stats": Object { + "items": Object { + "properties": Object { + "base_stat": Object { + "type": Array [ + "null", + "integer", + ], + }, + "effort": Object { + "type": Array [ + "null", + "integer", + ], + }, + "stat": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "types": Object { + "items": Object { + "properties": Object { + "slot": Object { + "type": Array [ + "null", + "integer", + ], + }, + "type": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "weight": Object { + "type": Array [ + "null", + "integer", + ], + }, + }, + "type": "object", + }, + "name": "pokemon", + "sourceDefinedPrimaryKey": Array [], + "supportedSyncModes": Array [ + "full_refresh", + ], + }, + }, + ], + }, + "transformations": Array [], + }, + ], + "current": Object { + "name": "Scrafty <> Heroku Postgres", + "namespaceDefinition": "source", + "namespaceFormat": "\${SOURCE_NAMESPACE}", + "normalization": "basic", + "prefix": "", + "scheduleData": null, + "syncCatalog": Object { + "streams": Array [ + Object { + "config": Object { + "aliasName": "pokemon", + "cursorField": Array [], + "destinationSyncMode": "overwrite", + "primaryKey": Array [], + "selected": true, + "syncMode": "full_refresh", + }, + "id": "0", + "stream": Object { + "defaultCursorField": Array [], + "jsonSchema": Object { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": Object { + "abilities": Object { + "items": Object { + "properties": Object { + "ability": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "is_hidden": Object { + "type": Array [ + "null", + "boolean", + ], + }, + "slot": Object { + "type": Array [ + "null", + "integer", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "base_experience": Object { + "type": Array [ + "null", + "integer", + ], + }, + "forms": Object { + "items": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "game_indices": Object { + "items": Object { + "properties": Object { + "game_index": Object { + "type": Array [ + "null", + "integer", + ], + }, + "version": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "height": Object { + "type": Array [ + "null", + "integer", + ], + }, + "held_items": Object { + "items": Object { + "properties": Object { + "item": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_details": Object { + "items": Object { + "properties": Object { + "rarity": Object { + "type": Array [ + "null", + "integer", + ], + }, + "version": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "id": Object { + "type": Array [ + "null", + "integer", + ], + }, + "is_default ": Object { + "type": Array [ + "null", + "boolean", + ], + }, + "location_area_encounters": Object { + "type": Array [ + "null", + "string", + ], + }, + "moves": Object { + "items": Object { + "properties": Object { + "move": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_group_details": Object { + "items": Object { + "properties": Object { + "level_learned_at": Object { + "type": Array [ + "null", + "integer", + ], + }, + "move_learn_method": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "version_group": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "order": Object { + "type": Array [ + "null", + "integer", + ], + }, + "species": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "sprites": Object { + "properties": Object { + "back_default": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_shiny": Object { + "type": Array [ + "null", + "string", + ], + }, + "back_shiny_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_default": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_female": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_shiny": Object { + "type": Array [ + "null", + "string", + ], + }, + "front_shiny_female": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "stats": Object { + "items": Object { + "properties": Object { + "base_stat": Object { + "type": Array [ + "null", + "integer", + ], + }, + "effort": Object { + "type": Array [ + "null", + "integer", + ], + }, + "stat": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "types": Object { + "items": Object { + "properties": Object { + "slot": Object { + "type": Array [ + "null", + "integer", + ], + }, + "type": Object { + "properties": Object { + "name": Object { + "type": Array [ + "null", + "string", + ], + }, + "url": Object { + "type": Array [ + "null", + "string", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + }, + "type": Array [ + "null", + "object", + ], + }, + "type": Array [ + "null", + "array", + ], + }, + "weight": Object { + "type": Array [ + "null", + "integer", + ], + }, + }, + "type": "object", + }, + "name": "pokemon", + "sourceDefinedPrimaryKey": Array [], + "supportedSyncModes": Array [ + "full_refresh", + ], + }, + }, + ], + }, + "transformations": Array [], + }, + "error": undefined, +} +`; diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.test.ts b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.test.ts index 54fa6416a640..3d28d75b928e 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.test.ts +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.test.ts @@ -1,12 +1,20 @@ import { renderHook } from "@testing-library/react-hooks"; +import mockDestinationDefinition from "test-utils/mock-data//mockDestinationDefinition.json"; +import mockConnection from "test-utils/mock-data/mockConnection.json"; +import { TestWrapper as wrapper } from "test-utils/testutils"; import { frequencyConfig } from "config/frequencyConfig"; -import { ConnectionScheduleTimeUnit } from "core/request/AirbyteClient"; -import { TestWrapper as wrapper } from "utils/testutils"; +import { NormalizationType } from "core/domain/connection"; +import { + ConnectionScheduleTimeUnit, + DestinationDefinitionSpecificationRead, + OperationRead, + WebBackendConnectionRead, +} from "core/request/AirbyteClient"; -import { useFrequencyDropdownData } from "./formConfig"; +import { mapFormPropsToOperation, useFrequencyDropdownData, useInitialValues } from "./formConfig"; -describe("useFrequencyDropdownData", () => { +describe("#useFrequencyDropdownData", () => { it("should return only default frequencies when no additional frequency is provided", () => { const { result } = renderHook(() => useFrequencyDropdownData(undefined), { wrapper }); expect(result.current.map((item) => item.value)).toEqual(frequencyConfig); @@ -36,3 +44,165 @@ describe("useFrequencyDropdownData", () => { expect(result.current).toContainEqual({ label: "Every 7 minutes", value: { units: 7, timeUnit: "minutes" } }); }); }); + +describe("#mapFormPropsToOperation", () => { + const workspaceId = "asdf"; + const normalization: OperationRead = { + workspaceId, + operationId: "asdf", + name: "asdf", + operatorConfiguration: { + operatorType: "normalization", + }, + }; + + it("should add any included transformations", () => { + expect( + mapFormPropsToOperation( + { + transformations: [normalization], + }, + undefined, + "asdf" + ) + ).toEqual([normalization]); + }); + + it("should add a basic normalization if normalization is set to basic", () => { + expect( + mapFormPropsToOperation( + { + normalization: NormalizationType.basic, + }, + + undefined, + workspaceId + ) + ).toEqual([ + { + name: "Normalization", + operatorConfiguration: { + normalization: { + option: "basic", + }, + operatorType: "normalization", + }, + workspaceId, + }, + ]); + }); + + it("should include any provided initial operations and not include the basic normalization operation when normalization type is basic", () => { + expect( + mapFormPropsToOperation( + { + normalization: NormalizationType.basic, + }, + [normalization], + workspaceId + ) + ).toEqual([normalization]); + }); + + it("should not include any provided initial operations and not include the basic normalization operation when normalization type is raw", () => { + expect( + mapFormPropsToOperation( + { + normalization: NormalizationType.raw, + }, + [normalization], + workspaceId + ) + ).toEqual([]); + }); + + it("should include provided transformations when normalization type is raw, but not any provided normalizations", () => { + expect( + mapFormPropsToOperation( + { + normalization: NormalizationType.raw, + transformations: [normalization], + }, + [normalization], + workspaceId + ) + ).toEqual([normalization]); + }); + + it("should include provided transformations and normalizations when normalization type is basic", () => { + expect( + mapFormPropsToOperation( + { + normalization: NormalizationType.basic, + transformations: [normalization], + }, + [normalization], + workspaceId + ) + ).toEqual([normalization, normalization]); + }); + + it("should include provided transformations and default normalization when normalization type is basic and no normalizations have been provided", () => { + expect( + mapFormPropsToOperation( + { + normalization: NormalizationType.basic, + transformations: [normalization], + }, + undefined, + workspaceId + ) + ).toEqual([ + { + name: "Normalization", + operatorConfiguration: { + normalization: { + option: "basic", + }, + operatorType: "normalization", + }, + workspaceId, + }, + normalization, + ]); + }); +}); + +describe("#useInitialValues", () => { + it("should generate initial values w/ no edit mode", () => { + const { result } = renderHook(() => + useInitialValues( + mockConnection as WebBackendConnectionRead, + mockDestinationDefinition as DestinationDefinitionSpecificationRead + ) + ); + expect(result).toMatchSnapshot(); + }); + + it("should generate initial values w/ edit mode: false", () => { + const { result } = renderHook(() => + useInitialValues( + mockConnection as WebBackendConnectionRead, + mockDestinationDefinition as DestinationDefinitionSpecificationRead, + false + ) + ); + expect(result).toMatchSnapshot(); + }); + + it("should generate initial values w/ edit mode: true", () => { + const { result } = renderHook(() => + useInitialValues( + mockConnection as WebBackendConnectionRead, + mockDestinationDefinition as DestinationDefinitionSpecificationRead, + true + ) + ); + expect(result).toMatchSnapshot(); + }); + + // This is a low-priority test + it.todo( + "should test for supportsDbt+initialValues.transformations and supportsNormalization+initialValues.normalization" + ); +}); diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/ServiceForm.test.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/ServiceForm.test.tsx index 3449734f5729..2084950fc201 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/ServiceForm.test.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/ServiceForm.test.tsx @@ -3,9 +3,9 @@ import { getByTestId, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import React from "react"; import selectEvent from "react-select-event"; +import { render, useMockIntersectionObserver } from "test-utils/testutils"; import { AirbyteJSONSchema } from "core/jsonSchema"; -import { render, useMockIntersectionObserver } from "utils/testutils"; import { ServiceForm } from "views/Connector/ServiceForm"; import { DestinationDefinitionSpecificationRead } from "../../../core/request/AirbyteClient"; diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.test.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.test.tsx index e93ec5cc7ce9..1cc982280d55 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.test.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.test.tsx @@ -1,6 +1,6 @@ import { screen, render } from "@testing-library/react"; +import { TestWrapper } from "test-utils/testutils"; -import { TestWrapper } from "utils/testutils"; import { useFormikOauthAdapter } from "views/Connector/ServiceForm/components/Sections/auth/useOauthFlowAdapter"; import { useServiceForm } from "views/Connector/ServiceForm/serviceFormContext"; From 4ef54ad62f15efee1337fb5560ec9cad7b9125a8 Mon Sep 17 00:00:00 2001 From: Anne <102554163+alovew@users.noreply.github.com> Date: Wed, 7 Sep 2022 12:07:56 -0700 Subject: [PATCH 064/200] Sync stats migration (#16285) * Sync stats migration --- .../airbyte/bootloader/BootloaderAppTest.java | 2 +- .../V0_40_3_001__CreateSyncStats.java | 64 +++++++++++++++++++ .../resources/jobs_database/schema_dump.txt | 23 +++++++ .../persistence/DefaultJobPersistence.java | 4 +- .../DefaultJobPersistenceTest.java | 6 +- 5 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_40_3_001__CreateSyncStats.java diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java index 8ec0eee3752b..32bd764e5a47 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderAppTest.java @@ -128,7 +128,7 @@ void testBootloaderAppBlankDb() throws Exception { bootloader.load(); val jobsMigrator = new JobsDatabaseMigrator(jobDatabase, jobsFlyway); - assertEquals("0.35.62.001", jobsMigrator.getLatestMigration().getVersion().getVersion()); + assertEquals("0.40.3.001", jobsMigrator.getLatestMigration().getVersion().getVersion()); val configsMigrator = new ConfigsDatabaseMigrator(configDatabase, configsFlyway); // this line should change with every new migration diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_40_3_001__CreateSyncStats.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_40_3_001__CreateSyncStats.java new file mode 100644 index 000000000000..55844114bdeb --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_40_3_001__CreateSyncStats.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.jobs.migrations; + +import static org.jooq.impl.DSL.currentOffsetDateTime; +import static org.jooq.impl.DSL.foreignKey; +import static org.jooq.impl.DSL.primaryKey; + +import java.time.OffsetDateTime; +import java.util.UUID; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class V0_40_3_001__CreateSyncStats extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_40_3_001__CreateSyncStats.class); + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + final DSLContext ctx = DSL.using(context.getConnection()); + createSyncStatsTable(ctx); + } + + private static void createSyncStatsTable(final DSLContext ctx) { + final Field id = DSL.field("id", SQLDataType.UUID.nullable(false)); + final Field attemptId = DSL.field("attempt_id", SQLDataType.INTEGER.nullable(false)); + final Field recordsEmitted = DSL.field("records_emitted", SQLDataType.BIGINT.nullable(true)); + final Field bytesEmitted = DSL.field("bytes_emitted", SQLDataType.BIGINT.nullable(true)); + final Field sourceStateMessagesEmitted = DSL.field("source_state_messages_emitted", SQLDataType.BIGINT.nullable(true)); + final Field destinationStateMessagesEmitted = DSL.field("destination_state_messages_emitted", SQLDataType.BIGINT.nullable(true)); + final Field recordsCommitted = DSL.field("records_committed", SQLDataType.BIGINT.nullable(true)); + final Field meanSecondsBeforeSourceStateMessageEmitted = + DSL.field("mean_seconds_before_source_state_message_emitted", SQLDataType.BIGINT.nullable(true)); + final Field maxSecondsBeforeSourceStateMessageEmitted = + DSL.field("max_seconds_before_source_state_message_emitted", SQLDataType.BIGINT.nullable(true)); + final Field meanSecondsBetweenStateMessageEmittedandCommitted = + DSL.field("mean_seconds_between_state_message_emitted_and_committed", SQLDataType.BIGINT.nullable(true)); + final Field maxSecondsBetweenStateMessageEmittedandCommitted = + DSL.field("max_seconds_between_state_message_emitted_and_committed", SQLDataType.BIGINT.nullable(true)); + final Field createdAt = + DSL.field("created_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + final Field updatedAt = + DSL.field("updated_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false).defaultValue(currentOffsetDateTime())); + + ctx.createTableIfNotExists("sync_stats") + .columns(id, attemptId, recordsEmitted, bytesEmitted, sourceStateMessagesEmitted, destinationStateMessagesEmitted, recordsCommitted, + meanSecondsBeforeSourceStateMessageEmitted, maxSecondsBeforeSourceStateMessageEmitted, meanSecondsBetweenStateMessageEmittedandCommitted, + maxSecondsBetweenStateMessageEmittedandCommitted, createdAt, updatedAt) + .constraints(primaryKey(id), foreignKey(attemptId).references("attempts", "id").onDeleteCascade()) + .execute(); + + ctx.createIndex("attempt_id_idx").on("sync_stats", "attempt_id").execute(); + } + +} diff --git a/airbyte-db/db-lib/src/main/resources/jobs_database/schema_dump.txt b/airbyte-db/db-lib/src/main/resources/jobs_database/schema_dump.txt index b8c1d305a926..f5b53f16f3dc 100644 --- a/airbyte-db/db-lib/src/main/resources/jobs_database/schema_dump.txt +++ b/airbyte-db/db-lib/src/main/resources/jobs_database/schema_dump.txt @@ -49,6 +49,27 @@ create table "public"."jobs"( constraint "jobs_pkey" primary key ("id") ); +create table "public"."sync_stats"( + "id" uuid not null, + "attempt_id" int4 not null, + "records_emitted" int8 null, + "bytes_emitted" int8 null, + "source_state_messages_emitted" int8 null, + "destination_state_messages_emitted" int8 null, + "records_committed" int8 null, + "mean_seconds_before_source_state_message_emitted" int8 null, + "max_seconds_before_source_state_message_emitted" int8 null, + "mean_seconds_between_state_message_emitted_and_committed" int8 null, + "max_seconds_between_state_message_emitted_and_committed" int8 null, + "created_at" timestamptz(35) not null default null, + "updated_at" timestamptz(35) not null default null, + constraint "sync_stats_pkey" + primary key ("id") +); +alter table "public"."sync_stats" + add constraint "sync_stats_attempt_id_fkey" + foreign key ("attempt_id") + references "public"."attempts" ("id"); create unique index "airbyte_jobs_migrations_pk" on "public"."airbyte_jobs_migrations"("installed_rank" asc); create index "airbyte_jobs_migrations_s_idx" on "public"."airbyte_jobs_migrations"("success" asc); create unique index "airbyte_metadata_pkey" on "public"."airbyte_metadata"("key" asc); @@ -60,3 +81,5 @@ create unique index "job_attempt_idx" on "public"."attempts"( create index "jobs_config_type_idx" on "public"."jobs"("config_type" asc); create unique index "jobs_pkey" on "public"."jobs"("id" asc); create index "jobs_scope_idx" on "public"."jobs"("scope" asc); +create index "attempt_id_idx" on "public"."sync_stats"("attempt_id" asc); +create unique index "sync_stats_pkey" on "public"."sync_stats"("id" asc); diff --git a/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java b/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java index 5f5b625d4d50..a909c49f85a6 100644 --- a/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java +++ b/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java @@ -118,7 +118,7 @@ public DefaultJobPersistence(final Database jobDatabase) { this(jobDatabase, Instant::now, 30, 500, 10); } - private static String jobSelectAndJoin(String jobsSubquery) { + private static String jobSelectAndJoin(final String jobsSubquery) { return "SELECT\n" + "jobs.id AS job_id,\n" + "jobs.config_type AS config_type,\n" @@ -802,7 +802,7 @@ private static void truncateTable(final DSLContext ctx, final String schema, fin final Table backupTableSql = getTable(backupSchema, tableName); ctx.dropTableIfExists(backupTableSql).execute(); ctx.createTable(backupTableSql).as(DSL.select(DSL.asterisk()).from(tableSql)).withData().execute(); - ctx.truncateTable(tableSql).restartIdentity().execute(); + ctx.truncateTable(tableSql).restartIdentity().cascade().execute(); } /** diff --git a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobPersistenceTest.java b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobPersistenceTest.java index c2791ce9bbdd..15ec947dda39 100644 --- a/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobPersistenceTest.java +++ b/airbyte-scheduler/scheduler-persistence/src/test/java/io/airbyte/scheduler/persistence/DefaultJobPersistenceTest.java @@ -198,9 +198,9 @@ void tearDown() throws Exception { private void resetDb() throws SQLException { // todo (cgardens) - truncate whole db. - jobDatabase.query(ctx -> ctx.truncateTable(JOBS).execute()); - jobDatabase.query(ctx -> ctx.truncateTable(ATTEMPTS).execute()); - jobDatabase.query(ctx -> ctx.truncateTable(AIRBYTE_METADATA).execute()); + jobDatabase.query(ctx -> ctx.truncateTable(JOBS).cascade().execute()); + jobDatabase.query(ctx -> ctx.truncateTable(ATTEMPTS).cascade().execute()); + jobDatabase.query(ctx -> ctx.truncateTable(AIRBYTE_METADATA).cascade().execute()); } private Result getJobRecord(final long jobId) throws SQLException { From d8567310297eb685fe0b81d4721393af82cfb851 Mon Sep 17 00:00:00 2001 From: Anne <102554163+alovew@users.noreply.github.com> Date: Wed, 7 Sep 2022 13:20:32 -0700 Subject: [PATCH 065/200] Implement more Error Prone PMD rules (#15491) * AvoidFieldNameMatchingTypeName rule * AvoidInstanceofChecksInCatchClause * compareObjectsWithEquals * DoNotTerminateVM and ConstructorCallsOverridableMethod * EmptyIfStmt and EmptyStatementNotInLoop * ImplicitSwitchFallThrough, InvalidLogMessageFormat, and MoreThanOneLogger --- .../io/airbyte/analytics/TrackingIdentity.java | 1 + .../analytics/SegmentTrackingClientTest.java | 5 +++-- .../io/airbyte/api/client/PatchedLogsApi.java | 6 +++++- .../io/airbyte/commons/io/FileTtlManager.java | 3 ++- .../commons/version/AirbyteVersion.java | 1 + .../java/io/airbyte/config/EnvConfigs.java | 2 +- .../config/helpers/StateMessageHelper.java | 1 + .../config/persistence/StatePersistence.java | 1 + .../split_secrets/VaultSecretPersistence.java | 4 +++- .../persistence/StatePersistenceTest.java | 2 +- .../config/init/RemoteDefinitionsProvider.java | 13 +++++++++---- .../ContainerOrchestratorApp.java | 2 +- .../main/java/io/airbyte/db/DataTypeUtils.java | 1 + .../java/io/airbyte/db/IncrementalUtils.java | 1 + .../db/bigquery/BigQuerySourceOperations.java | 2 +- .../V0_30_22_001__Store_last_sync_state.java | 1 + ...40_001__MigrateFailureReasonEnumValues.java | 1 + .../java/io/airbyte/db/jdbc/JdbcUtils.java | 1 + .../db/jdbc/streaming/BaseSizeEstimator.java | 1 + .../io/airbyte/db/mongodb/MongoDatabase.java | 2 +- .../java/io/airbyte/db/mongodb/MongoUtils.java | 1 + .../protocol/models/CatalogHelpers.java | 2 +- .../airbyte/protocol/models/CommonField.java | 4 ++-- .../java/io/airbyte/scheduler/models/Job.java | 1 + .../persistence/DefaultJobPersistence.java | 2 +- .../scheduler/persistence/JobCleaner.java | 1 + .../SentryExceptionHelper.java | 1 + .../main/java/io/airbyte/server/ServerApp.java | 3 ++- .../server/handlers/ConnectionsHandler.java | 8 ++------ .../server/handlers/OperationsHandler.java | 18 +++++++++--------- .../server/services/AirbyteGithubStore.java | 3 ++- .../server/handlers/ArchiveHandlerTest.java | 2 +- .../test/acceptance/CdcAcceptanceTests.java | 1 + .../ImportApi.java | 6 +++++- .../workers/general/DefaultGetSpecWorker.java | 2 +- .../general/DefaultReplicationWorker.java | 2 ++ .../workers/helper/EntrypointEnvChecker.java | 1 + .../internal/DefaultAirbyteStreamFactory.java | 1 + .../workers/internal/StateMetricsTracker.java | 1 + .../DefaultNormalizationRunner.java | 5 +++-- .../NormalizationAirbyteStreamFactory.java | 7 ++++--- .../workers/process/KubePodProcess.java | 1 + .../java/io/airbyte/workers/run/WorkerRun.java | 1 + tools/gradle/pmd/rules.xml | 12 ------------ 44 files changed, 82 insertions(+), 55 deletions(-) diff --git a/airbyte-analytics/src/main/java/io/airbyte/analytics/TrackingIdentity.java b/airbyte-analytics/src/main/java/io/airbyte/analytics/TrackingIdentity.java index 54703c9a27e8..25fc66db8528 100644 --- a/airbyte-analytics/src/main/java/io/airbyte/analytics/TrackingIdentity.java +++ b/airbyte-analytics/src/main/java/io/airbyte/analytics/TrackingIdentity.java @@ -9,6 +9,7 @@ import java.util.Optional; import java.util.UUID; +@SuppressWarnings("PMD.CompareObjectsWithEquals") public class TrackingIdentity { private final AirbyteVersion airbyteVersion; diff --git a/airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java b/airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java index 2d91aba22b41..190cf60672e4 100644 --- a/airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java +++ b/airbyte-analytics/src/test/java/io/airbyte/analytics/SegmentTrackingClientTest.java @@ -136,10 +136,11 @@ void testTrackWithMetadata() { } private static ImmutableMap filterTrackedAtProperty(final Map properties) { - assertTrue(properties.containsKey("tracked_at")); + final String trackedAtKey = "tracked_at"; + assertTrue(properties.containsKey(trackedAtKey)); final Builder builder = ImmutableMap.builder(); properties.forEach((key, value) -> { - if (!"tracked_at".equals(key)) { + if (!trackedAtKey.equals(key)) { builder.put(key, value); } }); diff --git a/airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java b/airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java index 221107722f80..10831fce4e13 100644 --- a/airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java +++ b/airbyte-api/src/main/java/io/airbyte/api/client/PatchedLogsApi.java @@ -75,7 +75,7 @@ public ApiResponse getLogsWithHttpInfo(final LogsRequestBody logsRequestBo if (memberVarResponseInterceptor != null) { memberVarResponseInterceptor.accept(localVarResponse); } - if (localVarResponse.statusCode() / 100 != 2) { + if (isErrorResponse(localVarResponse)) { throw new ApiException(localVarResponse.statusCode(), "getLogs call received non-success response", localVarResponse.headers(), @@ -100,6 +100,10 @@ public ApiResponse getLogsWithHttpInfo(final LogsRequestBody logsRequestBo } } + private Boolean isErrorResponse(final HttpResponse httpResponse) { + return httpResponse.statusCode() / 100 != 2; + } + private HttpRequest.Builder getLogsRequestBuilder(final LogsRequestBody logsRequestBody) throws ApiException { // verify the required parameter 'logsRequestBody' is set if (logsRequestBody == null) { diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/io/FileTtlManager.java b/airbyte-commons/src/main/java/io/airbyte/commons/io/FileTtlManager.java index fc162bb1da56..7ab4ad7134d0 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/io/FileTtlManager.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/io/FileTtlManager.java @@ -75,7 +75,8 @@ private void reportCacheStatus() { } }); sb.append("---\n"); - LOGGER.info(sb.toString()); + final String toLog = sb.toString(); + LOGGER.info(toLog); } } diff --git a/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java b/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java index 09845f205e52..078d40b487c2 100644 --- a/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java +++ b/airbyte-commons/src/main/java/io/airbyte/commons/version/AirbyteVersion.java @@ -10,6 +10,7 @@ /** * The AirbyteVersion identifies the version of the database used internally by Airbyte services. */ +@SuppressWarnings("PMD.ConstructorCallsOverridableMethod") public class AirbyteVersion { public static final String DEV_VERSION_PREFIX = "dev"; diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java index 5d10b1f14ef5..c92920f641e3 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -33,7 +33,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@SuppressWarnings({"PMD.LongVariable", "PMD.CyclomaticComplexity", "PMD.AvoidReassigningParameters"}) +@SuppressWarnings({"PMD.LongVariable", "PMD.CyclomaticComplexity", "PMD.AvoidReassigningParameters", "PMD.ConstructorCallsOverridableMethod"}) public class EnvConfigs implements Configs { private static final Logger LOGGER = LoggerFactory.getLogger(EnvConfigs.class); diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/StateMessageHelper.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/StateMessageHelper.java index 257a5441c343..e5c9857b04f7 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/StateMessageHelper.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/helpers/StateMessageHelper.java @@ -28,6 +28,7 @@ public static class AirbyteStateMessageListTypeReference extends TypeReference getTypedState(final JsonNode state, final boolean useStreamCapableState) { if (state == null) { return Optional.empty(); diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/StatePersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/StatePersistence.java index 47d828c2ff32..a393bec7d4eb 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/StatePersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/StatePersistence.java @@ -217,6 +217,7 @@ static void writeStateToDb(final DSLContext ctx, * @return the StateType of the records * @throws IllegalStateException If StateRecords have inconsistent types */ + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") private static io.airbyte.db.instance.configs.jooq.generated.enums.StateType getStateType( final UUID connectionId, final List records) { diff --git a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java index 066f06f109a6..43ee603fc8a8 100644 --- a/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java +++ b/airbyte-config/config-persistence/src/main/java/io/airbyte/config/persistence/split_secrets/VaultSecretPersistence.java @@ -31,7 +31,9 @@ public Optional read(final SecretCoordinate coordinate) { val response = vault.logical().read(pathPrefix + coordinate.getFullCoordinate()); val restResponse = response.getRestResponse(); val responseCode = restResponse.getStatus(); - if (responseCode != 200) { + final Boolean isErrorResponse = responseCode / 100 != 2; + + if (isErrorResponse) { log.error("Vault failed on read. Response code: " + responseCode); return Optional.empty(); } diff --git a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java index ef0e34057f4a..925573317a97 100644 --- a/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java +++ b/airbyte-config/config-persistence/src/test/java/io/airbyte/config/persistence/StatePersistenceTest.java @@ -275,7 +275,7 @@ void testGlobalFullReset() throws IOException { .withStreamState(null), new AirbyteStreamState() .withStreamDescriptor(new StreamDescriptor().withName("s1")) - .withStreamState(null)))));; + .withStreamState(null))))); statePersistence.updateOrCreateState(connectionId, state0); statePersistence.updateOrCreateState(connectionId, fullReset); diff --git a/airbyte-config/init/src/main/java/io/airbyte/config/init/RemoteDefinitionsProvider.java b/airbyte-config/init/src/main/java/io/airbyte/config/init/RemoteDefinitionsProvider.java index c7ff5ee68544..aaa159c37269 100644 --- a/airbyte-config/init/src/main/java/io/airbyte/config/init/RemoteDefinitionsProvider.java +++ b/airbyte-config/init/src/main/java/io/airbyte/config/init/RemoteDefinitionsProvider.java @@ -58,7 +58,7 @@ public void initialize() throws InterruptedException, IOException { @Override public StandardSourceDefinition getSourceDefinition(final UUID definitionId) throws ConfigNotFoundException { - StandardSourceDefinition definition = this.sourceDefinitions.get(definitionId); + final StandardSourceDefinition definition = this.sourceDefinitions.get(definitionId); if (definition == null) { throw new ConfigNotFoundException(SeedType.STANDARD_SOURCE_DEFINITION.name(), definitionId.toString()); } @@ -72,7 +72,7 @@ public List getSourceDefinitions() { @Override public StandardDestinationDefinition getDestinationDefinition(final UUID definitionId) throws ConfigNotFoundException { - StandardDestinationDefinition definition = this.destinationDefinitions.get(definitionId); + final StandardDestinationDefinition definition = this.destinationDefinitions.get(definitionId); if (definition == null) { throw new ConfigNotFoundException(SeedType.STANDARD_DESTINATION_DEFINITION.name(), definitionId.toString()); } @@ -84,15 +84,20 @@ public List getDestinationDefinitions() { return new ArrayList<>(this.destinationDefinitions.values()); } - private static CombinedConnectorCatalog getRemoteDefinitionCatalog(URI catalogUrl, Duration timeout) throws IOException, InterruptedException { + private static CombinedConnectorCatalog getRemoteDefinitionCatalog(final URI catalogUrl, final Duration timeout) + throws IOException, InterruptedException { final HttpRequest request = HttpRequest.newBuilder(catalogUrl).timeout(timeout).header("accept", "application/json").build(); final HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - if (response.statusCode() >= 400) { + if (errorStatusCode(response)) { throw new IOException( "getRemoteDefinitionCatalog request ran into status code error: " + response.statusCode() + " with message: " + response.getClass()); } return Jsons.deserialize(response.body(), CombinedConnectorCatalog.class); } + private static Boolean errorStatusCode(final HttpResponse response) { + return response.statusCode() >= 400; + } + } diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java index cd0570bfc535..e38e72b0929d 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/ContainerOrchestratorApp.java @@ -53,7 +53,7 @@ * future this will need to independently interact with cloud storage. */ @Slf4j -@SuppressWarnings("PMD.AvoidCatchingThrowable") +@SuppressWarnings({"PMD.AvoidCatchingThrowable", "PMD.DoNotTerminateVM"}) public class ContainerOrchestratorApp { public static final int MAX_SECONDS_TO_WAIT_FOR_FILE_COPY = 60; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/DataTypeUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/DataTypeUtils.java index 6e82d77a8fc4..10d83e1a3bb3 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/DataTypeUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/DataTypeUtils.java @@ -66,6 +66,7 @@ public static String toISO8601StringWithMicroseconds(final Instant instant) { return dateWithMilliseconds.substring(0, 23) + calculateMicrosecondsString(instant.getNano()) + dateWithMilliseconds.substring(23); } + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") private static String calculateMicrosecondsString(final int nano) { final var microSeconds = (nano / 1000) % 1000; final String result; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/IncrementalUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/IncrementalUtils.java index db3b10ef1547..6a73a03cc6bb 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/IncrementalUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/IncrementalUtils.java @@ -11,6 +11,7 @@ public class IncrementalUtils { private static final String PROPERTIES = "properties"; + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") public static String getCursorField(final ConfiguredAirbyteStream stream) { if (stream.getCursorField().size() == 0) { throw new IllegalStateException("No cursor field specified for stream attempting to do incremental."); diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQuerySourceOperations.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQuerySourceOperations.java index af8a4d1c923f..fd1f36877e27 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQuerySourceOperations.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/bigquery/BigQuerySourceOperations.java @@ -99,7 +99,7 @@ private void setJsonField(final Field field, final FieldValue fieldValue, final } } } catch (final UnsupportedOperationException e) { - LOGGER.error("Failed to parse Object field with name: ", fieldName, e.getMessage()); + LOGGER.error("Failed to parse Object field with name: {}, {}", fieldName, e.getMessage()); } } } diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_30_22_001__Store_last_sync_state.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_30_22_001__Store_last_sync_state.java index 412c7d974ee9..667db91594bf 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_30_22_001__Store_last_sync_state.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_30_22_001__Store_last_sync_state.java @@ -106,6 +106,7 @@ static void copyData(final DSLContext ctx, final Set standard * data from the job database). */ @VisibleForTesting + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") static Optional getJobsDatabase(final String databaseUser, final String databasePassword, final String databaseUrl) { try { if (databaseUrl == null || "".equals(databaseUrl.trim())) { diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_35_40_001__MigrateFailureReasonEnumValues.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_35_40_001__MigrateFailureReasonEnumValues.java index b8b96aad7908..88ea9b915919 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_35_40_001__MigrateFailureReasonEnumValues.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/jobs/migrations/V0_35_40_001__MigrateFailureReasonEnumValues.java @@ -24,6 +24,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@SuppressWarnings({"PMD.AvoidLiteralsInIfCondition", "PMD.CompareObjectsWithEquals"}) public class V0_35_40_001__MigrateFailureReasonEnumValues extends BaseJavaMigration { private static final Logger LOGGER = LoggerFactory.getLogger(V0_35_40_001__MigrateFailureReasonEnumValues.class); diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java index 3ff4a6ac1b20..2e1aea25d215 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/JdbcUtils.java @@ -87,6 +87,7 @@ public static Map parseJdbcParameters(final String jdbcPropertie return parseJdbcParameters(jdbcPropertiesString, "&"); } + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") public static Map parseJdbcParameters(final String jdbcPropertiesString, final String delimiter) { final Map parameters = new HashMap<>(); if (!jdbcPropertiesString.isBlank()) { diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimator.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimator.java index 4892576b9e87..f736d5c4a0b6 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimator.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/jdbc/streaming/BaseSizeEstimator.java @@ -52,6 +52,7 @@ public static long getEstimatedByteSize(final Object rowData) { * This method ensures that the fetch size is between {@code minFetchSize} and {@code maxFetchSize}, * inclusively. */ + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") protected int getBoundedFetchSize() { if (maxRowByteSize <= 0.0) { return defaultFetchSize; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoDatabase.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoDatabase.java index 680008b0c29b..6a0eccef589b 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoDatabase.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoDatabase.java @@ -117,7 +117,7 @@ public Stream read(final String collectionName, final List col }); } catch (final Exception e) { - LOGGER.error("Exception attempting to read data from collection: ", collectionName, e.getMessage()); + LOGGER.error("Exception attempting to read data from collection: {}, {}", collectionName, e.getMessage()); throw new RuntimeException(e); } } diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java index d4a3e20a2702..ab6d24ba8a85 100644 --- a/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/mongodb/MongoUtils.java @@ -278,6 +278,7 @@ private static List getTypes(final MongoCollection collection, return listOfTypes; } + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") private static BsonType getUniqueType(final List types) { if (types.size() != 1) { return BsonType.STRING; diff --git a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java index e781d8586ced..f1b779757f8f 100644 --- a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java +++ b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CatalogHelpers.java @@ -287,7 +287,7 @@ private static boolean isOneOfField(final JsonNode schema) { } private static boolean isObjectWithSubFields(final Field field) { - return field.getType() == JsonSchemaType.OBJECT && field.getSubFields() != null && !field.getSubFields().isEmpty(); + return field.getType().equals(JsonSchemaType.OBJECT) && field.getSubFields() != null && !field.getSubFields().isEmpty(); } public static StreamDescriptor extractStreamDescriptor(final AirbyteStream airbyteStream) { diff --git a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CommonField.java b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CommonField.java index 4c12fa5b986b..c5b096f786d4 100644 --- a/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CommonField.java +++ b/airbyte-protocol/protocol-models/src/main/java/io/airbyte/protocol/models/CommonField.java @@ -20,7 +20,7 @@ public CommonField(final String name, final T type) { this.properties = null; } - public CommonField(final String name, final T type, List> properties) { + public CommonField(final String name, final T type, final List> properties) { this.name = name; this.type = type; this.properties = properties; @@ -45,7 +45,7 @@ public boolean equals(final Object o) { final CommonField field = (CommonField) o; return name.equals(field.name) && - type == field.type && Objects.equals(properties, field.properties); + type.equals(field.type) && Objects.equals(properties, field.properties); } @Override diff --git a/airbyte-scheduler/scheduler-models/src/main/java/io/airbyte/scheduler/models/Job.java b/airbyte-scheduler/scheduler-models/src/main/java/io/airbyte/scheduler/models/Job.java index ed2f1de729d9..9447e5713d81 100644 --- a/airbyte-scheduler/scheduler-models/src/main/java/io/airbyte/scheduler/models/Job.java +++ b/airbyte-scheduler/scheduler-models/src/main/java/io/airbyte/scheduler/models/Job.java @@ -91,6 +91,7 @@ public long getUpdatedAtInSecond() { return updatedAtInSecond; } + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") public Optional getSuccessfulAttempt() { final List successfulAttempts = getAttempts() .stream() diff --git a/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java b/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java index a909c49f85a6..98e3fe2fac61 100644 --- a/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java +++ b/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/DefaultJobPersistence.java @@ -643,7 +643,7 @@ public void setDeployment(final UUID deployment) throws IOException { .orElse(deployment); // if no record was returned that means that the new deployment id was used. if (!deployment.equals(committedDeploymentId)) { - LOGGER.warn("Attempted to set a deployment id %s, but deployment id %s already set. Retained original value."); + LOGGER.warn("Attempted to set a deployment id {}, but deployment id {} already set. Retained original value.", deployment, deployment); } } diff --git a/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/JobCleaner.java b/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/JobCleaner.java index 2002fa92f3ab..f4f4d470fc5d 100644 --- a/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/JobCleaner.java +++ b/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/JobCleaner.java @@ -84,6 +84,7 @@ private void deleteOldFiles() throws IOException { }); } + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") private void deleteOnSize() throws IOException { final Set nonTerminalJobIds = new HashSet<>(); final Sets.SetView nonTerminalStatuses = Sets.difference(Set.of(JobStatus.values()), JobStatus.TERMINAL_STATUSES); diff --git a/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/job_error_reporter/SentryExceptionHelper.java b/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/job_error_reporter/SentryExceptionHelper.java index 076eabe06a79..28846a841682 100644 --- a/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/job_error_reporter/SentryExceptionHelper.java +++ b/airbyte-scheduler/scheduler-persistence/src/main/java/io/airbyte/scheduler/persistence/job_error_reporter/SentryExceptionHelper.java @@ -20,6 +20,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@SuppressWarnings("PMD.AvoidLiteralsInIfCondition") public class SentryExceptionHelper { private static final Logger LOGGER = LoggerFactory.getLogger(SentryExceptionHelper.class); diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index f3b3264c50fe..dde9592ab6d5 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -100,6 +100,7 @@ public ServerApp(final AirbyteVersion airbyteVersion, } @Override + @SuppressWarnings("PMD.InvalidLogMessageFormat") public void start() throws Exception { final Server server = new Server(PORT); @@ -138,7 +139,7 @@ public void start() throws Exception { Runtime.getRuntime().addShutdownHook(new Thread(() -> { try { server.stop(); - } catch (Exception ex) { + } catch (final Exception ex) { // silently fail at this stage because server is terminating. LOGGER.warn("exception: " + ex); } diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java index cfda5cdf69cb..7c8faa9fa766 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/ConnectionsHandler.java @@ -317,12 +317,8 @@ public Set getConfigurationDiff(final AirbyteCatalog oldCatalo newStreams.forEach(((streamDescriptor, airbyteStreamConfiguration) -> { final AirbyteStreamConfiguration oldConfig = oldStreams.get(streamDescriptor); - if (oldConfig == null) { - // The stream is a new one, the config has not change and it needs to be in the schema change list. - } else { - if (haveConfigChange(oldConfig, airbyteStreamConfiguration)) { - streamWithDifferentConf.add(streamDescriptor); - } + if (oldConfig != null && haveConfigChange(oldConfig, airbyteStreamConfiguration)) { + streamWithDifferentConf.add(streamDescriptor); } })); diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/OperationsHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/OperationsHandler.java index 3ea292642276..ae6ab3be08b7 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/OperationsHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/OperationsHandler.java @@ -74,12 +74,12 @@ private static StandardSyncOperation toStandardSyncOperation(final OperationCrea .withName(operationCreate.getName()) .withOperatorType(Enums.convertTo(operationCreate.getOperatorConfiguration().getOperatorType(), OperatorType.class)) .withTombstone(false); - if (operationCreate.getOperatorConfiguration().getOperatorType() == io.airbyte.api.model.generated.OperatorType.NORMALIZATION) { + if ((io.airbyte.api.model.generated.OperatorType.NORMALIZATION).equals(operationCreate.getOperatorConfiguration().getOperatorType())) { Preconditions.checkArgument(operationCreate.getOperatorConfiguration().getNormalization() != null); standardSyncOperation.withOperatorNormalization(new OperatorNormalization() .withOption(Enums.convertTo(operationCreate.getOperatorConfiguration().getNormalization().getOption(), Option.class))); } - if (operationCreate.getOperatorConfiguration().getOperatorType() == io.airbyte.api.model.generated.OperatorType.DBT) { + if ((io.airbyte.api.model.generated.OperatorType.DBT).equals(operationCreate.getOperatorConfiguration().getOperatorType())) { Preconditions.checkArgument(operationCreate.getOperatorConfiguration().getDbt() != null); standardSyncOperation.withOperatorDbt(new OperatorDbt() .withGitRepoUrl(operationCreate.getOperatorConfiguration().getDbt().getGitRepoUrl()) @@ -91,10 +91,10 @@ private static StandardSyncOperation toStandardSyncOperation(final OperationCrea } private void validateOperation(final OperatorConfiguration operatorConfiguration) { - if (operatorConfiguration.getOperatorType() == io.airbyte.api.model.generated.OperatorType.NORMALIZATION) { + if ((io.airbyte.api.model.generated.OperatorType.NORMALIZATION).equals(operatorConfiguration.getOperatorType())) { Preconditions.checkArgument(operatorConfiguration.getNormalization() != null); } - if (operatorConfiguration.getOperatorType() == io.airbyte.api.model.generated.OperatorType.DBT) { + if ((io.airbyte.api.model.generated.OperatorType.DBT).equals(operatorConfiguration.getOperatorType())) { Preconditions.checkArgument(operatorConfiguration.getDbt() != null); } } @@ -115,14 +115,14 @@ public static StandardSyncOperation updateOperation(final OperationUpdate operat standardSyncOperation .withName(operationUpdate.getName()) .withOperatorType(Enums.convertTo(operationUpdate.getOperatorConfiguration().getOperatorType(), OperatorType.class)); - if (operationUpdate.getOperatorConfiguration().getOperatorType() == io.airbyte.api.model.generated.OperatorType.NORMALIZATION) { + if ((io.airbyte.api.model.generated.OperatorType.NORMALIZATION).equals(operationUpdate.getOperatorConfiguration().getOperatorType())) { Preconditions.checkArgument(operationUpdate.getOperatorConfiguration().getNormalization() != null); standardSyncOperation.withOperatorNormalization(new OperatorNormalization() .withOption(Enums.convertTo(operationUpdate.getOperatorConfiguration().getNormalization().getOption(), Option.class))); } else { standardSyncOperation.withOperatorNormalization(null); } - if (operationUpdate.getOperatorConfiguration().getOperatorType() == io.airbyte.api.model.generated.OperatorType.DBT) { + if ((io.airbyte.api.model.generated.OperatorType.DBT).equals(operationUpdate.getOperatorConfiguration().getOperatorType())) { Preconditions.checkArgument(operationUpdate.getOperatorConfiguration().getDbt() != null); standardSyncOperation.withOperatorDbt(new OperatorDbt() .withGitRepoUrl(operationUpdate.getOperatorConfiguration().getDbt().getGitRepoUrl()) @@ -174,7 +174,7 @@ public void deleteOperationsForConnection(final StandardSync standardSync, final boolean sharedOperation = false; for (final StandardSync sync : configRepository.listStandardSyncsUsingOperation(operationId)) { // Check if other connections are using the same operation - if (sync.getConnectionId() != standardSync.getConnectionId()) { + if (!sync.getConnectionId().equals(standardSync.getConnectionId())) { sharedOperation = true; break; } @@ -216,12 +216,12 @@ private OperationRead buildOperationRead(final UUID operationId) private static OperationRead buildOperationRead(final StandardSyncOperation standardSyncOperation) { final OperatorConfiguration operatorConfiguration = new OperatorConfiguration() .operatorType(Enums.convertTo(standardSyncOperation.getOperatorType(), io.airbyte.api.model.generated.OperatorType.class)); - if (standardSyncOperation.getOperatorType() == OperatorType.NORMALIZATION) { + if ((OperatorType.NORMALIZATION).equals(standardSyncOperation.getOperatorType())) { Preconditions.checkArgument(standardSyncOperation.getOperatorNormalization() != null); operatorConfiguration.normalization(new io.airbyte.api.model.generated.OperatorNormalization() .option(Enums.convertTo(standardSyncOperation.getOperatorNormalization().getOption(), OptionEnum.class))); } - if (standardSyncOperation.getOperatorType() == OperatorType.DBT) { + if ((OperatorType.DBT).equals(standardSyncOperation.getOperatorType())) { Preconditions.checkArgument(standardSyncOperation.getOperatorDbt() != null); operatorConfiguration.dbt(new io.airbyte.api.model.generated.OperatorDbt() .gitRepoUrl(standardSyncOperation.getOperatorDbt().getGitRepoUrl()) diff --git a/airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java b/airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java index b9683acd7b6d..7c252dee1df9 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java +++ b/airbyte-server/src/main/java/io/airbyte/server/services/AirbyteGithubStore.java @@ -81,7 +81,8 @@ String getFile(final String filePathWithSlashPrefix) throws IOException, Interru .header("accept", "*/*") // accept any file type .build(); final var resp = httpClient.send(request, BodyHandlers.ofString()); - if (resp.statusCode() >= 400) { + final Boolean isErrorResponse = resp.statusCode() / 100 != 2; + if (isErrorResponse) { throw new IOException("getFile request ran into status code error: " + resp.statusCode() + "with message: " + resp.getClass()); } return resp.body(); diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java index 02e80e50082d..808bb438e2ec 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/ArchiveHandlerTest.java @@ -135,7 +135,7 @@ void setup() throws Exception { jsonSecretsProcessor = JsonSecretsProcessor.builder() .maskSecrets(false) .copySecrets(false) - .build();; + .build(); configPersistence = new DatabaseConfigPersistence(jobDatabase, jsonSecretsProcessor); configPersistence.replaceAllConfigs(Collections.emptyMap(), false); configPersistence.loadData(seedPersistence); diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java index bb2bd5c349d8..429652c2a0c5 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/CdcAcceptanceTests.java @@ -505,6 +505,7 @@ private SourceRead createCdcSource() throws ApiException { Jsons.jsonNode(sourceDbConfigMap)); } + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") private void assertDestinationMatches(final String streamName, final List expectedDestRecordMatchers) throws Exception { final List destRecords = testHarness.retrieveRawDestinationRecords(new SchemaTableNamePair(SCHEMA_NAME, streamName)); diff --git a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/ImportApi.java b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/ImportApi.java index 6a614e76b353..0b00ba9d7543 100644 --- a/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/ImportApi.java +++ b/airbyte-tests/src/automaticMigrationAcceptanceTest/java/io/airbyte/test/automaticMigrationAcceptance/ImportApi.java @@ -62,7 +62,7 @@ public ApiResponse importArchiveWithHttpInfo(final File body) throws if (memberVarResponseInterceptor != null) { memberVarResponseInterceptor.accept(localVarResponse); } - if (localVarResponse.statusCode() / 100 != 2) { + if (errorResponse(localVarResponse)) { throw new ApiException(localVarResponse.statusCode(), "importArchive call received non-success response", localVarResponse.headers(), @@ -81,6 +81,10 @@ public ApiResponse importArchiveWithHttpInfo(final File body) throws } } + private Boolean errorResponse(final HttpResponse localVarResponse) { + return localVarResponse.statusCode() / 100 != 2; + } + private HttpRequest.Builder importArchiveRequestBuilder(final File body) throws ApiException { // verify the required parameter 'body' is set if (body == null) { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultGetSpecWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultGetSpecWorker.java index 711ce2ca18b7..745b1ccbaf37 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultGetSpecWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultGetSpecWorker.java @@ -73,7 +73,7 @@ public ConnectorJobOutput run(final JobGetSpecConfig config, final Path jobRoot) final Optional spec = messagesByType .getOrDefault(Type.SPEC, new ArrayList<>()).stream() .map(AirbyteMessage::getSpec) - .findFirst();; + .findFirst(); final int exitCode = process.exitValue(); if (exitCode == 0) { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java index b91cde3c4c64..28be935591ed 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/general/DefaultReplicationWorker.java @@ -307,6 +307,7 @@ else if (hasFailed.get()) { } + @SuppressWarnings("PMD.AvoidInstanceofChecksInCatchClause") private static Runnable getReplicationRunnable(final AirbyteSource source, final AirbyteDestination destination, final AtomicBoolean cancelled, @@ -419,6 +420,7 @@ private static void validateSchema(final RecordSchemaValidator recordSchemaValid } } + @SuppressWarnings("PMD.AvoidInstanceofChecksInCatchClause") private static Runnable getDestinationOutputRunnable(final AirbyteDestination destination, final AtomicBoolean cancelled, final MessageTracker messageTracker, diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java b/airbyte-workers/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java index b4e360f3437f..9b59867ada27 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/helper/EntrypointEnvChecker.java @@ -27,6 +27,7 @@ public class EntrypointEnvChecker { * @return the entrypoint in the env variable AIRBYTE_ENTRYPOINT * @throws RuntimeException if there is ambiguous output from the container */ + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") public static String getEntrypointEnvVariable(final ProcessFactory processFactory, final String jobId, final int jobAttempt, diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java index ecf4f993476c..d871755ec67d 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/internal/DefaultAirbyteStreamFactory.java @@ -25,6 +25,7 @@ * AirbyteMessage will still be parsed. If there are multiple AirbyteMessage records on the same * line, only the first will be parsed. */ +@SuppressWarnings("PMD.MoreThanOneLogger") public class DefaultAirbyteStreamFactory implements AirbyteStreamFactory { private static final Logger LOGGER = LoggerFactory.getLogger(DefaultAirbyteStreamFactory.class); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/internal/StateMetricsTracker.java b/airbyte-workers/src/main/java/io/airbyte/workers/internal/StateMetricsTracker.java index 30ad22ccd2a0..fe33d638ea72 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/internal/StateMetricsTracker.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/internal/StateMetricsTracker.java @@ -101,6 +101,7 @@ void addStateMessageToStreamToStateHashTimestampTracker(final AirbyteStateMessag remainingCapacity -= 1; } + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") void updateMaxAndMeanSeconds(final LocalDateTime startingTime, final LocalDateTime timeCommitted) { final Long secondsUntilCommit = calculateSecondsBetweenStateEmittedAndCommitted(startingTime, timeCommitted); if (maxSecondsBetweenStateMessageEmittedandCommitted < secondsUntilCommit) { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java index 0b5e3fa813f4..2e1b88349f18 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/DefaultNormalizationRunner.java @@ -126,6 +126,7 @@ public boolean normalize(final String jobId, "--catalog", WorkerConstants.DESTINATION_CATALOG_JSON_FILENAME); } + @SuppressWarnings("PMD.AvoidLiteralsInIfCondition") private boolean runProcess(final String jobId, final int attempt, final Path jobRoot, @@ -160,7 +161,7 @@ private boolean runProcess(final String jobId, dbtErrorStack = String.join("\n", streamFactory.getDbtErrors()); if (!"".equals(dbtErrorStack)) { - AirbyteMessage dbtTraceMessage = new AirbyteMessage() + final AirbyteMessage dbtTraceMessage = new AirbyteMessage() .withType(Type.TRACE) .withTrace(new AirbyteTraceMessage() .withType(AirbyteTraceMessage.Type.ERROR) @@ -213,7 +214,7 @@ public Stream getTraceMessages() { } private String buildInternalErrorMessageFromDbtStackTrace() { - Map errorMap = SentryExceptionHelper.getUsefulErrorMessageAndTypeFromDbtError(dbtErrorStack); + final Map errorMap = SentryExceptionHelper.getUsefulErrorMessageAndTypeFromDbtError(dbtErrorStack); return errorMap.get(SentryExceptionHelper.ERROR_MAP_KEYS.ERROR_MAP_MESSAGE_KEY); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java index 451af9bd46f3..3cd8385fdd4f 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/normalization/NormalizationAirbyteStreamFactory.java @@ -29,6 +29,7 @@ * AirbyteMessage will still be parsed. If there are multiple AirbyteMessage records on the same * line, only the first will be parsed. */ +@SuppressWarnings("PMD.MoreThanOneLogger") public class NormalizationAirbyteStreamFactory implements AirbyteStreamFactory { private static final Logger LOGGER = LoggerFactory.getLogger(NormalizationAirbyteStreamFactory.class); @@ -64,7 +65,7 @@ public Stream create(final BufferedReader bufferedReader) { }); } - private Stream filterOutAndHandleNonJsonLines(String line) { + private Stream filterOutAndHandleNonJsonLines(final String line) { final Optional jsonLine = Jsons.tryDeserialize(line); if (jsonLine.isEmpty()) { // we log as info all the lines that are not valid json. @@ -81,7 +82,7 @@ private Stream filterOutAndHandleNonJsonLines(String line) { return jsonLine.stream(); } - private Stream filterOutAndHandleNonAirbyteMessageLines(JsonNode jsonLine) { + private Stream filterOutAndHandleNonAirbyteMessageLines(final JsonNode jsonLine) { final Optional m = Jsons.tryObject(jsonLine, AirbyteMessage.class); if (m.isEmpty()) { // valid JSON but not an AirbyteMessage, so we assume this is a dbt json log @@ -106,7 +107,7 @@ private Stream filterOutAndHandleNonAirbyteMessageLines(JsonNode return m.stream(); } - private void logAndCollectErrorMessage(String logMsg) { + private void logAndCollectErrorMessage(final String logMsg) { logger.error(logMsg); dbtErrors.add(logMsg); } diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubePodProcess.java b/airbyte-workers/src/main/java/io/airbyte/workers/process/KubePodProcess.java index 649dcd3afbec..793b0f0dc47b 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/process/KubePodProcess.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/process/KubePodProcess.java @@ -346,6 +346,7 @@ private Toleration[] buildPodTolerations(final List tolerations) .toArray(Toleration[]::new); } + @SuppressWarnings("PMD.InvalidLogMessageFormat") public KubePodProcess(final boolean isOrchestrator, final String processRunnerHost, final KubernetesClient fabricClient, diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/run/WorkerRun.java b/airbyte-workers/src/main/java/io/airbyte/workers/run/WorkerRun.java index bf46faaaa30d..33dd11a69306 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/run/WorkerRun.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/run/WorkerRun.java @@ -19,6 +19,7 @@ * outputs are passed to the selected worker. It also makes sure that the outputs of the worker are * persisted to the db. */ +@SuppressWarnings("PMD.AvoidFieldNameMatchingTypeName") public class WorkerRun implements Callable> { private static final Logger LOGGER = LoggerFactory.getLogger(WorkerRun.class); diff --git a/tools/gradle/pmd/rules.xml b/tools/gradle/pmd/rules.xml index 1d2244ca3d3c..982518c448f5 100644 --- a/tools/gradle/pmd/rules.xml +++ b/tools/gradle/pmd/rules.xml @@ -101,20 +101,8 @@ - - - - - - - - - - - - From b9d83a86c039605dc0543b9d9f4df16fb38cbead Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Wed, 7 Sep 2022 18:04:24 -0700 Subject: [PATCH 066/200] exclude file to fix build (#16423) --- build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle b/build.gradle index d05701b31c0b..4592da69de8e 100644 --- a/build.gradle +++ b/build.gradle @@ -118,6 +118,7 @@ def createSpotlessTarget = { pattern -> 'charts', // Helm charts often have injected template strings that will fail general linting. Helm linting is done separately. 'resources/seed/*_specs.yaml', // Do not remove - this is necessary to prevent diffs in our github workflows, as the file diff check runs between the Format step and the Build step, the latter of which generates the file. 'airbyte-integrations/connectors/source-amplitude/unit_tests/api_data/zipped.json', // Zipped file presents as non-UTF-8 making spotless sad + 'airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinition.json', // Prettier styling is conflicting with spotless styling in this file making the build fail if not excluded ] if (System.getenv().containsKey("SUB_BUILD")) { From 41b5797f9d917eb819b0afc38480d4251c033f41 Mon Sep 17 00:00:00 2001 From: Baz Date: Thu, 8 Sep 2022 14:03:09 +0300 Subject: [PATCH 067/200] Source Zendesk Chat: update releaseStage to GA (#16431) --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e5e34117215b..8f4d2ed20d19 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1104,7 +1104,7 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/zendesk-chat icon: zendesk.svg sourceType: api - releaseStage: beta + releaseStage: generally_available - name: Zendesk Sunshine sourceDefinitionId: 325e0640-e7b3-4e24-b823-3361008f603f dockerRepository: airbyte/source-zendesk-sunshine From 78313390d3578a35a4747ee799fc427b3119a2f5 Mon Sep 17 00:00:00 2001 From: Augustin Date: Thu, 8 Sep 2022 16:09:44 +0200 Subject: [PATCH 068/200] Source-instagram: fix incompatible metrics error for reels. (#16428) --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-instagram/Dockerfile | 2 +- .../schemas/media_insights.json | 15 +++++++++ .../source_instagram/streams.py | 8 +++-- docs/integrations/sources/instagram.md | 31 ++++++++++--------- 6 files changed, 40 insertions(+), 20 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 8f4d2ed20d19..3455482a45b0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -465,7 +465,7 @@ - name: Instagram sourceDefinitionId: 6acf6b55-4f1e-4fca-944e-1a3caef8aba8 dockerRepository: airbyte/source-instagram - dockerImageTag: 0.1.10 + dockerImageTag: 0.1.11 documentationUrl: https://docs.airbyte.com/integrations/sources/instagram icon: instagram.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 251388fedf5e..a90706a8bbc5 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4525,7 +4525,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-instagram:0.1.10" +- dockerImage: "airbyte/source-instagram:0.1.11" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/instagram" changelogUrl: "https://docs.airbyte.io/integrations/sources/instagram" diff --git a/airbyte-integrations/connectors/source-instagram/Dockerfile b/airbyte-integrations/connectors/source-instagram/Dockerfile index 9e9bb4270ebb..af483888c48e 100644 --- a/airbyte-integrations/connectors/source-instagram/Dockerfile +++ b/airbyte-integrations/connectors/source-instagram/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.10 +LABEL io.airbyte.version=0.1.11 LABEL io.airbyte.name=airbyte/source-instagram diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media_insights.json b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media_insights.json index 863f783781cf..8a1759549e83 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media_insights.json +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/schemas/media_insights.json @@ -36,6 +36,21 @@ }, "carousel_album_saved": { "type": ["null", "integer"] + }, + "comments": { + "type": ["null", "integer"] + }, + "likes": { + "type": ["null", "integer"] + }, + "shares": { + "type": ["null", "integer"] + }, + "total_interactions": { + "type": ["null", "integer"] + }, + "plays": { + "type": ["null", "integer"] } } } diff --git a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py index 3169cd0bf60d..e77e29164562 100644 --- a/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py +++ b/airbyte-integrations/connectors/source-instagram/source_instagram/streams.py @@ -320,6 +320,7 @@ class MediaInsights(Media): MEDIA_METRICS = ["engagement", "impressions", "reach", "saved"] CAROUSEL_ALBUM_METRICS = ["carousel_album_engagement", "carousel_album_impressions", "carousel_album_reach", "carousel_album_saved"] + REELS_METRICS = ["comments", "likes", "reach", "saved", "shares", "total_interactions", "plays"] def read_records( self, @@ -330,7 +331,7 @@ def read_records( ) -> Iterable[Mapping[str, Any]]: account = stream_slice["account"] ig_account = account["instagram_business_account"] - media = ig_account.get_media(params=self.request_params(), fields=["media_type"]) + media = ig_account.get_media(params=self.request_params(), fields=["media_type", "media_product_type"]) for ig_media in media: account_id = ig_account.get("id") insights = self._get_insights(ig_media, account_id) @@ -344,10 +345,13 @@ def read_records( def _get_insights(self, item, account_id) -> Optional[MutableMapping[str, Any]]: """Get insights for specific media""" - if item.get("media_type") == "VIDEO": + if item.get("media_product_type") == "REELS": + metrics = self.REELS_METRICS + elif item.get("media_type") == "VIDEO": metrics = self.MEDIA_METRICS + ["video_views"] elif item.get("media_type") == "CAROUSEL_ALBUM": metrics = self.CAROUSEL_ALBUM_METRICS + else: metrics = self.MEDIA_METRICS diff --git a/docs/integrations/sources/instagram.md b/docs/integrations/sources/instagram.md index 065daeac11cd..1adf2db75afb 100644 --- a/docs/integrations/sources/instagram.md +++ b/docs/integrations/sources/instagram.md @@ -75,10 +75,10 @@ For more information, see the [Instagram API](https://developers.facebook.com/do ### Features -| Feature | Supported?\(Yes/No\) | Notes | -| :--- | :--- | :--- | -| Full Refresh Sync | Yes | | -| Incremental Sync | Yes | only User Insights | +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :----------------- | +| Full Refresh Sync | Yes | | +| Incremental Sync | Yes | only User Insights | ### Rate Limiting & Performance Considerations @@ -90,19 +90,20 @@ See Facebook's [documentation on rate limiting](https://developers.facebook.com/ ## Data type map | Integration Type | Airbyte Type | -| :--- | :--- | -| `string` | `string` | -| `number` | `number` | -| `array` | `array` | -| `object` | `object` | +| :--------------- | :----------- | +| `string` | `string` | +| `number` | `number` | +| `array` | `array` | +| `object` | `object` | ## Changelog -| Version | Date | Pull Request | Subject | -|:--------|:-----------|:-------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- | +| 0.1.11 | 2022-09-08 | [16428](https://github.com/airbytehq/airbyte/pull/16428) | Fix requests metrics for Reels media product type | | 0.1.10 | 2022-09-05 | [16340](https://github.com/airbytehq/airbyte/pull/16340) | Update to latest version of the CDK (v0.1.81) | -| 0.1.9 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | -| 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | added check for empty state and fixed tests. | -| 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous format of STATE. | -| 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling. - fix sync fail with HTTP status 400. - integrate SAT. | +| 0.1.9 | 2021-09-30 | [6438](https://github.com/airbytehq/airbyte/pull/6438) | Annotate Oauth2 flow initialization parameters in connector specification | +| 0.1.8 | 2021-08-11 | [5354](https://github.com/airbytehq/airbyte/pull/5354) | added check for empty state and fixed tests. | +| 0.1.7 | 2021-07-19 | [4805](https://github.com/airbytehq/airbyte/pull/4805) | Add support for previous format of STATE. | +| 0.1.6 | 2021-07-07 | [4210](https://github.com/airbytehq/airbyte/pull/4210) | Refactor connector to use CDK: - improve error handling. - fix sync fail with HTTP status 400. - integrate SAT. | From 510309622369f54dd5286d2140f42ee4c82e0ddf Mon Sep 17 00:00:00 2001 From: Edmundo Ruiz Ghanem <168664+edmundito@users.noreply.github.com> Date: Thu, 8 Sep 2022 10:30:33 -0400 Subject: [PATCH 069/200] Add vscode settings to autoformat scss files (#16134) Update lint-staged to lint files --- .vscode/frontend.code-workspace | 13 +++++++++++-- .vscode/settings.json | 8 ++++++++ airbyte-webapp/package.json | 8 +++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.vscode/frontend.code-workspace b/.vscode/frontend.code-workspace index 578ef1c697e4..c55ade0551d9 100644 --- a/.vscode/frontend.code-workspace +++ b/.vscode/frontend.code-workspace @@ -18,9 +18,9 @@ "extensions": { "recommendations": [ "dbaeumer.vscode-eslint", + "stylelint.vscode-stylelint", "esbenp.prettier-vscode", - "ms-vsliveshare.vsliveshare", - "eamodio.gitlens" + "eamodio.gitlens", ] }, "settings": { @@ -33,6 +33,8 @@ "editor.detectIndentation": true, "eslint.format.enable": true, "eslint.run": "onType", + "stylelint.enable": true, + "stylelint.validate": ["css", "scss"], "[javascript]": { "editor.formatOnSave": true, "editor.defaultFormatter": "dbaeumer.vscode-eslint", @@ -57,6 +59,13 @@ "[json]": { "editor.formatOnSave": true, "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[scss]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.codeActionsOnSave": { + "source.fixAll.stylelint": true + } } } } diff --git a/.vscode/settings.json b/.vscode/settings.json index ad9b648063a2..e52044ed1069 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,6 +8,7 @@ "editor.detectIndentation": true, "eslint.format.enable": true, "eslint.run": "onType", + "stylelint.enable": true, "stylelint.validate": ["css", "scss"], "[javascript]": { "editor.formatOnSave": true, @@ -33,5 +34,12 @@ "[json]": { "editor.formatOnSave": true, "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "[scss]": { + "editor.formatOnSave": true, + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.codeActionsOnSave": { + "source.fixAll.stylelint": true + } } } diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index b5888ae6ddc9..453e0ca3a0a8 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -137,8 +137,14 @@ } }, "lint-staged": { - "src/**/*.{js,jsx,ts,tsx,json,css,scss,md}": [ + "src/**/*.{js,jsx,ts,tsx,json}": [ + "eslint --fix" + ], + "src/**/*.{css,scss,md}": [ "prettier --write" + ], + "src/**/*.{css,scss}": [ + "stylelint --fix" ] }, "browserslist": { From 8866c8ef1f84dae3b59d4c20c60b780f9c5864ed Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Thu, 8 Sep 2022 12:47:48 -0400 Subject: [PATCH 070/200] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=90=9B=20Prevent?= =?UTF-8?q?=20false=20triggers=20of=20"some=20streams=20have=20changed"=20?= =?UTF-8?q?modal=20(#16443)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add missing sort * sort both for good measure --- .../ConnectionItemPage/components/ReplicationView.tsx | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx index 21cfbe7ea479..575e9131e6ad 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ReplicationView.tsx @@ -19,7 +19,7 @@ import { useUpdateConnection, ValuesProps, } from "hooks/services/useConnectionHook"; -import { equal } from "utils/objects"; +import { equal, naturalComparatorBy } from "utils/objects"; import { CatalogDiffModal } from "views/Connection/CatalogDiffModal/CatalogDiffModal"; import { ConnectionForm, ConnectionFormSubmitResult } from "views/Connection/ConnectionForm"; @@ -142,8 +142,13 @@ export const ReplicationView: React.FC = ({ onAfterSaveSch // This could be due to user changes (e.g. in the sync mode) or due to new/removed // streams due to a "refreshed source schema". const hasCatalogChanged = !equal( - values.syncCatalog.streams.filter((s) => s.config?.selected), - initialConnection.syncCatalog.streams.filter((s) => s.config?.selected) + values.syncCatalog.streams + .filter((s) => s.config?.selected) + .sort(naturalComparatorBy((syncStream) => syncStream.stream?.name ?? "")), + initialConnection.syncCatalog.streams + .filter((s) => s.config?.selected) + + .sort(naturalComparatorBy((syncStream) => syncStream.stream?.name ?? "")) ); // Whenever the catalog changed show a warning to the user, that we're about to reset their data. // Given them a choice to opt-out in which case we'll be sending skipRefresh: true to the update From c30e77f0202e03c40c06dbdb02000b2ee10bc698 Mon Sep 17 00:00:00 2001 From: Anne <102554163+alovew@users.noreply.github.com> Date: Thu, 8 Sep 2022 10:32:21 -0700 Subject: [PATCH 071/200] bump e2e connector versions (#16404) * bump e2e test source connector version --- .../destination-e2e-test/src/main/resources/spec.json | 2 +- airbyte-integrations/connectors/source-e2e-test/Dockerfile | 2 +- .../connectors/source-e2e-test/src/main/resources/spec.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json b/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json index 8dc16fb5ad50..b35585285001 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/resources/spec.json @@ -4,7 +4,7 @@ "supportsNormalization": false, "supportsDBT": false, "supported_destination_sync_modes": ["overwrite", "append"], - "protocol_version": "1.0.0", + "protocol_version": "0.2.1", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "E2E Test Destination Spec", diff --git a/airbyte-integrations/connectors/source-e2e-test/Dockerfile b/airbyte-integrations/connectors/source-e2e-test/Dockerfile index 6d77c2b77af0..b38a53fab502 100644 --- a/airbyte-integrations/connectors/source-e2e-test/Dockerfile +++ b/airbyte-integrations/connectors/source-e2e-test/Dockerfile @@ -17,5 +17,5 @@ ENV ENABLE_SENTRY true COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.1.1 +LABEL io.airbyte.version=2.1.2 LABEL io.airbyte.name=airbyte/source-e2e-test diff --git a/airbyte-integrations/connectors/source-e2e-test/src/main/resources/spec.json b/airbyte-integrations/connectors/source-e2e-test/src/main/resources/spec.json index ec639b3e3d21..bd7bb680d662 100644 --- a/airbyte-integrations/connectors/source-e2e-test/src/main/resources/spec.json +++ b/airbyte-integrations/connectors/source-e2e-test/src/main/resources/spec.json @@ -1,6 +1,6 @@ { "documentationUrl": "https://docs.airbyte.io/integrations/sources/e2e-test", - "protocol_version": "1.0.0", + "protocol_version": "0.2.1", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", "title": "E2E Test Source Spec", From 01723602d4d7996ed781ea4f69fe35eb92695d15 Mon Sep 17 00:00:00 2001 From: Riley Brook Date: Thu, 8 Sep 2022 11:18:33 -0700 Subject: [PATCH 072/200] Update typo (Google Authentication) (#16458) Typo reads "Instagram" where it should read "Google" --- docs/integrations/destinations/google-sheets.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/destinations/google-sheets.md b/docs/integrations/destinations/google-sheets.md index ac15c5d60627..1d135ba23d95 100644 --- a/docs/integrations/destinations/google-sheets.md +++ b/docs/integrations/destinations/google-sheets.md @@ -32,7 +32,7 @@ Visit the [Google Support](https://support.google.com/accounts/answer/27441?hl=e 2. In the left navigation bar, click **Destinations**. In the top-right corner, click **+ new destination**. 3. On the source setup page, select **Google Sheets** from the Source type dropdown and enter a name for this connector. 4. Select `Sign in with Google`. -5. Log in and Authorize to the Instagram account and click `Set up source`. +5. Log in and Authorize to the Google account and click `Set up source`. **For Airbyte Open Source:** From a198fcd5896de304480da4aee997d61250ccde0a Mon Sep 17 00:00:00 2001 From: Edmundo Ruiz Ghanem <168664+edmundito@users.noreply.github.com> Date: Thu, 8 Sep 2022 14:28:50 -0400 Subject: [PATCH 073/200] =?UTF-8?q?=F0=9F=AA=9F=20=F0=9F=8E=A8=20?= =?UTF-8?q?=F0=9F=A7=B9=20Migrate=20`Input`=20and=20`TextArea`=20to=20SCSS?= =?UTF-8?q?=20=20(#16378)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Migrate TextArea component to SCSS and add Storybook * Move Input styles to SCSS, add Storybook * Fix Input stylelint issues * Fix hover selector on Input container to avoid hovering on focus * Fix Input focus test by using style file * Add missing & to Textarea style * Fix styleint inssue in Input * Move input testid before props --- .../components/base/Input/Input.module.scss | 78 +++++++++++++ .../src/components/base/Input/Input.test.tsx | 6 +- .../src/components/base/Input/Input.tsx | 105 +++++------------- .../components/base/Input/index.stories.tsx | 19 ++++ .../base/TextArea/TextArea.module.scss | 52 +++++++++ .../src/components/base/TextArea/TextArea.tsx | 59 ++++------ .../base/TextArea/index.stories.tsx | 15 +++ 7 files changed, 215 insertions(+), 119 deletions(-) create mode 100644 airbyte-webapp/src/components/base/Input/Input.module.scss create mode 100644 airbyte-webapp/src/components/base/Input/index.stories.tsx create mode 100644 airbyte-webapp/src/components/base/TextArea/TextArea.module.scss create mode 100644 airbyte-webapp/src/components/base/TextArea/index.stories.tsx diff --git a/airbyte-webapp/src/components/base/Input/Input.module.scss b/airbyte-webapp/src/components/base/Input/Input.module.scss new file mode 100644 index 000000000000..85d151d423c3 --- /dev/null +++ b/airbyte-webapp/src/components/base/Input/Input.module.scss @@ -0,0 +1,78 @@ +@use "../../../scss/colors"; + +.container { + width: 100%; + position: relative; + background-color: colors.$grey-50; + border: 1px solid colors.$grey-50; + border-radius: 4px; + + &.light { + background-color: colors.$white; + } + + &.error { + background-color: colors.$grey-100; + border-color: colors.$red; + } + + &:not(.disabled, .focused):hover { + background-color: colors.$grey-100; + border-color: colors.$grey-100; + + &.light { + background-color: colors.$white; + } + + &.error { + border-color: colors.$red; + } + } + + &.focused { + background-color: colors.$primaryColor12; + border-color: colors.$blue; + + &.light { + background-color: colors.$white; + } + } +} + +.input { + outline: none; + width: 100%; + padding: 7px 8px; + font-size: 14px; + line-height: 20px; + font-weight: normal; + border: none; + background: none; + color: colors.$dark-blue; + caret-color: colors.$blue; + + &:not(.disabled).password { + width: calc(100% - 22px); + } + + &::placeholder { + color: colors.$grey-300; + } + + &.disabled { + pointer-events: none; + color: colors.$grey-400; + } +} + +button.visibilityButton { + position: absolute; + right: 0; + top: 0; + display: flex; + height: 100%; + width: 30px; + align-items: center; + justify-content: center; + border: none; +} diff --git a/airbyte-webapp/src/components/base/Input/Input.test.tsx b/airbyte-webapp/src/components/base/Input/Input.test.tsx index fcd3c8d7e6de..2d2345a8aa79 100644 --- a/airbyte-webapp/src/components/base/Input/Input.test.tsx +++ b/airbyte-webapp/src/components/base/Input/Input.test.tsx @@ -3,6 +3,8 @@ import { act } from "react-dom/test-utils"; import { render } from "test-utils/testutils"; import { Input } from "./Input"; +// eslint-disable-next-line css-modules/no-unused-class +import styles from "./Input.module.scss"; describe("", () => { test("renders text input", async () => { @@ -117,7 +119,7 @@ describe("", () => { fireEvent.focus(inputEl); fireEvent.focus(inputEl); - expect(getByTestId("input-container")).toHaveClass("input-container--focused"); + expect(getByTestId("input-container")).toHaveClass(styles.focused); }); test("does not have focused class after blur", async () => { @@ -128,7 +130,7 @@ describe("", () => { fireEvent.blur(inputEl); fireEvent.blur(inputEl); - expect(getByTestId("input-container")).not.toHaveClass("input-container--focused"); + expect(getByTestId("input-container")).not.toHaveClass(styles.focused); }); test("calls onFocus if passed as prop", async () => { diff --git a/airbyte-webapp/src/components/base/Input/Input.tsx b/airbyte-webapp/src/components/base/Input/Input.tsx index 73eacb136580..70022c42ea13 100644 --- a/airbyte-webapp/src/components/base/Input/Input.tsx +++ b/airbyte-webapp/src/components/base/Input/Input.tsx @@ -4,85 +4,16 @@ import classNames from "classnames"; import React, { useCallback, useRef, useState } from "react"; import { useIntl } from "react-intl"; import { useToggle } from "react-use"; -import styled from "styled-components"; -import { Theme } from "theme"; import Button from "../Button"; - -type IStyleProps = InputProps & { theme: Theme }; - -const getBackgroundColor = (props: IStyleProps) => { - if (props.error) { - return props.theme.greyColor10; - } else if (props.light) { - return props.theme.whiteColor; - } - - return props.theme.greyColor0; -}; +import styles from "./Input.module.scss"; export interface InputProps extends React.InputHTMLAttributes { error?: boolean; light?: boolean; } -const InputContainer = styled.div` - width: 100%; - position: relative; - background: ${(props) => getBackgroundColor(props)}; - border: 1px solid ${(props) => (props.error ? props.theme.dangerColor : props.theme.greyColor0)}; - border-radius: 4px; - - ${({ disabled, theme, light, error }) => - !disabled && - ` - &:hover { - background: ${light ? theme.whiteColor : theme.greyColor20}; - border-color: ${error ? theme.dangerColor : theme.greyColor20}; - } - `} - - &.input-container--focused { - background: ${({ theme, light }) => (light ? theme.whiteColor : theme.primaryColor12)}; - border-color: ${({ theme }) => theme.primaryColor}; - } -`; - -const InputComponent = styled.input` - outline: none; - width: ${({ isPassword, disabled }) => (isPassword && !disabled ? "calc(100% - 22px)" : "100%")}; - padding: 7px 8px 7px 8px; - font-size: 14px; - line-height: 20px; - font-weight: normal; - border: none; - background: none; - color: ${({ theme }) => theme.textColor}; - caret-color: ${({ theme }) => theme.primaryColor}; - - &::placeholder { - color: ${({ theme }) => theme.greyColor40}; - } - - &:disabled { - pointer-events: none; - color: ${({ theme }) => theme.greyColor55}; - } -`; - -const VisibilityButton = styled(Button)` - position: absolute; - right: 0px; - top: 0; - display: flex; - height: 100%; - width: 30px; - align-items: center; - justify-content: center; - border: none; -`; - -const Input: React.FC = ({ ...props }) => { +export const Input: React.FC = ({ light, error, ...props }) => { const { formatMessage } = useIntl(); const inputRef = useRef(null); @@ -137,15 +68,34 @@ const Input: React.FC = ({ ...props }) => { }; return ( - - + {isVisibilityButtonVisible ? ( - { @@ -159,11 +109,10 @@ const Input: React.FC = ({ ...props }) => { data-testid="toggle-password-visibility-button" > - + ) : null} - +
    ); }; export default Input; -export { Input }; diff --git a/airbyte-webapp/src/components/base/Input/index.stories.tsx b/airbyte-webapp/src/components/base/Input/index.stories.tsx new file mode 100644 index 000000000000..cedfa6988a8f --- /dev/null +++ b/airbyte-webapp/src/components/base/Input/index.stories.tsx @@ -0,0 +1,19 @@ +import { ComponentStory, ComponentMeta } from "@storybook/react"; + +import Input from "./Input"; + +export default { + title: "Ui/Input", + component: Input, + argTypes: { + disabled: { control: "boolean" }, + type: { control: { type: "select", options: ["text", "number", "password"] } }, + }, +} as ComponentMeta; + +const Template: ComponentStory = (args) => ; + +export const Primary = Template.bind({}); +Primary.args = { + placeholder: "Enter text here...", +}; diff --git a/airbyte-webapp/src/components/base/TextArea/TextArea.module.scss b/airbyte-webapp/src/components/base/TextArea/TextArea.module.scss new file mode 100644 index 000000000000..0841f0864278 --- /dev/null +++ b/airbyte-webapp/src/components/base/TextArea/TextArea.module.scss @@ -0,0 +1,52 @@ +@use "../../../scss/colors"; +@use "../../../scss/variables"; + +.textarea { + outline: none; + resize: none; + width: 100%; + padding: 7px 8px; + border-radius: 4px; + font-size: 14px; + line-height: 20px; + font-weight: normal; + border: 1px solid colors.$grey-50; + background-color: colors.$grey-50; + color: colors.$dark-blue; + caret-color: colors.$blue; + + &.error { + border-color: colors.$red; + } + + &::placeholder { + color: colors.$grey-300; + } + + &:hover { + background-color: colors.$grey-100; + border-color: colors.$grey-100; + + &.light { + background-color: colors.$white; + } + + &.error { + border-color: colors.$red; + } + } + + &:focus { + background-color: colors.$primaryColor12; + border-color: colors.$blue; + + &.light { + background-color: colors.$white; + } + } + + &:disabled { + pointer-events: none; + color: colors.$grey-400; + } +} diff --git a/airbyte-webapp/src/components/base/TextArea/TextArea.tsx b/airbyte-webapp/src/components/base/TextArea/TextArea.tsx index a699c41aa838..52b79d316a03 100644 --- a/airbyte-webapp/src/components/base/TextArea/TextArea.tsx +++ b/airbyte-webapp/src/components/base/TextArea/TextArea.tsx @@ -1,44 +1,25 @@ +import classNames from "classnames"; import React from "react"; -import styled from "styled-components"; -type TextAreaProps = { +import styles from "./TextArea.module.scss"; + +interface TextAreaProps extends React.TextareaHTMLAttributes { error?: boolean; light?: boolean; -} & React.TextareaHTMLAttributes; - -const TextArea = styled.textarea` - outline: none; - resize: none; - width: 100%; - padding: 7px 8px; - border-radius: 4px; - font-size: 14px; - line-height: 20px; - font-weight: normal; - border: 1px solid ${(props) => (props.error ? props.theme.dangerColor : props.theme.greyColor0)}; - background: ${({ theme }) => theme.greyColor0}; - color: ${({ theme }) => theme.textColor}; - caret-color: ${({ theme }) => theme.primaryColor}; - - &::placeholder { - color: ${({ theme }) => theme.greyColor40}; - } - - &:hover { - background: ${({ theme, light }) => (light ? theme.whiteColor : theme.greyColor20)}; - border-color: ${(props) => (props.error ? props.theme.dangerColor : props.theme.greyColor20)}; - } - - &:focus { - background: ${({ theme, light }) => (light ? theme.whiteColor : theme.primaryColor12)}; - border-color: ${({ theme }) => theme.primaryColor}; - } - - &:disabled { - pointer-events: none; - color: ${({ theme }) => theme.greyColor55}; - } -`; +} -export { TextArea }; -export type { TextAreaProps }; +export const TextArea: React.FC = ({ error, light, children, className, ...textAreaProps }) => ( + +); diff --git a/airbyte-webapp/src/components/base/TextArea/index.stories.tsx b/airbyte-webapp/src/components/base/TextArea/index.stories.tsx new file mode 100644 index 000000000000..fc69c3a8fd8c --- /dev/null +++ b/airbyte-webapp/src/components/base/TextArea/index.stories.tsx @@ -0,0 +1,15 @@ +import { ComponentStory, ComponentMeta } from "@storybook/react"; + +import { TextArea } from "./TextArea"; + +export default { + title: "Ui/TextArea", + component: TextArea, +} as ComponentMeta; + +const Template: ComponentStory = (args) =>