From 36d3fe3737437c5c7c4e98c1fb8e84ac556ba54f Mon Sep 17 00:00:00 2001 From: Mark Ortleb Date: Wed, 14 Jun 2023 10:25:51 -0400 Subject: [PATCH] adding new Salesloft streams --- .../integration_tests/abnormal_state.json | 42 ++++ .../integration_tests/configured_catalog.json | 127 ++++++++++++ .../incremental_catalog.json | 126 ++++++++++++ .../schemas/call_data_records.json | 90 +++++++++ .../schemas/call_dispositions.json | 20 ++ .../schemas/call_sentiments.json | 20 ++ .../schemas/custom_fields.json | 26 +++ .../source_salesloft/schemas/meetings.json | 191 ++++++++++++++++++ .../source_salesloft/schemas/searches.json | 105 ++++++++++ .../source_salesloft/source.py | 52 +++++ 10 files changed, 799 insertions(+) create mode 100644 airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_data_records.json create mode 100644 airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_dispositions.json create mode 100644 airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_sentiments.json create mode 100644 airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/custom_fields.json create mode 100644 airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/meetings.json create mode 100644 airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/searches.json diff --git a/airbyte-integrations/connectors/source-salesloft/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-salesloft/integration_tests/abnormal_state.json index 81a81225bad3..475ff3c63ccf 100644 --- a/airbyte-integrations/connectors/source-salesloft/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-salesloft/integration_tests/abnormal_state.json @@ -89,5 +89,47 @@ "stream_state": { "updated_at": "2122-01-18T21:18:20.000Z" }, "stream_descriptor": { "name": "email_templates" } } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2122-01-18T21:18:20.000Z" }, + "stream_descriptor": { "name": "custom_fields" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2122-01-18T21:18:20.000Z" }, + "stream_descriptor": { "name": "call_data_records" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2122-01-18T21:18:20.000Z" }, + "stream_descriptor": { "name": "call_dispositions" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2122-01-18T21:18:20.000Z" }, + "stream_descriptor": { "name": "call_sentiments" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2122-01-18T21:18:20.000Z" }, + "stream_descriptor": { "name": "meetings" } + } + }, + { + "type": "STREAM", + "stream": { + "stream_state": { "updated_at": "2122-01-18T21:18:20.000Z" }, + "stream_descriptor": { "name": "searches" } + } } ] diff --git a/airbyte-integrations/connectors/source-salesloft/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-salesloft/integration_tests/configured_catalog.json index 99a5e4c4847d..e6440b4200ea 100644 --- a/airbyte-integrations/connectors/source-salesloft/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-salesloft/integration_tests/configured_catalog.json @@ -420,6 +420,133 @@ }, "destination_sync_mode": "overwrite", "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "custom_fields", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "updated_at" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + + { + "stream": { + "name": "call_data_records", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "updated_at" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "call_dispositions", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "updated_at" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "call_sentiments", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "updated_at" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "meetings", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "updated_at" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "searches", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "updated_at" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" } ] } diff --git a/airbyte-integrations/connectors/source-salesloft/integration_tests/incremental_catalog.json b/airbyte-integrations/connectors/source-salesloft/integration_tests/incremental_catalog.json index 0998800674d1..518b2c3c1051 100644 --- a/airbyte-integrations/connectors/source-salesloft/integration_tests/incremental_catalog.json +++ b/airbyte-integrations/connectors/source-salesloft/integration_tests/incremental_catalog.json @@ -260,6 +260,132 @@ }, "destination_sync_mode": "overwrite", "sync_mode": "incremental" + }, + { + "stream": { + "name": "custom_fields", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "updated_at" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + }, + { + "stream": { + "name": "call_data_records", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "updated_at" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + }, + { + "stream": { + "name": "call_dispositions", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "updated_at" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + }, + { + "stream": { + "name": "call_sentiments", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "updated_at" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + }, + { + "stream": { + "name": "meetings", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "updated_at" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + }, + { + "stream": { + "name": "searches", + "json_schema": {}, + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], + "source_defined_cursor": true, + "default_cursor_field": [ + "updated_at" + ], + "source_defined_primary_key": [ + [ + "id" + ] + ] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" } ] } diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_data_records.json b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_data_records.json new file mode 100644 index 000000000000..cae0117e8d17 --- /dev/null +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_data_records.json @@ -0,0 +1,90 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "to": { + "type": ["null", "string"] + }, + "from": { + "type": ["null", "string"] + }, + "duration": { + "type": ["null", "string"] + }, + "direction": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "call_type": { + "type": ["null", "string"] + }, + "call_uuid": { + "type": ["null", "string"] + }, + "recording": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "url": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "recording_status": { + "type": ["null", "string"] + } + } + }, + "call": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "_href": { + "type": ["null", "string"] + } + } + }, + "user": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "_href": { + "type": ["null", "string"] + } + } + }, + "called_person": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "_href": { + "type": ["null", "string"] + } + } + } + + } +} diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_dispositions.json b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_dispositions.json new file mode 100644 index 000000000000..5678bb6321ed --- /dev/null +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_dispositions.json @@ -0,0 +1,20 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_sentiments.json b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_sentiments.json new file mode 100644 index 000000000000..5678bb6321ed --- /dev/null +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/call_sentiments.json @@ -0,0 +1,20 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/custom_fields.json b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/custom_fields.json new file mode 100644 index 000000000000..f5c96903818c --- /dev/null +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/custom_fields.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "name": { + "type": ["null", "string"] + }, + "field_type": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "updated_at": { + "type": "string", + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/meetings.json b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/meetings.json new file mode 100644 index 000000000000..ba11439b18da --- /dev/null +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/meetings.json @@ -0,0 +1,191 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "title": { + "type": ["null", "string"] + }, + "start_time": { + "type": "string", + "format": "date-time" + }, + "end_time": { + "type": "string", + "format": "date-time" + }, + "calendar_id": { + "type": ["null", "string"] + }, + "calendar_type": { + "type": ["null", "string"] + }, + "meeting_type": { + "type": ["null", "string"] + }, + "recipient_name": { + "type": ["null", "string"] + }, + "recipient_email": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "event_id": { + "type": ["null", "string"] + }, + "account_id": { + "type": ["null", "string"] + }, + "task_id": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "guests": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "attendees": { + "type": ["null", "array"], + "items": { + "type": "object", + "additionalProperties": true, + "properties": { + "email": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "organizer": { + "type": ["null", "boolean"] + }, + "status": { + "type": ["null", "string"] + }, + "status_changed": { + "type": ["null", "boolean"] + }, + "deleted_at": { + "type": "string", + "format": "date-time" + } + } + } + }, + "person": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "_href": { + "type": ["null", "string"] + } + } + }, + "cadence": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "_href": { + "type": ["null", "string"] + } + } + }, + "step": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "_href": { + "type": ["null", "string"] + } + } + }, + "booked_by_user": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "_href": { + "type": ["null", "string"] + } + } + }, + "crm_references": { + "type": ["null", "object"], + "additionalProperties": true + }, + "event_source": { + "type": ["null", "string"] + }, + "canceled_at": { + "type": "string", + "format": "date-time" + }, + "all_day": { + "type": ["null", "boolean"] + }, + "no_show": { + "type": ["null", "boolean"] + }, + "crm_custom_fields": { + "type": ["null", "object"], + "additionalProperties": true + }, + "strict_attribution": { + "type": ["null", "boolean"] + }, + "i_cal_uid": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "reschedule_status": { + "type": ["null", "string"] + }, + "owned_by_meetings_settings": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "email_address": ["null", "integer"] + } + } + }, + "booked_by_meetings_settings": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "email_address": ["null", "integer"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/searches.json b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/searches.json new file mode 100644 index 000000000000..8a57d7e948b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/schemas/searches.json @@ -0,0 +1,105 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "user": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "id": { + "type": ["null", "integer"] + }, + "_href": { + "type": ["null", "string"] + } + } + }, + "user_slug": { + "type": ["null", "string"] + }, + "primary_calendar_id": { + "type": ["null", "string"] + }, + "primary_calendar_name": { + "type": ["null", "string"] + }, + "email_address": { + "type": ["null", "string"] + }, + "user_details": { + "type": ["null", "string"] + }, + "calendar_type": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "string"] + }, + "default_meeting_length": { + "type": "integer" + }, + "availability_limit_enabled": { + "type": ["null", "boolean"] + }, + "availability_limit": { + "type": "integer" + }, + "schedule_delay": { + "type": "integer" + }, + "buffer_time_duration": { + "type": "integer" + }, + "schedule_buffer_enabled": { + "type": ["null", "boolean"] + }, + "times_available": { + "type": ["null", "object"], + "additionalProperties": true + }, + "allow_booking_on_behalf": { + "type": ["null", "boolean"] + }, + "allow_booking_overtime": { + "type": ["null", "boolean"] + }, + "allow_event_overlap": { + "type": ["null", "boolean"] + }, + "allow_event_detail": { + "type": ["null", "boolean"] + }, + "enable_dynamic_location": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "updated_at": { + "type": "string", + "format": "date-time" + }, + "time_zone": { + "type": ["null", "string"] + }, + "primary_calendar_connection_failed": { + "type": ["null", "boolean"] + }, + "enable_calendar_sync": { + "type": ["null", "boolean"] + }, + "reschedule_meetings_enabled": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-salesloft/source_salesloft/source.py b/airbyte-integrations/connectors/source-salesloft/source_salesloft/source.py index 51136a94e6db..d54c9d3f9018 100644 --- a/airbyte-integrations/connectors/source-salesloft/source_salesloft/source.py +++ b/airbyte-integrations/connectors/source-salesloft/source_salesloft/source.py @@ -226,6 +226,52 @@ def path(self, **kwargs) -> str: return "groups" +class CustomFields(IncrementalSalesloftStream): + created_at_field = "updated_at" + + def path(self, **kwargs) -> str: + return "custom_fields" + + +class CallDataRecords(IncrementalSalesloftStream): + created_at_field = "updated_at" + + def path(self, **kwargs) -> str: + return "call_data_records" + + +class CallDispositions(IncrementalSalesloftStream): + created_at_field = "updated_at" + + def path(self, **kwargs) -> str: + return "call_dispositions" + + +class CallSentiments(IncrementalSalesloftStream): + created_at_field = "updated_at" + + def path(self, **kwargs) -> str: + return "call_sentiments" + + +class Meetings(IncrementalSalesloftStream): + created_at_field = "updated_at" + + def path(self, **kwargs) -> str: + return "meetings" + + +class Searches(IncrementalSalesloftStream): + created_at_field = "updated_at" + + def path(self, **kwargs) -> str: + return "meetings/settings/searches" + + @property + def http_method(self) -> str: + return "POST" + + # Source class SourceSalesloft(AbstractSource): def _create_authenticator(self, config) -> AuthBase: @@ -272,4 +318,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Groups(*args), Successes(*args), EmailTemplateAttachments(*args), + CustomFields(*args), + CallDataRecords(*args), + CallDispositions(*args), + CallSentiments(*args), + Meetings(*args), + Searches(*args) ]