From 8ca6394f1f3fdf0dc883a4fe199f85258249f693 Mon Sep 17 00:00:00 2001 From: Oliver Meyer <42039965+olivermeyer@users.noreply.github.com> Date: Fri, 25 Mar 2022 00:54:36 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20=20Source=20BambooHR:=20Add=20cu?= =?UTF-8?q?stom=20reports=20endpoint=20(#11326)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Migrate existing code to newer CDK * Additional changes * Update doc * Fixes for tests * fix primary key * bump connector version and update doc Co-authored-by: marcosmarxm --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-bamboo-hr/Dockerfile | 2 +- .../integration_tests/configured_catalog.json | 23 +- .../sample_files/configured_catalog.json | 811 +----------------- .../schemas/custom_reports_stream.json | 246 ++++++ .../schemas/employees_directory_stream.json | 63 ++ .../source_bamboo_hr/source.py | 259 +++--- .../source_bamboo_hr/spec.json | 10 + .../source-bamboo-hr/unit_tests/unit_test.py | 43 +- docs/integrations/sources/bamboo-hr.md | 7 +- 11 files changed, 557 insertions(+), 911 deletions(-) create mode 100644 airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/schemas/custom_reports_stream.json create mode 100644 airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/schemas/employees_directory_stream.json diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index bdb1ac2c5d94..4f16b10d9ce4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -70,7 +70,7 @@ - name: BambooHR sourceDefinitionId: 90916976-a132-4ce9-8bce-82a03dd58788 dockerRepository: airbyte/source-bamboo-hr - dockerImageTag: 0.1.0 + dockerImageTag: 0.2.0 documentationUrl: https://docs.airbyte.io/integrations/sources/bamboo-hr icon: bamboohr.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 46bef1883a95..b4c57165d5d2 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -643,7 +643,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-bamboo-hr:0.1.0" +- dockerImage: "airbyte/source-bamboo-hr:0.2.0" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/bamboo-hr" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-bamboo-hr/Dockerfile b/airbyte-integrations/connectors/source-bamboo-hr/Dockerfile index 8a4cd61d69f5..3a93a7100b8e 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/Dockerfile +++ b/airbyte-integrations/connectors/source-bamboo-hr/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-bamboo-hr diff --git a/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/configured_catalog.json index eb6044b93377..9f5d06b234ca 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-bamboo-hr/integration_tests/configured_catalog.json @@ -2,13 +2,34 @@ "streams": [ { "stream": { - "name": "employee", + "name": "employees_directory_stream", "json_schema": {}, "supported_sync_modes": ["full_refresh"], "supported_destination_sync_modes": ["overwrite", "append_dedup"] }, "sync_mode": "full_refresh", "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "custom_reports_stream", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "zipcode": { + "type": ["null", "string"] + }, + "terminationDate": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh"], + "supported_destination_sync_modes": ["overwrite", "append_dedup"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" } ] } diff --git a/airbyte-integrations/connectors/source-bamboo-hr/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-bamboo-hr/sample_files/configured_catalog.json index b21ac15c28bc..2486e6964fb1 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-bamboo-hr/sample_files/configured_catalog.json @@ -2,824 +2,83 @@ "streams": [ { "stream": { - "name": "employee", + "name": "employees_directory_stream", "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "401(k) - Company pays": { + "id": { "type": ["null", "string"] }, - "401(k) - Company pays - Currency code": { + "displayName": { "type": ["null", "string"] }, - "401(k) - Coverage": { - "type": ["null", "string"] - }, - "401(k) - Effective date": { - "type": ["null", "string"] - }, - "401(k) - Eligibility date": { - "type": ["null", "string"] - }, - "401(k) - Employee pays": { - "type": ["null", "string"] - }, - "401(k) - Employee pays - Currency code": { - "type": ["null", "string"] - }, - "401(k) - Status": { - "type": ["null", "string"] - }, - "Accrual Level Start Date": { - "type": ["null", "string"] - }, - "address1": { - "type": ["null", "string"] - }, - "address2": { - "type": ["null", "string"] - }, - "allergies": { - "type": ["null", "string"] - }, - "Asset Category": { - "type": ["null", "string"] - }, - "Asset Description": { - "type": ["null", "string"] - }, - "BambooHR Advantage Package Demo Video - Completed": { - "type": ["null", "string"] - }, - "BambooHR Advantage Package Demo Video - Due Date": { - "type": ["null", "string"] - }, - "BambooHR Advantage Package Demo Video - Expires": { - "type": ["null", "string"] - }, - "Benefit Groups": { - "type": ["null", "string"] - }, - "Benefit History": { - "type": ["null", "string"] - }, - "Bereavement - Accrual Start Date": { - "type": ["null", "string"] - }, - "Bereavement - Adjustments (YTD)": { - "type": ["null", "string"] - }, - "Bereavement - Available Balance": { - "type": ["null", "string"] - }, - "Bereavement - Current balance": { - "type": ["null", "string"] - }, - "Bereavement - Days scheduled": { - "type": ["null", "string"] - }, - "Bereavement - Days taken (YTD)": { - "type": ["null", "string"] - }, - "Bereavement - Policy": { - "type": ["null", "string"] - }, - "Bereavement - Policy Assigned": { - "type": ["null", "string"] - }, - "dateOfBirth": { - "type": ["null", "string"] - }, - "CA Health Insurance - Company pays": { - "type": ["null", "string"] - }, - "CA Health Insurance - Company pays - Currency code": { - "type": ["null", "string"] - }, - "CA Health Insurance - Coverage": { - "type": ["null", "string"] - }, - "CA Health Insurance - Effective date": { - "type": ["null", "string"] - }, - "CA Health Insurance - Eligibility date": { - "type": ["null", "string"] - }, - "CA Health Insurance - Employee pays": { - "type": ["null", "string"] - }, - "CA Health Insurance - Employee pays - Currency code": { - "type": ["null", "string"] - }, - "CA Health Insurance - Status": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "College/Institution": { - "type": ["null", "string"] - }, - "Comp/In Lieu Time - Accrual Start Date": { - "type": ["null", "string"] - }, - "Comp/In Lieu Time - Adjustments (YTD)": { - "type": ["null", "string"] - }, - "Comp/In Lieu Time - Available Balance": { - "type": ["null", "string"] - }, - "Comp/In Lieu Time - Current balance": { - "type": ["null", "string"] - }, - "Comp/In Lieu Time - Hours scheduled": { - "type": ["null", "string"] - }, - "Comp/In Lieu Time - Hours taken (YTD)": { - "type": ["null", "string"] - }, - "Comp/In Lieu Time - Policy": { - "type": ["null", "string"] - }, - "Comp/In Lieu Time - Policy Assigned": { - "type": ["null", "string"] - }, - "Company Handbook Quiz - Completed": { - "type": ["null", "string"] - }, - "Company Handbook Quiz - Due Date": { - "type": ["null", "string"] - }, - "Company Handbook Quiz - Expires": { - "type": ["null", "string"] - }, - "payRateEffectiveDate": { - "type": ["null", "string"] - }, - "payChangeReason": { - "type": ["null", "string"] - }, - "Compensation comments": { - "type": ["null", "string"] - }, - "Completed - Category": { - "type": ["null", "string"] - }, - "Completed - Completed": { - "type": ["null", "string"] - }, - "Completed - Cost": { - "type": ["null", "string"] - }, - "Completed - Credits": { - "type": ["null", "string"] - }, - "Completed - Hours": { - "type": ["null", "string"] - }, - "Completed - Instructor": { - "type": ["null", "string"] - }, - "Completed - Notes": { - "type": ["null", "string"] - }, - "Completed - Title": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "COVID-19 Related Absence - Accrual Start Date": { - "type": ["null", "string"] - }, - "COVID-19 Related Absence - Adjustments (YTD)": { - "type": ["null", "string"] - }, - "COVID-19 Related Absence - Available Balance": { - "type": ["null", "string"] - }, - "COVID-19 Related Absence - Current balance": { - "type": ["null", "string"] - }, - "COVID-19 Related Absence - Hours scheduled": { - "type": ["null", "string"] - }, - "COVID-19 Related Absence - Hours taken (YTD)": { - "type": ["null", "string"] - }, - "COVID-19 Related Absence - Policy": { - "type": ["null", "string"] - }, - "COVID-19 Related Absence - Policy Assigned": { - "type": ["null", "string"] - }, - "CPR - Completed": { - "type": ["null", "string"] - }, - "CPR - Due Date": { - "type": ["null", "string"] - }, - "CPR - Expires": { - "type": ["null", "string"] - }, - "Date Assigned": { - "type": ["null", "string"] - }, - "Date Returned": { - "type": ["null", "string"] - }, - "Degree": { - "type": ["null", "string"] - }, - "Dental Insurance - Company pays": { - "type": ["null", "string"] - }, - "Dental Insurance - Company pays - Currency code": { - "type": ["null", "string"] - }, - "Dental Insurance - Coverage": { - "type": ["null", "string"] - }, - "Dental Insurance - Effective date": { - "type": ["null", "string"] - }, - "Dental Insurance - Eligibility date": { - "type": ["null", "string"] - }, - "Dental Insurance - Employee pays": { - "type": ["null", "string"] - }, - "Dental Insurance - Employee pays - Currency code": { - "type": ["null", "string"] - }, - "Dental Insurance - Status": { - "type": ["null", "string"] - }, - "department": { - "type": ["null", "string"] - }, - "Dependent Birth Date": { - "type": ["null", "string"] - }, - "Dependent City": { - "type": ["null", "string"] - }, - "Dependent Country": { - "type": ["null", "string"] - }, - "Dependent First Name": { - "type": ["null", "string"] - }, - "Dependent Full-Time Student": { - "type": ["null", "string"] - }, - "Dependent Gender": { - "type": ["null", "string"] - }, - "Dependent Home Phone": { - "type": ["null", "string"] - }, - "Dependent Last Name": { - "type": ["null", "string"] - }, - "Dependent Middle Name": { - "type": ["null", "string"] - }, - "Dependent Relationship": { - "type": ["null", "string"] - }, - "Dependent SSN": { - "type": ["null", "string"] - }, - "Dependent State": { - "type": ["null", "string"] - }, - "Dependent Street 1": { - "type": ["null", "string"] - }, - "Dependent Street 2": { - "type": ["null", "string"] - }, - "Dependent US Citizen": { - "type": ["null", "string"] - }, - "Dependent ZIP": { - "type": ["null", "string"] - }, - "dietaryRestrictions": { - "type": ["null", "string"] - }, - "Disability - Company pays": { - "type": ["null", "string"] - }, - "Disability - Company pays - Currency code": { - "type": ["null", "string"] - }, - "Disability - Coverage": { - "type": ["null", "string"] - }, - "Disability - Effective date": { - "type": ["null", "string"] - }, - "Disability - Eligibility date": { - "type": ["null", "string"] - }, - "Disability - Employee pays": { - "type": ["null", "string"] - }, - "Disability - Employee pays - Currency code": { - "type": ["null", "string"] - }, - "Disability - Status": { - "type": ["null", "string"] - }, - "division": { - "type": ["null", "string"] - }, - "eeo": { - "type": ["null", "string"] - }, - "Eligible For Re-hire": { - "type": ["null", "string"] - }, - "Emergency Contact City": { - "type": ["null", "string"] - }, - "Emergency Contact Country": { - "type": ["null", "string"] - }, - "Emergency Contact Email": { - "type": ["null", "string"] - }, - "Emergency Contact Home Phone": { - "type": ["null", "string"] - }, - "Emergency Contact Mobile Phone": { - "type": ["null", "string"] - }, - "Emergency Contact Name": { - "type": ["null", "string"] - }, - "Emergency Contact Primary": { - "type": ["null", "string"] - }, - "Emergency Contact Relationship": { - "type": ["null", "string"] - }, - "Emergency Contacts": { - "type": ["null", "string"] - }, - "Emergency Contact State": { - "type": ["null", "string"] - }, - "Emergency Contact Street 1": { - "type": ["null", "string"] - }, - "Emergency Contact Street 2": { - "type": ["null", "string"] - }, - "Emergency Contact Work Ext": { - "type": ["null", "string"] - }, - "Emergency Contact Work Phone": { - "type": ["null", "string"] - }, - "Emergency Contact ZIP Code": { - "type": ["null", "string"] - }, - "employeeNumber": { - "type": ["null", "string"] - }, - "Employee Education: End Date": { - "type": ["null", "string"] - }, - "Employee Education: Start Date": { - "type": ["null", "string"] - }, - "employmentHistoryStatus": { - "type": ["null", "string"] - }, - "acaStatus": { - "type": ["null", "string"] - }, - "employeeStatusDate": { - "type": ["null", "string"] - }, - "Employment status comments": { - "type": ["null", "string"] - }, - "ethnicity": { - "type": ["null", "string"] - }, - "Expiration": { - "type": ["null", "string"] - }, - "facebook": { - "type": ["null", "string"] - }, - "First Aid - Completed": { - "type": ["null", "string"] - }, - "First Aid - Due Date": { - "type": ["null", "string"] - }, - "First Aid - Expires": { - "type": ["null", "string"] - }, - "firstName": { - "type": ["null", "string"] - }, - "FMLA - Accrual Start Date": { - "type": ["null", "string"] - }, - "FMLA - Adjustments (YTD)": { - "type": ["null", "string"] - }, - "FMLA - Available Balance": { - "type": ["null", "string"] - }, - "FMLA - Current balance": { - "type": ["null", "string"] - }, - "FMLA - Hours scheduled": { - "type": ["null", "string"] - }, - "FMLA - Hours taken (YTD)": { - "type": ["null", "string"] - }, - "FMLA - Policy": { - "type": ["null", "string"] - }, - "FMLA - Policy Assigned": { - "type": ["null", "string"] - }, - "gender": { - "type": ["null", "string"] - }, - "Getting Started in BambooHR - Completed": { - "type": ["null", "string"] - }, - "Getting Started in BambooHR - Due Date": { - "type": ["null", "string"] - }, - "Getting Started in BambooHR - Expires": { - "type": ["null", "string"] - }, - "GPA": { - "type": ["null", "string"] - }, - "Health Insurance - CA - Company pays": { - "type": ["null", "string"] - }, - "Health Insurance - CA - Company pays - Currency code": { - "type": ["null", "string"] - }, - "Health Insurance - CA - Coverage": { - "type": ["null", "string"] - }, - "Health Insurance - CA - Effective date": { - "type": ["null", "string"] - }, - "Health Insurance - CA - Eligibility date": { - "type": ["null", "string"] - }, - "Health Insurance - CA - Employee pays": { - "type": ["null", "string"] - }, - "Health Insurance - CA - Employee pays - Currency code": { - "type": ["null", "string"] - }, - "Health Insurance - CA - Status": { - "type": ["null", "string"] - }, - "hireDate": { - "type": ["null", "string"] - }, - "homeEmail": { - "type": ["null", "string"] - }, - "homePhone": { - "type": ["null", "string"] - }, - "instagram": { - "type": ["null", "string"] - }, - "Issued": { - "type": ["null", "string"] - }, - "Issuing country": { - "type": ["null", "string"] - }, - "Job Information: Date": { - "type": ["null", "string"] - }, - "jobTitle": { + "firstName": { "type": ["null", "string"] }, "lastName": { "type": ["null", "string"] }, - "Life Insurance - Company pays": { - "type": ["null", "string"] - }, - "Life Insurance - Company pays - Currency code": { - "type": ["null", "string"] - }, - "Life Insurance - Coverage": { - "type": ["null", "string"] - }, - "Life Insurance - Effective date": { - "type": ["null", "string"] - }, - "Life Insurance - Eligibility date": { - "type": ["null", "string"] - }, - "Life Insurance - Employee pays": { - "type": ["null", "string"] - }, - "Life Insurance - Employee pays - Currency code": { - "type": ["null", "string"] - }, - "Life Insurance - Status": { - "type": ["null", "string"] - }, - "linkedIn": { - "type": ["null", "string"] - }, - "location": { - "type": ["null", "string"] - }, - "Major/Specialization": { - "type": ["null", "string"] - }, - "maritalStatus": { - "type": ["null", "string"] - }, - "middleName": { - "type": ["null", "string"] - }, - "mobilePhone": { - "type": ["null", "string"] - }, - "New Hire Training - Completed": { - "type": ["null", "string"] - }, - "New Hire Training - Due Date": { - "type": ["null", "string"] - }, - "New Hire Training - Expires": { - "type": ["null", "string"] - }, - "customNIN1": { - "type": ["null", "string"] - }, - "Note": { - "type": ["null", "string"] - }, - "originalHireDate": { - "type": ["null", "string"] - }, - "overtimeRate": { - "type": ["null", "string"] - }, - "exempt": { - "type": ["null", "string"] - }, - "payPer": { - "type": ["null", "string"] - }, - "payRate": { - "type": ["null", "string"] - }, - "payPeriod": { - "type": ["null", "string"] - }, - "paySchedule": { - "type": ["null", "string"] - }, - "payType": { - "type": ["null", "string"] - }, - "pinterest": { - "type": ["null", "string"] - }, - "nickname": { - "type": ["null", "string"] - }, "preferredName": { "type": ["null", "string"] }, - "Reporting to": { - "type": ["null", "string"] - }, - "customSecondaryLanguage1": { - "type": ["null", "string"] - }, - "Self-service access": { - "type": ["null", "string"] - }, - "Serial #": { - "type": ["null", "string"] - }, - "customShirtsize": { - "type": ["null", "string"] - }, - "Sick - Accrual Start Date": { - "type": ["null", "string"] - }, - "Sick - Adjustments (YTD)": { - "type": ["null", "string"] - }, - "Sick - Available Balance": { - "type": ["null", "string"] - }, - "Sick - Current balance": { - "type": ["null", "string"] - }, - "Sick - Hours scheduled": { - "type": ["null", "string"] - }, - "Sick - Hours taken (YTD)": { - "type": ["null", "string"] - }, - "Sick - Policy": { - "type": ["null", "string"] - }, - "Sick - Policy Assigned": { - "type": ["null", "string"] - }, - "Sick Full-Time policy - Assigned": { - "type": ["null", "string"] - }, - "Sick Full-Time policy - Started on": { - "type": ["null", "string"] - }, - "ssn": { - "type": ["null", "string"] - }, - "state": { - "type": ["null", "string"] - }, - "employmentStatus": { - "type": ["null", "string"] - }, - "status": { - "type": ["null", "string"] - }, - "customTaxFileNumber1": { - "type": ["null", "string"] - }, - "Termination Reason": { - "type": ["null", "string"] - }, - "Termination Type": { - "type": ["null", "string"] - }, - "twitterFeed": { - "type": ["null", "string"] - }, - "UK Private Medical Insurance (PMI) - Company pays": { - "type": ["null", "string"] - }, - "UK Private Medical Insurance (PMI) - Company pays - Currency code": { - "type": ["null", "string"] - }, - "UK Private Medical Insurance (PMI) - Coverage": { - "type": ["null", "string"] - }, - "UK Private Medical Insurance (PMI) - Effective date": { - "type": ["null", "string"] - }, - "UK Private Medical Insurance (PMI) - Eligibility date": { - "type": ["null", "string"] - }, - "UK Private Medical Insurance (PMI) - Employee pays": { - "type": ["null", "string"] - }, - "UK Private Medical Insurance (PMI) - Employee pays - Currency code": { - "type": ["null", "string"] - }, - "UK Private Medical Insurance (PMI) - Status": { - "type": ["null", "string"] - }, - "Unlawful Harassment - Completed": { - "type": ["null", "string"] - }, - "Unlawful Harassment - Due Date": { - "type": ["null", "string"] - }, - "Unlawful Harassment - Expires": { - "type": ["null", "string"] - }, - "US Health Insurance - FSA Eligible - Company pays": { - "type": ["null", "string"] - }, - "US Health Insurance - FSA Eligible - Company pays - Currency code": { - "type": ["null", "string"] - }, - "US Health Insurance - FSA Eligible - Coverage": { - "type": ["null", "string"] - }, - "US Health Insurance - FSA Eligible - Effective date": { - "type": ["null", "string"] - }, - "US Health Insurance - FSA Eligible - Eligibility date": { - "type": ["null", "string"] - }, - "US Health Insurance - FSA Eligible - Employee pays": { - "type": ["null", "string"] - }, - "US Health Insurance - FSA Eligible - Employee pays - Currency code": { - "type": ["null", "string"] - }, - "US Health Insurance - FSA Eligible - Status": { - "type": ["null", "string"] - }, - "US Health Insurance - HSA Eligible - Company pays": { - "type": ["null", "string"] - }, - "US Health Insurance - HSA Eligible - Company pays - Currency code": { - "type": ["null", "string"] - }, - "US Health Insurance - HSA Eligible - Coverage": { - "type": ["null", "string"] - }, - "US Health Insurance - HSA Eligible - Effective date": { - "type": ["null", "string"] - }, - "US Health Insurance - HSA Eligible - Eligibility date": { - "type": ["null", "string"] - }, - "US Health Insurance - HSA Eligible - Employee pays": { - "type": ["null", "string"] - }, - "US Health Insurance - HSA Eligible - Employee pays - Currency code": { - "type": ["null", "string"] - }, - "US Health Insurance - HSA Eligible - Status": { - "type": ["null", "string"] - }, - "Vacation - Accrual Start Date": { - "type": ["null", "string"] - }, - "Vacation - Adjustments (YTD)": { - "type": ["null", "string"] - }, - "Vacation - Available Balance": { - "type": ["null", "string"] - }, - "Vacation - Current balance": { - "type": ["null", "string"] - }, - "Vacation - Hours scheduled": { - "type": ["null", "string"] - }, - "Vacation - Hours taken (YTD)": { - "type": ["null", "string"] - }, - "Vacation - Policy": { + "jobTitle": { "type": ["null", "string"] }, - "Vacation - Policy Assigned": { + "workPhone": { "type": ["null", "string"] }, - "Vacation Full-Time policy - Assigned": { + "mobilePhone": { "type": ["null", "string"] }, - "Vacation Full-Time policy - Started on": { + "workEmail": { "type": ["null", "string"] }, - "Veteran Status": { + "department": { "type": ["null", "string"] }, - "Visa": { + "location": { "type": ["null", "string"] }, - "Visa:Date": { + "division": { "type": ["null", "string"] }, - "Visa Information": { + "linkedIn": { "type": ["null", "string"] }, - "workEmail": { + "pronouns": { "type": ["null", "string"] }, "workPhoneExtension": { "type": ["null", "string"] }, - "Working from home during COVID-19 - Completed": { - "type": ["null", "string"] - }, - "Working from home during COVID-19 - Due Date": { + "supervisor": { "type": ["null", "string"] }, - "Working from home during COVID-19 - Expires": { - "type": ["null", "string"] + "photoUploaded": { + "type": ["null", "boolean"] }, - "workPhone": { + "photoUrl": { "type": ["null", "string"] }, + "canUploadPhoto": { + "type": ["null", "boolean"] + } + } + }, + "supported_sync_modes": ["full_refresh"], + "supported_destination_sync_modes": ["overwrite", "append_dedup"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "custom_reports_stream", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { "zipcode": { "type": ["null", "string"] }, diff --git a/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/schemas/custom_reports_stream.json b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/schemas/custom_reports_stream.json new file mode 100644 index 000000000000..ab07b0291e59 --- /dev/null +++ b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/schemas/custom_reports_stream.json @@ -0,0 +1,246 @@ +{ + "type": ["null", "object"], + "required": [], + "properties": { + "acaStatus": { + "type": ["null", "string"] + }, + "acaStatusCategory":{ + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "address2":{ + "type": ["null", "string"] + }, + "age": { + "type": ["null", "string"] + }, + "bestEmail":{ + "type": ["null", "string"] + }, + "birthday": { + "type": ["null", "string"] + }, + "bonusAmount":{ + "type": ["null", "string"] + }, + "bonusComment": { + "type": ["null", "string"] + }, + "bonusDate":{ + "type": ["null", "string"] + }, + "bonusReason": { + "type": ["null", "string"] + }, + "city":{ + "type": ["null", "string"] + }, + "commissionAmount": { + "type": ["null", "string"] + }, + "commissionComment":{ + "type": ["null", "string"] + }, + "commissionDate": { + "type": ["null", "string"] + }, + "commisionDate":{ + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "createdByUserId":{ + "type": ["null", "string"] + }, + "dateOfBirth": { + "type": ["null", "string"] + }, + "department":{ + "type": ["null", "string"] + }, + "division": { + "type": ["null", "string"] + }, + "eeo":{ + "type": ["null", "string"] + }, + "employeeNumber": { + "type": ["null", "string"] + }, + "employmentHistoryStatus":{ + "type": ["null", "string"] + }, + "ethnicity": { + "type": ["null", "string"] + }, + "exempt":{ + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "flsaCode":{ + "type": ["null", "string"] + }, + "fullName1": { + "type": ["null", "string"] + }, + "fullName2":{ + "type": ["null", "string"] + }, + "fullName3": { + "type": ["null", "string"] + }, + "fullName4":{ + "type": ["null", "string"] + }, + "fullName5": { + "type": ["null", "string"] + }, + "displayName":{ + "type": ["null", "string"] + }, + "gender": { + "type": ["null", "string"] + }, + "hireDate":{ + "type": ["null", "string"] + }, + "originalHireDate": { + "type": ["null", "string"] + }, + "homeEmail":{ + "type": ["null", "string"] + }, + "homePhone": { + "type": ["null", "string"] + }, + "id":{ + "type": ["null", "string"] + }, + "isPhotoUploaded": { + "type": ["null", "string"] + }, + "jobTitle":{ + "type": ["null", "string"] + }, + "lastChanged": { + "type": ["null", "string"] + }, + "lastName":{ + "type": ["null", "string"] + }, + "location": { + "type": ["null", "string"] + }, + "maritalStatus":{ + "type": ["null", "string"] + }, + "middleName": { + "type": ["null", "string"] + }, + "mobilePhone":{ + "type": ["null", "string"] + }, + "nationalId": { + "type": ["null", "string"] + }, + "nationality":{ + "type": ["null", "string"] + }, + "nin": { + "type": ["null", "string"] + }, + "payChangeReason":{ + "type": ["null", "string"] + }, + "payGroup": { + "type": ["null", "string"] + }, + "payGroupId":{ + "type": ["null", "string"] + }, + "payRate": { + "type": ["null", "string"] + }, + "payRateEffectiveDate":{ + "type": ["null", "string"] + }, + "payType": { + "type": ["null", "string"] + }, + "paidPer":{ + "type": ["null", "string"] + }, + "paySchedule": { + "type": ["null", "string"] + }, + "payScheduleId":{ + "type": ["null", "string"] + }, + "payFrequency": { + "type": ["null", "string"] + }, + "includeInPayroll":{ + "type": ["null", "string"] + }, + "timeTrackingEnabled": { + "type": ["null", "string"] + }, + "preferredName":{ + "type": ["null", "string"] + }, + "ssn": { + "type": ["null", "string"] + }, + "sin":{ + "type": ["null", "string"] + }, + "standardHoursPerWeek": { + "type": ["null", "string"] + }, + "state":{ + "type": ["null", "string"] + }, + "stateCode": { + "type": ["null", "string"] + }, + "status":{ + "type": ["null", "string"] + }, + "supervisor": { + "type": ["null", "string"] + }, + "supervisorId":{ + "type": ["null", "string"] + }, + "supervisorEId": { + "type": ["null", "string"] + }, + "supervisorEmail":{ + "type": ["null", "string"] + }, + "terminationDate": { + "type": ["null", "string"] + }, + "workEmail":{ + "type": ["null", "string"] + }, + "workPhone": { + "type": ["null", "string"] + }, + "workPhonePlusExtension":{ + "type": ["null", "string"] + }, + "workPhoneExtension": { + "type": ["null", "string"] + }, + "zipcode":{ + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/schemas/employees_directory_stream.json b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/schemas/employees_directory_stream.json new file mode 100644 index 000000000000..0cd1291f31f3 --- /dev/null +++ b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/schemas/employees_directory_stream.json @@ -0,0 +1,63 @@ +{ + "type": ["null", "object"], + "required": [], + "properties": { + "id": { + "type": ["null", "string"] + }, + "displayName": { + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "preferredName": { + "type": ["null", "string"] + }, + "jobTitle": { + "type": ["null", "string"] + }, + "workPhone": { + "type": ["null", "string"] + }, + "mobilePhone": { + "type": ["null", "string"] + }, + "workEmail": { + "type": ["null", "string"] + }, + "department": { + "type": ["null", "string"] + }, + "location": { + "type": ["null", "string"] + }, + "division": { + "type": ["null", "string"] + }, + "linkedIn": { + "type": ["null", "string"] + }, + "pronouns": { + "type": ["null", "string"] + }, + "workPhoneExtension": { + "type": ["null", "string"] + }, + "supervisor": { + "type": ["null", "string"] + }, + "photoUploaded": { + "type": ["null", "boolean"] + }, + "photoUrl": { + "type": ["null", "string"] + }, + "canUploadPhoto": { + "type": ["null", "number"] + } + } +} diff --git a/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/source.py b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/source.py index a36af251dd62..93156a5cd97c 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/source.py +++ b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/source.py @@ -4,147 +4,154 @@ import base64 -import json -from datetime import datetime -from typing import Dict, Generator +import logging +from abc import ABC +from typing import Any, Iterable, List, Mapping, Optional, Tuple import requests -from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import ( - AirbyteCatalog, - AirbyteConnectionStatus, - AirbyteMessage, - AirbyteRecordMessage, - AirbyteStream, - ConfiguredAirbyteCatalog, - Status, - Type, -) -from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode -from airbyte_cdk.sources import Source -from requests.models import Response - - -class BambooHrClient(object): - def __init__(self, config: json) -> None: - self.api_key = config["api_key"] - self.subdomain = config["subdomain"] - - def _prepare_request_auth(self): - return base64.b64encode("{}:x".format(self.api_key).encode("utf-8")).decode("utf-8") - - def request(self, uri: str, method: str = "GET", data={}, **kwargs) -> Response: - url = "https://api.bamboohr.com/api/gateway.php/{}/v1/{}".format(self.subdomain, uri) - headers = kwargs.get("headers", {}) - headers.update( - { - "Authorization": "Basic {}".format(self._prepare_request_auth()), - "Accept": "application/json", - "Content-Type": kwargs.get("content_type") or "application/json", - } - ) - - if data and not isinstance(data, str) and headers["Content-Type"] == "application/json": - data = json.dumps(data) - - response = requests.request(method=method, url=url, headers=headers) - response.raise_for_status() - return response - - -class SourceBambooHr(Source): - def check(self, logger: AirbyteLogger, config: json) -> AirbyteConnectionStatus: - """ - Tests if the input configuration can be used to successfully connect to the integration +from airbyte_cdk.models.airbyte_protocol import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator - :param logger: Logging object to display debug/info/error to the logs - (logs will not be accessible via airbyte UI if they are not passed to this logger) - :param config: Json object containing the configuration of this source, content of this json is as specified in - the properties of the spec.json file - :return: AirbyteConnectionStatus indicating a Success or Failure - """ - try: - bamboo = BambooHrClient(config) - bamboo.request("employees/directory") - return AirbyteConnectionStatus(status=Status.SUCCEEDED) - except Exception as e: - return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {str(e)}") +class BambooHrStream(HttpStream, ABC): + def __init__(self, config: Mapping[str, Any]) -> None: + self.config = config + super().__init__(authenticator=config["authenticator"]) + + @property + def url_base(self) -> str: + return f"https://api.bamboohr.com/api/gateway.php/{self.config['subdomain']}/v1/" - def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + return {"Accept": "application/json"} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ - Returns an AirbyteCatalog representing the available streams and fields in this integration. - For example, given valid credentials to a Postgres database, - returns an Airbyte catalog where each postgres table is a stream, and each table column is a field. - - :param logger: Logging object to display debug/info/error to the logs - (logs will not be accessible via airbyte UI if they are not passed to this logger) - :param config: Json object containing the configuration of this source, content of this json is as specified in - the properties of the spec.json file - - :return: AirbyteCatalog is an object describing a list of all available streams in this source. - A stream is an AirbyteStream object that includes: - - its stream name (or table name in the case of Postgres) - - json_schema providing the specifications of expected schema for this stream (a list of columns described - by their names and types) + BambooHR does not support pagination. """ - streams = [] + pass + + +class MetaFieldsStream(BambooHrStream): + primary_key = None + + def path(self, **kwargs) -> str: + return "meta/fields" - bamboo = BambooHrClient(config) - fields = bamboo.request("meta/fields").json() - properties = {} + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield from response.json() - for field in fields: - # All fields are nullable strings - # https://documentation.bamboohr.com/docs/field-types - properties[field.get("alias", field["name"])] = {"type": ["null", "string"]} - stream_name = "employee" - json_schema = { +class EmployeesDirectoryStream(BambooHrStream): + primary_key = "id" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield from response.json()["employees"] + + def path(self, **kwargs) -> str: + return "employees/directory" + + +class CustomReportsStream(BambooHrStream): + primary_key = None + + def __init__(self, *args, **kwargs): + self._schema = None + super().__init__(*args, **kwargs) + + @property + def schema(self): + if not self._schema: + self._schema = self.get_json_schema() + return self._schema + + def _get_json_schema_from_config(self): + if self.config.get("custom_reports_fields"): + properties = {field.strip(): {"type": ["null", "string"]} for field in self.config.get("custom_reports_fields").split(",")} + else: + properties = {} + return { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": properties, } - streams.append( - AirbyteStream( - name=stream_name, - json_schema=json_schema, - supported_sync_modes=[SyncMode.full_refresh], - supported_destination_sync_modes=[DestinationSyncMode.overwrite, DestinationSyncMode.append_dedup], - ) - ) - return AirbyteCatalog(streams=streams) - - def read( - self, logger: AirbyteLogger, config: json, catalog: ConfiguredAirbyteCatalog, state: Dict[str, any] - ) -> Generator[AirbyteMessage, None, None]: + + def _get_json_schema_from_file(self): + return super().get_json_schema() + + @staticmethod + def _union_schemas(schema1, schema2): + schema1["properties"] = {**schema1["properties"], **schema2["properties"]} + return schema1 + + def get_json_schema(self) -> Mapping[str, Any]: """ - Returns a generator of the AirbyteMessages generated by reading the source with the given configuration, - catalog, and state. - - :param logger: Logging object to display debug/info/error to the logs - (logs will not be accessible via airbyte UI if they are not passed to this logger) - :param config: Json object containing the configuration of this source, content of this json is as specified in - the properties of the spec.json file - :param catalog: The input catalog is a ConfiguredAirbyteCatalog which is almost the same as AirbyteCatalog - returned by discover(), but - in addition, it's been configured in the UI! For each particular stream and field, there may have been provided - with extra modifications such as: filtering streams and/or columns out, renaming some entities, etc - :param state: When a Airbyte reads data from a source, it might need to keep a checkpoint cursor to resume - replication in the future from that saved checkpoint. - This is the object that is provided with state from previous runs and avoid replicating the entire set of - data everytime. - - :return: A generator that produces a stream of AirbyteRecordMessage contained in AirbyteMessage object. + Returns the JSON schema. + + The final schema is constructed by first generating a schema for the fields + in the config and, if default fields should be included, adding these to the + schema. """ - stream_name = "employee" + schema = self._get_json_schema_from_config() + if self.config.get("custom_reports_include_default_fields"): + default_schema = self._get_json_schema_from_file() + schema = self._union_schemas(default_schema, schema) + return schema + + def path(self, **kwargs) -> str: + return "reports/custom" + + @property + def http_method(self) -> str: + return "POST" + + def request_body_json(self, **kwargs) -> Optional[Mapping]: + return {"title": "Airbyte", "fields": list(self.schema["properties"].keys())} - bamboo = BambooHrClient(config) - data = bamboo.request("employees/directory").json() - employees = data["employees"] + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + yield from response.json()["employees"] - for employee in employees: - yield AirbyteMessage( - type=Type.RECORD, - record=AirbyteRecordMessage(stream=stream_name, data=employee, emitted_at=int(datetime.now().timestamp()) * 1000), - ) + +class SourceBambooHr(AbstractSource): + @staticmethod + def _get_authenticator(api_key): + """ + Returns a TokenAuthenticator. + + The API token is concatenated with `:x` and the resulting string is base-64 encoded. + See https://documentation.bamboohr.com/docs#authentication + """ + return TokenAuthenticator(token=base64.b64encode(f"{api_key}:x".encode("utf-8")).decode("utf-8"), auth_method="Basic") + + @staticmethod + def add_authenticator_to_config(config: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Adds an authenticator entry to the config and returns the config. + """ + config["authenticator"] = SourceBambooHr._get_authenticator(config["api_key"]) + return config + + def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: + """ + Verifies the config and attempts to fetch the fields from the meta/fields endpoint. + """ + config = SourceBambooHr.add_authenticator_to_config(config) + if not config.get("custom_reports_fields") and not config.get("custom_reports_include_default_fields"): + return False, AttributeError("`custom_reports_fields` cannot be empty if `custom_reports_include_default_fields` is false") + try: + next(MetaFieldsStream(config).read_records(sync_mode=SyncMode.full_refresh)) + return True, None + except Exception as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + config = SourceBambooHr.add_authenticator_to_config(config) + return [ + EmployeesDirectoryStream(config), + CustomReportsStream(config), + ] diff --git a/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/spec.json b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/spec.json index ddcadf5aac74..e74378af6837 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/spec.json +++ b/airbyte-integrations/connectors/source-bamboo-hr/source_bamboo_hr/spec.json @@ -15,6 +15,16 @@ "type": "string", "description": "Api key of bamboo hr", "airbyte_secret": true + }, + "custom_reports_fields": { + "type": "string", + "default": "", + "description": "Comma-separated list of fields to include in custom reports." + }, + "custom_reports_include_default_fields": { + "type": "boolean", + "default": true, + "description": "If true, the custom reports endpoint will include the default fields defined here: https://documentation.bamboohr.com/docs/list-of-field-names." } } } diff --git a/airbyte-integrations/connectors/source-bamboo-hr/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-bamboo-hr/unit_tests/unit_test.py index 7adf4fd46507..aa593ce3bba0 100644 --- a/airbyte-integrations/connectors/source-bamboo-hr/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-bamboo-hr/unit_tests/unit_test.py @@ -2,12 +2,51 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +import pytest from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.models import Status -from source_bamboo_hr.source import SourceBambooHr +from source_bamboo_hr.source import CustomReportsStream, EmployeesDirectoryStream, SourceBambooHr -def test_client_wrong_credentials(): +@pytest.fixture +def config(): + return {"api_key": "foo", "subdomain": "bar", "authenticator": "baz", "custom_reports_include_default_fields": True} + + +def test_source_bamboo_hr_client_wrong_credentials(): source = SourceBambooHr() result = source.check(logger=AirbyteLogger, config={"subdomain": "test", "api_key": "blah-blah"}) assert result.status == Status.FAILED + + +def test_employees_directory_stream_url_base(config): + stream = EmployeesDirectoryStream(config) + assert stream.url_base == "https://api.bamboohr.com/api/gateway.php/bar/v1/" + + +def test_custom_reports_stream_get_json_schema_from_config(config): + config["custom_reports_fields"] = "one,two , three" + assert CustomReportsStream(config)._get_json_schema_from_config() == { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "one": {"type": ["null", "string"]}, + "two": {"type": ["null", "string"]}, + "three": {"type": ["null", "string"]}, + }, + } + + +def test_custom_reports_stream_union_schemas(): + schema1 = {"properties": {"one": 1, "two": 2}} + schema2 = {"properties": {"two": 2, "three": 3}} + assert CustomReportsStream._union_schemas(schema1, schema2) == {"properties": {"one": 1, "two": 2, "three": 3}} + + +def test_custom_reports_stream_request_body_json(config): + stream = CustomReportsStream(config) + stream._schema = {"properties": {"one": 1, "two": 2}} + assert stream.request_body_json() == { + "title": "Airbyte", + "fields": ["one", "two"], + } diff --git a/docs/integrations/sources/bamboo-hr.md b/docs/integrations/sources/bamboo-hr.md index ec3584496d95..df299e3912f8 100644 --- a/docs/integrations/sources/bamboo-hr.md +++ b/docs/integrations/sources/bamboo-hr.md @@ -9,6 +9,7 @@ The BambooHr source supports Full Refresh sync. You can choose if this connector This connector outputs the following streams: * [Employees](https://documentation.bamboohr.com/reference#get-employees-directory-1) +* [Custom Reports](https://documentation.bamboohr.com/reference/request-custom-report-1) ### Features @@ -33,6 +34,6 @@ BambooHR has the [rate limits](https://documentation.bamboohr.com/docs/api-detai ## Changelog | Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.0 | 2021-08-27 | [5054](https://github.com/airbytehq/airbyte/pull/5054) | Initial release with Employees API | - +|:--------| :--- | :--- | :--- | +| 0.2.0 | 2022-03-24 | [11326](https://github.com/airbytehq/airbyte/pull/11326) | Added support for Custom Reports endpoint | +| 0.1.0 | 2021-08-27 | [5054](https://github.com/airbytehq/airbyte/pull/5054) | Initial release with Employees API |