bump source-file-secure (#14704)

* bump source-file-secure * Bump more * bump requirements * Fix source-file * auto-bump connector version * correct unit test spec for file secure + format files Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com> Co-authored-by: marcosmarxm <marcosmarxm@gmail.com>
airbytehq · Jul 15, 2022 · 25ee6f6 · 25ee6f6
1 parent bf9bfd1
commit 25ee6f6
Show file tree

Hide file tree

Showing 7 changed files with 31 additions and 40 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -271,7 +271,7 @@
 - name: File
   sourceDefinitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77
   dockerRepository: airbyte/source-file
-  dockerImageTag: 0.2.13
+  dockerImageTag: 0.2.14
   documentationUrl: https://docs.airbyte.io/integrations/sources/file
   icon: file.svg
   sourceType: file

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -2261,7 +2261,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-file:0.2.13"
+- dockerImage: "airbyte/source-file:0.2.14"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/file"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-file-secure/Dockerfile b/airbyte-integrations/connectors/source-file-secure/Dockerfile
@@ -1,4 +1,4 @@
-FROM airbyte/source-file:0.2.9
+FROM airbyte/source-file:0.2.14
 
 WORKDIR /airbyte/integration_code
 COPY source_file_secure ./source_file_secure
@@ -9,5 +9,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.2
+LABEL io.airbyte.version=0.2.14
 LABEL io.airbyte.name=airbyte/source-file-secure
diff --git a/airbyte-integrations/connectors/source-file-secure/integration_tests/spec.json b/airbyte-integrations/connectors/source-file-secure/integration_tests/spec.json
@@ -14,7 +14,7 @@
       },
       "format": {
         "type": "string",
-        "enum": ["csv", "json", "jsonl", "excel", "feather", "parquet"],
+        "enum": ["csv", "json", "jsonl", "excel", "feather", "parquet", "yaml"],
         "default": "csv",
         "title": "File Format",
         "description": "The Format of the file which should be replicated (Warning: some formats may be experimental, please refer to the docs)."
@@ -23,7 +23,7 @@
         "type": "string",
         "title": "Reader Options",
         "description": "This should be a string in JSON format. It depends on the chosen file format to provide additional options and tune its behavior.",
-        "examples": ["{}", "{'sep': ' '}"]
+        "examples": ["{}", "{\"sep\": \" \"}"]
       },
       "url": {
         "type": "string",
@@ -40,9 +40,12 @@
             "title": "HTTPS: Public Web",
             "required": ["storage"],
             "properties": {
-              "storage": {
-                "type": "string",
-                "const": "HTTPS"
+              "storage": { "type": "string", "const": "HTTPS" },
+              "user_agent": {
+                "type": "boolean",
+                "title": "User-Agent",
+                "default": false,
+                "description": "Add User-Agent to request"
               }
             }
           },
@@ -121,23 +124,19 @@
                 "title": "Storage",
                 "const": "SSH"
               },
-              "user": {
-                "type": "string",
-                "title": "User"
-              },
+              "user": { "type": "string", "title": "User", "description": "" },
               "password": {
                 "type": "string",
                 "title": "Password",
+                "description": "",
                 "airbyte_secret": true
               },
-              "host": {
-                "type": "string",
-                "title": "Host"
-              },
+              "host": { "type": "string", "title": "Host", "description": "" },
               "port": {
                 "type": "string",
                 "title": "Port",
-                "default": "22"
+                "default": "22",
+                "description": ""
               }
             }
           },
@@ -150,23 +149,19 @@
                 "title": "Storage",
                 "const": "SCP"
               },
-              "user": {
-                "type": "string",
-                "title": "User"
-              },
+              "user": { "type": "string", "title": "User", "description": "" },
               "password": {
                 "type": "string",
                 "title": "Password",
+                "description": "",
                 "airbyte_secret": true
               },
-              "host": {
-                "type": "string",
-                "title": "Host"
-              },
+              "host": { "type": "string", "title": "Host", "description": "" },
               "port": {
                 "type": "string",
                 "title": "Port",
-                "default": "22"
+                "default": "22",
+                "description": ""
               }
             }
           },
@@ -179,23 +174,19 @@
                 "title": "Storage",
                 "const": "SFTP"
               },
-              "user": {
-                "type": "string",
-                "title": "User"
-              },
+              "user": { "type": "string", "title": "User", "description": "" },
               "password": {
                 "type": "string",
                 "title": "Password",
+                "description": "",
                 "airbyte_secret": true
               },
-              "host": {
-                "type": "string",
-                "title": "Host"
-              },
+              "host": { "type": "string", "title": "Host", "description": "" },
               "port": {
                 "type": "string",
                 "title": "Port",
-                "default": "22"
+                "default": "22",
+                "description": ""
               }
             }
           }

diff --git a/airbyte-integrations/connectors/source-file-secure/setup.py b/airbyte-integrations/connectors/source-file-secure/setup.py
@@ -10,14 +10,14 @@
     "gcsfs==0.7.1",
     "genson==1.2.2",
     "google-cloud-storage==1.35.0",
-    "pandas==1.2.0",
+    "pandas==1.4.3",
     "paramiko==2.7.2",
     "s3fs==0.4.2",
     "smart-open[all]==4.1.2",
     "lxml==4.6.5",
     "html5lib==1.1",
     "beautifulsoup4==4.9.3",
-    "pyarrow==3.0.0",
+    "pyarrow==8.0.0",
     "xlrd==2.0.1",
     "openpyxl==3.0.6",
     "pyxlsb==1.0.8",

diff --git a/airbyte-integrations/connectors/source-file/Dockerfile b/airbyte-integrations/connectors/source-file/Dockerfile
@@ -17,5 +17,5 @@ COPY source_file ./source_file
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.13
+LABEL io.airbyte.version=0.2.14
 LABEL io.airbyte.name=airbyte/source-file
diff --git a/airbyte-integrations/connectors/source-file/source_file/client.py b/airbyte-integrations/connectors/source-file/source_file/client.py
@@ -117,7 +117,7 @@ def _open(self, binary):
             return smart_open.open(uri, transport_params=transport_params, mode=mode)
         elif storage in ("https://", "http://"):
             transport_params = None
-            if self._provider["user_agent"]:
+            if "user_agent" in self._provider and self._provider["user_agent"]:
                 airbyte_version = environ.get("AIRBYTE_VERSION", "0.0")
                 transport_params = {"headers": {"Accept-Encoding": "identity", "User-Agent": f"Airbyte/{airbyte_version}"}}
             logger.info(f"TransportParams: {transport_params}")