Skip to content

Commit

Permalink
S3 destination: updating specs regarding certification
Browse files Browse the repository at this point in the history
  • Loading branch information
@sashaNeshcheret
sashaNeshcheret committed Apr 12, 2022
1 parent b5966e9 commit 3baf426
Showing 1 changed file with 58 additions and 49 deletions.
107 changes: 58 additions & 49 deletions airbyte-integrations/connectors/destination-s3/src/main/resources/spec.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,34 +16,19 @@
],
"additionalProperties": false,
"properties": {
"s3_endpoint": {
"title": "Endpoint",
"type": "string",
"default": "",
"description": "This is your S3 endpoint url.(if you are working with AWS S3, just leave empty).",
"examples": ["http://localhost:9000"],
"order": 0
},
"s3_bucket_name": {
"title": "S3 Bucket Name",
"type": "string",
"description": "The name of the S3 bucket.",
"examples": ["airbyte_sync"],
"order": 1
"order": 0
},
"s3_bucket_path": {
"title": "S3 Bucket Path",
"description": "Directory under the S3 bucket where data will be written.",
"type": "string",
"examples": ["data_sync/test"],
"order": 2
},
"s3_path_format": {
"description": "Format string on how data will be organized inside the S3 bucket directory",
"type": "string",
"examples": [
"${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_"
],
"order": 3
"order": 1
},
"s3_bucket_region": {
"title": "S3 Bucket Region",
Expand Down Expand Up @@ -78,26 +63,10 @@
"us-gov-east-1",
"us-gov-west-1"
],
"order": 4
},
"access_key_id": {
"type": "string",
"description": "The access key id to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket, if not set, Airbyte will rely on Instance Profile.",
"title": "S3 Key Id",
"airbyte_secret": true,
"examples": ["A012345678910EXAMPLE"],
"order": 5
},
"secret_access_key": {
"type": "string",
"description": "The corresponding secret to the access key id, if S3 Key Id is set, then S3 Access Key must also be provided",
"title": "S3 Access Key",
"airbyte_secret": true,
"examples": ["a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"],
"order": 6
"order": 2
},
"format": {
"title": "Output Format",
"title": "Output Format *",
"type": "object",
"description": "Output data format",
"oneOf": [
Expand All @@ -106,12 +75,14 @@
"required": ["format_type", "compression_codec"],
"properties": {
"format_type": {
"title": "Format Type *",
"type": "string",
"enum": ["Avro"],
"default": "Avro"
"default": "Avro",
"order": 0
},
"compression_codec": {
"title": "Compression Codec",
"title": "Compression Codec *",
"description": "The compression algorithm used to compress data. Default to no compression.",
"type": "object",
"oneOf": [
Expand Down Expand Up @@ -211,14 +182,16 @@
}
}
}
]
],
"order": 1
},
"part_size_mb": {
"title": "Block Size (MB) for Amazon S3 multipart upload",
"title": "dddBlock Size (MB) for Amazon S3 multipart upload (Optional)",
"description": "This is the size of a \"Part\" being buffered in memory. It limits the memory usage when writing. Larger values will allow to upload a bigger files and improve the speed, but consumes9 more memory. Allowed values: min=5MB, max=525MB Default: 5MB.",
"type": "integer",
"default": 5,
"examples": [5]
"examples": [5],
"order": 2
}
}
},
Expand All @@ -227,6 +200,7 @@
"required": ["format_type", "flattening"],
"properties": {
"format_type": {
"title": "Format Type *",
"type": "string",
"enum": ["CSV"],
"default": "CSV"
Expand All @@ -239,7 +213,7 @@
"enum": ["No flattening", "Root level flattening"]
},
"part_size_mb": {
"title": "Block Size (MB) for Amazon S3 multipart upload",
"title": "Block Size (MB) for Amazon S3 multipart upload (Optional)",
"description": "This is the size of a \"Part\" being buffered in memory. It limits the memory usage when writing. Larger values will allow to upload a bigger files and improve the speed, but consumes9 more memory. Allowed values: min=5MB, max=525MB Default: 5MB.",
"type": "integer",
"default": 5,
Expand All @@ -252,12 +226,13 @@
"required": ["format_type"],
"properties": {
"format_type": {
"title": "Format Type *",
"type": "string",
"enum": ["JSONL"],
"default": "JSONL"
},
"part_size_mb": {
"title": "Block Size (MB) for Amazon S3 multipart upload",
"title": "Block Size (MB) for Amazon S3 multipart upload (Optional)",
"description": "This is the size of a \"Part\" being buffered in memory. It limits the memory usage when writing. Larger values will allow to upload a bigger files and improve the speed, but consumes9 more memory. Allowed values: min=5MB, max=525MB Default: 5MB.",
"type": "integer",
"default": 5,
Expand All @@ -270,12 +245,13 @@
"required": ["format_type"],
"properties": {
"format_type": {
"title": "Format Type *",
"type": "string",
"enum": ["Parquet"],
"default": "Parquet"
},
"compression_codec": {
"title": "Compression Codec",
"title": "Compression Codec (Optional)",
"description": "The compression algorithm used to compress data pages.",
"type": "string",
"enum": [
Expand All @@ -290,42 +266,75 @@
"default": "UNCOMPRESSED"
},
"block_size_mb": {
"title": "Block Size (Row Group Size) (MB)",
"title": "Block Size (Row Group Size) (MB) (Optional)",
"description": "This is the size of a row group being buffered in memory. It limits the memory usage when writing. Larger values will improve the IO when reading, but consume more memory when writing. Default: 128 MB.",
"type": "integer",
"default": 128,
"examples": [128]
},
"max_padding_size_mb": {
"title": "Max Padding Size (MB)",
"title": "Max Padding Size (MB) (Optional)",
"description": "Maximum size allowed as padding to align row groups. This is also the minimum size of a row group. Default: 8 MB.",
"type": "integer",
"default": 8,
"examples": [8]
},
"page_size_kb": {
"title": "Page Size (KB)",
"title": "Page Size (KB) (Optional)",
"description": "The page size is for compression. A block is composed of pages. A page is the smallest unit that must be read fully to access a single record. If this value is too small, the compression will deteriorate. Default: 1024 KB.",
"type": "integer",
"default": 1024,
"examples": [1024]
},
"dictionary_page_size_kb": {
"title": "Dictionary Page Size (KB)",
"title": "Dictionary Page Size (KB) (Optional)",
"description": "There is one dictionary page per column per row group when dictionary encoding is used. The dictionary page size works like the page size but for dictionary. Default: 1024 KB.",
"type": "integer",
"default": 1024,
"examples": [1024]
},
"dictionary_encoding": {
"title": "Dictionary Encoding",
"title": "Dictionary Encoding (Optional)",
"description": "Default: true.",
"type": "boolean",
"default": true
}
}
}
],
"order": 3
},
"s3_endpoint": {
"title": "Endpoint (Optional)",
"type": "string",
"default": "",
"description": "This is your S3 endpoint url.(if you are working with AWS S3, just leave empty).",
"examples": ["http://localhost:9000"],
"order": 4
},
"access_key_id": {
"type": "string",
"description": "The access key id to access the S3 bucket. Airbyte requires Read and Write permissions to the given bucket, if not set, Airbyte will rely on Instance Profile.",
"title": "S3 Key Id (Optional)",
"airbyte_secret": true,
"examples": ["A012345678910EXAMPLE"],
"order": 5
},
"secret_access_key": {
"type": "string",
"description": "The corresponding secret to the access key id, if S3 Key Id is set, then S3 Access Key must also be provided",
"title": "S3 Access Key (Optional)",
"airbyte_secret": true,
"examples": ["a012345678910ABCDEFGH/AbCdEfGhEXAMPLEKEY"],
"order": 6
},
"s3_path_format": {
"title": "s3_path_format (Optional)",
"description": "Format string on how data will be organized inside the S3 bucket directory",
"type": "string",
"examples": [
"${NAMESPACE}/${STREAM_NAME}/${YEAR}_${MONTH}_${DAY}_${EPOCH}_"
],
"order": 7
}
}
Expand Down

1 comment on commit 3baf426

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SonarQube Report

SonarQube report for Airbyte Connectors Destination S3(#11917)

Measures

Name Value Name Value Name Value
Reliability Rating C Lines of Code 2974 Bugs 1
Duplicated Blocks 3 Duplicated Lines (%) 1.2 Security Rating A
Quality Gate Status OK Lines to Cover 1338 Coverage 0.0
Code Smells 49 Vulnerabilities 0 Blocker Issues 0
Critical Issues 4 Major Issues 40 Minor Issues 6

Detected Issues

Rule File Description Message
java:S112 (MAJOR) s3/S3Destination.java:83 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S112 (MAJOR) s3/S3Destination.java:97 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S1155 (MINOR) s3/S3StorageOperations.java:111 Collection.isEmpty() should be used to test for emptiness Use isEmpty() to check whether the collection is empty or not.
java:S112 (MAJOR) s3/BlobStorageOperations.java:19 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S112 (MAJOR) s3/BlobStorageOperations.java:31 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S117 (MINOR) s3/BlobStorageOperations.java:33 Local variable and method parameter names should comply with a naming convention Rename this local variable to match the regular expression '^[a-z][a-zA-Z0-9]*$'.
java:S1192 (CRITICAL) s3/S3StorageOperations.java:229 String literals should not be duplicated Define a constant instead of duplicating this literal "[0-9]{2}" 5 times.
java:S119 (MINOR) credential/BlobStorageCredentialConfig.java:7 Type parameter names should comply with a naming convention Rename this generic name to match the regular expression '^[A-Z][0-9]?$'.
java:S1172 (MAJOR) s3/S3ConsumerFactory.java:73 Unused method parameters should be removed Remove this unused method parameter "namingResolver".
java:S1611 (MINOR) s3/S3ConsumerFactory.java:152 Parentheses should be removed from a single lambda input parameter when its type is inferred Remove the parentheses around the "hasFailed" parameter (sonar.java.source not set. Assuming 8 or greater.)
java:S1118 (MAJOR) s3/SerializedBufferFactory.java:25 Utility classes should not have public constructors Add a private constructor to hide the implicit public one.
java:S112 (MAJOR) s3/SerializedBufferFactory.java:72 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S112 (MAJOR) avro/AvroSerializedBuffer.java:34 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
common-java:DuplicatedBlocks (MAJOR) avro/S3AvroWriter.java Source files should not have any duplicated blocks 1 duplicated blocks of code must be removed.
common-java:DuplicatedBlocks (MAJOR) csv/S3CsvWriter.java Source files should not have any duplicated blocks 1 duplicated blocks of code must be removed.
java:S112 (MAJOR) jsonl/JsonLSerializedBuffer.java:33 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S1172 (MAJOR) jsonl/JsonLSerializedBuffer.java:63 Unused method parameters should be removed Remove this unused method parameter "config".
common-java:DuplicatedBlocks (MAJOR) jsonl/S3JsonlWriter.java Source files should not have any duplicated blocks 1 duplicated blocks of code must be removed.
java:S1118 (MAJOR) util/StreamTransferManagerHelper.java:12 Utility classes should not have public constructors Add a private constructor to hide the implicit public one.
java:S3358 (MAJOR) avro/JsonToAvroSchemaConverter.java:179 Ternary operators should not be nested Extract this nested ternary operation into an independent statement.
java:S112 (MAJOR) s3/BlobStorageOperations.java:26 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S112 (MAJOR) s3/S3StorageOperations.java:124 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S112 (MAJOR) s3/S3StorageOperations.java:148 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S112 (MAJOR) s3/S3StorageOperations.java:158 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S112 (MAJOR) csv/CsvSerializedBuffer.java:33 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S112 (MAJOR) csv/CsvSerializedBuffer.java:47 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S6213 (MAJOR) avro/S3AvroWriter.java:106 Restricted Identifiers should not be used as Identifiers Rename this variable to not match a restricted identifier.
java:S2259 (MAJOR) avro/JsonToAvroSchemaConverter.java:164 Null pointers should not be dereferenced A "NullPointerException" could be thrown; "properties" is nullable here.
java:S3776 (CRITICAL) avro/JsonToAvroSchemaConverter.java:338 Cognitive Complexity of methods should not be too high Refactor this method to reduce its Cognitive Complexity from 22 to the 15 allowed.
java:S3252 (CRITICAL) avro/JsonToAvroSchemaConverter.java:372 "static" base class members should not be accessed via derived types Use static access with "!Unknown!" for "Entry".
java:S1121 (MAJOR) avro/JsonToAvroSchemaConverter.java:261 Assignments should not be made from within sub-expressions Extract the assignment out of this expression.
java:S107 (MAJOR) s3/S3DestinationConfig.java:42 Methods should not have too many parameters Constructor has 9 parameters, which is greater than 7 authorized.
java:S107 (MAJOR) csv/S3CsvWriter.java:40 Methods should not have too many parameters Constructor has 9 parameters, which is greater than 7 authorized.
java:S3358 (MAJOR) s3/S3Destination.java:141 Ternary operators should not be nested Extract this nested ternary operation into an independent statement.
java:S5361 (CRITICAL) s3/S3Destination.java:142 "String#replace" should be preferred to "String#replaceAll" Replace this call to "replaceAll()" by a call to the "replace()" method.
java:S1121 (MAJOR) avro/JsonToAvroSchemaConverter.java:217 Assignments should not be made from within sub-expressions Extract the assignment out of this expression.
java:S1118 (MAJOR) avro/AvroConstants.java:10 Utility classes should not have public constructors Add a private constructor to hide the implicit public one.
java:S125 (MAJOR) avro/JsonToAvroSchemaConverter.java:209 Sections of code should not be commented out This block of commented-out lines of code should be removed.
java:S1068 (MAJOR) jsonl/S3JsonlWriter.java:37 Unused "private" fields should be removed Remove this unused "WRITER" private field.
java:S1118 (MAJOR) util/AvroRecordHelper.java:17 Utility classes should not have public constructors Add a private constructor to hide the implicit public one.
java:S1700 (MAJOR) avro/JsonSchemaType.java:23 A field should not duplicate the name of its containing class Rename field "jsonSchemaType"
java:S1118 (MAJOR) parquet/S3ParquetConstants.java:9 Utility classes should not have public constructors Add a private constructor to hide the implicit public one.
java:S1118 (MAJOR) util/S3OutputPathHelper.java:13 Utility classes should not have public constructors Add a private constructor to hide the implicit public one.
java:S112 (MAJOR) writer/ProductionWriterFactory.java:59 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S112 (MAJOR) writer/S3WriterFactory.java:21 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S1118 (MAJOR) s3/S3FormatConfigs.java:16 Utility classes should not have public constructors Add a private constructor to hide the implicit public one.
java:S112 (MAJOR) s3/S3FormatConfigs.java:39 Generic exceptions should never be thrown Define and throw a dedicated exception instead of using a generic one.
java:S1118 (MAJOR) csv/CsvSheetGenerator.java:25 Utility classes should not have public constructors Add a private constructor to hide the implicit public one.
java:S2094 (MINOR) csv/CsvSheetGenerators.java:7 Classes should not be empty Remove this empty class, write its code or make it an "interface".
java:S1116 (MINOR) csv/RootLevelFlatteningSheetGenerator.java:25 Empty statements should be removed Remove this empty statement.

Coverage (0.0%)

File Coverage File Coverage
src/main/java/io/airbyte/integrations/destination/s3/avro/AvroConstants.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/avro/AvroNameTransformer.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/avro/AvroRecordFactory.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/avro/AvroSerializedBuffer.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/avro/JsonFieldNameUpdater.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/avro/JsonSchemaType.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/avro/JsonToAvroSchemaConverter.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroFormatConfig.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/avro/S3AvroWriter.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/credential/S3AccessKeyCredentialConfig.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/credential/S3InstanceProfileCredentialConfig.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/csv/BaseSheetGenerator.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSerializedBuffer.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/csv/CsvSheetGenerator.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/csv/NoFlatteningSheetGenerator.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/csv/RootLevelFlatteningSheetGenerator.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvFormatConfig.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/csv/S3CsvWriter.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/csv/StagingDatabaseCsvSheetGenerator.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/jsonl/JsonLSerializedBuffer.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlFormatConfig.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/jsonl/S3JsonlWriter.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/parquet/ParquetSerializedBuffer.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetConstants.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetFormatConfig.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/parquet/S3ParquetWriter.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/S3ConsumerFactory.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/S3Destination.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfig.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConfigFactory.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/S3DestinationConstants.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/S3Format.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfig.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/S3FormatConfigs.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/S3StorageOperations.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/SerializedBufferFactory.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/util/AvroRecordHelper.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/util/S3NameTransformer.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/util/S3OutputPathHelper.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/util/StreamTransferManagerHelper.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/WriteConfig.java 0.0 src/main/java/io/airbyte/integrations/destination/s3/writer/BaseS3Writer.java 0.0
src/main/java/io/airbyte/integrations/destination/s3/writer/ProductionWriterFactory.java 0.0

Please sign in to comment.