From d01c691c2815c9e06ab5a5c6d7bcbf5372841a56 Mon Sep 17 00:00:00 2001 From: "Bala.FA" Date: Wed, 29 Apr 2020 22:18:53 +0530 Subject: [PATCH] fix pylint errors in minio.select --- Makefile | 2 +- examples/select_object_content.py | 2 +- minio/__init__.py | 2 +- minio/select/__init__.py | 15 ++- minio/select/helpers.py | 1 - minio/select/options.py | 94 +++++++++++-------- minio/select/reader.py | 11 ++- minio/xml_marshal.py | 146 ++++++++++++++++++++---------- tests/functional/tests.py | 26 +++--- tests/unit/generate_xml_test.py | 27 +++--- 10 files changed, 201 insertions(+), 125 deletions(-) diff --git a/Makefile b/Makefile index 64d455dd0..c46d898ef 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ check: @which pylint >/dev/null || pip install --user --upgrade pylint - @if python --version | grep -qi 'python 3'; then pylint --reports=no minio/definitions.py; fi + @if python --version | grep -qi 'python 3'; then pylint --reports=no minio/definitions.py minio/select; fi @which isort >/dev/null || pip install --user --upgrade isort @isort --diff --recursive . diff --git a/examples/select_object_content.py b/examples/select_object_content.py index 4a7e2d177..b32e45ca3 100644 --- a/examples/select_object_content.py +++ b/examples/select_object_content.py @@ -56,7 +56,7 @@ QuoteEscapeCharacter='"', ), - # json = JsonOutput(RecordDelimiter="\n") + # json = JSONOutput(RecordDelimiter="\n") ), request_progress=RequestProgress( enabled="False" diff --git a/minio/__init__.py b/minio/__init__.py index b182ecc5b..5bbb96a2a 100644 --- a/minio/__init__.py +++ b/minio/__init__.py @@ -29,7 +29,7 @@ __title__ = 'minio-py' __author__ = 'MinIO, Inc.' -__version__ = '5.0.11' +__version__ = '6.0.0' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2015, 2016, 2017, 2018, 2019 MinIO, Inc.' diff --git a/minio/select/__init__.py b/minio/select/__init__.py index 77a51ec40..fe6c9e632 100644 --- a/minio/select/__init__.py +++ b/minio/select/__init__.py @@ -21,11 +21,16 @@ __title__ = 'minio-py' __author__ = 'MinIO, Inc.' -__version__ = '0.0.1' +__version__ = '0.1.0' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2019 MinIO, Inc.' -from .errors import * -from .helpers import * -from .options import * -from .reader import * +from .errors import (SelectCRCValidationError, # pylint: disable=unused-import + SelectMessageError) +from .helpers import (byte_int, calculate_crc, # pylint: disable=unused-import + validate_crc) +from .options import (CSVInput, CSVOutput, # pylint: disable=unused-import + InputSerialization, JSONInput, JSONOutput, + OutputSerialization, ParquetInput, RequestProgress, + SelectObjectOptions) +from .reader import SelectObjectReader # pylint: disable=unused-import diff --git a/minio/select/helpers.py b/minio/select/helpers.py index 34d9f7515..6d0147a9e 100644 --- a/minio/select/helpers.py +++ b/minio/select/helpers.py @@ -28,7 +28,6 @@ import codecs from binascii import crc32 -SQL = 'SQL' # Value for ExpressionType EVENT_RECORDS = 'Records' # Event Type is Records EVENT_PROGRESS = 'Progress' # Event Type Progress EVENT_STATS = 'Stats' # Event Type Stats diff --git a/minio/select/options.py b/minio/select/options.py index 04d9006b2..26f500148 100644 --- a/minio/select/options.py +++ b/minio/select/options.py @@ -25,25 +25,26 @@ """ -from .helpers import SQL - class CSVInput: """ CSVInput: Input Format as CSV. """ - def __init__(self, FileHeaderInfo=None, RecordDelimiter="\n", - FieldDelimiter=",", QuoteCharacter='"', - QuoteEscapeCharacter='"', Comments="#", - AllowQuotedRecordDelimiter=False): - self.FileHeaderInfo = FileHeaderInfo - self.RecordDelimiter = RecordDelimiter - self.FieldDelimiter = FieldDelimiter - self.QuoteCharacter = QuoteCharacter - self.QuoteEscapeCharacter = QuoteEscapeCharacter - self.Comments = Comments - self.AllowQuotedRecordDelimiter = AllowQuotedRecordDelimiter + def __init__(self, file_header_info=None, record_delimiter="\n", + field_delimiter=",", quote_character='"', + quote_escape_character='"', comments="#", + allow_quoted_record_delimiter=False): + if file_header_info not in [None, "USE", "IGNORE", "NONE"]: + ValueError("invalid file header info {0}".format(file_header_info)) + + self.file_header_info = file_header_info + self.record_delimiter = record_delimiter + self.field_delimiter = field_delimiter + self.quote_character = quote_character + self.quote_escape_character = quote_escape_character + self.comments = comments + self.allow_quoted_record_delimiter = allow_quoted_record_delimiter class JSONInput: @@ -51,8 +52,11 @@ class JSONInput: JSONInput: Input format as JSON. """ - def __init__(self, Type=None): - self.Type = Type + def __init__(self, json_type=None): + if json_type not in [None, "DOCUMENT", "LINES"]: + ValueError("invalid type {0}".format(json_type)) + + self.json_type = json_type class ParquetInput: @@ -66,11 +70,21 @@ class InputSerialization: InputSerialization: nput Format. """ - def __init__(self, compression_type="NONE", csv=None, json=None, par=None): + def __init__(self, compression_type="NONE", csv=None, json=None, + parquet=None): + if compression_type not in ["NONE", "GZIP", "BZIP2"]: + ValueError("invalid compression type {0}".format(compression_type)) + self.compression_type = compression_type - self.csv_input = csv - self.json_input = json - self.parquet_input = par + if (csv is not None) ^ (json is not None) ^ (parquet is not None): + self.csv = csv + self.json = json + self.parquet = parquet + else: + ValueError( + "only one csv, json or parquet input serialization " + "must be provided" + ) class CSVOutput: @@ -79,23 +93,25 @@ class CSVOutput: """ - def __init__(self, QuoteFields="ASNEEDED", RecordDelimiter="\n", - FieldDelimiter=",", QuoteCharacter='"', - QuoteEscapeCharacter='"'): - self.QuoteFields = QuoteFields - self.RecordDelimiter = RecordDelimiter - self.FieldDelimiter = FieldDelimiter - self.QuoteCharacter = QuoteCharacter - self.QuoteEscapeCharacter = QuoteEscapeCharacter + def __init__(self, quote_fields="ASNEEDED", record_delimiter="\n", + field_delimiter=",", quote_character='"', + quote_escape_character='"'): + if quote_fields not in ["ALWAYS", "ASNEEDED"]: + ValueError("invalid quote fields {0}".format(quote_fields)) + self.quote_fields = quote_fields + self.record_delimiter = record_delimiter + self.field_delimiter = field_delimiter + self.quote_character = quote_character + self.quote_escape_character = quote_escape_character -class JsonOutput: +class JSONOutput: """ - JsonOutput- Output as JSON. + JSONOutput- Output as JSON. """ - def __init__(self, RecordDelimiter="\n"): - self.RecordDelimiter = RecordDelimiter + def __init__(self, record_delimiter="\n"): + self.record_delimiter = record_delimiter class OutputSerialization: @@ -103,9 +119,12 @@ class OutputSerialization: OutputSerialization: Output Format. """ - def __init__(self, csv=None, json=None): - self.csv_output = csv - self.json_output = json + def __init__(self, csv=None, json=None): + if (csv is not None) ^ (json is not None): + self.csv = csv + self.json = json + else: + ValueError("csv or json output serialization must be provided") class RequestProgress: @@ -121,11 +140,10 @@ class SelectObjectOptions: """ SelectObjectOptions: Options for select object """ - expression_type = SQL def __init__(self, expression, input_serialization, output_serialization, request_progress): self.expression = expression - self.in_ser = input_serialization - self.out_ser = output_serialization - self.req_progress = request_progress + self.input_serialization = input_serialization + self.output_serialization = output_serialization + self.request_progress = request_progress diff --git a/minio/select/reader.py b/minio/select/reader.py index f65148fac..e63fc638f 100644 --- a/minio/select/reader.py +++ b/minio/select/reader.py @@ -83,7 +83,7 @@ def _parse_stats(stats): return stat -class SelectObjectReader(object): +class SelectObjectReader: """ SelectObjectReader returns a Reader that upon read returns queried data, but stops when the response ends. @@ -96,19 +96,24 @@ def __init__(self, response): self.stat = {} self.prog = {} - def readable(self): + def readable(self): # pylint: disable=no-self-use + """Return this is readable.""" return True - def writeable(self): + def writeable(self): # pylint: disable=no-self-use + """Return this is not writeable.""" return False def close(self): + """Close response.""" self.response.close() def stats(self): + """Get stats information.""" return self.stat def progress(self): + """Get progress information.""" return self.prog def __extract_message(self): diff --git a/minio/xml_marshal.py b/minio/xml_marshal.py index a691567dc..d7a441fbe 100644 --- a/minio/xml_marshal.py +++ b/minio/xml_marshal.py @@ -120,54 +120,104 @@ def xml_marshal_bucket_constraint(region): return get_xml_data(root) -def xml_marshal_select(opts): - root = Element('SelectObjectContentRequest') - SubElement(root, 'Expression', opts.expression) - SubElement(root, 'ExpressionType', 'SQL') - - input_serialization = SubElement(root, 'InputSerialization') - SubElement(input_serialization, 'CompressionType', - opts.in_ser.compression_type) - - if opts.in_ser.csv_input: - csv = SubElement(input_serialization, 'CSV') - SubElement(csv, 'FileHeaderInfo', opts.in_ser.csv_input.FileHeaderInfo) - SubElement(csv, 'RecordDelimiter', - opts.in_ser.csv_input.RecordDelimiter) - SubElement(csv, 'FieldDelimiter', opts.in_ser.csv_input.FieldDelimiter) - SubElement(csv, 'QuoteCharacter', opts.in_ser.csv_input.QuoteCharacter) - SubElement(csv, 'QuoteEscapeCharacter', - opts.in_ser.csv_input.QuoteEscapeCharacter) - SubElement(csv, 'Comments', opts.in_ser.csv_input.Comments) - SubElement(csv, 'AllowQuotedRecordDelimiter', - opts.in_ser.csv_input.AllowQuotedRecordDelimiter.lower()) - - if opts.in_ser.json_input: - SubElement(SubElement(input_serialization, 'JSON'), 'Type', - opts.in_ser.json_input.Type) - - if opts.in_ser.parquet_input: - SubElement(input_serialization, 'Parquet') - - output_serialization = SubElement(root, 'OutputSerialization') - if opts.out_ser.csv_output: - csv = SubElement(output_serialization, 'CSV') - SubElement(csv, 'QuoteFields', opts.out_ser.csv_output.QuoteFields) - SubElement(csv, 'RecordDelimiter', - opts.out_ser.csv_output.RecordDelimiter) - SubElement(csv, 'FieldDelimiter', - opts.out_ser.csv_output.FieldDelimiter) - SubElement(csv, 'QuoteCharacter', - opts.out_ser.csv_output.QuoteCharacter) - SubElement(csv, 'QuoteEscapeCharacter', - opts.out_ser.csv_output.QuoteEscapeCharacter) - - if opts.out_ser.json_output: - SubElement(SubElement(output_serialization, 'JSON'), 'RecordDelimiter', - opts.out_ser.json_output.RecordDelimiter) - - SubElement(SubElement(root, 'RequestProgress'), 'Enabled', - opts.req_progress.enabled.lower()) +def xml_marshal_select(req): + def bool_to_str(value): + return "true" if value else "false" + + root = Element("SelectObjectContentRequest") + SubElement(root, "Expression", req.expression) + SubElement(root, "ExpressionType", "SQL") + + input_serialization = SubElement(root, "InputSerialization") + SubElement( + input_serialization, + "CompressionType", + req.input_serialization.compression_type, + ) + + if req.input_serialization.csv: + csv = SubElement(input_serialization, "CSV") + SubElement( + csv, + "FileHeaderInfo", + req.input_serialization.csv.file_header_info, + ) + SubElement( + csv, + "RecordDelimiter", + req.input_serialization.csv.record_delimiter, + ) + SubElement( + csv, "FieldDelimiter", req.input_serialization.csv.field_delimiter, + ) + SubElement( + csv, "QuoteCharacter", req.input_serialization.csv.quote_character, + ) + SubElement( + csv, + "QuoteEscapeCharacter", + req.input_serialization.csv.quote_escape_character, + ) + SubElement(csv, "Comments", req.input_serialization.csv.comments) + SubElement( + csv, + "AllowQuotedRecordDelimiter", + bool_to_str( + req.input_serialization.csv.allow_quoted_record_delimiter, + ), + ) + + if req.input_serialization.json: + SubElement( + SubElement(input_serialization, "JSON"), + "Type", + req.input_serialization.json.json_type, + ) + + if req.input_serialization.parquet: + SubElement(input_serialization, "Parquet") + + output_serialization = SubElement(root, "OutputSerialization") + if req.output_serialization.csv: + csv = SubElement(output_serialization, "CSV") + SubElement( + csv, + "QuoteFields", + req.output_serialization.csv.quote_fields, + ) + SubElement( + csv, + "RecordDelimiter", + req.output_serialization.csv.record_delimiter, + ) + SubElement( + csv, + "FieldDelimiter", + req.output_serialization.csv.field_delimiter, + ) + SubElement( + csv, + "QuoteCharacter", + req.output_serialization.csv.quote_character, + ) + SubElement( + csv, + "QuoteEscapeCharacter", + req.output_serialization.csv.quote_escape_character, + ) + + if req.output_serialization.json: + SubElement( + SubElement(output_serialization, "JSON"), + "RecordDelimiter", + req.output_serialization.json.record_delimiter, + ) + + SubElement( + SubElement(root, "RequestProgress"), + "Enabled", + bool_to_str(req.request_progress.enabled), + ) return get_xml_data(root) diff --git a/tests/functional/tests.py b/tests/functional/tests.py index 312397ae2..2e11a7c67 100644 --- a/tests/functional/tests.py +++ b/tests/functional/tests.py @@ -293,22 +293,22 @@ def test_select_object_content(log_entry): expression="select * from s3object", input_serialization=InputSerialization( compression_type="NONE", - csv=CSVInput(FileHeaderInfo="NONE", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter='"', - QuoteEscapeCharacter='"', - Comments="#", - AllowQuotedRecordDelimiter="FALSE") + csv=CSVInput(file_header_info="NONE", + record_delimiter="\n", + field_delimiter=",", + quote_character='"', + quote_escape_character='"', + comments="#", + allow_quoted_record_delimiter=False), ), output_serialization=OutputSerialization( - csv=CSVOutput(QuoteFields="ASNEEDED", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter='"', - QuoteEscapeCharacter='"') + csv=CSVOutput(quote_fields="ASNEEDED", + record_delimiter="\n", + field_delimiter=",", + quote_character='"', + quote_escape_character='"') ), - request_progress=RequestProgress(enabled="False") + request_progress=RequestProgress(enabled=False) ) data = client.select_object_content(bucket_name, csvfile, options) diff --git a/tests/unit/generate_xml_test.py b/tests/unit/generate_xml_test.py index ffc728a87..64e48da1a 100644 --- a/tests/unit/generate_xml_test.py +++ b/tests/unit/generate_xml_test.py @@ -95,24 +95,23 @@ def test_xml_marshal_select(self): expression="select * from s3object", input_serialization=InputSerialization( compression_type="NONE", - csv=CSVInput(FileHeaderInfo="USE", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter='"', - QuoteEscapeCharacter='"', - Comments="#", - AllowQuotedRecordDelimiter="FALSE"), + csv=CSVInput(file_header_info="USE", + record_delimiter="\n", + field_delimiter=",", + quote_character='"', + quote_escape_character='"', + comments="#", + allow_quoted_record_delimiter=False), ), - output_serialization=OutputSerialization( - csv=CSVOutput(QuoteFields="ASNEEDED", - RecordDelimiter="\n", - FieldDelimiter=",", - QuoteCharacter='"', - QuoteEscapeCharacter='"') + csv=CSVOutput(quote_fields="ASNEEDED", + record_delimiter="\n", + field_delimiter=",", + quote_character='"', + quote_escape_character='"') ), request_progress=RequestProgress( - enabled="TRUE" + enabled=True ) ) actual_string = xml_marshal_select(options)