fix pylint errors in minio.select

Makefile

@@ -2,7 +2,7 @@
 
 check:
 	@which pylint >/dev/null || pip install --user --upgrade pylint
-	@if python --version | grep -qi 'python 3'; then pylint --reports=no minio/definitions.py; fi
+	@if python --version | grep -qi 'python 3'; then pylint --reports=no minio/definitions.py minio/select; fi
 
 	@which isort >/dev/null || pip install --user --upgrade isort
 	@isort --diff --recursive .

examples/select_object_content.py

@@ -56,7 +56,7 @@
             QuoteEscapeCharacter='"',
         ),
 
-        # json = JsonOutput(RecordDelimiter="\n")
+        # json = JSONOutput(RecordDelimiter="\n")
     ),
     request_progress=RequestProgress(
         enabled="False"

minio/__init__.py

@@ -29,7 +29,7 @@
 
 __title__ = 'minio-py'
 __author__ = 'MinIO, Inc.'
-__version__ = '5.0.11'
+__version__ = '6.0.0'
 __license__ = 'Apache 2.0'
 __copyright__ = 'Copyright 2015, 2016, 2017, 2018, 2019 MinIO, Inc.'
 

minio/select/__init__.py

@@ -21,11 +21,16 @@
 
 __title__ = 'minio-py'
 __author__ = 'MinIO, Inc.'
-__version__ = '0.0.1'
+__version__ = '0.1.0'
 __license__ = 'Apache 2.0'
 __copyright__ = 'Copyright 2019 MinIO, Inc.'
 
-from .errors import *
-from .helpers import *
-from .options import *
-from .reader import *
+from .errors import (SelectCRCValidationError,  # pylint: disable=unused-import
+                     SelectMessageError)
+from .helpers import (byte_int, calculate_crc,  # pylint: disable=unused-import
+                      validate_crc)
+from .options import (CSVInput, CSVOutput,  # pylint: disable=unused-import
+                      InputSerialization, JSONInput, JSONOutput,
+                      OutputSerialization, ParquetInput, RequestProgress,
+                      SelectObjectOptions)
+from .reader import SelectObjectReader  # pylint: disable=unused-import

minio/select/helpers.py

@@ -28,7 +28,6 @@
 import codecs
 from binascii import crc32
 
-SQL = 'SQL'  # Value for ExpressionType
 EVENT_RECORDS = 'Records'  # Event Type is Records
 EVENT_PROGRESS = 'Progress'  # Event Type Progress
 EVENT_STATS = 'Stats'  # Event Type Stats

minio/select/options.py

@@ -25,34 +25,38 @@
 
 """
 
-from .helpers import SQL
-
 
 class CSVInput:
     """
     CSVInput: Input Format as CSV.
     """
 
-    def __init__(self, FileHeaderInfo=None, RecordDelimiter="\n",
-                 FieldDelimiter=",", QuoteCharacter='"',
-                 QuoteEscapeCharacter='"', Comments="#",
-                 AllowQuotedRecordDelimiter=False):
-        self.FileHeaderInfo = FileHeaderInfo
-        self.RecordDelimiter = RecordDelimiter
-        self.FieldDelimiter = FieldDelimiter
-        self.QuoteCharacter = QuoteCharacter
-        self.QuoteEscapeCharacter = QuoteEscapeCharacter
-        self.Comments = Comments
-        self.AllowQuotedRecordDelimiter = AllowQuotedRecordDelimiter
+    def __init__(self, file_header_info=None, record_delimiter="\n",
+                 field_delimiter=",", quote_character='"',
+                 quote_escape_character='"', comments="#",
+                 allow_quoted_record_delimiter=False):
+        if file_header_info not in [None, "USE", "IGNORE", "NONE"]:
+            ValueError("invalid file header info {0}".format(file_header_info))
+
+        self.file_header_info = file_header_info
+        self.record_delimiter = record_delimiter
+        self.field_delimiter = field_delimiter
+        self.quote_character = quote_character
+        self.quote_escape_character = quote_escape_character
+        self.comments = comments
+        self.allow_quoted_record_delimiter = allow_quoted_record_delimiter
 
 
 class JSONInput:
     """
     JSONInput: Input format as JSON.
     """
 
-    def __init__(self, Type=None):
-        self.Type = Type
+    def __init__(self, json_type=None):
+        if json_type not in [None, "DOCUMENT", "LINES"]:
+            ValueError("invalid type {0}".format(json_type))
+
+        self.json_type = json_type
 
 
 class ParquetInput:
@@ -66,11 +70,21 @@ class InputSerialization:
     InputSerialization: nput Format.
     """
 
-    def __init__(self, compression_type="NONE", csv=None, json=None, par=None):
+    def __init__(self, compression_type="NONE", csv=None, json=None,
+                 parquet=None):
+        if compression_type not in ["NONE", "GZIP", "BZIP2"]:
+            ValueError("invalid compression type {0}".format(compression_type))
+
         self.compression_type = compression_type
-        self.csv_input = csv
-        self.json_input = json
-        self.parquet_input = par
+        if (csv is not None) ^ (json is not None) ^ (parquet is not None):
+            self.csv = csv
+            self.json = json
+            self.parquet = parquet
+        else:
+            ValueError(
+                "only one csv, json or parquet input serialization "
+                "must be provided"
+            )
 
 
 class CSVOutput:
@@ -79,33 +93,38 @@ class CSVOutput:
 
     """
 
-    def __init__(self, QuoteFields="ASNEEDED", RecordDelimiter="\n",
-                 FieldDelimiter=",", QuoteCharacter='"',
-                 QuoteEscapeCharacter='"'):
-        self.QuoteFields = QuoteFields
-        self.RecordDelimiter = RecordDelimiter
-        self.FieldDelimiter = FieldDelimiter
-        self.QuoteCharacter = QuoteCharacter
-        self.QuoteEscapeCharacter = QuoteEscapeCharacter
+    def __init__(self, quote_fields="ASNEEDED", record_delimiter="\n",
+                 field_delimiter=",", quote_character='"',
+                 quote_escape_character='"'):
+        if quote_fields not in ["ALWAYS", "ASNEEDED"]:
+            ValueError("invalid quote fields {0}".format(quote_fields))
+        self.quote_fields = quote_fields
+        self.record_delimiter = record_delimiter
+        self.field_delimiter = field_delimiter
+        self.quote_character = quote_character
+        self.quote_escape_character = quote_escape_character
 
 
-class JsonOutput:
+class JSONOutput:
     """
-    JsonOutput- Output as JSON.
+    JSONOutput- Output as JSON.
     """
 
-    def __init__(self, RecordDelimiter="\n"):
-        self.RecordDelimiter = RecordDelimiter
+    def __init__(self, record_delimiter="\n"):
+        self.record_delimiter = record_delimiter
 
 
 class OutputSerialization:
     """
     OutputSerialization: Output Format.
     """
 
-    def __init__(self,  csv=None, json=None):
-        self.csv_output = csv
-        self.json_output = json
+    def __init__(self, csv=None, json=None):
+        if (csv is not None) ^ (json is not None):
+            self.csv = csv
+            self.json = json
+        else:
+            ValueError("csv or json output serialization must be provided")
 
 
 class RequestProgress:
@@ -121,11 +140,10 @@ class SelectObjectOptions:
     """
     SelectObjectOptions: Options for select object
     """
-    expression_type = SQL
 
     def __init__(self, expression, input_serialization,
                  output_serialization, request_progress):
         self.expression = expression
-        self.in_ser = input_serialization
-        self.out_ser = output_serialization
-        self.req_progress = request_progress
+        self.input_serialization = input_serialization
+        self.output_serialization = output_serialization
+        self.request_progress = request_progress

minio/select/reader.py

@@ -83,7 +83,7 @@ def _parse_stats(stats):
     return stat
 
 
-class SelectObjectReader(object):
+class SelectObjectReader:
     """
     SelectObjectReader returns a Reader that upon read
     returns queried data, but stops when the response ends.
@@ -96,19 +96,24 @@ def __init__(self, response):
         self.stat = {}
         self.prog = {}
 
-    def readable(self):
+    def readable(self):  # pylint: disable=no-self-use
+        """Return this is readable."""
         return True
 
-    def writeable(self):
+    def writeable(self):  # pylint: disable=no-self-use
+        """Return this is not writeable."""
         return False
 
     def close(self):
+        """Close response."""
         self.response.close()
 
     def stats(self):
+        """Get stats information."""
         return self.stat
 
     def progress(self):
+        """Get progress information."""
         return self.prog
 
     def __extract_message(self):

minio/xml_marshal.py

@@ -120,54 +120,104 @@ def xml_marshal_bucket_constraint(region):
     return get_xml_data(root)
 
 
-def xml_marshal_select(opts):
-    root = Element('SelectObjectContentRequest')
-    SubElement(root, 'Expression', opts.expression)
-    SubElement(root, 'ExpressionType', 'SQL')
-
-    input_serialization = SubElement(root, 'InputSerialization')
-    SubElement(input_serialization, 'CompressionType',
-               opts.in_ser.compression_type)
-
-    if opts.in_ser.csv_input:
-        csv = SubElement(input_serialization, 'CSV')
-        SubElement(csv, 'FileHeaderInfo', opts.in_ser.csv_input.FileHeaderInfo)
-        SubElement(csv, 'RecordDelimiter',
-                   opts.in_ser.csv_input.RecordDelimiter)
-        SubElement(csv, 'FieldDelimiter', opts.in_ser.csv_input.FieldDelimiter)
-        SubElement(csv, 'QuoteCharacter', opts.in_ser.csv_input.QuoteCharacter)
-        SubElement(csv, 'QuoteEscapeCharacter',
-                   opts.in_ser.csv_input.QuoteEscapeCharacter)
-        SubElement(csv, 'Comments', opts.in_ser.csv_input.Comments)
-        SubElement(csv, 'AllowQuotedRecordDelimiter',
-                   opts.in_ser.csv_input.AllowQuotedRecordDelimiter.lower())
-
-    if opts.in_ser.json_input:
-        SubElement(SubElement(input_serialization, 'JSON'), 'Type',
-                   opts.in_ser.json_input.Type)
-
-    if opts.in_ser.parquet_input:
-        SubElement(input_serialization, 'Parquet')
-
-    output_serialization = SubElement(root, 'OutputSerialization')
-    if opts.out_ser.csv_output:
-        csv = SubElement(output_serialization, 'CSV')
-        SubElement(csv, 'QuoteFields', opts.out_ser.csv_output.QuoteFields)
-        SubElement(csv, 'RecordDelimiter',
-                   opts.out_ser.csv_output.RecordDelimiter)
-        SubElement(csv, 'FieldDelimiter',
-                   opts.out_ser.csv_output.FieldDelimiter)
-        SubElement(csv, 'QuoteCharacter',
-                   opts.out_ser.csv_output.QuoteCharacter)
-        SubElement(csv, 'QuoteEscapeCharacter',
-                   opts.out_ser.csv_output.QuoteEscapeCharacter)
-
-    if opts.out_ser.json_output:
-        SubElement(SubElement(output_serialization, 'JSON'), 'RecordDelimiter',
-                   opts.out_ser.json_output.RecordDelimiter)
-
-    SubElement(SubElement(root, 'RequestProgress'), 'Enabled',
-               opts.req_progress.enabled.lower())
+def xml_marshal_select(req):
+    def bool_to_str(value):
+        return "true" if value else "false"
+
+    root = Element("SelectObjectContentRequest")
+    SubElement(root, "Expression", req.expression)
+    SubElement(root, "ExpressionType", "SQL")
+
+    input_serialization = SubElement(root, "InputSerialization")
+    SubElement(
+        input_serialization,
+        "CompressionType",
+        req.input_serialization.compression_type,
+    )
+
+    if req.input_serialization.csv:
+        csv = SubElement(input_serialization, "CSV")
+        SubElement(
+            csv,
+            "FileHeaderInfo",
+            req.input_serialization.csv.file_header_info,
+        )
+        SubElement(
+            csv,
+            "RecordDelimiter",
+            req.input_serialization.csv.record_delimiter,
+        )
+        SubElement(
+            csv, "FieldDelimiter", req.input_serialization.csv.field_delimiter,
+        )
+        SubElement(
+            csv, "QuoteCharacter", req.input_serialization.csv.quote_character,
+        )
+        SubElement(
+            csv,
+            "QuoteEscapeCharacter",
+            req.input_serialization.csv.quote_escape_character,
+        )
+        SubElement(csv, "Comments", req.input_serialization.csv.comments)
+        SubElement(
+            csv,
+            "AllowQuotedRecordDelimiter",
+            bool_to_str(
+                req.input_serialization.csv.allow_quoted_record_delimiter,
+            ),
+        )
+
+    if req.input_serialization.json:
+        SubElement(
+            SubElement(input_serialization, "JSON"),
+            "Type",
+            req.input_serialization.json.json_type,
+        )
+
+    if req.input_serialization.parquet:
+        SubElement(input_serialization, "Parquet")
+
+    output_serialization = SubElement(root, "OutputSerialization")
+    if req.output_serialization.csv:
+        csv = SubElement(output_serialization, "CSV")
+        SubElement(
+            csv,
+            "QuoteFields",
+            req.output_serialization.csv.quote_fields,
+        )
+        SubElement(
+            csv,
+            "RecordDelimiter",
+            req.output_serialization.csv.record_delimiter,
+        )
+        SubElement(
+            csv,
+            "FieldDelimiter",
+            req.output_serialization.csv.field_delimiter,
+        )
+        SubElement(
+            csv,
+            "QuoteCharacter",
+            req.output_serialization.csv.quote_character,
+        )
+        SubElement(
+            csv,
+            "QuoteEscapeCharacter",
+            req.output_serialization.csv.quote_escape_character,
+        )
+
+    if req.output_serialization.json:
+        SubElement(
+            SubElement(output_serialization, "JSON"),
+            "RecordDelimiter",
+            req.output_serialization.json.record_delimiter,
+        )
+
+    SubElement(
+        SubElement(root, "RequestProgress"),
+        "Enabled",
+        bool_to_str(req.request_progress.enabled),
+    )
 
     return get_xml_data(root)