diff --git a/docs/API.md b/docs/API.md index 5e434a75a..52c19a34f 100644 --- a/docs/API.md +++ b/docs/API.md @@ -40,8 +40,8 @@ s3Client = Minio( | [`get_bucket_versioning`](#get_bucket_versioning) | [`remove_objects`](#remove_objects) | | [`remove_all_bucket_notification`](#remove_all_bucket_notification) | | [`set_bucket_versioning`](#set_bucket_versioning) | [`fput_object`](#fput_object) | | [`listen_bucket_notification`](#listen_bucket_notification) | | [`delete_bucket_replication`](#delete_bucket_replication) | [`fget_object`](#fget_object) | | [`get_bucket_encryption`](#get_bucket_encryption) | -| [`get_bucket_replication`](#get_bucket_replication) | [`remove_objects`](#remove_objects) | | [`remove_all_bucket_notification`](#remove_all_bucket_notification) | -| [`set_bucket_replication`](#set_bucket_replication) | [`select_object_content`](#select_object_content) | | [`put_bucket_encryption`](#put_bucket_encryption) | +| [`get_bucket_replication`](#get_bucket_replication) | [`select_object_content`](#select_object_content) | | [`remove_all_bucket_notification`](#remove_all_bucket_notification) | +| [`set_bucket_replication`](#set_bucket_replication) | | | [`put_bucket_encryption`](#put_bucket_encryption) | | | | | [`delete_bucket_encryption`](#delete_bucket_encryption) | ## 1. Constructor @@ -753,11 +753,11 @@ Select content of an object by SQL expression. __Parameters__ -| Param | Type | Description | -|:--------------|:----------------------|:---------------------------| -| `bucket_name` | _str_ | Name of the bucket. | -| `object_name` | _str_ | Object name in the bucket. | -| `opts` | _SelectObjectOptions_ | Options for select object. | +| Param | Type | Description | +|:--------------|:----------------|:---------------------------| +| `bucket_name` | _str_ | Name of the bucket. | +| `object_name` | _str_ | Object name in the bucket. | +| `request` | _SelectRequest_ | Select request. | __Return Value__ @@ -768,40 +768,18 @@ __Return Value__ __Example__ ```py -options = SelectObjectOptions( - expression=" select * from s3object", - input_serialization=InputSerialization( - compression_type="NONE", - csv=CSVInput(file_header_info="USE", - record_delimiter="\n", - field_delimiter=",", - quote_character='"', - quote_escape_character='"', - comments="#", - allow_quoted_record_delimiter="FALSE", - ), - ), - - output_serialization=OutputSerialization( - csv=CSVOutput(quote_fields="ASNEEDED", - record_delimiter="\n", - field_delimiter=",", - quote_character='"', - quote_escape_character='"',) - ), - request_progress=RequestProgress( - enabled="FALSE" - ) - ) - -data = client.select_object_content('my-bucket', 'my-object', options) -# Get the records +request = SelectRequest( + "select * from s3object", + CSVInputSerialization(), + CSVOutputSerialization(), + request_progress=True, +) +data = client.select_object_content('my-bucket', 'my-object', request) with open('my-record-file', 'w') as record_data: - for d in data.stream(10*1024): - record_data.write(d) - - # Get the stats - print(data.stats()) + for d in data.stream(10*1024): + record_data.write(d) + # Get the stats + print(data.stats()) ``` diff --git a/examples/select_object_content.py b/examples/select_object_content.py index 4f45db582..8ebfd1ed2 100644 --- a/examples/select_object_content.py +++ b/examples/select_object_content.py @@ -16,69 +16,22 @@ from minio import Minio -from minio.error import ResponseError -from minio.select.errors import SelectCRCValidationError, SelectMessageError -from minio.select.options import (CSVInput, CSVOutput, InputSerialization, - OutputSerialization, RequestProgress, - SelectObjectOptions) - -# from minio.select.options import JSONOutput -# from minio.select.options import JsonInput -# from minio.select.options import ParquetInput +from minio.selectrequest import (CSVInputSerialization, CSVOutputSerialization, + SelectRequest) client = Minio('s3.amazonaws.com', access_key='YOUR-ACCESSKEY', secret_key='YOUR-SECRETKEY') -options = SelectObjectOptions( - expression="select * from s3object", - input_serialization=InputSerialization( - compression_type="NONE", - csv=CSVInput( - file_header_info="USE", - record_delimiter="\n", - field_delimiter=",", - quote_character='"', - quote_escape_character='"', - comments="#", - allow_quoted_record_delimiter="FALSE", - ), - # If input is JSON - # json=JSONInput(json_type="DOCUMENT") - ), - - output_serialization=OutputSerialization( - csv=CSVOutput( - quote_fields="ASNEEDED", - record_delimiter="\n", - field_delimiter=",", - quote_character='"', - quote_escape_character='"', - ), - - # json = JSONOutput(record_delimiter="\n") - ), - request_progress=RequestProgress( - enabled="False" - ) +request = SelectRequest( + "select * from s3object", + CSVInputSerialization(), + CSVOutputSerialization(), + request_progress=True, ) - -try: - data = client.select_object_content('your-bucket', 'your-object', options) - - # Get the records - with open('my-record-file', 'w') as record_data: - for d in data.stream(10*1024): - record_data.write(d) - +data = client.select_object_content('my-bucket', 'my-object', request) +with open('my-record-file', 'w') as record_data: + for d in data.stream(10*1024): + record_data.write(d) # Get the stats print(data.stats()) - -except SelectMessageError as err: - print(err) - -except SelectCRCValidationError as err: - print(err) - -except ResponseError as err: - print(err) diff --git a/minio/api.py b/minio/api.py index b4d91870d..73cc22f87 100644 --- a/minio/api.py +++ b/minio/api.py @@ -61,6 +61,7 @@ parse_new_multipart_upload, parse_versioning_config) from .replicationconfig import ReplicationConfig from .select import SelectObjectReader +from .selectrequest import SelectRequest from .signer import (AMZ_DATE_FORMAT, SIGN_V4_ALGORITHM, get_credential_string, post_presign_v4, presign_v4, sign_v4_s3) from .sse import SseCustomerKey @@ -71,8 +72,7 @@ marshal_versioning_config, xml_marshal_bucket_constraint, xml_marshal_bucket_encryption, - xml_marshal_delete_objects, xml_marshal_select, - xml_to_dict) + xml_marshal_delete_objects, xml_to_dict) try: from json.decoder import JSONDecodeError @@ -533,21 +533,29 @@ def disable_virtual_style_endpoint(self): """Disables virtual style endpoint.""" self._base_url.virtual_style_flag = False - def select_object_content(self, bucket_name, object_name, opts): + def select_object_content(self, bucket_name, object_name, request): """ Select content of an object by SQL expression. :param bucket_name: Name of the bucket. :param object_name: Object name in the bucket. - :param opts: Options for select object. + :param request: :class:`SelectRequest ` object. :return: A reader contains requested records and progress information. Example:: - data = client.select_object_content('foo', 'test.csv', options) + request = SelectRequest( + "select * from s3object", + CSVInputSerialization(), + CSVOutputSerialization(), + request_progress=True, + ) + data = client.select_object_content('foo', 'test.csv', request) """ check_bucket_name(bucket_name) check_non_empty_string(object_name) - body = xml_marshal_select(opts) + if not isinstance(request, SelectRequest): + raise ValueError("request must be SelectRequest type") + body = marshal(request) response = self._execute( "POST", bucket_name=bucket_name, diff --git a/minio/select/__init__.py b/minio/select/__init__.py index 7fbca443c..4da89ef5f 100644 --- a/minio/select/__init__.py +++ b/minio/select/__init__.py @@ -29,8 +29,4 @@ from .errors import SelectCRCValidationError, SelectMessageError from .helpers import (byte_int, calculate_crc, # pylint: disable=unused-import validate_crc) -from .options import (CSVInput, CSVOutput, # pylint: disable=unused-import - InputSerialization, JSONInput, JSONOutput, - OutputSerialization, ParquetInput, RequestProgress, - SelectObjectOptions) from .reader import SelectObjectReader # pylint: disable=unused-import diff --git a/minio/select/options.py b/minio/select/options.py deleted file mode 100644 index 26f500148..000000000 --- a/minio/select/options.py +++ /dev/null @@ -1,149 +0,0 @@ -# -*- coding: utf-8 -*- -# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) -# 2019 MinIO, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -minio.select.options -~~~~~~~~~~~~~~~ - -This module implements the SelectOption definition for SelectObject API. - -:copyright: (c) 2019 by MinIO, Inc. -:license: Apache 2.0, see LICENSE for more details. - -""" - - -class CSVInput: - """ - CSVInput: Input Format as CSV. - """ - - def __init__(self, file_header_info=None, record_delimiter="\n", - field_delimiter=",", quote_character='"', - quote_escape_character='"', comments="#", - allow_quoted_record_delimiter=False): - if file_header_info not in [None, "USE", "IGNORE", "NONE"]: - ValueError("invalid file header info {0}".format(file_header_info)) - - self.file_header_info = file_header_info - self.record_delimiter = record_delimiter - self.field_delimiter = field_delimiter - self.quote_character = quote_character - self.quote_escape_character = quote_escape_character - self.comments = comments - self.allow_quoted_record_delimiter = allow_quoted_record_delimiter - - -class JSONInput: - """ - JSONInput: Input format as JSON. - """ - - def __init__(self, json_type=None): - if json_type not in [None, "DOCUMENT", "LINES"]: - ValueError("invalid type {0}".format(json_type)) - - self.json_type = json_type - - -class ParquetInput: - """ - ParquetInput: Input format as Parquet - """ - - -class InputSerialization: - """ - InputSerialization: nput Format. - """ - - def __init__(self, compression_type="NONE", csv=None, json=None, - parquet=None): - if compression_type not in ["NONE", "GZIP", "BZIP2"]: - ValueError("invalid compression type {0}".format(compression_type)) - - self.compression_type = compression_type - if (csv is not None) ^ (json is not None) ^ (parquet is not None): - self.csv = csv - self.json = json - self.parquet = parquet - else: - ValueError( - "only one csv, json or parquet input serialization " - "must be provided" - ) - - -class CSVOutput: - """ - CSVOutput: Output as CSV. - - """ - - def __init__(self, quote_fields="ASNEEDED", record_delimiter="\n", - field_delimiter=",", quote_character='"', - quote_escape_character='"'): - if quote_fields not in ["ALWAYS", "ASNEEDED"]: - ValueError("invalid quote fields {0}".format(quote_fields)) - self.quote_fields = quote_fields - self.record_delimiter = record_delimiter - self.field_delimiter = field_delimiter - self.quote_character = quote_character - self.quote_escape_character = quote_escape_character - - -class JSONOutput: - """ - JSONOutput- Output as JSON. - """ - - def __init__(self, record_delimiter="\n"): - self.record_delimiter = record_delimiter - - -class OutputSerialization: - """ - OutputSerialization: Output Format. - """ - - def __init__(self, csv=None, json=None): - if (csv is not None) ^ (json is not None): - self.csv = csv - self.json = json - else: - ValueError("csv or json output serialization must be provided") - - -class RequestProgress: - """ - RequestProgress: Sends progress message. - """ - - def __init__(self, enabled=False): - self.enabled = enabled - - -class SelectObjectOptions: - """ - SelectObjectOptions: Options for select object - """ - - def __init__(self, expression, input_serialization, - output_serialization, request_progress): - self.expression = expression - self.input_serialization = input_serialization - self.output_serialization = output_serialization - self.request_progress = request_progress diff --git a/minio/selectrequest.py b/minio/selectrequest.py new file mode 100644 index 000000000..6e4156d81 --- /dev/null +++ b/minio/selectrequest.py @@ -0,0 +1,275 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Request/response of PutBucketReplication and GetBucketReplication APIs.""" + +from __future__ import absolute_import + +from abc import ABCMeta + +from .xml import Element, SubElement + +COMPRESSION_TYPE_NONE = "NONE" +COMPRESSION_TYPE_GZIP = "GZIP" +COMPRESSION_TYPE_BZIP2 = "BZIP2" + +FILE_HEADER_INFO_USE = "USE" +FILE_HEADER_INFO_IGNORE = "IGNORE" +FILE_HEADER_INFO_NONE = "NONE" + +JSON_TYPE_DOCUMENT = "DOCUMENT" +JSON_TYPE_LINES = "LINES" + +QUOTE_FIELDS_ALWAYS = "ALWAYS" +QUOTE_FIELDS_ASNEEDED = "ASNEEDED" + + +class InputSerialization: + """Input serialization.""" + + __metaclass__ = ABCMeta + + def __init__(self, compression_type): + if ( + compression_type is not None and + compression_type not in [ + COMPRESSION_TYPE_NONE, + COMPRESSION_TYPE_GZIP, + COMPRESSION_TYPE_BZIP2, + ] + ): + raise ValueError( + "compression type must be {0}, {1} or {2}".format( + COMPRESSION_TYPE_NONE, + COMPRESSION_TYPE_GZIP, + COMPRESSION_TYPE_BZIP2, + ), + ) + self._compression_type = compression_type + + def toxml(self, element): + """Convert to XML.""" + if self._compression_type is not None: + SubElement(element, "CompressionType") + return element + + +class CSVInputSerialization(InputSerialization): + """CSV input serialization.""" + + def __init__(self, compression_type=None, + allow_quoted_record_delimiter=None, comments=None, + field_delimiter=None, file_header_info=None, + quote_character=None, quote_escape_character=None, + record_delimiter=None): + super().__init__(compression_type) + self._allow_quoted_record_delimiter = allow_quoted_record_delimiter + self._comments = comments + self._field_delimiter = field_delimiter + if ( + file_header_info is not None and + file_header_info not in [ + FILE_HEADER_INFO_USE, + FILE_HEADER_INFO_IGNORE, + FILE_HEADER_INFO_NONE, + ] + ): + raise ValueError( + "file header info must be {0}, {1} or {2}".format( + FILE_HEADER_INFO_USE, + FILE_HEADER_INFO_IGNORE, + FILE_HEADER_INFO_NONE, + ), + ) + self._file_header_info = file_header_info + self._quote_character = quote_character + self._quote_escape_character = quote_escape_character + self._record_delimiter = record_delimiter + + def toxml(self, element): + """Convert to XML.""" + super().toxml(element) + element = SubElement(element, "CSV") + if self._allow_quoted_record_delimiter is not None: + SubElement( + element, + "AllowQuotedRecordDelimiter", + self._allow_quoted_record_delimiter, + ) + if self._comments is not None: + SubElement(element, "Comments", self._comments) + if self._field_delimiter is not None: + SubElement(element, "FieldDelimiter", self._field_delimiter) + if self._file_header_info is not None: + SubElement(element, "FileHeaderInfo", self._file_header_info) + if self._quote_character is not None: + SubElement(element, "QuoteCharacter", self._quote_character) + if self._quote_escape_character is not None: + SubElement( + element, + "QuoteEscapeCharacter", + self._quote_escape_character, + ) + if self._record_delimiter is not None: + SubElement(element, "RecordDelimiter", self._record_delimiter) + + +class JSONInputSerialization(InputSerialization): + """JSON input serialization.""" + + def __init__(self, compression_type=None, json_type=None): + super().__init__(compression_type) + if ( + json_type is not None and + json_type not in [JSON_TYPE_DOCUMENT, JSON_TYPE_LINES] + ): + raise ValueError( + "json type must be {0} or {1}".format( + JSON_TYPE_DOCUMENT, JSON_TYPE_LINES, + ), + ) + self._json_type = json_type + + def toxml(self, element): + """Convert to XML.""" + super().toxml(element) + element = SubElement(element, "JSON") + if self._json_type is not None: + SubElement(element, "Type", self._json_type) + + +class ParquetInputSerialization(InputSerialization): + """Parquet input serialization.""" + + def __init__(self, compression_type=None): + super().__init__(compression_type) + + def toxml(self, element): + """Convert to XML.""" + super().toxml(element) + return SubElement(element, "Parquet") + + +class CSVOutputSerialization: + """CSV output serialization.""" + + def __init__(self, field_delimiter=None, quote_character=None, + quote_escape_character=None, quote_fields=None, + record_delimiter=None): + self._field_delimiter = field_delimiter + self._quote_character = quote_character + self._quote_escape_character = quote_escape_character + if ( + quote_fields is not None and + quote_fields not in [ + QUOTE_FIELDS_ALWAYS, QUOTE_FIELDS_ASNEEDED, + ] + ): + raise ValueError( + "quote fields must be {0} or {1}".format( + QUOTE_FIELDS_ALWAYS, QUOTE_FIELDS_ASNEEDED, + ), + ) + self._quote_fields = quote_fields + self._record_delimiter = record_delimiter + + def toxml(self, element): + """Convert to XML.""" + element = SubElement(element, "CSV") + if self._field_delimiter is not None: + SubElement(element, "FieldDelimiter", self._field_delimiter) + if self._quote_character is not None: + SubElement(element, "QuoteCharacter", self._quote_character) + if self._quote_escape_character is not None: + SubElement( + element, + "QuoteEscapeCharacter", + self._quote_escape_character, + ) + if self._quote_fields is not None: + SubElement(element, "QuoteFields", self._quote_fields) + if self._record_delimiter is not None: + SubElement(element, "RecordDelimiter", self._record_delimiter) + + +class JSONOutputSerialization: + """JSON output serialization.""" + + def __init__(self, record_delimiter=None): + self._record_delimiter = record_delimiter + + def toxml(self, element): + """Convert to XML.""" + element = SubElement(element, "JSON") + if self._record_delimiter is not None: + SubElement(element, "RecordDelimiter", self._record_delimiter) + + +class SelectRequest: + """Select object content request.""" + + def __init__(self, expression, input_serialization, output_serialization, + request_progress=False, scan_start_range=None, + scan_end_range=None): + self._expession = expression + if not isinstance( + input_serialization, + ( + CSVInputSerialization, + JSONInputSerialization, + ParquetInputSerialization, + ), + ): + raise ValueError( + "input serialization must be CSVInputSerialization, " + "JSONInputSerialization or ParquetInputSerialization type", + ) + self._input_serialization = input_serialization + if not isinstance( + output_serialization, + (CSVOutputSerialization, JSONOutputSerialization), + ): + raise ValueError( + "output serialization must be CSVOutputSerialization or " + "JSONOutputSerialization type", + ) + self._output_serialization = output_serialization + self._request_progress = request_progress + self._scan_start_range = scan_start_range + self._scan_end_range = scan_end_range + + def toxml(self, element): + """Convert to XML.""" + element = Element("SelectObjectContentRequest") + SubElement(element, "Expression", self._expession) + SubElement(element, "ExpressionType", "SQL") + self._input_serialization.toxml( + SubElement(element, "InputSerialization"), + ) + self._output_serialization.toxml( + SubElement(element, "OutputSerialization"), + ) + if self._request_progress: + SubElement( + SubElement(element, "RequestProgress"), "Enabled", "true", + ) + if self._scan_start_range or self._scan_end_range: + tag = SubElement(element, "ScanRange") + if self._scan_start_range: + SubElement(tag, "Start", self._scan_start_range) + if self._scan_end_range: + SubElement(tag, "End", self._scan_end_range) + return element diff --git a/minio/xml_marshal.py b/minio/xml_marshal.py index 969212b87..5fb0bf8b6 100644 --- a/minio/xml_marshal.py +++ b/minio/xml_marshal.py @@ -123,110 +123,6 @@ def xml_marshal_bucket_constraint(region): return _get_xml_data(root) -def xml_marshal_select(req): - """Encode select request to XML.""" - - def bool_to_str(value): - return "true" if value else "false" - - root = Element("SelectObjectContentRequest") - SubElement(root, "Expression", req.expression) - SubElement(root, "ExpressionType", "SQL") - - input_serialization = SubElement(root, "InputSerialization") - SubElement( - input_serialization, - "CompressionType", - req.input_serialization.compression_type, - ) - - if req.input_serialization.csv: - csv = SubElement(input_serialization, "CSV") - SubElement( - csv, - "FileHeaderInfo", - req.input_serialization.csv.file_header_info, - ) - SubElement( - csv, - "RecordDelimiter", - req.input_serialization.csv.record_delimiter, - ) - SubElement( - csv, "FieldDelimiter", req.input_serialization.csv.field_delimiter, - ) - SubElement( - csv, "QuoteCharacter", req.input_serialization.csv.quote_character, - ) - SubElement( - csv, - "QuoteEscapeCharacter", - req.input_serialization.csv.quote_escape_character, - ) - SubElement(csv, "Comments", req.input_serialization.csv.comments) - SubElement( - csv, - "AllowQuotedRecordDelimiter", - bool_to_str( - req.input_serialization.csv.allow_quoted_record_delimiter, - ), - ) - - if req.input_serialization.json: - SubElement( - SubElement(input_serialization, "JSON"), - "Type", - req.input_serialization.json.json_type, - ) - - if req.input_serialization.parquet: - SubElement(input_serialization, "Parquet") - - output_serialization = SubElement(root, "OutputSerialization") - if req.output_serialization.csv: - csv = SubElement(output_serialization, "CSV") - SubElement( - csv, - "QuoteFields", - req.output_serialization.csv.quote_fields, - ) - SubElement( - csv, - "RecordDelimiter", - req.output_serialization.csv.record_delimiter, - ) - SubElement( - csv, - "FieldDelimiter", - req.output_serialization.csv.field_delimiter, - ) - SubElement( - csv, - "QuoteCharacter", - req.output_serialization.csv.quote_character, - ) - SubElement( - csv, - "QuoteEscapeCharacter", - req.output_serialization.csv.quote_escape_character, - ) - - if req.output_serialization.json: - SubElement( - SubElement(output_serialization, "JSON"), - "RecordDelimiter", - req.output_serialization.json.record_delimiter, - ) - - SubElement( - SubElement(root, "RequestProgress"), - "Enabled", - bool_to_str(req.request_progress.enabled), - ) - - return _get_xml_data(root) - - def marshal_complete_multipart(uploaded_parts): """ Marshal's complete multipart upload request based on *uploaded_parts*. diff --git a/tests/functional/tests.py b/tests/functional/tests.py index 91cf0ae69..11805029b 100644 --- a/tests/functional/tests.py +++ b/tests/functional/tests.py @@ -41,9 +41,8 @@ from minio import CopyConditions, Minio, PostPolicy, VersioningConfig from minio.error import S3Error from minio.select.helpers import calculate_crc -from minio.select.options import (CSVInput, CSVOutput, InputSerialization, - OutputSerialization, RequestProgress, - SelectObjectOptions) +from minio.selectrequest import (CSVInputSerialization, CSVOutputSerialization, + SelectRequest) from minio.sse import SseCustomerKey if sys.version_info[0] == 2: @@ -292,29 +291,13 @@ def test_select_object_content(log_entry): _CLIENT.put_object(bucket_name, csvfile, content, len(content.getvalue())) - options = SelectObjectOptions( - expression="select * from s3object", - input_serialization=InputSerialization( - compression_type="NONE", - csv=CSVInput(file_header_info="NONE", - record_delimiter="\n", - field_delimiter=",", - quote_character='"', - quote_escape_character='"', - comments="#", - allow_quoted_record_delimiter=False), - ), - output_serialization=OutputSerialization( - csv=CSVOutput(quote_fields="ASNEEDED", - record_delimiter="\n", - field_delimiter=",", - quote_character='"', - quote_escape_character='"') - ), - request_progress=RequestProgress(enabled=False) + request = SelectRequest( + "select * from s3object", + CSVInputSerialization(), + CSVOutputSerialization(), + request_progress=True, ) - - data = _CLIENT.select_object_content(bucket_name, csvfile, options) + data = _CLIENT.select_object_content(bucket_name, csvfile, request) # Get the records records = io.BytesIO() for data_bytes in data.stream(10*KB): diff --git a/tests/unit/generate_xml_test.py b/tests/unit/generate_xml_test.py index 4b31ed50c..63d5b1987 100644 --- a/tests/unit/generate_xml_test.py +++ b/tests/unit/generate_xml_test.py @@ -19,12 +19,8 @@ from nose.tools import eq_ from minio.definitions import Part -from minio.select.options import (CSVInput, CSVOutput, InputSerialization, - OutputSerialization, RequestProgress, - SelectObjectOptions) from minio.xml_marshal import (marshal_complete_multipart, - xml_marshal_bucket_constraint, - xml_marshal_select) + xml_marshal_bucket_constraint) class GenerateRequestTest(TestCase): @@ -57,56 +53,3 @@ def test_generate_complete_multipart_upload(self): ] actual_string = marshal_complete_multipart(etags) eq_(expected_string, actual_string) - - def test_xml_marshal_select(self): - expected_string = (b'' - b'select * from s3object' - b'SQL' - b'' - b'NONE' - b'USE' - b'\n' - b',' - b'"' - b'"' - b'#' - b'false' - b'' - b'' - b'' - b'ASNEEDED' - b'\n' - b',' - b'"' - b'"' - b'' - b'' - b'true' - b'' - b'') - - options = SelectObjectOptions( - expression="select * from s3object", - input_serialization=InputSerialization( - compression_type="NONE", - csv=CSVInput(file_header_info="USE", - record_delimiter="\n", - field_delimiter=",", - quote_character='"', - quote_escape_character='"', - comments="#", - allow_quoted_record_delimiter=False), - ), - output_serialization=OutputSerialization( - csv=CSVOutput(quote_fields="ASNEEDED", - record_delimiter="\n", - field_delimiter=",", - quote_character='"', - quote_escape_character='"') - ), - request_progress=RequestProgress( - enabled=True - ) - ) - actual_string = xml_marshal_select(options) - eq_(expected_string, actual_string)