Skip to content

Commit

Permalink
Refactor XML handling of select_object_content() API
Browse files Browse the repository at this point in the history
  • Loading branch information
@balamurugana
balamurugana committed Oct 5, 2020
1 parent 9400fc6 commit a56a073
Show file tree
Hide file tree
Showing 9 changed files with 327 additions and 444 deletions.
58 changes: 18 additions & 40 deletions docs/API.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ s3Client = Minio(
| [`get_bucket_versioning`](#get_bucket_versioning) | [`remove_objects`](#remove_objects) | | [`remove_all_bucket_notification`](#remove_all_bucket_notification) |
| [`set_bucket_versioning`](#set_bucket_versioning) | [`fput_object`](#fput_object) | | [`listen_bucket_notification`](#listen_bucket_notification) |
| [`delete_bucket_replication`](#delete_bucket_replication) | [`fget_object`](#fget_object) | | [`get_bucket_encryption`](#get_bucket_encryption) |
| [`get_bucket_replication`](#get_bucket_replication) | [`remove_objects`](#remove_objects) | | [`remove_all_bucket_notification`](#remove_all_bucket_notification) |
| [`set_bucket_replication`](#set_bucket_replication) | [`select_object_content`](#select_object_content) | | [`put_bucket_encryption`](#put_bucket_encryption) |
| [`get_bucket_replication`](#get_bucket_replication) | [`select_object_content`](#select_object_content) | | [`remove_all_bucket_notification`](#remove_all_bucket_notification) |
| [`set_bucket_replication`](#set_bucket_replication) | | | [`put_bucket_encryption`](#put_bucket_encryption) |
| | | | [`delete_bucket_encryption`](#delete_bucket_encryption) |

## 1. Constructor
Expand Down Expand Up @@ -745,11 +745,11 @@ Select content of an object by SQL expression.

__Parameters__

| Param | Type | Description |
|:--------------|:----------------------|:---------------------------|
| `bucket_name` | _str_ | Name of the bucket. |
| `object_name` | _str_ | Object name in the bucket. |
| `opts` | _SelectObjectOptions_ | Options for select object. |
| Param | Type | Description |
|:--------------|:----------------|:---------------------------|
| `bucket_name` | _str_ | Name of the bucket. |
| `object_name` | _str_ | Object name in the bucket. |
| `request` | _SelectRequest_ | Select request. |

__Return Value__

Expand All @@ -760,40 +760,18 @@ __Return Value__
__Example__

```py
options = SelectObjectOptions(
expression=" select * from s3object",
input_serialization=InputSerialization(
compression_type="NONE",
csv=CSVInput(file_header_info="USE",
record_delimiter="\n",
field_delimiter=",",
quote_character='"',
quote_escape_character='"',
comments="#",
allow_quoted_record_delimiter="FALSE",
),
),

output_serialization=OutputSerialization(
csv=CSVOutput(quote_fields="ASNEEDED",
record_delimiter="\n",
field_delimiter=",",
quote_character='"',
quote_escape_character='"',)
),
request_progress=RequestProgress(
enabled="FALSE"
)
)

data = client.select_object_content('my-bucket', 'my-object', options)
# Get the records
request = SelectRequest(
"select * from s3object",
CSVInputSerialization(),
CSVOutputSerialization(),
request_progress=True,
)
data = client.select_object_content('my-bucket', 'my-object', request)
with open('my-record-file', 'w') as record_data:
for d in data.stream(10*1024):
record_data.write(d)

# Get the stats
print(data.stats())
for d in data.stream(10*1024):
record_data.write(d)
# Get the stats
print(data.stats())
```

<a name="fget_object"></a>
Expand Down
69 changes: 11 additions & 58 deletions examples/select_object_content.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,69 +16,22 @@


from minio import Minio
from minio.error import ResponseError
from minio.select.errors import SelectCRCValidationError, SelectMessageError
from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
OutputSerialization, RequestProgress,
SelectObjectOptions)

# from minio.select.options import JSONOutput
# from minio.select.options import JsonInput
# from minio.select.options import ParquetInput
from minio.selectrequest import (CSVInputSerialization, CSVOutputSerialization,
SelectRequest)

client = Minio('s3.amazonaws.com',
access_key='YOUR-ACCESSKEY',
secret_key='YOUR-SECRETKEY')

options = SelectObjectOptions(
expression="select * from s3object",
input_serialization=InputSerialization(
compression_type="NONE",
csv=CSVInput(
file_header_info="USE",
record_delimiter="\n",
field_delimiter=",",
quote_character='"',
quote_escape_character='"',
comments="#",
allow_quoted_record_delimiter="FALSE",
),
# If input is JSON
# json=JSONInput(json_type="DOCUMENT")
),

output_serialization=OutputSerialization(
csv=CSVOutput(
quote_fields="ASNEEDED",
record_delimiter="\n",
field_delimiter=",",
quote_character='"',
quote_escape_character='"',
),

# json = JSONOutput(record_delimiter="\n")
),
request_progress=RequestProgress(
enabled="False"
)
request = SelectRequest(
"select * from s3object",
CSVInputSerialization(),
CSVOutputSerialization(),
request_progress=True,
)

try:
data = client.select_object_content('your-bucket', 'your-object', options)

# Get the records
with open('my-record-file', 'w') as record_data:
for d in data.stream(10*1024):
record_data.write(d)

data = client.select_object_content('my-bucket', 'my-object', request)
with open('my-record-file', 'w') as record_data:
for d in data.stream(10*1024):
record_data.write(d)
# Get the stats
print(data.stats())

except SelectMessageError as err:
print(err)

except SelectCRCValidationError as err:
print(err)

except ResponseError as err:
print(err)
20 changes: 14 additions & 6 deletions minio/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
parse_new_multipart_upload, parse_versioning_config)
from .replicationconfig import ReplicationConfig
from .select import SelectObjectReader
from .selectrequest import SelectRequest
from .signer import (AMZ_DATE_FORMAT, SIGN_V4_ALGORITHM, get_credential_string,
post_presign_v4, presign_v4, sign_v4_s3)
from .sse import SseCustomerKey
Expand All @@ -71,8 +72,7 @@
marshal_versioning_config,
xml_marshal_bucket_constraint,
xml_marshal_bucket_encryption,
xml_marshal_delete_objects, xml_marshal_select,
xml_to_dict)
xml_marshal_delete_objects, xml_to_dict)

try:
from json.decoder import JSONDecodeError
Expand Down Expand Up @@ -533,21 +533,29 @@ def disable_virtual_style_endpoint(self):
"""Disables virtual style endpoint."""
self._base_url.virtual_style_flag = False

def select_object_content(self, bucket_name, object_name, opts):
def select_object_content(self, bucket_name, object_name, request):
"""
Select content of an object by SQL expression.
:param bucket_name: Name of the bucket.
:param object_name: Object name in the bucket.
:param opts: Options for select object.
:param request: :class:`SelectRequest <SelectRequest>` object.
:return: A reader contains requested records and progress information.
Example::
data = client.select_object_content('foo', 'test.csv', options)
request = SelectRequest(
"select * from s3object",
CSVInputSerialization(),
CSVOutputSerialization(),
request_progress=True,
)
data = client.select_object_content('foo', 'test.csv', request)
"""
check_bucket_name(bucket_name)
check_non_empty_string(object_name)
body = xml_marshal_select(opts)
if not isinstance(request, SelectRequest):
raise ValueError("request must be SelectRequest type")
body = marshal(request)
response = self._execute(
"POST",
bucket_name=bucket_name,
Expand Down
4 changes: 0 additions & 4 deletions minio/select/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,4 @@
from .errors import SelectCRCValidationError, SelectMessageError
from .helpers import (byte_int, calculate_crc, # pylint: disable=unused-import
validate_crc)
from .options import (CSVInput, CSVOutput, # pylint: disable=unused-import
InputSerialization, JSONInput, JSONOutput,
OutputSerialization, ParquetInput, RequestProgress,
SelectObjectOptions)
from .reader import SelectObjectReader # pylint: disable=unused-import
149 changes: 0 additions & 149 deletions minio/select/options.py
View file

This file was deleted.

Loading

0 comments on commit a56a073

Please sign in to comment.