Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Make 'encryption_key' an attribute of the Blob. #2507

Merged
merged 2 commits into from
Oct 6, 2016
Merged

Make 'encryption_key' an attribute of the Blob. #2507

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2da75d4
Make 'encryption_key' an attribute of the Blob.
tseaver Oct 6, 2016
cf451cf
Pass 'encryption_key' through 'Bucket.blob' factory.
tseaver Oct 6, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 44 additions & 64 deletions storage/google/cloud/storage/blob.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,19 +61,25 @@ class Blob(_PropertyMixin):
:param chunk_size: The size of a chunk of data whenever iterating (1 MB).
This must be a multiple of 256 KB per the API
specification.

:type encryption_key: bytes
:param encryption_key:
Optional 32 byte encryption key for customer-supplied encryption.
See https://cloud.google.com/storage/docs/encryption#customer-supplied
"""

_chunk_size = None # Default value for each instance.

_CHUNK_SIZE_MULTIPLE = 256 * 1024
"""Number (256 KB, in bytes) that must divide the chunk size."""

def __init__(self, name, bucket, chunk_size=None):
def __init__(self, name, bucket, chunk_size=None, encryption_key=None):
super(Blob, self).__init__(name=name)

self.chunk_size = chunk_size # Check that setter accepts value.
self.bucket = bucket
self._acl = ObjectACL(self)
self._encryption_key = encryption_key

@property
def chunk_size(self):
Expand Down Expand Up @@ -284,7 +290,7 @@ def delete(self, client=None):
"""
return self.bucket.delete_blob(self.name, client=client)

def download_to_file(self, file_obj, encryption_key=None, client=None):
def download_to_file(self, file_obj, client=None):
"""Download the contents of this blob into a file-like object.

.. note::
Expand All @@ -301,10 +307,10 @@ def download_to_file(self, file_obj, encryption_key=None, client=None):
>>> client = storage.Client(project='my-project')
>>> bucket = client.get_bucket('my-bucket')
>>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19'
>>> blob = Blob('secure-data', bucket)
>>> blob = Blob('secure-data', bucket,
... encryption_key=encryption_key)
>>> with open('/tmp/my-secure-file', 'wb') as file_obj:
>>> blob.download_to_file(file_obj,
... encryption_key=encryption_key)
>>> blob.download_to_file(file_obj)

The ``encryption_key`` should be a str or bytes with a length of at
least 32.
Expand All @@ -315,10 +321,6 @@ def download_to_file(self, file_obj, encryption_key=None, client=None):
:type file_obj: file
:param file_obj: A file handle to which to write the blob's data.

:type encryption_key: str or bytes
:param encryption_key: Optional 32 byte encryption key for
customer-supplied encryption.

:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
Expand All @@ -338,9 +340,7 @@ def download_to_file(self, file_obj, encryption_key=None, client=None):
if self.chunk_size is not None:
download.chunksize = self.chunk_size

headers = {}
if encryption_key:
_set_encryption_headers(encryption_key, headers)
headers = _get_encryption_headers(self._encryption_key)

request = Request(download_url, 'GET', headers)

Expand All @@ -352,16 +352,12 @@ def download_to_file(self, file_obj, encryption_key=None, client=None):
# it has all three (http, API_BASE_URL and build_api_url).
download.initialize_download(request, client._connection.http)

def download_to_filename(self, filename, encryption_key=None, client=None):
def download_to_filename(self, filename, client=None):
"""Download the contents of this blob into a named file.

:type filename: string
:param filename: A filename to be passed to ``open``.

:type encryption_key: str or bytes
:param encryption_key: Optional 32 byte encryption key for
customer-supplied encryption.

:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
Expand All @@ -370,19 +366,14 @@ def download_to_filename(self, filename, encryption_key=None, client=None):
:raises: :class:`google.cloud.exceptions.NotFound`
"""
with open(filename, 'wb') as file_obj:
self.download_to_file(file_obj, encryption_key=encryption_key,
client=client)
self.download_to_file(file_obj, client=client)

mtime = time.mktime(self.updated.timetuple())
os.utime(file_obj.name, (mtime, mtime))

def download_as_string(self, encryption_key=None, client=None):
def download_as_string(self, client=None):
"""Download the contents of this blob as a string.

:type encryption_key: str or bytes
:param encryption_key: Optional 32 byte encryption key for
customer-supplied encryption.

:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
Expand All @@ -393,8 +384,7 @@ def download_as_string(self, encryption_key=None, client=None):
:raises: :class:`google.cloud.exceptions.NotFound`
"""
string_buffer = BytesIO()
self.download_to_file(string_buffer, encryption_key=encryption_key,
client=client)
self.download_to_file(string_buffer, client=client)
return string_buffer.getvalue()

@staticmethod
Expand All @@ -409,8 +399,7 @@ def _check_response_error(request, http_response):

# pylint: disable=too-many-locals
def upload_from_file(self, file_obj, rewind=False, size=None,
encryption_key=None, content_type=None, num_retries=6,
client=None):
content_type=None, num_retries=6, client=None):
"""Upload the contents of this blob from a file-like object.

The content type of the upload will either be
Expand All @@ -437,10 +426,10 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
>>> client = storage.Client(project='my-project')
>>> bucket = client.get_bucket('my-bucket')
>>> encryption_key = 'aa426195405adee2c8081bb9e7e74b19'
>>> blob = Blob('secure-data', bucket)
>>> blob = Blob('secure-data', bucket,
... encryption_key=encryption_key)
>>> with open('my-file', 'rb') as my_file:
>>> blob.upload_from_file(my_file,
... encryption_key=encryption_key)
>>> blob.upload_from_file(my_file)

The ``encryption_key`` should be a str or bytes with a length of at
least 32.
Expand All @@ -461,10 +450,6 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
:func:`os.fstat`. (If the file handle is not from the
filesystem this won't be possible.)

:type encryption_key: str or bytes
:param encryption_key: Optional 32 byte encryption key for
customer-supplied encryption.

:type content_type: string or ``NoneType``
:param content_type: Optional type of content being uploaded.

Expand Down Expand Up @@ -510,8 +495,7 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
'User-Agent': connection.USER_AGENT,
}

if encryption_key:
_set_encryption_headers(encryption_key, headers)
headers.update(_get_encryption_headers(self._encryption_key))

upload = Upload(file_obj, content_type, total_bytes,
auto_transfer=False)
Expand Down Expand Up @@ -561,8 +545,7 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
self._set_properties(json.loads(response_content))
# pylint: enable=too-many-locals

def upload_from_filename(self, filename, content_type=None,
encryption_key=None, client=None):
def upload_from_filename(self, filename, content_type=None, client=None):
"""Upload this blob's contents from the content of a named file.

The content type of the upload will either be
Expand All @@ -587,10 +570,6 @@ def upload_from_filename(self, filename, content_type=None,
:type content_type: string or ``NoneType``
:param content_type: Optional type of content being uploaded.

:type encryption_key: str or bytes
:param encryption_key: Optional 32 byte encryption key for
customer-supplied encryption.

:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
Expand All @@ -601,11 +580,10 @@ def upload_from_filename(self, filename, content_type=None,
content_type, _ = mimetypes.guess_type(filename)

with open(filename, 'rb') as file_obj:
self.upload_from_file(file_obj, content_type=content_type,
encryption_key=encryption_key, client=client)
self.upload_from_file(
file_obj, content_type=content_type, client=client)

def upload_from_string(self, data, content_type='text/plain',
encryption_key=None, client=None):
def upload_from_string(self, data, content_type='text/plain', client=None):
"""Upload contents of this blob from the provided string.

.. note::
Expand All @@ -627,10 +605,6 @@ def upload_from_string(self, data, content_type='text/plain',
:param content_type: Optional type of content being uploaded. Defaults
to ``'text/plain'``.

:type encryption_key: str or bytes
:param encryption_key: Optional 32 byte encryption key for
customer-supplied encryption.

:type client: :class:`~google.cloud.storage.client.Client` or
``NoneType``
:param client: Optional. The client to use. If not passed, falls back
Expand All @@ -640,9 +614,9 @@ def upload_from_string(self, data, content_type='text/plain',
data = data.encode('utf-8')
string_buffer = BytesIO()
string_buffer.write(data)
self.upload_from_file(file_obj=string_buffer, rewind=True,
size=len(data), content_type=content_type,
encryption_key=encryption_key, client=client)
self.upload_from_file(
file_obj=string_buffer, rewind=True, size=len(data),
content_type=content_type, client=client)

def make_public(self, client=None):
"""Make this blob public giving all users read access.
Expand Down Expand Up @@ -964,19 +938,25 @@ def __init__(self, bucket_name, object_name):
self._relative_path = ''


def _set_encryption_headers(key, headers):
def _get_encryption_headers(key):
"""Builds customer encryption key headers

:type key: str or bytes
:type key: bytes
:param key: 32 byte key to build request key and hash.

:type headers: dict
:param headers: dict of HTTP headers being sent in request.
:rtype: dict
:returns: dict of HTTP headers being sent in request.
"""
if key is None:
return {}

key = _to_bytes(key)
sha256_key = hashlib.sha256(key).digest()
key_hash = base64.b64encode(sha256_key).rstrip()
encoded_key = base64.b64encode(key).rstrip()
headers['X-Goog-Encryption-Algorithm'] = 'AES256'
headers['X-Goog-Encryption-Key'] = _bytes_to_unicode(encoded_key)
headers['X-Goog-Encryption-Key-Sha256'] = _bytes_to_unicode(key_hash)
key_hash = hashlib.sha256(key).digest()
key_hash = base64.b64encode(key_hash).rstrip()
key = base64.b64encode(key).rstrip()

return {
'X-Goog-Encryption-Algorithm': 'AES256',
'X-Goog-Encryption-Key': _bytes_to_unicode(key),
'X-Goog-Encryption-Key-Sha256': _bytes_to_unicode(key_hash),
}
30 changes: 19 additions & 11 deletions storage/unit_tests/test_blob.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ def _makeOne(self, *args, **kw):
blob._properties = properties or {}
return blob

def test_ctor(self):
def test_ctor_wo_encryption_key(self):
BLOB_NAME = 'blob-name'
bucket = _Bucket()
properties = {'key': 'value'}
Expand All @@ -34,6 +34,14 @@ def test_ctor(self):
self.assertEqual(blob._properties, properties)
self.assertFalse(blob._acl.loaded)
self.assertIs(blob._acl.blob, blob)
self.assertEqual(blob._encryption_key, None)

def test_ctor_w_encryption_key(self):
KEY = b'01234567890123456789012345678901' # 32 bytes
BLOB_NAME = 'blob-name'
bucket = _Bucket()
blob = self._makeOne(BLOB_NAME, bucket=bucket, encryption_key=KEY)
self.assertEqual(blob._encryption_key, KEY)

def test_chunk_size_ctor(self):
from google.cloud.storage.blob import Blob
Expand Down Expand Up @@ -391,7 +399,7 @@ def test_download_to_filename_w_key(self):
from google.cloud._testing import _NamedTemporaryFile

BLOB_NAME = 'blob-name'
KEY = 'aa426195405adee2c8081bb9e7e74b19'
KEY = b'aa426195405adee2c8081bb9e7e74b19'
HEADER_KEY_VALUE = 'YWE0MjYxOTU0MDVhZGVlMmM4MDgxYmI5ZTdlNzRiMTk='
HEADER_KEY_HASH_VALUE = 'V3Kwe46nKc3xLv96+iJ707YfZfFvlObta8TQcx2gpm0='
chunk1_response = {'status': PARTIAL_CONTENT,
Expand All @@ -407,12 +415,13 @@ def test_download_to_filename_w_key(self):
MEDIA_LINK = 'http://example.com/media/'
properties = {'mediaLink': MEDIA_LINK,
'updated': '2014-12-06T13:13:50.690Z'}
blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties)
blob = self._makeOne(BLOB_NAME, bucket=bucket, properties=properties,
encryption_key=KEY)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 3

with _NamedTemporaryFile() as temp:
blob.download_to_filename(temp.name, encryption_key=KEY)
blob.download_to_filename(temp.name)
with open(temp.name, 'rb') as file_obj:
wrote = file_obj.read()
mtime = os.path.getmtime(temp.name)
Expand Down Expand Up @@ -835,7 +844,7 @@ def test_upload_from_filename_w_key(self):
BLOB_NAME = 'blob-name'
UPLOAD_URL = 'http://example.com/upload/name/key'
DATA = b'ABCDEF'
KEY = 'aa426195405adee2c8081bb9e7e74b19'
KEY = b'aa426195405adee2c8081bb9e7e74b19'
HEADER_KEY_VALUE = 'YWE0MjYxOTU0MDVhZGVlMmM4MDgxYmI5ZTdlNzRiMTk='
HEADER_KEY_HASH_VALUE = 'V3Kwe46nKc3xLv96+iJ707YfZfFvlObta8TQcx2gpm0='
EXPECTED_CONTENT_TYPE = 'foo/bar'
Expand All @@ -852,16 +861,15 @@ def test_upload_from_filename_w_key(self):
client = _Client(connection)
bucket = _Bucket(client)
blob = self._makeOne(BLOB_NAME, bucket=bucket,
properties=properties)
properties=properties, encryption_key=KEY)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 5

with _NamedTemporaryFile(suffix='.jpeg') as temp:
with open(temp.name, 'wb') as file_obj:
file_obj.write(DATA)
blob.upload_from_filename(temp.name,
content_type=EXPECTED_CONTENT_TYPE,
encryption_key=KEY)
content_type=EXPECTED_CONTENT_TYPE)

rq = connection.http._requested
self.assertEqual(len(rq), 1)
Expand Down Expand Up @@ -1040,7 +1048,7 @@ def test_upload_from_string_text_w_key(self):
from six.moves.urllib.parse import urlsplit
from google.cloud.streaming import http_wrapper
BLOB_NAME = 'blob-name'
KEY = 'aa426195405adee2c8081bb9e7e74b19'
KEY = b'aa426195405adee2c8081bb9e7e74b19'
HEADER_KEY_VALUE = 'YWE0MjYxOTU0MDVhZGVlMmM4MDgxYmI5ZTdlNzRiMTk='
HEADER_KEY_HASH_VALUE = 'V3Kwe46nKc3xLv96+iJ707YfZfFvlObta8TQcx2gpm0='
UPLOAD_URL = 'http://example.com/upload/name/key'
Expand All @@ -1057,10 +1065,10 @@ def test_upload_from_string_text_w_key(self):
)
client = _Client(connection)
bucket = _Bucket(client=client)
blob = self._makeOne(BLOB_NAME, bucket=bucket)
blob = self._makeOne(BLOB_NAME, bucket=bucket, encryption_key=KEY)
blob._CHUNK_SIZE_MULTIPLE = 1
blob.chunk_size = 5
blob.upload_from_string(DATA, encryption_key=KEY)
blob.upload_from_string(DATA)
rq = connection.http._requested
self.assertEqual(len(rq), 1)
self.assertEqual(rq[0]['method'], 'POST')
Expand Down