Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(storage): add 'ARCHIVE' storage class #9533

Merged
merged 5 commits into from
Jan 2, 2020
Merged

feat(storage): add 'ARCHIVE' storage class #9533

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d31eeb9
feat(storage): add 'ARCHIVE' storage class
tseaver Oct 24, 2019
8272f9f
docs: link to API ref for storage classes in constant docstrings
tseaver Oct 24, 2019
03ba437
docs: use final / non-redirected URL
tseaver Oct 25, 2019
f4d5ff1
tests(storage): use constants for storage class names
tseaver Oct 28, 2019
be69fb7
tests(storage): add systest for creating bucketw/ ARCHIVE storage class
tseaver Oct 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions storage/google/cloud/storage/blob.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,12 @@
from google.cloud.storage._signing import generate_signed_url_v4
from google.cloud.storage.acl import ACL
from google.cloud.storage.acl import ObjectACL
from google.cloud.storage.constants import STANDARD_STORAGE_CLASS
from google.cloud.storage.constants import NEARLINE_STORAGE_CLASS
from google.cloud.storage.constants import ARCHIVE_STORAGE_CLASS
from google.cloud.storage.constants import COLDLINE_STORAGE_CLASS
from google.cloud.storage.constants import MULTI_REGIONAL_LEGACY_STORAGE_CLASS
from google.cloud.storage.constants import NEARLINE_STORAGE_CLASS
from google.cloud.storage.constants import REGIONAL_LEGACY_STORAGE_CLASS
from google.cloud.storage.constants import STANDARD_STORAGE_CLASS

_STORAGE_HOST = _get_storage_host()

Expand Down Expand Up @@ -143,6 +144,7 @@ class Blob(_PropertyMixin):
STANDARD_STORAGE_CLASS,
NEARLINE_STORAGE_CLASS,
COLDLINE_STORAGE_CLASS,
ARCHIVE_STORAGE_CLASS,
MULTI_REGIONAL_LEGACY_STORAGE_CLASS,
REGIONAL_LEGACY_STORAGE_CLASS,
)
Expand Down Expand Up @@ -1656,6 +1658,7 @@ def update_storage_class(self, new_class, client=None):
new storage class for the object. One of:
:attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`,
or
Expand Down Expand Up @@ -1951,6 +1954,7 @@ def kms_key_name(self):
:attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.DURABLE_REDUCED_AVAILABILITY_STORAGE_CLASS`,
Expand Down
4 changes: 4 additions & 0 deletions storage/google/cloud/storage/bucket.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
from google.cloud.storage.acl import BucketACL
from google.cloud.storage.acl import DefaultObjectACL
from google.cloud.storage.blob import Blob
from google.cloud.storage.constants import ARCHIVE_STORAGE_CLASS
from google.cloud.storage.constants import COLDLINE_STORAGE_CLASS
from google.cloud.storage.constants import DUAL_REGION_LOCATION_TYPE
from google.cloud.storage.constants import (
Expand Down Expand Up @@ -402,6 +403,7 @@ class Bucket(_PropertyMixin):
STANDARD_STORAGE_CLASS,
NEARLINE_STORAGE_CLASS,
COLDLINE_STORAGE_CLASS,
ARCHIVE_STORAGE_CLASS,
MULTI_REGIONAL_LEGACY_STORAGE_CLASS, # legacy
REGIONAL_LEGACY_STORAGE_CLASS, # legacy
DURABLE_REDUCED_AVAILABILITY_LEGACY_STORAGE_CLASS, # legacy
Expand Down Expand Up @@ -1634,6 +1636,7 @@ def storage_class(self):
If set, one of
:attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`,
Expand All @@ -1654,6 +1657,7 @@ def storage_class(self, value):
One of
:attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`,
Expand Down
25 changes: 22 additions & 3 deletions storage/google/cloud/storage/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,28 @@
# Storage classes

STANDARD_STORAGE_CLASS = "STANDARD"
"""Storage class for objects accessed more than once per month."""
"""Storage class for objects accessed more than once per month.

See: https://cloud.google.com/storage/docs/storage-classes
"""

NEARLINE_STORAGE_CLASS = "NEARLINE"
"""Storage class for objects accessed at most once per month."""
"""Storage class for objects accessed at most once per month.

See: https://cloud.google.com/storage/docs/storage-classes
"""

COLDLINE_STORAGE_CLASS = "COLDLINE"
"""Storage class for objects accessed at most once per year."""
"""Storage class for objects accessed at most once per year.

See: https://cloud.google.com/storage/docs/storage-classes
"""

ARCHIVE_STORAGE_CLASS = "ARCHIVE"
"""Storage class for objects accessed less frequently than once per year.

See: https://cloud.google.com/storage/docs/storage-classes
"""

MULTI_REGIONAL_LEGACY_STORAGE_CLASS = "MULTI_REGIONAL"
"""Legacy storage class.
Expand All @@ -32,6 +47,8 @@
Can only be used for objects in buckets whose
:attr:`~google.cloud.storage.bucket.Bucket.location_type` is
:attr:`~google.cloud.storage.bucket.Bucket.MULTI_REGION_LOCATION_TYPE`.

See: https://cloud.google.com/storage/docs/storage-classes
"""

REGIONAL_LEGACY_STORAGE_CLASS = "REGIONAL"
Expand All @@ -42,6 +59,8 @@
Can only be used for objects in buckets whose
:attr:`~google.cloud.storage.bucket.Bucket.location_type` is
:attr:`~google.cloud.storage.bucket.Bucket.REGION_LOCATION_TYPE`.

See: https://cloud.google.com/storage/docs/storage-classes
"""

DURABLE_REDUCED_AVAILABILITY_LEGACY_STORAGE_CLASS = "DURABLE_REDUCED_AVAILABILITY"
Expand Down
44 changes: 34 additions & 10 deletions storage/tests/system.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,21 +172,41 @@ def test_create_bucket(self):
self.case_buckets_to_delete.append(new_bucket_name)
self.assertEqual(created.name, new_bucket_name)

def test_bucket_create_w_alt_storage_class(self):
from google.cloud.storage import constants

new_bucket_name = "bucket-w-archive" + unique_resource_id("-")
self.assertRaises(
exceptions.NotFound, Config.CLIENT.get_bucket, new_bucket_name
)
bucket = Config.CLIENT.bucket(new_bucket_name)
bucket.storage_class = constants.ARCHIVE_STORAGE_CLASS
retry_429_503(bucket.create)()
self.case_buckets_to_delete.append(new_bucket_name)
created = Config.CLIENT.get_bucket(new_bucket_name)
self.assertEqual(created.storage_class, constants.ARCHIVE_STORAGE_CLASS)

def test_lifecycle_rules(self):
from google.cloud.storage import constants

new_bucket_name = "w-lifcycle-rules" + unique_resource_id("-")
self.assertRaises(
exceptions.NotFound, Config.CLIENT.get_bucket, new_bucket_name
)
bucket = Config.CLIENT.bucket(new_bucket_name)
bucket.add_lifecycle_delete_rule(age=42)
bucket.add_lifecycle_set_storage_class_rule(
"COLDLINE", is_live=False, matches_storage_class=["NEARLINE"]
constants.COLDLINE_STORAGE_CLASS,
is_live=False,
matches_storage_class=[constants.NEARLINE_STORAGE_CLASS],
)

expected_rules = [
LifecycleRuleDelete(age=42),
LifecycleRuleSetStorageClass(
"COLDLINE", is_live=False, matches_storage_class=["NEARLINE"]
constants.COLDLINE_STORAGE_CLASS,
is_live=False,
matches_storage_class=[constants.NEARLINE_STORAGE_CLASS],
),
]

Expand Down Expand Up @@ -1216,34 +1236,38 @@ def test_rewrite_rotate_with_user_project(self):

class TestStorageUpdateStorageClass(TestStorageFiles):
def test_update_storage_class_small_file(self):
from google.cloud.storage import constants

blob = self.bucket.blob("SmallFile")

file_data = self.FILES["simple"]
blob.upload_from_filename(file_data["path"])
self.case_blobs_to_delete.append(blob)

blob.update_storage_class("NEARLINE")
blob.update_storage_class(constants.NEARLINE_STORAGE_CLASS)
blob.reload()
self.assertEqual(blob.storage_class, "NEARLINE")
self.assertEqual(blob.storage_class, constants.NEARLINE_STORAGE_CLASS)

blob.update_storage_class("COLDLINE")
blob.update_storage_class(constants.COLDLINE_STORAGE_CLASS)
blob.reload()
self.assertEqual(blob.storage_class, "COLDLINE")
self.assertEqual(blob.storage_class, constants.COLDLINE_STORAGE_CLASS)

def test_update_storage_class_large_file(self):
from google.cloud.storage import constants

blob = self.bucket.blob("BigFile")

file_data = self.FILES["big"]
blob.upload_from_filename(file_data["path"])
self.case_blobs_to_delete.append(blob)

blob.update_storage_class("NEARLINE")
blob.update_storage_class(constants.NEARLINE_STORAGE_CLASS)
blob.reload()
self.assertEqual(blob.storage_class, "NEARLINE")
self.assertEqual(blob.storage_class, constants.NEARLINE_STORAGE_CLASS)

blob.update_storage_class("COLDLINE")
blob.update_storage_class(constants.COLDLINE_STORAGE_CLASS)
blob.reload()
self.assertEqual(blob.storage_class, "COLDLINE")
self.assertEqual(blob.storage_class, constants.COLDLINE_STORAGE_CLASS)


class TestStorageNotificationCRUD(unittest.TestCase):
Expand Down
9 changes: 9 additions & 0 deletions storage/tests/unit/test_bucket.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1773,6 +1773,15 @@ def test_storage_class_setter_COLDLINE(self):
self.assertEqual(bucket.storage_class, COLDLINE_STORAGE_CLASS)
self.assertTrue("storageClass" in bucket._changes)

def test_storage_class_setter_ARCHIVE(self):
from google.cloud.storage.constants import ARCHIVE_STORAGE_CLASS

NAME = "name"
bucket = self._make_one(name=NAME)
bucket.storage_class = ARCHIVE_STORAGE_CLASS
self.assertEqual(bucket.storage_class, ARCHIVE_STORAGE_CLASS)
self.assertTrue("storageClass" in bucket._changes)

def test_storage_class_setter_MULTI_REGIONAL(self):
from google.cloud.storage.constants import MULTI_REGIONAL_LEGACY_STORAGE_CLASS

Expand Down