Skip to content

Commit

Permalink
feat(storage): add 'ARCHIVE' storage class (#9533)
Browse files Browse the repository at this point in the history 
* feat(storage): add 'ARCHIVE' storage class

Closes #9532.
  • Loading branch information
@tseaver @crwilcox
tseaver authored and crwilcox committed Jan 2, 2020
1 parent 92d7b46 commit ac544b1
Show file tree
Hide file tree
Showing 5 changed files with 75 additions and 15 deletions.
8 changes: 6 additions & 2 deletions storage/google/cloud/storage/blob.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,12 @@
from google.cloud.storage._signing import generate_signed_url_v4
from google.cloud.storage.acl import ACL
from google.cloud.storage.acl import ObjectACL
from google.cloud.storage.constants import STANDARD_STORAGE_CLASS
from google.cloud.storage.constants import NEARLINE_STORAGE_CLASS
from google.cloud.storage.constants import ARCHIVE_STORAGE_CLASS
from google.cloud.storage.constants import COLDLINE_STORAGE_CLASS
from google.cloud.storage.constants import MULTI_REGIONAL_LEGACY_STORAGE_CLASS
from google.cloud.storage.constants import NEARLINE_STORAGE_CLASS
from google.cloud.storage.constants import REGIONAL_LEGACY_STORAGE_CLASS
from google.cloud.storage.constants import STANDARD_STORAGE_CLASS

_STORAGE_HOST = _get_storage_host()

Expand Down Expand Up @@ -145,6 +146,7 @@ class Blob(_PropertyMixin):
STANDARD_STORAGE_CLASS,
NEARLINE_STORAGE_CLASS,
COLDLINE_STORAGE_CLASS,
ARCHIVE_STORAGE_CLASS,
MULTI_REGIONAL_LEGACY_STORAGE_CLASS,
REGIONAL_LEGACY_STORAGE_CLASS,
)
Expand Down Expand Up @@ -1711,6 +1713,7 @@ def update_storage_class(self, new_class, client=None):
new storage class for the object. One of:
:attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`,
or
Expand Down Expand Up @@ -2007,6 +2010,7 @@ def kms_key_name(self):
:attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.DURABLE_REDUCED_AVAILABILITY_STORAGE_CLASS`,
Expand Down
4 changes: 4 additions & 0 deletions storage/google/cloud/storage/bucket.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
from google.cloud.storage.acl import BucketACL
from google.cloud.storage.acl import DefaultObjectACL
from google.cloud.storage.blob import Blob
from google.cloud.storage.constants import ARCHIVE_STORAGE_CLASS
from google.cloud.storage.constants import COLDLINE_STORAGE_CLASS
from google.cloud.storage.constants import DUAL_REGION_LOCATION_TYPE
from google.cloud.storage.constants import (
Expand Down Expand Up @@ -480,6 +481,7 @@ class Bucket(_PropertyMixin):
STANDARD_STORAGE_CLASS,
NEARLINE_STORAGE_CLASS,
COLDLINE_STORAGE_CLASS,
ARCHIVE_STORAGE_CLASS,
MULTI_REGIONAL_LEGACY_STORAGE_CLASS, # legacy
REGIONAL_LEGACY_STORAGE_CLASS, # legacy
DURABLE_REDUCED_AVAILABILITY_LEGACY_STORAGE_CLASS, # legacy
Expand Down Expand Up @@ -1713,6 +1715,7 @@ def storage_class(self):
If set, one of
:attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`,
Expand All @@ -1733,6 +1736,7 @@ def storage_class(self, value):
One of
:attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`,
:attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`,
Expand Down
25 changes: 22 additions & 3 deletions storage/google/cloud/storage/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,28 @@
# Storage classes

STANDARD_STORAGE_CLASS = "STANDARD"
"""Storage class for objects accessed more than once per month."""
"""Storage class for objects accessed more than once per month.
See: https://cloud.google.com/storage/docs/storage-classes
"""

NEARLINE_STORAGE_CLASS = "NEARLINE"
"""Storage class for objects accessed at most once per month."""
"""Storage class for objects accessed at most once per month.
See: https://cloud.google.com/storage/docs/storage-classes
"""

COLDLINE_STORAGE_CLASS = "COLDLINE"
"""Storage class for objects accessed at most once per year."""
"""Storage class for objects accessed at most once per year.
See: https://cloud.google.com/storage/docs/storage-classes
"""

ARCHIVE_STORAGE_CLASS = "ARCHIVE"
"""Storage class for objects accessed less frequently than once per year.
See: https://cloud.google.com/storage/docs/storage-classes
"""

MULTI_REGIONAL_LEGACY_STORAGE_CLASS = "MULTI_REGIONAL"
"""Legacy storage class.
Expand All @@ -32,6 +47,8 @@
Can only be used for objects in buckets whose
:attr:`~google.cloud.storage.bucket.Bucket.location_type` is
:attr:`~google.cloud.storage.bucket.Bucket.MULTI_REGION_LOCATION_TYPE`.
See: https://cloud.google.com/storage/docs/storage-classes
"""

REGIONAL_LEGACY_STORAGE_CLASS = "REGIONAL"
Expand All @@ -42,6 +59,8 @@
Can only be used for objects in buckets whose
:attr:`~google.cloud.storage.bucket.Bucket.location_type` is
:attr:`~google.cloud.storage.bucket.Bucket.REGION_LOCATION_TYPE`.
See: https://cloud.google.com/storage/docs/storage-classes
"""

DURABLE_REDUCED_AVAILABILITY_LEGACY_STORAGE_CLASS = "DURABLE_REDUCED_AVAILABILITY"
Expand Down
44 changes: 34 additions & 10 deletions storage/tests/system.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,21 +174,41 @@ def test_create_bucket(self):
self.case_buckets_to_delete.append(new_bucket_name)
self.assertEqual(created.name, new_bucket_name)

def test_bucket_create_w_alt_storage_class(self):
from google.cloud.storage import constants

new_bucket_name = "bucket-w-archive" + unique_resource_id("-")
self.assertRaises(
exceptions.NotFound, Config.CLIENT.get_bucket, new_bucket_name
)
bucket = Config.CLIENT.bucket(new_bucket_name)
bucket.storage_class = constants.ARCHIVE_STORAGE_CLASS
retry_429_503(bucket.create)()
self.case_buckets_to_delete.append(new_bucket_name)
created = Config.CLIENT.get_bucket(new_bucket_name)
self.assertEqual(created.storage_class, constants.ARCHIVE_STORAGE_CLASS)

def test_lifecycle_rules(self):
from google.cloud.storage import constants

new_bucket_name = "w-lifcycle-rules" + unique_resource_id("-")
self.assertRaises(
exceptions.NotFound, Config.CLIENT.get_bucket, new_bucket_name
)
bucket = Config.CLIENT.bucket(new_bucket_name)
bucket.add_lifecycle_delete_rule(age=42)
bucket.add_lifecycle_set_storage_class_rule(
"COLDLINE", is_live=False, matches_storage_class=["NEARLINE"]
constants.COLDLINE_STORAGE_CLASS,
is_live=False,
matches_storage_class=[constants.NEARLINE_STORAGE_CLASS],
)

expected_rules = [
LifecycleRuleDelete(age=42),
LifecycleRuleSetStorageClass(
"COLDLINE", is_live=False, matches_storage_class=["NEARLINE"]
constants.COLDLINE_STORAGE_CLASS,
is_live=False,
matches_storage_class=[constants.NEARLINE_STORAGE_CLASS],
),
]

Expand Down Expand Up @@ -1235,34 +1255,38 @@ def test_rewrite_rotate_with_user_project(self):

class TestStorageUpdateStorageClass(TestStorageFiles):
def test_update_storage_class_small_file(self):
from google.cloud.storage import constants

blob = self.bucket.blob("SmallFile")

file_data = self.FILES["simple"]
blob.upload_from_filename(file_data["path"])
self.case_blobs_to_delete.append(blob)

blob.update_storage_class("NEARLINE")
blob.update_storage_class(constants.NEARLINE_STORAGE_CLASS)
blob.reload()
self.assertEqual(blob.storage_class, "NEARLINE")
self.assertEqual(blob.storage_class, constants.NEARLINE_STORAGE_CLASS)

blob.update_storage_class("COLDLINE")
blob.update_storage_class(constants.COLDLINE_STORAGE_CLASS)
blob.reload()
self.assertEqual(blob.storage_class, "COLDLINE")
self.assertEqual(blob.storage_class, constants.COLDLINE_STORAGE_CLASS)

def test_update_storage_class_large_file(self):
from google.cloud.storage import constants

blob = self.bucket.blob("BigFile")

file_data = self.FILES["big"]
blob.upload_from_filename(file_data["path"])
self.case_blobs_to_delete.append(blob)

blob.update_storage_class("NEARLINE")
blob.update_storage_class(constants.NEARLINE_STORAGE_CLASS)
blob.reload()
self.assertEqual(blob.storage_class, "NEARLINE")
self.assertEqual(blob.storage_class, constants.NEARLINE_STORAGE_CLASS)

blob.update_storage_class("COLDLINE")
blob.update_storage_class(constants.COLDLINE_STORAGE_CLASS)
blob.reload()
self.assertEqual(blob.storage_class, "COLDLINE")
self.assertEqual(blob.storage_class, constants.COLDLINE_STORAGE_CLASS)


class TestStorageNotificationCRUD(unittest.TestCase):
Expand Down
9 changes: 9 additions & 0 deletions storage/tests/unit/test_bucket.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1901,6 +1901,15 @@ def test_storage_class_setter_COLDLINE(self):
self.assertEqual(bucket.storage_class, COLDLINE_STORAGE_CLASS)
self.assertTrue("storageClass" in bucket._changes)

def test_storage_class_setter_ARCHIVE(self):
from google.cloud.storage.constants import ARCHIVE_STORAGE_CLASS

NAME = "name"
bucket = self._make_one(name=NAME)
bucket.storage_class = ARCHIVE_STORAGE_CLASS
self.assertEqual(bucket.storage_class, ARCHIVE_STORAGE_CLASS)
self.assertTrue("storageClass" in bucket._changes)

def test_storage_class_setter_MULTI_REGIONAL(self):
from google.cloud.storage.constants import MULTI_REGIONAL_LEGACY_STORAGE_CLASS

Expand Down

0 comments on commit ac544b1

Please sign in to comment.