From d31eeb9fac3542286e487a9d26e42b1f26540688 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 24 Oct 2019 13:27:47 -0400 Subject: [PATCH 1/5] feat(storage): add 'ARCHIVE' storage class Closes #9532. --- storage/google/cloud/storage/blob.py | 8 ++++++-- storage/google/cloud/storage/bucket.py | 4 ++++ storage/google/cloud/storage/constants.py | 3 +++ storage/tests/unit/test_bucket.py | 9 +++++++++ 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/storage/google/cloud/storage/blob.py b/storage/google/cloud/storage/blob.py index cf22a9a77ae2..1b718520865d 100644 --- a/storage/google/cloud/storage/blob.py +++ b/storage/google/cloud/storage/blob.py @@ -59,11 +59,12 @@ from google.cloud.storage._signing import generate_signed_url_v4 from google.cloud.storage.acl import ACL from google.cloud.storage.acl import ObjectACL -from google.cloud.storage.constants import STANDARD_STORAGE_CLASS -from google.cloud.storage.constants import NEARLINE_STORAGE_CLASS +from google.cloud.storage.constants import ARCHIVE_STORAGE_CLASS from google.cloud.storage.constants import COLDLINE_STORAGE_CLASS from google.cloud.storage.constants import MULTI_REGIONAL_LEGACY_STORAGE_CLASS +from google.cloud.storage.constants import NEARLINE_STORAGE_CLASS from google.cloud.storage.constants import REGIONAL_LEGACY_STORAGE_CLASS +from google.cloud.storage.constants import STANDARD_STORAGE_CLASS _STORAGE_HOST = _get_storage_host() @@ -143,6 +144,7 @@ class Blob(_PropertyMixin): STANDARD_STORAGE_CLASS, NEARLINE_STORAGE_CLASS, COLDLINE_STORAGE_CLASS, + ARCHIVE_STORAGE_CLASS, MULTI_REGIONAL_LEGACY_STORAGE_CLASS, REGIONAL_LEGACY_STORAGE_CLASS, ) @@ -1656,6 +1658,7 @@ def update_storage_class(self, new_class, client=None): new storage class for the object. One of: :attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`, + :attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`, or @@ -1951,6 +1954,7 @@ def kms_key_name(self): :attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`, + :attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.DURABLE_REDUCED_AVAILABILITY_STORAGE_CLASS`, diff --git a/storage/google/cloud/storage/bucket.py b/storage/google/cloud/storage/bucket.py index 48ab09e23e4f..764bd385789b 100644 --- a/storage/google/cloud/storage/bucket.py +++ b/storage/google/cloud/storage/bucket.py @@ -39,6 +39,7 @@ from google.cloud.storage.acl import BucketACL from google.cloud.storage.acl import DefaultObjectACL from google.cloud.storage.blob import Blob +from google.cloud.storage.constants import ARCHIVE_STORAGE_CLASS from google.cloud.storage.constants import COLDLINE_STORAGE_CLASS from google.cloud.storage.constants import DUAL_REGION_LOCATION_TYPE from google.cloud.storage.constants import ( @@ -402,6 +403,7 @@ class Bucket(_PropertyMixin): STANDARD_STORAGE_CLASS, NEARLINE_STORAGE_CLASS, COLDLINE_STORAGE_CLASS, + ARCHIVE_STORAGE_CLASS, MULTI_REGIONAL_LEGACY_STORAGE_CLASS, # legacy REGIONAL_LEGACY_STORAGE_CLASS, # legacy DURABLE_REDUCED_AVAILABILITY_LEGACY_STORAGE_CLASS, # legacy @@ -1634,6 +1636,7 @@ def storage_class(self): If set, one of :attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`, + :attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`, @@ -1654,6 +1657,7 @@ def storage_class(self, value): One of :attr:`~google.cloud.storage.constants.NEARLINE_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.COLDLINE_STORAGE_CLASS`, + :attr:`~google.cloud.storage.constants.ARCHIVE_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.STANDARD_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.MULTI_REGIONAL_LEGACY_STORAGE_CLASS`, :attr:`~google.cloud.storage.constants.REGIONAL_LEGACY_STORAGE_CLASS`, diff --git a/storage/google/cloud/storage/constants.py b/storage/google/cloud/storage/constants.py index e93d3ab29546..fa281649da29 100644 --- a/storage/google/cloud/storage/constants.py +++ b/storage/google/cloud/storage/constants.py @@ -24,6 +24,9 @@ COLDLINE_STORAGE_CLASS = "COLDLINE" """Storage class for objects accessed at most once per year.""" +ARCHIVE_STORAGE_CLASS = "ARCHIVE" +"""Storage class for objects accessed less frequently than once per year.""" + MULTI_REGIONAL_LEGACY_STORAGE_CLASS = "MULTI_REGIONAL" """Legacy storage class. diff --git a/storage/tests/unit/test_bucket.py b/storage/tests/unit/test_bucket.py index de943339e200..2eb6f7d57561 100644 --- a/storage/tests/unit/test_bucket.py +++ b/storage/tests/unit/test_bucket.py @@ -1773,6 +1773,15 @@ def test_storage_class_setter_COLDLINE(self): self.assertEqual(bucket.storage_class, COLDLINE_STORAGE_CLASS) self.assertTrue("storageClass" in bucket._changes) + def test_storage_class_setter_ARCHIVE(self): + from google.cloud.storage.constants import ARCHIVE_STORAGE_CLASS + + NAME = "name" + bucket = self._make_one(name=NAME) + bucket.storage_class = ARCHIVE_STORAGE_CLASS + self.assertEqual(bucket.storage_class, ARCHIVE_STORAGE_CLASS) + self.assertTrue("storageClass" in bucket._changes) + def test_storage_class_setter_MULTI_REGIONAL(self): from google.cloud.storage.constants import MULTI_REGIONAL_LEGACY_STORAGE_CLASS From 8272f9f0d78761269032e23fd52b2785bf4f81ea Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Thu, 24 Oct 2019 13:56:04 -0400 Subject: [PATCH 2/5] docs: link to API ref for storage classes in constant docstrings --- storage/google/cloud/storage/constants.py | 24 +++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/storage/google/cloud/storage/constants.py b/storage/google/cloud/storage/constants.py index fa281649da29..5b3117d92fa4 100644 --- a/storage/google/cloud/storage/constants.py +++ b/storage/google/cloud/storage/constants.py @@ -16,16 +16,28 @@ # Storage classes STANDARD_STORAGE_CLASS = "STANDARD" -"""Storage class for objects accessed more than once per month.""" +"""Storage class for objects accessed more than once per month. + +See: https://cloud.google.com/storage/docs/per-object-storage-class +""" NEARLINE_STORAGE_CLASS = "NEARLINE" -"""Storage class for objects accessed at most once per month.""" +"""Storage class for objects accessed at most once per month. + +See: https://cloud.google.com/storage/docs/per-object-storage-class +""" COLDLINE_STORAGE_CLASS = "COLDLINE" -"""Storage class for objects accessed at most once per year.""" +"""Storage class for objects accessed at most once per year. + +See: https://cloud.google.com/storage/docs/per-object-storage-class +""" ARCHIVE_STORAGE_CLASS = "ARCHIVE" -"""Storage class for objects accessed less frequently than once per year.""" +"""Storage class for objects accessed less frequently than once per year. + +See: https://cloud.google.com/storage/docs/per-object-storage-class +""" MULTI_REGIONAL_LEGACY_STORAGE_CLASS = "MULTI_REGIONAL" """Legacy storage class. @@ -35,6 +47,8 @@ Can only be used for objects in buckets whose :attr:`~google.cloud.storage.bucket.Bucket.location_type` is :attr:`~google.cloud.storage.bucket.Bucket.MULTI_REGION_LOCATION_TYPE`. + +See: https://cloud.google.com/storage/docs/per-object-storage-class """ REGIONAL_LEGACY_STORAGE_CLASS = "REGIONAL" @@ -45,6 +59,8 @@ Can only be used for objects in buckets whose :attr:`~google.cloud.storage.bucket.Bucket.location_type` is :attr:`~google.cloud.storage.bucket.Bucket.REGION_LOCATION_TYPE`. + +See: https://cloud.google.com/storage/docs/per-object-storage-class """ DURABLE_REDUCED_AVAILABILITY_LEGACY_STORAGE_CLASS = "DURABLE_REDUCED_AVAILABILITY" From 03ba4372102488176b0f3d985b0275e8f89848f6 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Fri, 25 Oct 2019 11:02:46 -0400 Subject: [PATCH 3/5] docs: use final / non-redirected URL --- storage/google/cloud/storage/constants.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/storage/google/cloud/storage/constants.py b/storage/google/cloud/storage/constants.py index 5b3117d92fa4..faadff1f0702 100644 --- a/storage/google/cloud/storage/constants.py +++ b/storage/google/cloud/storage/constants.py @@ -18,25 +18,25 @@ STANDARD_STORAGE_CLASS = "STANDARD" """Storage class for objects accessed more than once per month. -See: https://cloud.google.com/storage/docs/per-object-storage-class +See: https://cloud.google.com/storage/docs/storage-classes """ NEARLINE_STORAGE_CLASS = "NEARLINE" """Storage class for objects accessed at most once per month. -See: https://cloud.google.com/storage/docs/per-object-storage-class +See: https://cloud.google.com/storage/docs/storage-classes """ COLDLINE_STORAGE_CLASS = "COLDLINE" """Storage class for objects accessed at most once per year. -See: https://cloud.google.com/storage/docs/per-object-storage-class +See: https://cloud.google.com/storage/docs/storage-classes """ ARCHIVE_STORAGE_CLASS = "ARCHIVE" """Storage class for objects accessed less frequently than once per year. -See: https://cloud.google.com/storage/docs/per-object-storage-class +See: https://cloud.google.com/storage/docs/storage-classes """ MULTI_REGIONAL_LEGACY_STORAGE_CLASS = "MULTI_REGIONAL" @@ -48,7 +48,7 @@ :attr:`~google.cloud.storage.bucket.Bucket.location_type` is :attr:`~google.cloud.storage.bucket.Bucket.MULTI_REGION_LOCATION_TYPE`. -See: https://cloud.google.com/storage/docs/per-object-storage-class +See: https://cloud.google.com/storage/docs/storage-classes """ REGIONAL_LEGACY_STORAGE_CLASS = "REGIONAL" @@ -60,7 +60,7 @@ :attr:`~google.cloud.storage.bucket.Bucket.location_type` is :attr:`~google.cloud.storage.bucket.Bucket.REGION_LOCATION_TYPE`. -See: https://cloud.google.com/storage/docs/per-object-storage-class +See: https://cloud.google.com/storage/docs/storage-classes """ DURABLE_REDUCED_AVAILABILITY_LEGACY_STORAGE_CLASS = "DURABLE_REDUCED_AVAILABILITY" From f4d5ff170142ab22641ba16d0c3d5df92977b158 Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 28 Oct 2019 12:29:15 -0400 Subject: [PATCH 4/5] tests(storage): use constants for storage class names --- storage/tests/system.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/storage/tests/system.py b/storage/tests/system.py index 65f4f976a41f..36b97b3db10c 100644 --- a/storage/tests/system.py +++ b/storage/tests/system.py @@ -173,6 +173,8 @@ def test_create_bucket(self): self.assertEqual(created.name, new_bucket_name) def test_lifecycle_rules(self): + from google.cloud.storage import constants + new_bucket_name = "w-lifcycle-rules" + unique_resource_id("-") self.assertRaises( exceptions.NotFound, Config.CLIENT.get_bucket, new_bucket_name @@ -180,13 +182,17 @@ def test_lifecycle_rules(self): bucket = Config.CLIENT.bucket(new_bucket_name) bucket.add_lifecycle_delete_rule(age=42) bucket.add_lifecycle_set_storage_class_rule( - "COLDLINE", is_live=False, matches_storage_class=["NEARLINE"] + constants.COLDLINE_STORAGE_CLASS, + is_live=False, + matches_storage_class=[constants.NEARLINE_STORAGE_CLASS], ) expected_rules = [ LifecycleRuleDelete(age=42), LifecycleRuleSetStorageClass( - "COLDLINE", is_live=False, matches_storage_class=["NEARLINE"] + constants.COLDLINE_STORAGE_CLASS, + is_live=False, + matches_storage_class=[constants.NEARLINE_STORAGE_CLASS], ), ] @@ -1216,34 +1222,38 @@ def test_rewrite_rotate_with_user_project(self): class TestStorageUpdateStorageClass(TestStorageFiles): def test_update_storage_class_small_file(self): + from google.cloud.storage import constants + blob = self.bucket.blob("SmallFile") file_data = self.FILES["simple"] blob.upload_from_filename(file_data["path"]) self.case_blobs_to_delete.append(blob) - blob.update_storage_class("NEARLINE") + blob.update_storage_class(constants.NEARLINE_STORAGE_CLASS) blob.reload() - self.assertEqual(blob.storage_class, "NEARLINE") + self.assertEqual(blob.storage_class, constants.NEARLINE_STORAGE_CLASS) - blob.update_storage_class("COLDLINE") + blob.update_storage_class(constants.COLDLINE_STORAGE_CLASS) blob.reload() - self.assertEqual(blob.storage_class, "COLDLINE") + self.assertEqual(blob.storage_class, constants.COLDLINE_STORAGE_CLASS) def test_update_storage_class_large_file(self): + from google.cloud.storage import constants + blob = self.bucket.blob("BigFile") file_data = self.FILES["big"] blob.upload_from_filename(file_data["path"]) self.case_blobs_to_delete.append(blob) - blob.update_storage_class("NEARLINE") + blob.update_storage_class(constants.NEARLINE_STORAGE_CLASS) blob.reload() - self.assertEqual(blob.storage_class, "NEARLINE") + self.assertEqual(blob.storage_class, constants.NEARLINE_STORAGE_CLASS) - blob.update_storage_class("COLDLINE") + blob.update_storage_class(constants.COLDLINE_STORAGE_CLASS) blob.reload() - self.assertEqual(blob.storage_class, "COLDLINE") + self.assertEqual(blob.storage_class, constants.COLDLINE_STORAGE_CLASS) class TestStorageNotificationCRUD(unittest.TestCase): From be69fb79eb21c1c5f7c4aeacb334a70094d7a9fd Mon Sep 17 00:00:00 2001 From: Tres Seaver Date: Mon, 28 Oct 2019 12:31:09 -0400 Subject: [PATCH 5/5] tests(storage): add systest for creating bucketw/ ARCHIVE storage class --- storage/tests/system.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/storage/tests/system.py b/storage/tests/system.py index 36b97b3db10c..eb24a6b6ab42 100644 --- a/storage/tests/system.py +++ b/storage/tests/system.py @@ -172,6 +172,20 @@ def test_create_bucket(self): self.case_buckets_to_delete.append(new_bucket_name) self.assertEqual(created.name, new_bucket_name) + def test_bucket_create_w_alt_storage_class(self): + from google.cloud.storage import constants + + new_bucket_name = "bucket-w-archive" + unique_resource_id("-") + self.assertRaises( + exceptions.NotFound, Config.CLIENT.get_bucket, new_bucket_name + ) + bucket = Config.CLIENT.bucket(new_bucket_name) + bucket.storage_class = constants.ARCHIVE_STORAGE_CLASS + retry_429_503(bucket.create)() + self.case_buckets_to_delete.append(new_bucket_name) + created = Config.CLIENT.get_bucket(new_bucket_name) + self.assertEqual(created.storage_class, constants.ARCHIVE_STORAGE_CLASS) + def test_lifecycle_rules(self): from google.cloud.storage import constants