Merge pull request #490 from SophieHallstedtQC/issue-488

Make cache timeout for GCS configurable

docs/source/deploying/configuration.rst

@@ -162,11 +162,13 @@ Quetz can store packages in Google Cloud Storage. To configure, use the followin
     token = ".."
     bucket_prefix="..."
     bucket_suffix="..."
+    cache_timeout="..."
 
 :project: The Google Cloud Project ID to work under
 :token: A token to pass the `gcsfs`. See the `gcsfs documention <https://gcsfs.readthedocs.io/en/latest/index.html#credentials>`_ for valid values.
 :bucket_prefix:
 :bucket_suffix: channel buckets on GCS are created with the following semantics: ``{bucket_prefix}{channel_name}{bucket_suffix}``
+:cache_timeout: Timeout in s after which local GCS cache entries are invalidated. Set to a value <=0 to disable caching completely. Default is that entries are never invalidated.
 
 ``local_store`` section
 ^^^^^^^^^^^^^^^^^^^^^^^

environment.yml

@@ -27,7 +27,7 @@ dependencies:
   - alembic
   - zchunk
   - s3fs
-  - gcsfs >=2021.10.1
+  - gcsfs >=2022.02
   - sphinx
   - sphinx-book-theme
   - tenacity

quetz/config.py

@@ -162,6 +162,7 @@ class Config:
                 ConfigEntry("token", str, default=""),
                 ConfigEntry("bucket_prefix", str, default=""),
                 ConfigEntry("bucket_suffix", str, default=""),
+                ConfigEntry("cache_timeout", int, default=None),
             ],
             required=False,
         ),
@@ -403,6 +404,7 @@ def get_package_store(self) -> pkgstores.PackageStore:
                     'token': self.gcs_token,
                     'bucket_prefix': self.gcs_bucket_prefix,
                     'bucket_suffix': self.gcs_bucket_suffix,
+                    'cache_timeout': self.gcs_cache_timeout,
                 }
             )
         else:

quetz/pkgstores.py

@@ -558,10 +558,12 @@ def __init__(self, config):
 
         self.project = config.get("project")
         self.token = config.get("token")
+        self.cache_timeout = config.get("cache_timeout")
 
         self.fs = gcsfs.GCSFileSystem(
             project=self.project,
             token=self.token if self.token else None,
+            cache_timeout=self.cache_timeout,
         )
 
         self.bucket_prefix = config['bucket_prefix']

setup.cfg

@@ -54,7 +54,7 @@ console_scripts =
 azure =
   adlfs
 gcs =
-  gcsfs >=2021.10.1
+  gcsfs >=2022.02
 pam = 
   pamela
 postgre =