From 65426c3b28c309ecc782ab08b925eab541fb58d9 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Mon, 25 Nov 2024 14:07:42 -0600 Subject: [PATCH] Use string paths with open_mfdataset --- tests/geospatial/workloads/cloud_optimize.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/geospatial/workloads/cloud_optimize.py b/tests/geospatial/workloads/cloud_optimize.py index 93b08bb611..140f51cff3 100644 --- a/tests/geospatial/workloads/cloud_optimize.py +++ b/tests/geospatial/workloads/cloud_optimize.py @@ -5,7 +5,7 @@ def cloud_optimize( - scale: Literal["small", "medium", "large"], s3fs: S3FileSystem, storage_url: str + scale: Literal["small", "medium", "large"], fs: S3FileSystem, storage_url: str ): models = [ "ACCESS-CM2", @@ -59,12 +59,11 @@ def cloud_optimize( # Get netCDF data files -- see https://registry.opendata.aws/nex-gddp-cmip6 # for dataset details. - file_list = [] + files = [] for model in models: for variable in variables: data_dir = f"s3://nex-gddp-cmip6/NEX-GDDP-CMIP6/{model}/historical/r1i1p1f1/{variable}/*.nc" - file_list += [f"s3://{path}" for path in s3fs.glob(data_dir)] - files = [s3fs.open(f) for f in file_list] + files += [f"s3://{path}" for path in fs.glob(data_dir)] print(f"Processing {len(files)} NetCDF files") # Load input NetCDF data files