Skip to content

Commit

Permalink
Use string paths with open_mfdataset
Browse files Browse the repository at this point in the history
  • Loading branch information
jrbourbeau committed Nov 25, 2024
1 parent 4c29fdf commit 65426c3
Showing 1 changed file with 3 additions and 4 deletions.
7 changes: 3 additions & 4 deletions tests/geospatial/workloads/cloud_optimize.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@


def cloud_optimize(
scale: Literal["small", "medium", "large"], s3fs: S3FileSystem, storage_url: str
scale: Literal["small", "medium", "large"], fs: S3FileSystem, storage_url: str
):
models = [
"ACCESS-CM2",
Expand Down Expand Up @@ -59,12 +59,11 @@ def cloud_optimize(

# Get netCDF data files -- see https://registry.opendata.aws/nex-gddp-cmip6
# for dataset details.
file_list = []
files = []
for model in models:
for variable in variables:
data_dir = f"s3://nex-gddp-cmip6/NEX-GDDP-CMIP6/{model}/historical/r1i1p1f1/{variable}/*.nc"
file_list += [f"s3://{path}" for path in s3fs.glob(data_dir)]
files = [s3fs.open(f) for f in file_list]
files += [f"s3://{path}" for path in fs.glob(data_dir)]
print(f"Processing {len(files)} NetCDF files")

# Load input NetCDF data files
Expand Down

0 comments on commit 65426c3

Please sign in to comment.