diff --git a/dist/aws/Dockerfile b/dist/aws/Dockerfile deleted file mode 100644 index b1ec82400..000000000 --- a/dist/aws/Dockerfile +++ /dev/null @@ -1,65 +0,0 @@ -FROM amazonlinux:latest - -ARG REF="master" - -# development tools -RUN yum update -y && yum -y install \ - groupinstall \ - development \ - gcc \ - gcc-c++ \ - git \ - zip \ - freetype-devel \ - yum-utils \ - findutils \ - openssl-devel \ - && yum clean all - -# Mock current AWS Lambda docker image -# NOTE: this is still Py3.7, need to be careful about version management -RUN yum -y install \ - python3 \ - python3-pip \ - python3-devel \ - && yum clean all - -# clone the podpac repository and checkout the requested tag -# for developers looking to create a custom deployment package or dependencies, -# comment this block and un-comment the next block -RUN git clone https://github.com/creare-com/podpac.git /podpac/ &&\ - pushd /podpac/ && \ - git fetch --all && \ - git checkout $REF && \ - popd - -# # uncomment this block to create a custom deployment package or dependencies archive -# # based on your local copy of the PODPAC repository -# # this command assumes you are building the Dockerfile using `build_lambda.sh` (which runs from the root of the PODPAC repository ) -# ADD . /podpac/ - -# Install core, datatype and aws optional dependencies -RUN mkdir /tmp/vendored/ && \ - cd /podpac/ && rm -rf .git/ doc/ .github/ && \ - pip3 install . -t /tmp/vendored/ --upgrade && \ - pip3 install .[datatype] -t /tmp/vendored/ --upgrade && \ - pip3 install .[aws] -t /tmp/vendored/ --upgrade && \ - pip3 install .[algorithms] -t /tmp/vendored/ --upgrade - -# need to add some __init__ files -RUN cd /tmp/vendored/ && touch pydap/__init__.py && \ - touch pydap/responses/__init__.py && \ - touch pydap/handlers/__init__.py && \ - touch pydap/parsers/__init__.py - -# copy handler and _mk_dist: -RUN cp /podpac/dist/aws/handler.py /tmp/vendored/handler.py && \ - cp /podpac/dist/aws/_mk_dist.py /tmp/vendored/_mk_dist.py - -RUN cd /tmp/vendored && \ - find * -maxdepth 0 -type f | grep ".zip" -v | grep -v ".pyc" | xargs zip -9 -rqy podpac_dist.zip -RUN cd /tmp/vendored && \ - find * -maxdepth 0 -type d -exec zip -9 -rqy {}.zip {} \; -RUN cd /tmp/vendored && du -s *.zip > zip_package_sizes.txt -RUN cd /tmp/vendored && du -s * | grep .zip -v > package_sizes.txt -RUN cd /tmp/vendored && python3 _mk_dist.py diff --git a/dist/aws/README.md b/dist/aws/README.md deleted file mode 100644 index 5c74d6d42..000000000 --- a/dist/aws/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# AWS - -## Public PODPAC distribution - -PODPAC is compiled into two distribution .zip archives for building serverless functions. -PODPAC is compile for each version: - -- `podpac_dist.zip`: Archive containing PODPAC core distribution -- `podpac_deps.zip`: Archive containing PODPAC dependencies - -These archives are posted publically in the S3 bucket `podpac-dist`. -This bucket has one directory for each podpac version. -The bucket itself is private, but each directory is made public individually. - -### Creating new distribution - -The following process is used to create new PODPAC distribution in the `podpac-dist` bucket -when a new version of PODPAC is released. - -- Run `build_lambda.sh` -- Run `upload_lambda.sh` -- Navigate to `podpac-dist` (or input bucket) and make the archives public diff --git a/dist/aws/_mk_dist.py b/dist/aws/_mk_dist.py deleted file mode 100644 index 6fdb7a6eb..000000000 --- a/dist/aws/_mk_dist.py +++ /dev/null @@ -1,65 +0,0 @@ -#!python - -# Make dist and deps zip archives -# for podpac lambda function -# -# Used in the Dockerfile - -import subprocess - -with open("zip_package_sizes.txt", "r") as fid: - zps = fid.read() - -with open("package_sizes.txt", "r") as fid: - ps = fid.read() - - -def parse_ps(ps): - lns = ps.split("\n") - pkgs = {} - for ln in lns: - try: - parts = ln.split("\t") - pkgs[parts[1]] = int(parts[0]) - except: - pass - return pkgs - - -pgz = parse_ps(zps) -pg = parse_ps(ps) - -data = {} -for p, s in pgz.items(): - os = pg.get(p[:-4], 0) - data[p] = {"zip_size": s, "size": os, "ratio": os * 1.0 / s} - -sdata = sorted(data.items(), key=lambda t: t[1]["ratio"]) - -zipsize = data["podpac_dist.zip"]["zip_size"] -totsize = sum([pg[k] for k in pg if (k + ".zip") not in pgz.keys()]) -pkgs = [] -for val in sdata[::-1]: - if val[0] == "podpac_dist.zip" or "rasterio" in val[0] or "pyproj" in val[0]: - continue - key = val[0] - pkgs.append(key) - - zipsize += data[key]["zip_size"] - totsize += data[key]["size"] - - if zipsize > 50000 or totsize > 250000: - k = pkgs.pop() - zipsize -= data[k]["zip_size"] - totsize -= data[k]["size"] - -core = [k[:-4] for k in pkgs if k != "podpac_dist.zip"] -deps = [k[:-4] for k in data if k[:-4] not in core and k != "podpac_dist.zip"] -dep_size = sum([data[k + ".zip"]["size"] for k in deps]) -dep_size_zip = sum([data[k + ".zip"]["zip_size"] for k in deps]) - -# add core to podpac_dist.zip -cmd = ["zip", "-9", "-rq", "podpac_dist.zip"] + core -subprocess.call(cmd) -cmd = ["zip", "-9", "-rqy", "podpac_deps.zip"] + deps -subprocess.call(cmd) diff --git a/dist/aws/build_lambda.sh b/dist/aws/build_lambda.sh deleted file mode 100644 index 8fbf5ad8e..000000000 --- a/dist/aws/build_lambda.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# -# Build podpac lambda distribution and dependencies. -# Change $REF to specify a specific branch, tag, or commit in podpac to build from. -# -# Usage: -# -# $ bash build_lambda.sh -# -# Requires: -# - Docker - -# variables -REF="master" -# REF="tags/1.1.0" # Change $REF to the branch, tag, or commit in podpac you want to use -# REF="develop" - -DOCKER_NAME="podpac" -DOCKER_TAG=$REF - -echo "Creating docker image from podpac version ${REF}" -echo "${DOCKER_NAME}:${DOCKER_TAG}" - -# Navigate to root, build docker, and extract zips -pushd ../../ -docker build -f dist/aws/Dockerfile --no-cache --tag $DOCKER_NAME:$DOCKER_TAG --build-arg REF="${REF}" . -docker run --name "${DOCKER_NAME}" -itd $DOCKER_NAME:$DOCKER_TAG -docker cp "${DOCKER_NAME}":/tmp/vendored/podpac_dist.zip ./dist/aws -docker cp "${DOCKER_NAME}":/tmp/vendored/podpac_deps.zip ./dist/aws -docker stop "${DOCKER_NAME}" -docker rm "${DOCKER_NAME}" -popd - -echo "Built podpac deployment package: podpac_dist.zip" -echo "Built podpac dependencies: podpac_deps.zip" diff --git a/dist/aws/handler.py b/dist/aws/handler.py deleted file mode 100644 index c6bd924b0..000000000 --- a/dist/aws/handler.py +++ /dev/null @@ -1,313 +0,0 @@ -""" -PODPAC AWS Handler -""" - -import json -import subprocess -import sys -import urllib.parse as urllib -import os - -import boto3 -import botocore - -from six import string_types - - -def default_pipeline(pipeline=None): - """Get default pipeline definiton, merging with input pipline if supplied - - Parameters - ---------- - pipeline : dict, optional - Input pipline. Will fill in any missing defaults. - - Returns - ------- - dict - pipeline dict - """ - defaults = { - "pipeline": {}, - "settings": {}, - "output": {"format": "netcdf", "filename": None, "format_kwargs": {}}, - # API Gateway - "url": "", - "params": {}, - } - - # merge defaults with input pipelines, if supplied - if pipeline is not None: - pipeline = {**defaults, **pipeline} - pipeline["output"] = {**defaults["output"], **pipeline["output"]} - pipeline["settings"] = {**defaults["settings"], **pipeline["settings"]} - else: - pipeline = defaults - - # overwrite certain settings so that the function doesn't fail - pipeline["settings"]["ROOT_PATH"] = "/tmp" - pipeline["settings"]["LOG_FILE_PATH"] = "/tmp/podpac.log" - - return pipeline - - -def get_trigger(event): - """ - Helper method to determine the trigger for the lambda invocation - - Parameters - ---------- - event : dict - Event dict from AWS. See [TODO: add link reference] - - Returns - ------- - str - One of "S3", "eval", or "APIGateway" - """ - - if "Records" in event and event["Records"][0]["eventSource"] == "aws:s3": - return "S3" - elif "queryStringParameters" in event: - return "APIGateway" - else: - return "eval" - - -def parse_event(trigger, event): - """Parse pipeline, settings, and output details from event depending on trigger - - Parameters - ---------- - trigger : str - One of "S3", "eval", or "APIGateway" - event : dict - Event dict from AWS. See [TODO: add link reference] - """ - - if trigger == "eval": - print("Triggered by Invoke") - - # event is the pipeline, provide consistent pipeline defaults - pipeline = default_pipeline(event) - - return pipeline - - elif trigger == "S3": - print("Triggered from S3") - - # get boto s3 client - s3 = boto3.client("s3") - - # We always have to look to the bucket that triggered the event for the input - triggered_bucket = event["Records"][0]["s3"]["bucket"]["name"] - - # get the pipeline object and read - file_key = urllib.unquote_plus(event["Records"][0]["s3"]["object"]["key"]) - pipline_obj = s3.get_object(Bucket=triggered_bucket, Key=file_key) - pipeline = json.loads(pipline_obj["Body"].read().decode("utf-8")) - - # provide consistent pipeline defaults - pipeline = default_pipeline(pipeline) - - # create output filename - pipeline["output"]["filename"] = file_key.replace(".json", "." + pipeline["output"]["format"]).replace( - pipeline["settings"]["FUNCTION_S3_INPUT"], pipeline["settings"]["FUNCTION_S3_OUTPUT"] - ) - - if not pipeline["settings"]["FUNCTION_FORCE_COMPUTE"]: - - # get configured s3 bucket to check for cache - bucket = pipeline["settings"]["S3_BUCKET_NAME"] - - # We can return if there is a valid cached output to save compute time. - try: - s3.head_object(Bucket=bucket, Key=pipeline["output"]["filename"]) - return None - - # throws ClientError if no file is found - except botocore.exceptions.ClientError: - pass - - # return pipeline definition - return pipeline - - elif trigger == "APIGateway": - print("Triggered from API Gateway") - - pipeline = default_pipeline() - pipeline["url"] = event["queryStringParameters"] - if isinstance(pipeline["url"], string_types): - pipeline["url"] = urllib.parse_qs(urllib.urlparse(pipeline["url"]).query) - - # These are parameters not part of the OGC spec, which are stored in the "PARAMS" variable (which is part of the spec) - pipeline["params"] = event["queryStringParameters"].get("params", "{}") - if isinstance(pipeline["params"], string_types): - pipeline["params"] = json.loads(pipeline["params"]) - - # make all params lowercase - pipeline["params"] = [param.lower() for param in pipeline["params"]] - - # look for specific parameter definitions in query parameters, these are not part of the OGC spec - for param in pipeline["params"]: - # handle SETTINGS in query parameters - if param == "settings": - # Try loading this settings string into a dict to merge with default settings - try: - api_settings = pipeline["params"][param] - # If we get here, the api settings were loaded - pipeline["settings"] = {**pipeline["settings"], **api_settings} - except Exception as e: - print("Got an exception when attempting to load api settings: ", e) - print(pipeline) - - # handle OUTPUT in query parameters - elif param == "output": - pipeline["output"] = pipeline["params"][param] - # handle FORMAT in query parameters - elif param == "format": - pipeline["output"]["format"] = pipeline["params"][param].split("/")[-1] - # handle image returns - if pipeline["output"]["format"] in ["png", "jpg", "jpeg"]: - pipeline["output"]["format_kwargs"]["return_base64"] = True - - # Check for the FORMAT QS parameter, as it might be part of the OGC spec - for param in pipeline["url"]: - if param.lower() == "format": - pipeline["output"][param] = pipeline["url"][param].split("/")[-1] - # handle image returns - if pipeline["output"]["format"] in ["png", "jpg", "jpeg"]: - pipeline["output"]["format_kwargs"]["return_base64"] = True - - return pipeline - - else: - raise Exception("Unsupported trigger") - - -def handler(event, context): - """Lambda function handler - - Parameters - ---------- - event : dict - Description - context : TYPE - Description - get_deps : bool, optional - Description - ret_pipeline : bool, optional - Description - """ - print(event) - - # Add /tmp/ path to handle python path for dependencies - sys.path.append("/tmp/") - - # handle triggers - trigger = get_trigger(event) - - # parse event - pipeline = parse_event(trigger, event) - - # bail if we can't parse - if pipeline is None: - return - - # ----- - # TODO: remove when layers is configured - # get configured bucket to download dependencies - # If specified in the environmental variables, we cannot overwrite it. Otherwise it HAS to be - # specified in the settings. - bucket = os.environ.get("S3_BUCKET_NAME", pipeline["settings"].get("S3_BUCKET_NAME")) - - # get dependencies path - if "FUNCTION_DEPENDENCIES_KEY" in pipeline["settings"] or "FUNCTION_DEPENDENCIES_KEY" in os.environ: - dependencies = os.environ.get( - "FUNCTION_DEPENDENCIES_KEY", pipeline["settings"].get("FUNCTION_DEPENDENCIES_KEY") - ) - else: - dependencies = "podpac_deps_{}.zip".format( - os.environ.get("PODPAC_VERSION", pipeline["settings"].get("PODPAC_VERSION")) - ) - if "None" in dependencies: - dependencies = "podpac_deps.zip" # Development version of podpac - # this should be equivalent to version.semver() - - # Check to see if this function is "hot", in which case the dependencies have already been downloaded and are - # available for use right away. - if os.path.exists("/tmp/scipy"): - print( - "Scipy has been detected in the /tmp/ directory. Assuming this function is hot, dependencies will" - " not be downloaded." - ) - else: - # Download dependencies from specific bucket/object - print("Downloading and extracting dependencies from {} {}".format(bucket, dependencies)) - s3 = boto3.client("s3") - s3.download_file(bucket, dependencies, "/tmp/" + dependencies) - subprocess.call(["unzip", "/tmp/" + dependencies, "-d", "/tmp"]) - sys.path.append("/tmp/") - subprocess.call(["rm", "/tmp/" + dependencies]) - # ----- - - # Load PODPAC - - # Need to set matplotlib backend to 'Agg' before importing it elsewhere - import matplotlib - - matplotlib.use("agg") - from podpac import settings - from podpac.core.node import Node - from podpac.core.coordinates import Coordinates - from podpac.core.utils import JSONEncoder, _get_query_params_from_url - import podpac.datalib - - # update podpac settings with inputs from the trigger - settings.update(json.loads(os.environ.get("SETTINGS", "{}"))) - settings.update(pipeline["settings"]) - - # build the Node and Coordinates - if trigger in ("eval", "S3"): - node = Node.from_definition(pipeline["pipeline"]) - coords = Coordinates.from_json(json.dumps(pipeline["coordinates"], indent=4, cls=JSONEncoder)) - - # TODO: handle API Gateway better - is this always going to be WCS? - elif trigger == "APIGateway": - node = Node.from_url(pipeline["url"]) - coords = Coordinates.from_url(pipeline["url"]) - - # make sure pipeline is allowed to be run - if "PODPAC_RESTRICT_PIPELINES" in os.environ: - whitelist = json.loads(os.environ["PODPAC_RESTRICT_PIPELINES"]) - if node.hash not in whitelist: - raise ValueError("Node hash is not in the whitelist for this function") - - # run analysis - output = node.eval(coords) - - # convert to output format - body = output.to_format(pipeline["output"]["format"], **pipeline["output"]["format_kwargs"]) - - # Response - if trigger == "eval": - return body - - elif trigger == "S3": - s3.put_object(Bucket=settings["S3_BUCKET_NAME"], Key=pipeline["output"]["filename"], Body=body) - - elif trigger == "APIGateway": - - # TODO: can we handle the deserialization better? - try: - json.dumps(body) - except Exception as e: - print("Output body is not serializable, attempting to decode.") - body = body.decode() - - return { - "statusCode": 200, - "headers": {"Content-Type": pipeline["output"]["format"]}, - "isBase64Encoded": pipeline["output"]["format_kwargs"]["return_base64"], - "body": body, - } diff --git a/dist/aws/print_logs.sh b/dist/aws/print_logs.sh deleted file mode 100644 index 8b95fc094..000000000 --- a/dist/aws/print_logs.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -# -# Print cloudwatch logs to console for a user specific log group -# Defaults to showing the last 10 minutes of logs -# -# Usage: -# -# $ bash print_logs.sh [log-group-name] [amount of time] -# -# [amount of time] must be a parameter than can be passed to the `date --date ` function -# i.e. "10 minutes", "1 hour", "1 day", "1 week" -# -# Example: -# -# $ bash print_logs.sh /aws/lambda/podpac-dist # show the last 10 minutes of logs -# $ bash print_logs.sh /aws/lambda/podpac-dist "1 hour" # show the last hour of logs - -function dumpstreams() { - aws $AWSARGS logs describe-log-streams --order-by LastEventTime --log-group-name "$LOGGROUP" --output text | while read -a st; do - [ "${st[4]}" -lt "$starttime" ] && continue - stname="${st[1]}" - echo ${stname##*:} - done | while read stream; do - aws $AWSARGS logs get-log-events --start-from-head --start-time $starttime --log-group-name "$LOGGROUP" --log-stream-name $stream --output text - done -} - -AWSARGS="--region us-east-1" -LOGGROUP="$1" -DEFAULT_DT='10 minutes' -DT=${2:-${DEFAULT_DT}} -TAIL= -starttime=$(date --date "-${DT}" +%s)000 -nexttime=$(date +%s)000 -dumpstreams -if [ -n "$TAIL" ]; then - while true; do - starttime=$nexttime - nexttime=$(date +%s)000 - sleep 1 - dumpstreams - done -fi diff --git a/dist/aws/upload_lambda.sh b/dist/aws/upload_lambda.sh deleted file mode 100644 index 5a399ec12..000000000 --- a/dist/aws/upload_lambda.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/sh -# -# Upload podpac lambda distribution and dependencies -# Change $BUCKET or $DIR to control the S3 Bucket and Bucket path -# where zip archives are uploaded. -# -# Usage: -# -# $ bash upload_lambda.sh -# -# Requires: -# - AWS CLI: https://docs.aws.amazon.com/cli/ -# - AWS credentials must be configured using the `aws` cli. -# See https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html#cli-quick-configuration - - -BUCKET="podpac-dist" -DIR="dev" -# DIR="1.3.0" # for releases, upload to release path by semantic version - -AWSPATH="s3://$BUCKET/$DIR" -echo "Uploading podpac distribution to S3 path: ${AWSPATH}" - -# Upload zips to S3 -aws s3 cp podpac_deps.zip $AWSPATH/podpac_deps.zip -aws s3 cp podpac_dist.zip $AWSPATH/podpac_dist.zip - -echo "Navigate to your bucket $BUCKET, select the zip archives you just uploaded and make them public" diff --git a/dist/local_Windows_install/README.MD b/dist/local_Windows_install/README.MD deleted file mode 100644 index 89ac92589..000000000 --- a/dist/local_Windows_install/README.MD +++ /dev/null @@ -1,14 +0,0 @@ -# Quickstart - -You are using the PODPAC Windows 10 installation. - -To get started running Jupyter notebooks, double-click on the `run_podpac_jupyterlab.bat` file in this folder. - -To run an IPython session: -1. Open a Windows command prompt in this directory -2. Run the `run_ipython.bat` script - -For additional documentation, see the [README.MD](podpac/README.MD) file in the PODPAC folder. - -The official PODPAC documentation is also available here: https://podpac.org - diff --git a/dist/local_Windows_install/bin/activate_podpac_conda_env.bat b/dist/local_Windows_install/bin/activate_podpac_conda_env.bat deleted file mode 100644 index f587b2145..000000000 --- a/dist/local_Windows_install/bin/activate_podpac_conda_env.bat +++ /dev/null @@ -1,5 +0,0 @@ -@ECHO OFF -ECHO "Activating PODPAC environment." -REM This assumes that set_local_conda_path.bat has been called -SET CURL_CA_BUNDLE=%mypath%miniconda\envs\podpac\Library\ssl\cacert.pem -conda activate podpac diff --git a/dist/local_Windows_install/bin/fix_hardcoded_absolute_paths.bat b/dist/local_Windows_install/bin/fix_hardcoded_absolute_paths.bat deleted file mode 100644 index 014fd3ab9..000000000 --- a/dist/local_Windows_install/bin/fix_hardcoded_absolute_paths.bat +++ /dev/null @@ -1,21 +0,0 @@ -@ECHO OFF -ECHO "Fixing hard-coded paths included in local PODPAC conda installation." -REM Write/fix the kernel.json file -set kernels={"argv": [" -set kernele=miniconda\\envs\\podpac\\python.exe", "-m", "ipykernel_launcher", "-f", "{connection_file}" ], "display_name": "Python 3", "language": "python"} - -set mypathescaped=%mypath:\=\\% -set mypathfwd=%mypath:\=/% - -del "miniconda\envs\podpac\share\jupyter\kernels\python3\kernel.json" -echo %kernels%%mypathescaped%%kernele% >> miniconda\envs\podpac\share\jupyter\kernels\python3\kernel.json - -REM Write/Fix qt.conf - -del "miniconda\envs\podpac\qt.conf" -echo [Paths] >> "miniconda\envs\podpac\qt.conf" -echo Prefix = %mypathfwd%miniconda/envs/podpac/Library >> "miniconda\envs\podpac\qt.conf" -echo Binaries = %mypathfwd%miniconda/envs/podpac/Library/bin >> "miniconda\envs\podpac\qt.conf" -echo Libraries = %mypathfwd%miniconda/envs/podpac/Library/lib >> "miniconda\envs\podpac\qt.conf" -echo Headers = %mypathfwd%miniconda/envs/podpac/Library/include/qt >> "miniconda\envs\podpac\qt.conf" - diff --git a/dist/local_Windows_install/bin/set_local_conda_path.bat b/dist/local_Windows_install/bin/set_local_conda_path.bat deleted file mode 100644 index b5c6a7b44..000000000 --- a/dist/local_Windows_install/bin/set_local_conda_path.bat +++ /dev/null @@ -1,7 +0,0 @@ -@ECHO OFF -ECHO "Setting system path to use local PODPAC conda installation" -SET mypath=%~dp0..\ -SET CONDAPATH=%mypath%miniconda;%mypath%miniconda\Library\mingw-w64\bin;%mypath%miniconda\Library\usr\bin;%mypath%miniconda\Library\bin;%mypath%miniconda\Scripts -SET PATH=%CONDAPATH%;%PATH% -SET GDAL_DATA=%mypath%miniconda\envs\podpac\Lib\site-packages\osgeo\data\gdal -SET PYTHONPATH=%mypath%podpac; diff --git a/dist/local_Windows_install/run_ipython.bat b/dist/local_Windows_install/run_ipython.bat deleted file mode 100644 index f19a85ebf..000000000 --- a/dist/local_Windows_install/run_ipython.bat +++ /dev/null @@ -1,5 +0,0 @@ -@echo off -call bin\set_local_conda_path.bat -call bin\fix_hardcoded_absolute_paths.bat -call bin\activate_podpac_conda_env.bat -ipython \ No newline at end of file diff --git a/dist/local_Windows_install/run_podpac_jupyterlab.bat b/dist/local_Windows_install/run_podpac_jupyterlab.bat deleted file mode 100644 index 9a7ab05ab..000000000 --- a/dist/local_Windows_install/run_podpac_jupyterlab.bat +++ /dev/null @@ -1,9 +0,0 @@ -@ECHO OFF -ECHO "Launching PODPAC Jupyter Lab notebooks." -call bin\set_local_conda_path.bat -call bin\fix_hardcoded_absolute_paths.bat -call bin\activate_podpac_conda_env.bat - -cd podpac-examples -jupyter lab -cd .. diff --git a/dist/local_Windows_install/update_podpac.bat b/dist/local_Windows_install/update_podpac.bat deleted file mode 100644 index 91eae5657..000000000 --- a/dist/local_Windows_install/update_podpac.bat +++ /dev/null @@ -1,24 +0,0 @@ -@echo off -call bin\set_local_conda_path.bat -call bin\fix_hardcoded_absolute_paths.bat -call bin\activate_podpac_conda_env.bat - -cd podpac -echo "Updating PODPAC" -git fetch -for /f %%a in ('git describe --tags --abbrev^=0 origin/master') do git checkout %%a -cd .. -echo "Updating PODPAC EXAMPLES" -cd podpac-examples -git fetch -for /f %%a in ('git describe --tags --abbrev^=0 origin/master') do git checkout %%a -cd .. -cd podpac -cd dist -echo "Updating CONDA ENVIRONMENT" -conda env update -f windows_conda_environment.yml -cd .. -cd .. - - - diff --git a/dist/windows_conda_environment.json b/dist/windows_conda_environment.json deleted file mode 100644 index 729133bc0..000000000 --- a/dist/windows_conda_environment.json +++ /dev/null @@ -1,215 +0,0 @@ -# This file may be used to create an environment using: -# $ conda create --name --file -# platform: win-64 -@EXPLICIT -https://repo.anaconda.com/pkgs/main/win-64/blas-1.0-mkl.conda -https://repo.anaconda.com/pkgs/main/win-64/ca-certificates-2020.6.24-0.conda -https://repo.anaconda.com/pkgs/main/win-64/icc_rt-2019.0.0-h0cc432a_1.conda -https://repo.anaconda.com/pkgs/main/win-64/intel-openmp-2020.0-166.conda -https://repo.anaconda.com/pkgs/msys2/win-64/msys2-conda-epoch-20160418-1.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/pandoc-2.2.3.2-0.conda -https://conda.anaconda.org/conda-forge/noarch/poppler-data-0.4.9-1.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/vs2015_runtime-14.16.27012-hf0eaf9b_1.conda -https://repo.anaconda.com/pkgs/main/win-64/winpty-0.4.3-4.conda -https://repo.anaconda.com/pkgs/msys2/win-64/m2w64-expat-2.1.1-2.tar.bz2 -https://repo.anaconda.com/pkgs/msys2/win-64/m2w64-gmp-6.1.0-2.tar.bz2 -https://repo.anaconda.com/pkgs/msys2/win-64/m2w64-libiconv-1.14-6.tar.bz2 -https://repo.anaconda.com/pkgs/msys2/win-64/m2w64-libwinpthread-git-5.0.0.4634.697f757-2.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/mkl-2020.0-166.conda -https://repo.anaconda.com/pkgs/main/win-64/nodejs-10.13.0-0.conda -https://repo.anaconda.com/pkgs/main/win-64/vc-14.1-h0510ff6_4.conda -https://conda.anaconda.org/conda-forge/win-64/bzip2-1.0.8-hfa6e2cd_2.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/cfitsio-3.470-hfa6e2cd_2.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/expat-2.2.9-he025d50_2.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/geos-3.8.1-he025d50_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/icu-64.2-he025d50_1.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/jpeg-9c-hfa6e2cd_1001.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/libiconv-1.15-h1df5818_7.conda -https://repo.anaconda.com/pkgs/main/win-64/libsodium-1.0.16-h9d3ae62_0.conda -https://conda.anaconda.org/conda-forge/win-64/libwebp-base-1.1.0-hfa6e2cd_3.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/lz4-c-1.9.2-h33f27b4_0.tar.bz2 -https://repo.anaconda.com/pkgs/msys2/win-64/m2w64-gcc-libs-core-5.3.0-7.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/openssl-1.1.1g-he774522_0.conda -https://conda.anaconda.org/conda-forge/win-64/pcre-8.44-h6538335_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/tbb-2018.0.5-he980bc4_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/tk-8.6.8-hfa6e2cd_0.conda -https://conda.anaconda.org/conda-forge/win-64/xerces-c-3.2.2-h6538335_1004.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/xz-5.2.5-h62dcd97_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/yaml-0.2.5-he774522_0.conda -https://repo.anaconda.com/pkgs/main/win-64/zlib-1.2.11-h62dcd97_4.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/boost-cpp-1.72.0-h0caebb8_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/freexl-1.0.5-hd288d7e_1002.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/hdf4-4.2.13-hf8e6fe8_1003.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/hdf5-1.10.5-nompi_ha405e13_1104.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/krb5-1.17.1-hc04afaa_0.conda -https://repo.anaconda.com/pkgs/main/win-64/libpng-1.6.37-h2a8f88b_0.conda -https://conda.anaconda.org/conda-forge/win-64/libssh2-1.8.2-h642c060_2.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/libxml2-2.9.10-h9ce36c8_0.tar.bz2 -https://repo.anaconda.com/pkgs/msys2/win-64/m2w64-gcc-libgfortran-5.3.0-6.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/sqlite-3.31.1-h2a8f88b_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/zeromq-4.3.1-h33f27b4_3.conda -https://conda.anaconda.org/conda-forge/win-64/zstd-1.4.4-h9f78265_3.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/freetype-2.9.1-ha9979f8_1.conda -https://conda.anaconda.org/conda-forge/win-64/kealib-1.4.13-hd6dc3df_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/libcurl-7.69.1-h1dcc11c_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/libkml-1.3.0-h7e985d0_1011.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/libpq-12.2-hd9aa61d_1.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/libtiff-4.1.0-h885aae3_6.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/libxslt-1.1.33-h579f668_0.conda -https://repo.anaconda.com/pkgs/msys2/win-64/m2w64-gcc-libs-5.3.0-7.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/python-3.7.7-h60c2a47_2.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/qt-5.9.7-h506e8af_3.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/affine-2.3.0-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/appdirs-1.4.3-py37h28b3542_0.conda -https://repo.anaconda.com/pkgs/main/noarch/asciitree-0.3.3-py_2.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/asn1crypto-1.3.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/attrs-19.3.0-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/backcall-0.1.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/certifi-2020.6.20-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/chardet-3.0.4-py37_1003.conda -https://conda.anaconda.org/conda-forge/noarch/click-7.1.1-pyh8c360ce_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/cloudpickle-1.5.0-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/colorama-0.4.3-py_0.conda -https://conda.anaconda.org/conda-forge/win-64/curl-7.69.1-h1dcc11c_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/decorator-4.4.2-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/defusedxml-0.6.0-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/docutils-0.15.2-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/entrypoints-0.3-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/fsspec-0.7.1-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/heapdict-1.0.1-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/idna-2.9-py_1.conda -https://repo.anaconda.com/pkgs/main/win-64/ipython_genutils-0.2.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/jmespath-0.9.4-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/json5-0.9.4-py_0.conda -https://repo.anaconda.com/pkgs/main/win-64/kiwisolver-1.1.0-py37ha925a31_0.conda -https://conda.anaconda.org/conda-forge/win-64/libffi-3.2.1-h6538335_1007.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/locket-0.2.0-py37_1.conda -https://repo.anaconda.com/pkgs/main/win-64/lxml-4.5.0-py37h1350720_0.conda -https://repo.anaconda.com/pkgs/msys2/win-64/m2w64-gettext-0.19.7-2.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/markupsafe-1.1.1-py37he774522_0.conda -https://repo.anaconda.com/pkgs/main/win-64/mistune-0.8.4-py37he774522_0.conda -https://repo.anaconda.com/pkgs/main/noarch/monotonic-1.5-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/msgpack-python-1.0.0-py37h74a9793_1.conda -https://repo.anaconda.com/pkgs/main/win-64/olefile-0.46-py37_0.conda -https://conda.anaconda.org/conda-forge/win-64/openjpeg-2.3.1-h57dd2e7_3.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/pandocfilters-1.4.2-py37_1.conda -https://repo.anaconda.com/pkgs/main/noarch/parso-0.6.2-py_0.conda -https://repo.anaconda.com/pkgs/main/win-64/pickleshare-0.7.5-py37_0.conda -https://conda.anaconda.org/conda-forge/win-64/postgresql-12.2-he14cc48_1.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/proj-7.0.0-haa36216_3.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/prometheus_client-0.7.1-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/psutil-5.7.0-py37he774522_0.conda -https://repo.anaconda.com/pkgs/main/noarch/pycparser-2.20-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/pyparsing-2.4.6-py_0.conda -https://repo.anaconda.com/pkgs/main/win-64/pyreadline-2.1-py37_1.conda -https://repo.anaconda.com/pkgs/main/noarch/pyshp-2.1.0-py_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/python_abi-3.7-1_cp37m.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/pytz-2019.3-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/pywin32-227-py37he774522_1.conda -https://repo.anaconda.com/pkgs/main/win-64/pywinpty-0.5.7-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/pyyaml-5.3.1-py37he774522_1.conda -https://repo.anaconda.com/pkgs/main/win-64/pyzmq-18.1.1-py37ha925a31_0.conda -https://repo.anaconda.com/pkgs/main/win-64/send2trash-1.5.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/sip-4.19.8-py37h6538335_0.conda -https://repo.anaconda.com/pkgs/main/win-64/six-1.14.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/sortedcontainers-2.2.2-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/soupsieve-2.0-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/tblib-1.6.0-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/testpath-0.4.4-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/toolz-0.10.0-py_0.conda -https://repo.anaconda.com/pkgs/main/win-64/tornado-6.0.4-py37he774522_1.conda -https://repo.anaconda.com/pkgs/main/noarch/typing_extensions-3.7.4.2-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/wcwidth-0.1.9-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/webencodings-0.5.1-py37_1.conda -https://repo.anaconda.com/pkgs/main/win-64/win_inet_pton-1.1.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/wincertstore-0.2-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/zipp-2.2.0-py_0.conda -https://repo.anaconda.com/pkgs/main/win-64/beautifulsoup4-4.9.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/cffi-1.14.0-py37h7a1dbc1_0.conda -https://conda.anaconda.org/conda-forge/noarch/click-plugins-1.1.1-py_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/cligj-0.5.0-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/cycler-0.10.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/cytoolz-0.10.1-py37he774522_0.conda -https://repo.anaconda.com/pkgs/main/noarch/dask-core-2.20.0-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/fasteners-0.15-py_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/geotiff-1.5.1-h3d29ae3_10.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/gettext-0.19.8.1-hb01d8f6_1002.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/importlib_metadata-1.5.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/jedi-0.16.0-py37_1.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/libnetcdf-4.7.4-nompi_hc957ea6_101.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/libspatialite-4.3.0a-h51df0ed_1038.tar.bz2 -https://repo.anaconda.com/pkgs/msys2/win-64/m2w64-xz-5.2.2-2.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/mkl-service-2.3.0-py37hb782905_0.conda -https://repo.anaconda.com/pkgs/main/noarch/packaging-20.4-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/partd-1.1.0-py_0.conda -https://repo.anaconda.com/pkgs/main/win-64/pillow-7.0.0-py37hcc1f983_0.conda -https://conda.anaconda.org/conda-forge/win-64/pyproj-2.6.0-py37he833962_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/pyqt-5.9.2-py37h6538335_2.conda -https://repo.anaconda.com/pkgs/main/win-64/pyrsistent-0.16.0-py37he774522_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/pysocks-1.7.1-py37_0.conda -https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.7.5-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/setuptools-46.1.3-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/terminado-0.8.3-py37_0.conda -https://conda.anaconda.org/conda-forge/win-64/tiledb-1.7.7-h0b90766_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/traitlets-4.3.3-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/zict-2.0.0-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/bleach-3.1.4-py_0.conda -https://repo.anaconda.com/pkgs/main/win-64/cryptography-2.8-py37h7a1dbc1_0.conda -https://repo.anaconda.com/pkgs/main/win-64/distributed-2.20.0-py37_0.conda -https://conda.anaconda.org/conda-forge/win-64/glib-2.64.2-he4de6d7_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/jinja2-2.11.1-py_0.conda -https://repo.anaconda.com/pkgs/main/win-64/jsonschema-3.2.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/jupyter_core-4.6.3-py37_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/numpy-base-1.18.1-py37hc3f5095_1.conda -https://repo.anaconda.com/pkgs/main/noarch/pygments-2.6.1-py_0.conda -https://conda.anaconda.org/conda-forge/noarch/traittypes-0.2.1-py_1.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/wheel-0.34.2-py37_0.conda -https://conda.anaconda.org/conda-forge/noarch/branca-0.3.1-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/jupyter_client-6.1.2-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/nbformat-5.0.4-py_0.conda -https://repo.anaconda.com/pkgs/main/win-64/pip-20.0.2-py37_1.conda -https://conda.anaconda.org/conda-forge/win-64/poppler-0.67.0-h1707e21_8.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/prompt-toolkit-3.0.4-py_0.conda -https://repo.anaconda.com/pkgs/main/win-64/pyopenssl-19.1.0-py37_0.conda -https://conda.anaconda.org/conda-forge/win-64/libgdal-3.0.4-h821c9b7_6.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/nbconvert-5.6.1-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/prompt_toolkit-3.0.4-0.conda -https://repo.anaconda.com/pkgs/main/win-64/urllib3-1.25.8-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/botocore-1.15.39-py_0.conda -https://repo.anaconda.com/pkgs/main/win-64/ipython-7.13.0-py37h5ca1d4c_0.conda -https://repo.anaconda.com/pkgs/main/win-64/requests-2.23.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/ipykernel-5.1.4-py37h39e3cac_0.conda -https://repo.anaconda.com/pkgs/main/noarch/owslib-0.18.0-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/pyepsg-0.4.0-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/s3transfer-0.3.3-py37_0.conda -https://conda.anaconda.org/conda-forge/noarch/sat-stac-0.3.3-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/boto3-1.12.39-py_0.conda -https://repo.anaconda.com/pkgs/main/win-64/notebook-6.0.3-py37_0.conda -https://conda.anaconda.org/conda-forge/noarch/sat-search-0.2.3-pyh9f0ad1d_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/jupyterlab_server-1.1.0-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/s3fs-0.4.0-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/widgetsnbextension-3.5.1-py37_0.conda -https://repo.anaconda.com/pkgs/main/noarch/ipywidgets-7.5.1-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/jupyterlab-1.2.6-pyhf63ae98_0.conda -https://conda.anaconda.org/conda-forge/noarch/ipyleaflet-0.12.4-pyh9f0ad1d_0.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/ipympl-0.5.6-pyh9f0ad1d_1.tar.bz2 -https://conda.anaconda.org/conda-forge/noarch/snuggs-1.4.7-py_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/h5py-2.10.0-nompi_py37h422b98e_102.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/rasterio-1.1.3-py37h2617b1b_0.tar.bz2 -https://conda.anaconda.org/conda-forge/win-64/shapely-1.7.0-py37he1cf020_3.tar.bz2 -https://repo.anaconda.com/pkgs/main/win-64/bokeh-2.1.1-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/matplotlib-3.1.3-py37_0.conda -https://repo.anaconda.com/pkgs/main/win-64/matplotlib-base-3.1.3-py37h64f37c6_0.conda -https://repo.anaconda.com/pkgs/main/win-64/mkl_fft-1.0.15-py37h14836fe_0.conda -https://repo.anaconda.com/pkgs/main/win-64/mkl_random-1.1.0-py37h675688f_0.conda -https://repo.anaconda.com/pkgs/main/win-64/numpy-1.18.1-py37h93ca92e_0.conda -https://repo.anaconda.com/pkgs/main/win-64/numcodecs-0.6.4-py37ha925a31_0.conda -https://repo.anaconda.com/pkgs/main/win-64/numexpr-2.7.1-py37h25d0782_0.conda -https://repo.anaconda.com/pkgs/main/win-64/pandas-1.0.3-py37h47e9c7a_0.conda -https://repo.anaconda.com/pkgs/main/win-64/pykdtree-1.3.1-py37h8c2d366_2.conda -https://repo.anaconda.com/pkgs/main/win-64/scipy-1.4.1-py37h9439919_0.conda -https://conda.anaconda.org/conda-forge/win-64/cartopy-0.17.0-py37h21f5c67_1015.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/dask-2.20.0-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/xarray-0.15.1-py_0.conda -https://repo.anaconda.com/pkgs/main/noarch/zarr-2.3.2-py_0.tar.bz2 -https://repo.anaconda.com/pkgs/main/noarch/intake-0.6.0-py_0.conda diff --git a/dist/windows_conda_environment.yml b/dist/windows_conda_environment.yml deleted file mode 100644 index cae816a8f..000000000 --- a/dist/windows_conda_environment.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: podpac -channels: - - defaults -dependencies: - - python=3.7 - - numpy - - ipython - - matplotlib - - scipy - - requests - - xarray - - psutil - - traitlets - - rasterio - - pyproj - - lxml - - zarr - - h5py - - beautifulsoup4 - - s3fs - - boto3 - - numexpr - - nodejs - - ipywidgets - - jupyterlab - - ipyleaflet - - ipympl - - sat-search - diff --git a/doc/source/cache.md b/doc/source/cache.md index a9977ef70..02dcea4e7 100644 --- a/doc/source/cache.md +++ b/doc/source/cache.md @@ -2,71 +2,171 @@ This document describes the caching methodology used in PODPAC, and how to control it. PODPAC uses a central cache shared by all nodes. Retrieval from the cache is based on the node's definition (`node.json`), the coordinates, and a key. -Each node has a **Cache Control** (`cache_ctrl`) defined by default, and the **Cache Control** may contain multiple **Cache Stores** (.e.g 'disk', 'ram'). A **Cache Store** may also have a specific **Cache Container**. +Each node has a **Cache Control** (`cache_ctrl`) defined by default, and the **Cache Control** may contain multiple **Cache Stores** (.e.g 'disk', 'ram'). -## Default Cache +## Caching Outputs -By default, every node caches their outputs to memory (RAM). These settings can be controlled using `podpac.settings`. +By default, PODPAC caches evaluated node outputs to memory (RAM). When a node is evaluated with the same coordinates, the output is retrieved from the cache. -**Settings and their Defaults:** +The following example demonstrates that the output was retrieved from the cache on the second evaluation: -* DEFAULT_CACHE : list - * Defines a default list of cache stores in priority order. Defaults to `['ram']`. Can include ['ram', 'disk', 's3']. - * This can be over-written on an individual node by specifying `cache_ctrl` when creating the node. E.g. `node = podpac.Node(cache_ctrl=['disk'])` - * Authors of nodes may require certain caches always be available. For example, the `podpac.datalib.smap.SMAPDateFolder` node always requires a 'disk' cache, and will add it. -* DISK_CACHE_DIR : str - * Subdirectory to use for the disk cache. Defaults to ``'cache'`` in the podpac root directory. -* S3_CACHE_DIR : str - * Subdirectory to use for S3 cache (within the specified S3 bucket). Defaults to ``'cache'``. -* CACHE_OUTPUT_DEFAULT : bool - * Automatically cache node outputs to the default cache store(s). Outputs for nodes with `cache_output=False` will not be cached. Defaults to ``True``. -* RAM_CACHE_ENABLED: bool - * Enable caching to RAM. Note that if disabled, some nodes may fail. Defaults to ``True``. -* DISK_CACHE_ENABLED: bool - * Enable caching to disk. Note that if disabled, some nodes may fail. Defaults to ``True``. -* S3_CACHE_ENABLED: bool - * Enable caching to S3. Note that if disabled, some nodes may fail. Defaults to ``True``. +```python +[.] import podpac +[.] import podpac.datalib +[.] coords = podpac.Coordinates([podpac.clinspace(40, 39, 16), + podpac.clinspace(-100, -90, 16), + '2015-01-01T00', ['lat', 'lon', 'time']]) +[.] smap = podpac.datalib.smap.SMAP() +[.] o = smap1.eval(coords) +[.] smap._from_cache +False +[.] o = smap1.eval(coords) +[.] smap._from_cache +True +``` -## Clearing Cache -To globally clear cache use: +Importantly, different instances of the same node share a cache. The following example demonstrates that a different instance of a node will retrieve output from the cache as well: ```python -podpac.utils.clear_cache(mode) +[.] smap2 = podpac.datalib.smap.SMAP() +[.] o = smap2.eval(coords) +[.] smap2._from_cache +True ``` -where `mode` can be 'ram', 'disk', or 's3'. This will clean the entire cache store. -To clear cache for an individual node: +### Configure Output Caching -## Examples +Automatic caching of outputs can be controlled globally and in individual nodes. For example, to globally disable caching outputs: -To globally disable automatic caching of outputs use: ```python -import podpac podpac.settings["CACHE_OUTPUT_DEFAULT"] = False -podpac.settings.save() ``` -To overwrite this behavior for a particular node (i.e. making sure outputs are cached) use: +To disable output caching for a particular node: + ```python -smap = podpac.datalib.smap.SMAP(cache_output=True) +smap = podpac.datalib.smap.SMAP(cache_output=False) ``` -Different instances of the same node share a cache. For example: +## Disk Cache + +In addition to caching to memory (RAM), PODPAC provides a disk cache that persists across processes. For example, when the disk cache is used, a script that evaluates a node can be run multiple times and will retrieve node outputs from the disk cache on subsequent runs. + +Each node has a `cache_ctrl` that specifies which cache stores to use, in priority order. For example, to use the RAM cache and the disk cache: + ```python -[.] import podpac -[.] import podpac.datalib -[.] coords = podpac.Coordinates([podpac.clinspace(40, 39, 16), - podpac.clinspace(-100, -90, 16), - '2015-01-01T00', ['lat', 'lon', 'time']]) -[.] smap1 = podpac.datalib.smap.SMAP() -[.] o = smap1.eval(coords) -[.] smap1._from_cache -False -[.] del smap1 -[.] smap2 = podpac.datalib.smap.SMAP() -[.] o = smap2.eval(coords) -[.] smap2._from_cache -True +smap = podpac.datalib.smap.SMAP(cache_ctrl=['ram', 'disk']) +``` + +The default cache control can be set globally in the settings: + +```python +podpac.settings["DEFAULT_CACHE"] = ['ram', 'disk'] +``` + +### Configure Disk Caching + +The disk cache directory can be set using the `DISK_CACHE_DIR` setting. + +## S3 Cache + +PODPAC also provides caching to the cloud using AWS S3. Configure the S3 bucket and cache subdirectory using the `S3_BUCKET_NAME` and `S3_CACHE_DIR` settings. + +## Clearing the Cache + +To clear the entire cache use: + +```python +podpac.utils.clear_cache() +``` + +To clear the cache for a particular node: + +```python +smap.clear_cache() +``` + +You can also clear a particular cache store, for example clear the disk cache leaving the RAM cache in place: + +```python +# node +smap.clear_cache('disk') + +# entire cache +podpac.utils.clear_cache('disk') +``` + +## Cache Limits + +PODPAC provides a limit for each cache store in the podpac settings. + +``` +RAM_CACHE_MAX_BYTES +DISK_CACHE_MAX_BYTES +S3_CACHE_MAX_BYTES +``` + +When a cache store is full, new entries are ignored cached. + + +## Advanced Usage + +### Caching Other Objects + +Nodes can cache other data and objects using a cache key and, optionally, coordinates. The following example caches and retrieves data using the key `my_data`. + +```python +[.] smap.put_cache(10, 'my_data') +[.] smap.get_cache('my_data') +10 +``` + +In general, the node cache can be managed using the `Node.put_cache`, `Node.get_cache`, `Node.has_cache`, and `Node.rem_cache` methods. + + +### Cache Expiration + +Cached entries can optionally have an expiration date, after which the entry is considered invalid and automatically removed. + +To specify an expiration date + +```python +# specific datetime +node.put_cache(10, 'my_data', expires='2021-01-01T12:00:00') + +# timedelta, in 12 hours +node.put_cache(10, 'my_data', expires='12,h') +``` + +### Cached Node Properties + +PODPAC provides a `cached_property` decorator that enhances the builtin `property` decorator. + +By default, the `cached_property` stores the value as a private attribute in the object. To use the PODPAC cache so that the property persists across objects or processes according to the node node `cache_ctrl`: + +```python +class MyNode(podpac.Node): + @podpac.cached_property(use_cache_ctrl=True) + def my_cached_property(self): + return 10 +``` + +### Updating Existing Entries + +By default, a existing cache entries will be overwritten with new data. + +```python +[.] smap.put_cache(10, 'my_data') +[.] smap.put_cache(20, 'my_data') +[.] smap.get_cache('my_data') +20 +``` + +To prevent overwriting existing cache entries, use `overwrite=False`: + +```python +[.] smap.put_cache(100, 'my_data', overwrite=False) +podpac.core.node.NodeException: Cached data already exists for key 'my_data' and coordinates None ``` diff --git a/doc/source/interpolation.md b/doc/source/interpolation.md index dc757d9d6..4b804e271 100644 --- a/doc/source/interpolation.md +++ b/doc/source/interpolation.md @@ -5,7 +5,8 @@ PODPAC allows users to specify various different interpolation schemes for nodes with increased granularity, and even lets users write their own interpolators. By default PODPAC uses the `podpac.settings["DEFAULT_INTERPOLATION"] == "nearest"`, which may -be modified by users. +be modified by users. Users who wish to see raw datasources can also use `"none"` +for the interpolator type. Relevant example notebooks include: * [Advanced Interpolation](https://github.com/creare-com/podpac-examples/blob/master/notebooks/4-advanced/interpolation.ipynb) @@ -21,7 +22,8 @@ Consider a `DataSource` with `lat`, `lon`, `time` coordinates that we will insta ### ...as a string `interpolation='nearest'` -* **Descripition**: All dimensions are interpolated using nearest neighbor interpolation. This is the default, but available options can be found here: `podpac.core.interpolation.interpolation.INTERPOLATION_METHODS` . +<<<<<<< HEAD +* **Descripition**: All dimensions are interpolated using nearest neighbor interpolation. This is the default, but available options can be found here: `podpac.core.interpolation.interpolation.INTERPOLATION_METHODS`. In particular, for no interpolation, use `interpolation="none"`. *NOTE* the `none` interpolator ONLY considers the bounds of any evaluated coordinates. This means the data is returned at FULL resolution (no striding or sub-selection). * **Details**: PODPAC will automatically select appropriate interpolators based on the source coordinates and eval coordinates. Default interpolator orders can be found in `podpac.core.interpolation.interpolation.INTERPOLATION_METHODS_DICT` ### ...as a dictionary @@ -62,6 +64,7 @@ The first item in the list will be interpolated first. In this case, `lat`/`lon` ## Interpolators The list of available interpolators are as follows: +* `NoneInterpolator`: An interpolator that passes through the raw, source data at full resolution -- it does not do any interpolation. **Note**: This interpolator can be used for **some** of the dimension by specifying `interpolation` as a list. * `NearestNeighbor`: A custom implementation based on `scipy.cKDtree`, which handles nearly any combination of source and destination coordinates * `XarrayInterpolator`: A light-weight wrapper around `xarray`'s `DataArray.interp` method, which is itself a wrapper around `scipy` interpolation functions, but with a clean `xarray` interface * `RasterioInterpolator`: A wrapper around `rasterio`'s interpolation/reprojection routines. Appropriate for grid-to-grid interpolation. diff --git a/doc/source/overview.md b/doc/source/overview.md index 4022f125d..3dc919b52 100644 --- a/doc/source/overview.md +++ b/doc/source/overview.md @@ -59,6 +59,16 @@ node = podpac.datalib.TerrainTiles(tile_format='geotiff', zoom=8) # ... and more each release ``` +Retrieve the raw source data array at full/native resolution. **Note**: Some data source are too large to fit in RAM, and calling this function can crash Python. + +```python +# retrieve full source data +node.get_source_data() + +# retrieve bounded source data +node.get_source_data(bounds={'lat': (40, 45), 'lon': (-70, -75)}) +``` + ## Coordinates Define geospatial and temporal dataset coordinates. diff --git a/podpac/algorithm.py b/podpac/algorithm.py index 69b1de0e9..69cebe458 100644 --- a/podpac/algorithm.py +++ b/podpac/algorithm.py @@ -17,6 +17,7 @@ Variance, StandardDeviation, Skew, + Percentile, Kurtosis, DayOfYear, GroupReduce, diff --git a/podpac/authentication.py b/podpac/authentication.py index 873ce53e7..98df14703 100644 --- a/podpac/authentication.py +++ b/podpac/authentication.py @@ -5,4 +5,4 @@ # REMINDER: update api docs (doc/source/api.rst) to reflect changes to this file -from podpac.core.authentication import RequestsSessionMixin, S3Mixin, NASAURSSessionMixin +from podpac.core.authentication import RequestsSessionMixin, S3Mixin, NASAURSSessionMixin, set_credentials diff --git a/podpac/conftest.py b/podpac/conftest.py index 4d413bd57..6541f1b9b 100644 --- a/podpac/conftest.py +++ b/podpac/conftest.py @@ -51,7 +51,7 @@ def pytest_unconfigure(config): pass -original_settings = None +original_settings = {} def pytest_sessionstart(session): diff --git a/podpac/coordinates.py b/podpac/coordinates.py index 058341b5c..9f83145c2 100644 --- a/podpac/coordinates.py +++ b/podpac/coordinates.py @@ -8,6 +8,6 @@ from podpac.core.coordinates import Coordinates from podpac.core.coordinates import crange, clinspace from podpac.core.coordinates import Coordinates1d, ArrayCoordinates1d, UniformCoordinates1d -from podpac.core.coordinates import StackedCoordinates, RotatedCoordinates +from podpac.core.coordinates import StackedCoordinates, AffineCoordinates from podpac.core.coordinates import merge_dims, concat, union from podpac.core.coordinates import GroupCoordinates diff --git a/podpac/core/__init__.py b/podpac/core/__init__.py index caf44fdc9..4c74052df 100644 --- a/podpac/core/__init__.py +++ b/podpac/core/__init__.py @@ -21,6 +21,5 @@ def lazy_module(modname, *args, **kwargs): matplotlib = lazy_import.lazy_module("matplotlib") plt = lazy_import.lazy_module("matplotlib.pyplot") np = lazy_import.lazy_module("numpy") -sp = lazy_import.lazy_module("scipy") tl = lazy_import.lazy_module("traitlets") # xr = lazy_import.lazy_module("xarray") diff --git a/podpac/core/algorithm/algorithm.py b/podpac/core/algorithm/algorithm.py index 11024906f..6b05666d0 100644 --- a/podpac/core/algorithm/algorithm.py +++ b/podpac/core/algorithm/algorithm.py @@ -58,6 +58,9 @@ class Algorithm(BaseAlgorithm): Developers of new Algorithm nodes need to implement the `algorithm` method. """ + # not the best solution... hard to check for these attrs + # abstract = tl.Bool(default_value=True, allow_none=True).tag(attr=True, required=False, hidden=True) + def algorithm(self, inputs, coordinates): """ Arguments @@ -167,7 +170,7 @@ class UnaryAlgorithm(BaseAlgorithm): Developers of new Algorithm nodes need to implement the `eval` method. """ - source = NodeTrait().tag(attr=True) + source = NodeTrait().tag(attr=True, required=True) # list of attribute names, used by __repr__ and __str__ to display minimal info about the node _repr_keys = ["source"] @@ -175,3 +178,7 @@ class UnaryAlgorithm(BaseAlgorithm): @tl.default("outputs") def _default_outputs(self): return self.source.outputs + + @tl.default("style") + def _default_style(self): # Pass through source style by default + return self.source.style diff --git a/podpac/core/algorithm/coord_select.py b/podpac/core/algorithm/coord_select.py index 2c50397a6..63e10aedc 100644 --- a/podpac/core/algorithm/coord_select.py +++ b/podpac/core/algorithm/coord_select.py @@ -225,7 +225,7 @@ def get_modified_coordinates1d(self, coords, dim): # no selection in this dimension return coords1d - if len(selection) == 1: + if len(selection) == 1 or ((len(selection) == 2) and (selection[0] == selection[1])): # a single value coords1d = ArrayCoordinates1d(selection, **coords1d.properties) @@ -233,7 +233,13 @@ def get_modified_coordinates1d(self, coords, dim): # use available source coordinates within the selected bounds available_coordinates = self.coordinates_source.find_coordinates() if len(available_coordinates) != 1: - raise ValueError("Cannot select within bounds; too many available coordinates") + raise ValueError( + "SelectCoordinates Node cannot determine the step size between bounds for dimension" + + "{} because source node (source.find_coordinates()) has {} different coordinates.".format( + dim, len(available_coordinates) + ) + + "Please specify step-size for this dimension." + ) coords1d = available_coordinates[0][dim].select(selection) elif len(selection) == 3: diff --git a/podpac/core/algorithm/generic.py b/podpac/core/algorithm/generic.py index 081687ba0..7fde8b71e 100644 --- a/podpac/core/algorithm/generic.py +++ b/podpac/core/algorithm/generic.py @@ -18,7 +18,7 @@ from podpac import settings from podpac import Coordinates -from podpac.core.node import Node +from podpac.core.node import Node, NodeException from podpac.core.utils import NodeTrait from podpac.core.algorithm.algorithm import Algorithm @@ -31,7 +31,7 @@ class PermissionError(OSError): class GenericInputs(Algorithm): """Base class for Algorithms that accept generic named inputs.""" - inputs = tl.Dict(read_only=True) + inputs = tl.Dict(read_only=True, value_trait=NodeTrait(), key_trait=tl.Unicode()).tag(attr=True, required=True) _repr_keys = ["inputs"] @@ -62,8 +62,8 @@ class Arithmetic(GenericInputs): arith = Arithmetic(A=a, B=b, eqn = 'A * B + {offset}', params={'offset': 1}) """ - eqn = tl.Unicode().tag(attr=True) - params = tl.Dict().tag(attr=True) + eqn = tl.Unicode().tag(attr=True, required=True) + params = tl.Dict().tag(attr=True, required=True) _repr_keys = ["eqn"] @@ -71,7 +71,7 @@ def init(self): if not settings.allow_unsafe_eval: warnings.warn( "Insecure evaluation of Python code using Arithmetic node has not been allowed. If " - "this is an error, use: `podpac.settings.set_unsafe_eval(True)`. " + "this is an error, use: `podpac.settings.allow_unrestricted_code_execution(True)`. " "NOTE: Allowing unsafe evaluation enables arbitrary execution of Python code through PODPAC " "Node definitions." ) @@ -101,7 +101,7 @@ def algorithm(self, inputs, coordinates): if not settings.allow_unsafe_eval: raise PermissionError( "Insecure evaluation of Python code using Arithmetic node has not been allowed. If " - "this is an error, use: `podpac.settings.set_unsafe_eval(True)`. " + "this is an error, use: `podpac.settings.allow_unrestricted_code_execution(True)`. " "NOTE: Allowing unsafe evaluation enables arbitrary execution of Python code through PODPAC " "Node definitions." ) @@ -145,13 +145,13 @@ class Generic(GenericInputs): generic = Generic(code=code, a=a, b=b) """ - code = tl.Unicode().tag(attr=True, readonly=True) + code = tl.Unicode().tag(attr=True, readonly=True, required=True) def init(self): if not settings.allow_unsafe_eval: warnings.warn( "Insecure evaluation of Python code using Generic node has not been allowed. If this " - "this is an error, use: `podpac.settings.set_unsafe_eval(True)`. " + "this is an error, use: `podpac.settings.allow_unrestricted_code_execution(True)`. " "NOTE: Allowing unsafe evaluation enables arbitrary execution of Python code through PODPAC " "Node definitions." ) @@ -178,7 +178,7 @@ def algorithm(self, inputs, coordinates): if not settings.allow_unsafe_eval: raise PermissionError( "Insecure evaluation of Python code using Generic node has not been allowed. If this " - "this is an error, use: `podpac.settings.set_unsafe_eval(True)`. " + "this is an error, use: `podpac.settings.allow_unrestricted_code_execution(True)`. " "NOTE: Allowing unsafe evaluation enables arbitrary execution of Python code through PODPAC " "Node definitions." ) @@ -227,8 +227,8 @@ class Mask(Algorithm): """ - source = NodeTrait().tag(attr=True) - mask = NodeTrait().tag(attr=True) + source = NodeTrait().tag(attr=True, required=True) + mask = NodeTrait().tag(attr=True, required=True) masked_val = tl.Float(allow_none=True, default_value=None).tag(attr=True) bool_val = tl.Float(1).tag(attr=True) bool_op = tl.Enum(["==", "<", "<=", ">", ">="], default_value="==").tag(attr=True) diff --git a/podpac/core/algorithm/reprojection.py b/podpac/core/algorithm/reprojection.py index 7a6cb82fa..091c79e5b 100644 --- a/podpac/core/algorithm/reprojection.py +++ b/podpac/core/algorithm/reprojection.py @@ -15,6 +15,7 @@ from podpac.core.coordinates.coordinates import Coordinates, merge_dims from podpac.core.interpolation.interpolation import Interpolate from podpac.core.utils import NodeTrait, cached_property +from podpac import settings class Reproject(Interpolate): @@ -84,7 +85,15 @@ def _source_eval(self, coordinates, selector, output=None): # Better to evaluate in reproject coordinate crs than eval crs for next step of interpolation extra_eval_coords = extra_eval_coords.transform(coords.crs) coords = merge_dims([coords, extra_eval_coords]) - return self.source.eval(coords, output=output, _selector=selector) + if settings["MULTITHREADING"]: + # we have to do a new node here to avoid clashing with the source node. + # What happens is that the non-projected source gets evaluated + # at the projected source coordinates because we have to set + # self._requested_coordinates for the datasource to avoid floating point + # lat/lon disagreement issues + return Node.from_definition(self.source.definition).eval(coords, output=output, _selector=selector) + else: + return self.source.eval(coords, output=output, _selector=selector) @property def base_ref(self): diff --git a/podpac/core/algorithm/signal.py b/podpac/core/algorithm/signal.py index cfb8c8286..7029aa7f6 100644 --- a/podpac/core/algorithm/signal.py +++ b/podpac/core/algorithm/signal.py @@ -11,7 +11,7 @@ import scipy.signal from podpac.core.settings import settings -from podpac.core.coordinates import Coordinates, UniformCoordinates1d +from podpac.core.coordinates import Coordinates, UniformCoordinates1d, ArrayCoordinates1d from podpac.core.coordinates import add_coord from podpac.core.node import Node from podpac.core.algorithm.algorithm import UnaryAlgorithm @@ -23,25 +23,25 @@ COMMON_DOC[ "full_kernel" ] = """Kernel that contains all the dimensions of the input source, in the correct order. - + Returns ------- np.ndarray The dimensionally full convolution kernel""" COMMON_DOC[ "validate_kernel" -] = """Checks to make sure the kernel is valid. - +] = """Checks to make sure the kernel is valid. + Parameters ---------- proposal : np.ndarray The proposed kernel - + Returns ------- np.ndarray The valid kernel - + Raises ------ ValueError @@ -59,14 +59,15 @@ class Convolution(UnaryAlgorithm): Source node on which convolution will be performed. kernel : np.ndarray, optional The convolution kernel. This kernel must include the dimensions of source node outputs. The dimensions for this - array are labelled by `kernel_dims`. Any dimensions not in the soucr nodes outputs will be summed over. + array are labelled by `kernel_dims`. Any dimensions not in the source nodes outputs will be summed over. kernel_dims : list, optional - A list of the dimensions for the kernel axes. The dimensions in this list must match the - coordinates in the source, or contain additional dimensions, and the order does not need to match. - Any extra dimensions are summed out. + A list of the dimensions for the kernel axes. If the dimensions in this list do not match the + coordinates in the source, then any extra dimensions in the kernel are removed by adding all the values over that axis + dimensions in the source are not convolved with any kernel. + kernel_type : str, optional If kernel is not defined, kernel_type will create a kernel based on the inputs, and it will have the - same number of axes as kernel_ndim. + same number of axes as kernel_dims. The format for the created kernels is ', , '. Any kernel defined in `scipy.signal` as well as `mean` can be used. For example: kernel_type = 'mean, 8' or kernel_type = 'gaussian,16,8' are both valid. @@ -75,6 +76,8 @@ class Convolution(UnaryAlgorithm): kernel = ArrayTrait(dtype=float).tag(attr=True) kernel_dims = tl.List().tag(attr=True) + # Takes one or the other which is hard to implement in a GUI + kernel_type = tl.List().tag(attr=True) def _first_init(self, kernel=None, kernel_dims=None, kernel_type=None, kernel_ndim=None, **kwargs): if kernel_dims is None: @@ -117,7 +120,7 @@ def _eval(self, coordinates, output=None, _selector=None): {eval_return} """ # The size of this kernel is used to figure out the expanded size - full_kernel = self._get_full_kernel(coordinates) + full_kernel = self.kernel # expand the coordinates # The next line effectively drops extra coordinates, so we have to add those later in case the @@ -130,28 +133,48 @@ def _eval(self, coordinates, output=None, _selector=None): for dim in kernel_dims: coord = coordinates[dim] s = full_kernel.shape[self.kernel_dims.index(dim)] - if s == 1 or not isinstance(coord, UniformCoordinates1d): + if s == 1 or not isinstance(coord, (UniformCoordinates1d, ArrayCoordinates1d)): exp_coords.append(coord) exp_slice.append(slice(None)) continue - s_start = -s // 2 - s_end = max(s // 2 - ((s + 1) % 2), 1) - # The 1e-07 is for floating point error because if endpoint is slightly - # in front of step * N then the endpoint is excluded - exp_coords.append( - UniformCoordinates1d( - add_coord(coord.start, s_start * coord.step), - add_coord(coord.stop, s_end * coord.step + 1e-07 * coord.step), - coord.step, - **coord.properties + if isinstance(coord, UniformCoordinates1d): + s_start = -s // 2 + s_end = max(s // 2 - ((s + 1) % 2), 1) + # The 1e-14 is for floating point error because if endpoint is slightly + # in front of step * N then the endpoint is excluded + # ALSO: MUST use size instead of step otherwise floating point error + # makes the xarray arrays not align. The following HAS to be true: + # np.diff(coord.coordinates).mean() == coord.step + exp_coords.append( + UniformCoordinates1d( + add_coord(coord.start, s_start * coord.step), + add_coord(coord.stop, s_end * coord.step + 1e-14 * coord.step), + size=coord.size - s_start + s_end, # HAVE to use size, see note above + **coord.properties + ) ) - ) - exp_slice.append(slice(-s_start, -s_end)) + exp_slice.append(slice(-s_start, -s_end)) + elif isinstance(coord, ArrayCoordinates1d): + if not coord.is_monotonic or coord.size < 2: + exp_coords.append(coord) + exp_slice.append(slice(None)) + continue + + arr_coords = coord.coordinates + delta_start = arr_coords[1] - arr_coords[0] + extra_start = np.arange(arr_coords[0] - delta_start * (s // 2), arr_coords[0], delta_start) + delta_end = arr_coords[-1] - arr_coords[-2] + # The 1e-14 is for floating point error to make sure endpoint is included + extra_end = np.arange( + arr_coords[-1] + delta_end, arr_coords[-1] + delta_end * (s // 2) + delta_end * 1e-14, delta_end + ) + arr_coords = np.concatenate([extra_start, arr_coords, extra_end]) + exp_coords.append(ArrayCoordinates1d(arr_coords, **coord.properties)) + exp_slice.append(slice(extra_start.size, -extra_end.size)) # Add missing dims back in -- this is needed in case the source is a reduce node. exp_coords += [coordinates[d] for d in missing_dims] - # exp_slice += [slice(None) for d in missing_dims] # Create expanded coordinates exp_slice = tuple(exp_slice) @@ -163,25 +186,32 @@ def _eval(self, coordinates, output=None, _selector=None): # evaluate source using expanded coordinates, convolve, and then slice out original coordinates source = self.source.eval(expanded_coordinates, _selector=_selector) - # Check dimensions - if any([d not in kernel_dims for d in source.dims if d != "output"]): - raise ValueError( - "Kernel dims must contain all of the dimensions in source but not all of {} is in kernel_dims={}".format( - source.dims, kernel_dims - ) - ) - - full_kernel = self._get_full_kernel(coordinates) + kernel_dims_u = kernel_dims kernel_dims = self.kernel_dims sum_dims = [d for d in kernel_dims if d not in source.dims] # Sum out the extra dims full_kernel = full_kernel.sum(axis=tuple([kernel_dims.index(d) for d in sum_dims])) + exp_slice = [exp_slice[i] for i in range(len(kernel_dims_u)) if kernel_dims_u[i] not in sum_dims] kernel_dims = [d for d in kernel_dims if d in source.dims] # Put the kernel axes in the correct order # The (if d in kernel_dims) takes care of "output", which can be optionally present full_kernel = full_kernel.transpose([kernel_dims.index(d) for d in source.dims if (d in kernel_dims)]) + # Check for extra dimensions in the source and reshape the kernel appropriately + if any([d not in kernel_dims for d in source.dims if d != "output"]): + new_axis = [] + new_exp_slice = [] + for d in source.dims: + if d in kernel_dims: + new_axis.append(slice(None)) + new_exp_slice.append(exp_slice[kernel_dims.index(d)]) + else: + new_axis.append(None) + new_exp_slice.append(slice(None)) + full_kernel = full_kernel[tuple(new_axis)] + exp_slice = new_exp_slice + if np.any(np.isnan(source)): method = "direct" else: @@ -198,7 +228,7 @@ def _eval(self, coordinates, output=None, _selector=None): ], axis=source.dims.index("output"), ) - result = result[exp_slice] + result = result[tuple(exp_slice)] if output is None: missing_dims = [d for d in coordinates.dims if d not in source.dims] @@ -223,7 +253,3 @@ def _make_kernel(kernel_type, ndim): k = np.tensordot(k, k1d, 0) return k / k.sum() - - def _get_full_kernel(self, coordinates): - """{full_kernel}""" - return self.kernel diff --git a/podpac/core/algorithm/stats.py b/podpac/core/algorithm/stats.py index d55a2981b..019316668 100644 --- a/podpac/core/algorithm/stats.py +++ b/podpac/core/algorithm/stats.py @@ -40,7 +40,9 @@ class Reduce(UnaryAlgorithm): The source node that will be reduced. """ - dims = tl.List().tag(attr=True) + from podpac.core.utils import DimsTrait + + dims = DimsTrait(allow_none=True, default_value=None).tag(attr=True) _reduced_coordinates = tl.Instance(Coordinates, allow_none=True) _dims = tl.List(trait=tl.Unicode()) @@ -78,7 +80,7 @@ def chunk_size(self): chunk_size = podpac.settings["CHUNK_SIZE"] if chunk_size == "auto": - return 1024 ** 2 # TODO + return 1024**2 # TODO else: return chunk_size @@ -609,7 +611,7 @@ def reduce_chunked(self, xs, output): Nx = np.isfinite(x).sum(dim=self._dims) M1x = x.mean(dim=self._dims) Ex = x - M1x - Ex2 = Ex ** 2 + Ex2 = Ex**2 Ex3 = Ex2 * Ex M2x = (Ex2).sum(dim=self._dims) M3x = (Ex3).sum(dim=self._dims) @@ -630,13 +632,13 @@ def reduce_chunked(self, xs, output): n = Nb + Nx NNx = Nb * Nx - M3.data[b] += M3x + d ** 3 * NNx * (Nb - Nx) / n ** 2 + 3 * d * (Nb * M2x - Nx * M2b) / n - M2.data[b] += M2x + d ** 2 * NNx / n + M3.data[b] += M3x + d**3 * NNx * (Nb - Nx) / n**2 + 3 * d * (Nb * M2x - Nx * M2b) / n + M2.data[b] += M2x + d**2 * NNx / n M1.data[b] += d * Nx / n N.data[b] = n # calculate skew - skew = np.sqrt(N) * M3 / np.sqrt(M2 ** 3) + skew = np.sqrt(N) * M3 / np.sqrt(M2**3) return skew @@ -695,9 +697,9 @@ def reduce_chunked(self, xs, output): Nx = np.isfinite(x).sum(dim=self._dims) M1x = x.mean(dim=self._dims) Ex = x - M1x - Ex2 = Ex ** 2 + Ex2 = Ex**2 Ex3 = Ex2 * Ex - Ex4 = Ex2 ** 2 + Ex4 = Ex2**2 M2x = (Ex2).sum(dim=self._dims) M3x = (Ex3).sum(dim=self._dims) M4x = (Ex4).sum(dim=self._dims) @@ -722,18 +724,18 @@ def reduce_chunked(self, xs, output): M4.data[b] += ( M4x - + d ** 4 * NNx * (Nb ** 2 - NNx + Nx ** 2) / n ** 3 - + 6 * d ** 2 * (Nb ** 2 * M2x + Nx ** 2 * M2b) / n ** 2 + + d**4 * NNx * (Nb**2 - NNx + Nx**2) / n**3 + + 6 * d**2 * (Nb**2 * M2x + Nx**2 * M2b) / n**2 + 4 * d * (Nb * M3x - Nx * M3b) / n ) - M3.data[b] += M3x + d ** 3 * NNx * (Nb - Nx) / n ** 2 + 3 * d * (Nb * M2x - Nx * M2b) / n - M2.data[b] += M2x + d ** 2 * NNx / n + M3.data[b] += M3x + d**3 * NNx * (Nb - Nx) / n**2 + 3 * d * (Nb * M2x - Nx * M2b) / n + M2.data[b] += M2x + d**2 * NNx / n M1.data[b] += d * Nx / n N.data[b] = n # calculate kurtosis - kurtosis = N * M4 / M2 ** 2 - 3 + kurtosis = N * M4 / M2**2 - 3 return kurtosis diff --git a/podpac/core/algorithm/test/test_signal.py b/podpac/core/algorithm/test/test_signal.py index 26a478ce4..55e84a4e0 100644 --- a/podpac/core/algorithm/test/test_signal.py +++ b/podpac/core/algorithm/test/test_signal.py @@ -67,16 +67,6 @@ def test_eval(self): o = node2d.eval(Coordinates([lat, lon])) o = node3d.eval(Coordinates([lat, lon, time])) - with pytest.raises( - ValueError, match="Kernel dims must contain all of the dimensions in source but not all of " - ): - node2d.eval(Coordinates([lat, lon, time])) - - with pytest.raises( - ValueError, match="Kernel dims must contain all of the dimensions in source but not all of " - ): - node2d.eval(Coordinates([lat, time])) - def test_eval_multiple_outputs(self): lat = clinspace(45, 66, 30, name="lat") @@ -181,3 +171,51 @@ def test_coords_order(self): o1 = node.eval(coords1) o2 = node.eval(coords2) assert np.all(o2.data == o1.data.T) + + def test_missing_source_dims(self): + """When the kernel has more dimensions than the source, sum out the kernel for the missing dim""" + lat = clinspace(-0.25, 1.25, 7, name="lat") + lon = clinspace(-0.125, 1.125, 11, name="lon") + time = ["2012-05-19", "2016-01-31", "2018-06-20"] + coords = Coordinates([lat, lon, time], dims=["lat", "lon", "time"]) + coords2 = Coordinates([lat[[1, 2, 4]], lon, time], dims=["lat", "lon", "time"]) + + source = Array(source=np.random.random(coords.drop("time").shape), coordinates=coords.drop("time")) + node = Convolution( + source=source, kernel=[[[-1], [2], [-1]]], kernel_dims=["lat", "lon", "time"], force_eval=True + ) + o = node.eval(coords[:, 1:-1, :]) + expected = source.source[:, 1:-1] * 2 - source.source[:, 2:] - source.source[:, :-2] + assert np.abs(o.data - expected).max() < 1e-14 + + # Check when request has an ArrayCoordinates1d + node = Convolution(source=source, kernel_type="mean,3", kernel_dims=["lat", "lon", "time"], force_eval=True) + o = node.eval(coords2[:, 1:-1]) + expected = ( + source.source[[1, 2, 4], 1:-1] + + source.source[[0, 1, 2], 1:-1] + + source.source[[2, 4, 6], 1:-1] + + source.source[[1, 2, 4], :-2] + + source.source[[0, 1, 2], :-2] + + source.source[[2, 4, 6], :-2] + + source.source[[1, 2, 4], 2:] + + source.source[[0, 1, 2], 2:] + + source.source[[2, 4, 6], 2:] + ) / 9 + assert np.abs(o.data - expected).max() < 1e-14 + + # Check to make sure array coordinates for a single coordinate is ok... + o = node.eval(coords2[0, 1:-1]) + + def test_partial_source_convolution(self): + lat = clinspace(-0.25, 1.25, 7, name="lat") + lon = clinspace(-0.125, 1.125, 11, name="lon") + time = ["2012-05-19", "2016-01-31", "2018-06-20"] + coords = Coordinates([lat, lon, time], dims=["lat", "lon", "time"]) + + source = Array(source=np.random.random(coords.shape), coordinates=coords) + node = Convolution(source=source, kernel=[[-1, 2, -1]], kernel_dims=["lat", "lon"], force_eval=True) + o = node.eval(coords[:, 1:-1, :]) + expected = source.source[:, 1:-1] * 2 - source.source[:, 2:] - source.source[:, :-2] + + assert np.abs(o.data - expected).max() < 1e-14 diff --git a/podpac/core/algorithm/test/test_stats.py b/podpac/core/algorithm/test/test_stats.py index 08c4867a7..bf5a195c0 100644 --- a/podpac/core/algorithm/test/test_stats.py +++ b/podpac/core/algorithm/test/test_stats.py @@ -36,7 +36,7 @@ def setup_module(): class TestReduce(object): - """ Tests the Reduce class """ + """Tests the Reduce class""" def test_auto_chunk(self): # any reduce node would do here @@ -71,7 +71,7 @@ def reduce(self, x): class BaseTests(object): - """ Common tests for Reduce subclasses """ + """Common tests for Reduce subclasses""" def test_full(self): with podpac.settings: diff --git a/podpac/core/algorithm/utility.py b/podpac/core/algorithm/utility.py index a72eceb8b..484c84da9 100644 --- a/podpac/core/algorithm/utility.py +++ b/podpac/core/algorithm/utility.py @@ -11,6 +11,7 @@ # Internal dependencies from podpac.core.coordinates import Coordinates from podpac.core.algorithm.algorithm import Algorithm +from podpac.core.style import Style class Arange(Algorithm): @@ -44,7 +45,9 @@ class CoordData(Algorithm): Name of coordinate to extract (one of lat, lon, time, alt) """ - coord_name = tl.Enum(["time", "lat", "lon", "alt"], default_value="none", allow_none=False).tag(attr=True) + coord_name = tl.Enum(["time", "lat", "lon", "alt"], default_value="none", allow_none=False).tag( + attr=True, required=True + ) def algorithm(self, inputs, coordinates): """Extract coordinate from request and makes data available. @@ -74,6 +77,10 @@ def algorithm(self, inputs, coordinates): class SinCoords(Algorithm): """A simple test node that creates a data based on coordinates and trigonometric (sin) functions.""" + @tl.default("style") + def _default_style(self): + return Style(clim=[-1.0, 1.0], colormap="jet") + def algorithm(self, inputs, coordinates): """Computes sinusoids of all the coordinates. @@ -95,7 +102,7 @@ def algorithm(self, inputs, coordinates): i_time = list(out.coords.keys()).index("time") crds[i_time] = crds[i_time].astype("datetime64[h]").astype(float) except ValueError: - pass + pass # Value error indicates the source does not have time crds = np.meshgrid(*crds, indexing="ij") for crd in crds: diff --git a/podpac/core/authentication.py b/podpac/core/authentication.py index a80a30550..8d1f41e65 100644 --- a/podpac/core/authentication.py +++ b/podpac/core/authentication.py @@ -19,14 +19,14 @@ _log = logging.getLogger(__name__) -def set_credentials(hostname, username=None, password=None): +def set_credentials(hostname, uname=None, password=None): """Set authentication credentials for a remote URL in the :class:`podpac.settings`. Parameters ---------- hostname : str - Hostname for `username` and `password`. - username : str, optional + Hostname for `uname` and `password`. + uname : str, optional Username to store in settings for `hostname`. If no username is provided and the username does not already exist in the settings, the user will be prompted to enter one. @@ -44,7 +44,7 @@ def set_credentials(hostname, username=None, password=None): p_settings = settings.get("password@{}".format(hostname)) # get username from 1. function input 2. settings 3. python input() - u = username or u_settings or input("Username: ") + u = uname or u_settings or getpass.getpass("Username: ") p = password or p_settings or getpass.getpass() # set values in settings @@ -136,7 +136,7 @@ def set_credentials(self, username=None, password=None): If no password is provided and the password does not already exist in the settings, the user will be prompted to enter one. """ - return set_credentials(self.hostname, username=username, password=password) + return set_credentials(self.hostname, uname=username, password=password) def _create_session(self): """Creates a :class:`requests.Session` with username and password defined @@ -187,7 +187,7 @@ def _create_session(self): class S3Mixin(tl.HasTraits): - """ Mixin to add S3 credentials and access to a Node. """ + """Mixin to add S3 credentials and access to a Node.""" anon = tl.Bool(False).tag(attr=True) aws_access_key_id = tl.Unicode(allow_none=True) diff --git a/podpac/core/cache/cache_ctrl.py b/podpac/core/cache/cache_ctrl.py index 86c9f7293..b2607910e 100644 --- a/podpac/core/cache/cache_ctrl.py +++ b/podpac/core/cache/cache_ctrl.py @@ -107,7 +107,7 @@ def cache_stores(self): def _get_cache_stores_by_mode(self, mode="all"): return [c for c in self._cache_stores if mode in c.cache_modes] - def put(self, node, data, key, coordinates=None, expires=None, mode="all", update=True): + def put(self, node, data, item, coordinates=None, expires=None, mode="all", update=True): """Cache data for specified node. Parameters @@ -116,8 +116,8 @@ def put(self, node, data, key, coordinates=None, expires=None, mode="all", updat node requesting storage. data : any Data to cache - key : str - Cached object key, e.g. 'output'. + item : str + Cached object item or key, e.g. 'output'. coordinates : :class:`podpac.Coordinates`, optional Coordinates for which cached object should be retrieved, for coordinate-dependent data such as evaluation output mode : str @@ -131,8 +131,8 @@ def put(self, node, data, key, coordinates=None, expires=None, mode="all", updat if not isinstance(node, podpac.Node): raise TypeError("Invalid node (must be of type Node, not '%s')" % type(node)) - if not isinstance(key, six.string_types): - raise TypeError("Invalid key (must be a string, not '%s')" % (type(key))) + if not isinstance(item, six.string_types): + raise TypeError("Invalid item (must be a string, not '%s')" % (type(item))) if not isinstance(coordinates, podpac.Coordinates) and coordinates is not None: raise TypeError("Invalid coordinates (must be of type 'Coordinates', not '%s')" % type(coordinates)) @@ -140,21 +140,21 @@ def put(self, node, data, key, coordinates=None, expires=None, mode="all", updat if mode not in _CACHE_MODES: raise ValueError("Invalid mode (must be one of %s, not '%s')" % (_CACHE_MODES, mode)) - if key == "*": - raise ValueError("Invalid key ('*' is reserved)") + if item == "*": + raise ValueError("Invalid item ('*' is reserved)") for c in self._get_cache_stores_by_mode(mode): - c.put(node=node, data=data, key=key, coordinates=coordinates, expires=expires, update=update) + c.put(node=node, data=data, item=item, coordinates=coordinates, expires=expires, update=update) - def get(self, node, key, coordinates=None, mode="all"): + def get(self, node, item, coordinates=None, mode="all"): """Get cached data for this node. Parameters ------------ node : Node node requesting storage. - key : str - Cached object key, e.g. 'output'. + item : str + Cached object item or key, e.g. 'output'. coordinates : :class:`podpac.Coordinates`, optional Coordinates for which cached object should be retrieved, for coordinate-dependent data such as evaluation output mode : str @@ -174,8 +174,8 @@ def get(self, node, key, coordinates=None, mode="all"): if not isinstance(node, podpac.Node): raise TypeError("Invalid node (must be of type Node, not '%s')" % type(node)) - if not isinstance(key, six.string_types): - raise TypeError("Invalid key (must be a string, not '%s')" % (type(key))) + if not isinstance(item, six.string_types): + raise TypeError("Invalid item (must be a string, not '%s')" % (type(item))) if not isinstance(coordinates, podpac.Coordinates) and coordinates is not None: raise TypeError("Invalid coordinates (must be of type 'Coordinates', not '%s')" % type(coordinates)) @@ -183,23 +183,23 @@ def get(self, node, key, coordinates=None, mode="all"): if mode not in _CACHE_MODES: raise ValueError("Invalid mode (must be one of %s, not '%s')" % (_CACHE_MODES, mode)) - if key == "*": - raise ValueError("Invalid key ('*' is reserved)") + if item == "*": + raise ValueError("Invalid item ('*' is reserved)") for c in self._get_cache_stores_by_mode(mode): - if c.has(node=node, key=key, coordinates=coordinates): - return c.get(node=node, key=key, coordinates=coordinates) + if c.has(node=node, item=item, coordinates=coordinates): + return c.get(node=node, item=item, coordinates=coordinates) raise CacheException("Requested data is not in any cache stores.") - def has(self, node, key, coordinates=None, mode="all"): + def has(self, node, item, coordinates=None, mode="all"): """Check for cached data for this node Parameters ------------ node : Node node requesting storage. - key : str - Cached object key, e.g. 'output'. + item : str + Cached object item or key, e.g. 'output'. coordinates: Coordinate, optional Coordinates for which cached object should be checked mode : str @@ -214,8 +214,8 @@ def has(self, node, key, coordinates=None, mode="all"): if not isinstance(node, podpac.Node): raise TypeError("Invalid node (must be of type Node, not '%s')" % type(node)) - if not isinstance(key, six.string_types): - raise TypeError("Invalid key (must be a string, not '%s')" % (type(key))) + if not isinstance(item, six.string_types): + raise TypeError("Invalid item (must be a string, not '%s')" % (type(item))) if not isinstance(coordinates, podpac.Coordinates) and coordinates is not None: raise TypeError("Invalid coordinates (must be of type 'Coordinates', not '%s')" % type(coordinates)) @@ -223,24 +223,24 @@ def has(self, node, key, coordinates=None, mode="all"): if mode not in _CACHE_MODES: raise ValueError("Invalid mode (must be one of %s, not '%s')" % (_CACHE_MODES, mode)) - if key == "*": - raise ValueError("Invalid key ('*' is reserved)") + if item == "*": + raise ValueError("Invalid item ('*' is reserved)") for c in self._get_cache_stores_by_mode(mode): - if c.has(node=node, key=key, coordinates=coordinates): + if c.has(node=node, item=item, coordinates=coordinates): return True return False - def rem(self, node, key, coordinates=None, mode="all"): + def rem(self, node, item, coordinates=None, mode="all"): """Delete cached data for this node. Parameters ---------- node : Node, str node requesting storage. - key : str - Delete only cached objects with this key. Use `'*'` to match all keys. + item : str + Delete only cached objects with this item/key. Use `'*'` to match all keys. coordinates : :class:`podpac.Coordinates`, str Delete only cached objects for these coordinates. Use `'*'` to match all coordinates. mode : str @@ -250,8 +250,8 @@ def rem(self, node, key, coordinates=None, mode="all"): if not isinstance(node, podpac.Node): raise TypeError("Invalid node (must be of type Node, not '%s')" % type(node)) - if not isinstance(key, six.string_types): - raise TypeError("Invalid key (must be a string, not '%s')" % (type(key))) + if not isinstance(item, six.string_types): + raise TypeError("Invalid item (must be a string, not '%s')" % (type(item))) if not isinstance(coordinates, podpac.Coordinates) and coordinates is not None and coordinates != "*": raise TypeError("Invalid coordinates (must be '*' or of type 'Coordinates', not '%s')" % type(coordinates)) @@ -259,14 +259,14 @@ def rem(self, node, key, coordinates=None, mode="all"): if mode not in _CACHE_MODES: raise ValueError("Invalid mode (must be one of %s, not '%s')" % (_CACHE_MODES, mode)) - if key == "*": - key = CacheWildCard() + if item == "*": + item = CacheWildCard() if coordinates == "*": coordinates = CacheWildCard() for c in self._get_cache_stores_by_mode(mode): - c.rem(node=node, key=key, coordinates=coordinates) + c.rem(node=node, item=item, coordinates=coordinates) def clear(self, mode="all"): """ diff --git a/podpac/core/cache/cache_store.py b/podpac/core/cache/cache_store.py index 63ead4d04..8efaf28af 100644 --- a/podpac/core/cache/cache_store.py +++ b/podpac/core/cache/cache_store.py @@ -29,7 +29,7 @@ def size(self): raise NotImplementedError - def put(self, node, data, key, coordinates=None, expires=None, update=True): + def put(self, node, data, item, coordinates=None, expires=None, update=True): """Cache data for specified node. Parameters @@ -38,8 +38,8 @@ def put(self, node, data, key, coordinates=None, expires=None, update=True): node requesting storage. data : any Data to cache - key : str - Cached object key, e.g. 'output'. + item : str + Cached object item or key, e.g. 'output'. coordinates : :class:`podpac.Coordinates`, optional Coordinates for which cached object should be retrieved, for coordinate-dependent data such as evaluation output expires : float, datetime, timedelta @@ -49,15 +49,15 @@ def put(self, node, data, key, coordinates=None, expires=None, update=True): """ raise NotImplementedError - def get(self, node, key, coordinates=None): + def get(self, node, item, coordinates=None): """Get cached data for this node. Parameters ------------ node : Node node requesting storage. - key : str - Cached object key, e.g. 'output'. + item : str + Cached object item or key, e.g. 'output'. coordinates : :class:`podpac.Coordinates`, optional Coordinates for which cached object should be retrieved, for coordinate-dependent data such as evaluation output @@ -73,29 +73,29 @@ def get(self, node, key, coordinates=None): """ raise NotImplementedError - def rem(self, node=None, key=None, coordinates=None): + def rem(self, node=None, item=None, coordinates=None): """Delete cached data for this node. Parameters ------------ node : Node node requesting storage. - key : str, optional - Delete only cached objects with this key. + item : str, optional + Delete only cached objects with this item name or key. coordinates : :class:`podpac.Coordinates` Delete only cached objects for these coordinates. """ raise NotImplementedError - def has(self, node, key, coordinates=None): + def has(self, node, item, coordinates=None): """Check for cached data for this node Parameters ------------ node : Node node requesting storage. - key : str - Cached object key, e.g. 'output'. + item : str + Cached object item or key, e.g. 'output'. coordinates: Coordinate, optional Coordinates for which cached object should be checked diff --git a/podpac/core/cache/disk_cache_store.py b/podpac/core/cache/disk_cache_store.py index 19e95d271..187208c72 100644 --- a/podpac/core/cache/disk_cache_store.py +++ b/podpac/core/cache/disk_cache_store.py @@ -48,8 +48,8 @@ def size(self): # helper methods # ----------------------------------------------------------------------------------------------------------------- - def search(self, node, key=CacheWildCard(), coordinates=CacheWildCard()): - pattern = self._path_join(self._get_node_dir(node), self._get_filename_pattern(node, key, coordinates)) + def search(self, node, item=CacheWildCard(), coordinates=CacheWildCard()): + pattern = self._path_join(self._get_node_dir(node), self._get_filename_pattern(node, item, coordinates)) return [path for path in glob.glob(pattern) if not path.endswith(".meta")] # ----------------------------------------------------------------------------------------------------------------- diff --git a/podpac/core/cache/file_cache_store.py b/podpac/core/cache/file_cache_store.py index b720b4dde..23d343b91 100644 --- a/podpac/core/cache/file_cache_store.py +++ b/podpac/core/cache/file_cache_store.py @@ -20,10 +20,11 @@ from podpac.core.utils import is_json_serializable from podpac.core.cache.utils import CacheException, CacheWildCard, expiration_timestamp from podpac.core.cache.cache_store import CacheStore +from podpac.core.utils import hash_alg def _hash_string(s): - return hashlib.md5(s.encode()).hexdigest() + return hash_alg(s.encode()).hexdigest() class FileCacheStore(CacheStore): @@ -38,14 +39,14 @@ class FileCacheStore(CacheStore): # public cache API methods # ----------------------------------------------------------------------------------------------------------------- - def has(self, node, key, coordinates=None): + def has(self, node, item, coordinates=None): """Check for valid cached data for this node. Parameters ------------ node : Node node requesting storage. - key : str + item : str Cached object key, e.g. 'output'. coordinates: Coordinate, optional Coordinates for which cached object should be checked @@ -56,10 +57,10 @@ def has(self, node, key, coordinates=None): True if there is a valid cached object for this node for the given key and coordinates. """ - path = self.find(node, key, coordinates) + path = self.find(node, item, coordinates) return path is not None and not self._expired(path) - def put(self, node, data, key, coordinates=None, expires=None, update=True): + def put(self, node, data, item, coordinates=None, expires=None, update=True): """Cache data for specified node. Parameters @@ -68,8 +69,8 @@ def put(self, node, data, key, coordinates=None, expires=None, update=True): node requesting storage. data : any Data to cache - key : str - Cached object key, e.g. 'output'. + item : str + Cached object item, e.g. 'output'. coordinates : :class:`podpac.Coordinates`, optional Coordinates for which cached object should be retrieved, for coordinate-dependent data such as evaluation output expires : float, datetime, timedelta @@ -79,14 +80,14 @@ def put(self, node, data, key, coordinates=None, expires=None, update=True): """ # check for valid existing entry (expired entries are automatically ignored and overwritten) - if self.has(node, key, coordinates): + if self.has(node, item, coordinates): if not update: raise CacheException("Cache entry already exists. Use `update=True` to overwrite.") else: - self._remove(self.find(node, key, coordinates)) + self._remove(self.find(node, item, coordinates)) # serialize - root = self._get_filename(node, key, coordinates) + root = self._get_filename(node, item, coordinates) if isinstance(data, podpac.core.units.UnitsDataArray): ext = "uda.nc" @@ -150,15 +151,15 @@ def put(self, node, data, key, coordinates=None, expires=None, update=True): return True - def get(self, node, key, coordinates=None): + def get(self, node, item, coordinates=None): """Get cached data for this node. Parameters ------------ node : Node node requesting storage. - key : str - Cached object key, e.g. 'output'. + item : str + Cached object item, e.g. 'output'. coordinates : :class:`podpac.Coordinates`, optional Coordinates for which cached object should be retrieved, for coordinate-dependent data such as evaluation output @@ -173,7 +174,7 @@ def get(self, node, key, coordinates=None): If the data is not in the cache. """ - path = self.find(node, key, coordinates) + path = self.find(node, item, coordinates) if path is None: raise CacheException("Cache miss. Requested data not found.") @@ -209,21 +210,21 @@ def get(self, node, key, coordinates=None): return data - def rem(self, node, key=CacheWildCard(), coordinates=CacheWildCard()): + def rem(self, node, item=CacheWildCard(), coordinates=CacheWildCard()): """Delete cached data for this node. Parameters ------------ node : Node node requesting storage - key : str, CacheWildCard, optional - Delete cached objects with this key, or any key if `key` is a CacheWildCard. + item : str, CacheWildCard, optional + Delete cached objects item, or any item if `item` is a CacheWildCard. coordinates : :class:`podpac.Coordinates`, CacheWildCard, None, optional Delete only cached objects for these coordinates, or any coordinates if `coordinates` is a CacheWildCard. `None` specifically indicates entries that do not have coordinates. """ # delete matching cached objects - for path in self.search(node, key=key, coordinates=coordinates): + for path in self.search(node, item=item, coordinates=coordinates): self._remove(path) # remove empty node directories @@ -244,25 +245,25 @@ def clear(self): # helper methods # ----------------------------------------------------------------------------------------------------------------- - def search(self, node, key=CacheWildCard(), coordinates=CacheWildCard()): + def search(self, node, item=CacheWildCard(), coordinates=CacheWildCard()): """ Search for matching cached objects. """ raise NotImplementedError - def find(self, node, key, coordinates=None): + def find(self, node, item, coordinates=None): """ Find the path for a specific cached object. """ - paths = self.search(node, key=key, coordinates=coordinates) + paths = self.search(node, item=item, coordinates=coordinates) if len(paths) == 0: return None elif len(paths) == 1: return paths[0] elif len(paths) > 1: - return RuntimeError("Too many cached files matching '%s'" % rootpath) + return RuntimeError("Too many cached files matching '%s'" % self._root_dir_path) def _get_node_dir(self, node): fullclass = str(node.__class__)[8:-2] diff --git a/podpac/core/cache/ram_cache_store.py b/podpac/core/cache/ram_cache_store.py index c102fe6b0..4ad2ac07e 100644 --- a/podpac/core/cache/ram_cache_store.py +++ b/podpac/core/cache/ram_cache_store.py @@ -57,7 +57,7 @@ def size(self): process = psutil.Process(os.getpid()) return process.memory_info().rss # this is actually the total size of the process - def put(self, node, data, key, coordinates=None, expires=None, update=True): + def put(self, node, data, item, coordinates=None, expires=None, update=True): """Cache data for specified node. Parameters @@ -66,8 +66,8 @@ def put(self, node, data, key, coordinates=None, expires=None, update=True): node requesting storage. data : any Data to cache - key : str - Cached object key, e.g. 'output'. + item : str + Cached object item, e.g. 'output'. coordinates : :class:`podpac.Coordinates`, optional Coordinates for which cached object should be retrieved, for coordinate-dependent data such as evaluation output expires : float, datetime, timedelta @@ -79,12 +79,12 @@ def put(self, node, data, key, coordinates=None, expires=None, update=True): if not hasattr(_thread_local, "cache"): _thread_local.cache = {} - full_key = self._get_full_key(node, key, coordinates) + full_key = self._get_full_key(node, item, coordinates) - if not update and self.has(node, key, coordinates): + if not update and self.has(node, item, coordinates): raise CacheException("Cache entry already exists. Use update=True to overwrite.") - self.rem(node, key, coordinates) + self.rem(node, item, coordinates) # check size if self.max_size is not None: @@ -107,15 +107,15 @@ def put(self, node, data, key, coordinates=None, expires=None, update=True): _thread_local.cache[full_key] = entry - def get(self, node, key, coordinates=None): + def get(self, node, item, coordinates=None): """Get cached data for this node. Parameters ------------ node : Node node requesting storage. - key : str - Cached object key, e.g. 'output'. + item : str + Cached object item, e.g. 'output'. coordinates : :class:`podpac.Coordinates`, optional Coordinates for which cached object should be retrieved, for coordinate-dependent data such as evaluation output @@ -133,7 +133,7 @@ def get(self, node, key, coordinates=None): if not hasattr(_thread_local, "cache"): _thread_local.cache = {} - full_key = self._get_full_key(node, key, coordinates) + full_key = self._get_full_key(node, item, coordinates) if full_key not in _thread_local.cache: raise CacheException("Cache miss. Requested data not found.") @@ -144,15 +144,15 @@ def get(self, node, key, coordinates=None): self._set_metadata(full_key, "accessed", time.time()) return copy.deepcopy(_thread_local.cache[full_key]["data"]) - def has(self, node, key, coordinates=None): + def has(self, node, item, coordinates=None): """Check for cached data for this node Parameters ------------ node : Node node requesting storage. - key : str - Cached object key, e.g. 'output'. + item : str + Cached object item, e.g. 'output'. coordinates: Coordinate, optional Coordinates for which cached object should be checked @@ -165,10 +165,10 @@ def has(self, node, key, coordinates=None): if not hasattr(_thread_local, "cache"): _thread_local.cache = {} - full_key = self._get_full_key(node, key, coordinates) + full_key = self._get_full_key(node, item, coordinates) return full_key in _thread_local.cache and not self._expired(full_key) - def rem(self, node, key=CacheWildCard(), coordinates=CacheWildCard()): + def rem(self, node, item=CacheWildCard(), coordinates=CacheWildCard()): """Delete cached data for this node. Parameters @@ -194,7 +194,7 @@ def rem(self, node, key=CacheWildCard(), coordinates=CacheWildCard()): for nk, k, ck in _thread_local.cache.keys(): if nk != node_key: continue - if not isinstance(key, CacheWildCard) and k != key: + if not isinstance(item, CacheWildCard) and k != item: continue if not isinstance(coordinates, CacheWildCard) and ck != coordinates_key: continue @@ -205,13 +205,13 @@ def rem(self, node, key=CacheWildCard(), coordinates=CacheWildCard()): del _thread_local.cache[k] def clear(self): - """Remove all entries from the cache. """ + """Remove all entries from the cache.""" if hasattr(_thread_local, "cache"): _thread_local.cache.clear() def cleanup(self): - """ Remove all expired entries. """ + """Remove all expired entries.""" for full_key, entry in list(_thread_local.cache.items()): if entry["expires"] is not None and time.time() >= entry["expires"]: del _thread_local.cache[full_key] diff --git a/podpac/core/cache/s3_cache_store.py b/podpac/core/cache/s3_cache_store.py index b17bc6b73..40df9f661 100644 --- a/podpac/core/cache/s3_cache_store.py +++ b/podpac/core/cache/s3_cache_store.py @@ -90,13 +90,13 @@ def cleanup(self): # helper methods # ----------------------------------------------------------------------------------------------------------------- - def search(self, node, key=CacheWildCard(), coordinates=CacheWildCard()): + def search(self, node, item=CacheWildCard(), coordinates=CacheWildCard()): """Fileglob to match files that could be storing cached data for specified node,key,coordinates Parameters ---------- node : podpac.core.node.Node - key : str, CacheWildCard + item : str, CacheWildCard CacheWildCard indicates to match any key coordinates : podpac.core.coordinates.coordinates.Coordinates, CacheWildCard, None CacheWildCard indicates to macth any coordinates @@ -118,7 +118,7 @@ def search(self, node, key=CacheWildCard(), coordinates=CacheWildCard()): obj_names = [] node_dir = self._get_node_dir(node) - obj_names = fnmatch.filter(obj_names, self._get_filename_pattern(node, key, coordinates)) + obj_names = fnmatch.filter(obj_names, self._get_filename_pattern(node, item, coordinates)) paths = [self._path_join(node_dir, filename) for filename in obj_names] return paths diff --git a/podpac/core/cache/test/test_cache_ctrl.py b/podpac/core/cache/test/test_cache_ctrl.py index 282bfd768..bbc74d05e 100644 --- a/podpac/core/cache/test/test_cache_ctrl.py +++ b/podpac/core/cache/test/test_cache_ctrl.py @@ -145,7 +145,7 @@ def test_rem_wildcard_key(self): assert ctrl.has(NODE, "key") # rem other and check has - ctrl.rem(NODE, key="*") + ctrl.rem(NODE, item="*") assert not ctrl.has(NODE, "key") def test_rem_wildcard_coordinates(self): @@ -243,26 +243,26 @@ def test_invalid_key(self): ctrl = CacheCtrl(cache_stores=[RamCacheStore(), DiskCacheStore()]) # type - with pytest.raises(TypeError, match="Invalid key"): + with pytest.raises(TypeError, match="Invalid item"): ctrl.put(NODE, 10, 10) - with pytest.raises(TypeError, match="Invalid key"): + with pytest.raises(TypeError, match="Invalid item"): ctrl.get(NODE, 10) - with pytest.raises(TypeError, match="Invalid key"): + with pytest.raises(TypeError, match="Invalid item"): ctrl.has(NODE, 10) - with pytest.raises(TypeError, match="Invalid key"): + with pytest.raises(TypeError, match="Invalid item"): ctrl.rem(NODE, 10) # wildcard - with pytest.raises(ValueError, match="Invalid key"): + with pytest.raises(ValueError, match="Invalid item"): ctrl.put(NODE, 10, "*") - with pytest.raises(ValueError, match="Invalid key"): + with pytest.raises(ValueError, match="Invalid item"): ctrl.get(NODE, "*") - with pytest.raises(ValueError, match="Invalid key"): + with pytest.raises(ValueError, match="Invalid item"): ctrl.has(NODE, "*") # allowed diff --git a/podpac/core/cache/test/test_cache_stores.py b/podpac/core/cache/test/test_cache_stores.py index 6ab774874..a752306a8 100644 --- a/podpac/core/cache/test/test_cache_stores.py +++ b/podpac/core/cache/test/test_cache_stores.py @@ -112,8 +112,8 @@ def test_rem_object(self): store.put(NODE2, 110, "mykey1") store.put(NODE2, 120, "mykeyA", COORDS1) - store.rem(NODE1, key="mykey1") - store.rem(NODE1, key="mykeyA", coordinates=COORDS1) + store.rem(NODE1, item="mykey1") + store.rem(NODE1, item="mykeyA", coordinates=COORDS1) assert store.has(NODE1, "mykey1") is False assert store.has(NODE1, "mykey2") is True @@ -134,8 +134,8 @@ def test_rem_key(self): store.put(NODE2, 110, "mykey1") store.put(NODE2, 120, "mykeyA", COORDS1) - store.rem(NODE1, key="mykey1") - store.rem(NODE1, key="mykeyA") + store.rem(NODE1, item="mykey1") + store.rem(NODE1, item="mykeyA") assert store.has(NODE1, "mykey1") is False assert store.has(NODE1, "mykey2") is True diff --git a/podpac/core/compositor/__init__.py b/podpac/core/compositor/__init__.py index e69de29bb..c1a808acf 100644 --- a/podpac/core/compositor/__init__.py +++ b/podpac/core/compositor/__init__.py @@ -0,0 +1,3 @@ +from .compositor import BaseCompositor +from .ordered_compositor import OrderedCompositor +from .tile_compositor import TileCompositor diff --git a/podpac/core/compositor/compositor.py b/podpac/core/compositor/compositor.py index e0f905465..7f2a7fb03 100644 --- a/podpac/core/compositor/compositor.py +++ b/podpac/core/compositor/compositor.py @@ -48,7 +48,7 @@ class BaseCompositor(Node): for most use-cases. """ - sources = tl.List(trait=NodeTrait()).tag(attr=True) + sources = tl.List(trait=NodeTrait()).tag(attr=True, required=True) source_coordinates = tl.Instance(Coordinates, allow_none=True, default_value=None).tag(attr=True) multithreading = tl.Bool(False) @@ -221,6 +221,35 @@ def iteroutputs(self, coordinates, _selector=None): for src in sources: yield src.eval(coordinates, _selector=_selector) + @common_doc(COMMON_COMPOSITOR_DOC) + def eval(self, coordinates, **kwargs): + """ + Wraps the super Node.eval method in order to cache with the correct coordinates. + + The output is independent of any extra dimensions, so this removes extra dimensions before caching in the + super eval method. + """ + + super_coordinates = coordinates + + # remove extra dimensions + if self.dims: + extra = [ + c.name + for c in coordinates.values() + if (isinstance(c, Coordinates1d) and c.name not in self.dims) + or (isinstance(c, StackedCoordinates) and all(dim not in self.dims for dim in c.dims)) + ] + super_coordinates = super_coordinates.drop(extra) + + # note: super().eval (not self._eval) + output = super().eval(super_coordinates, **kwargs) + + if settings["DEBUG"]: + self._requested_coordinates = coordinates + + return output + @common_doc(COMMON_COMPOSITOR_DOC) def _eval(self, coordinates, output=None, _selector=None): """Evaluates this nodes using the supplied coordinates. @@ -239,18 +268,6 @@ def _eval(self, coordinates, output=None, _selector=None): {eval_return} """ - self._requested_coordinates = coordinates - - # remove extra dimensions - if self.dims: - extra = [ - c.name - for c in coordinates.values() - if (isinstance(c, Coordinates1d) and c.name not in self.dims) - or (isinstance(c, StackedCoordinates) and all(dim not in self.dims for dim in c.dims)) - ] - coordinates = coordinates.drop(extra) - self._evaluated_coordinates = coordinates outputs = self.iteroutputs(coordinates, _selector) output = self.composite(coordinates, outputs, output) diff --git a/podpac/core/compositor/test/test_tiled_compositor.py b/podpac/core/compositor/test/test_tiled_compositor.py index bf424caa6..df91f764b 100644 --- a/podpac/core/compositor/test/test_tiled_compositor.py +++ b/podpac/core/compositor/test/test_tiled_compositor.py @@ -47,3 +47,27 @@ def test_composition_stacked_multiindex_names(self): np.testing.assert_array_equal(output["lat"], [3, 4, 5, 6]) np.testing.assert_array_equal(output["lon"], [3, 4, 5, 6]) np.testing.assert_array_equal(output, [103, 104, 200, 201]) + + def test_get_source_data(self): + a = ArrayRaw(source=np.arange(5) + 100, coordinates=podpac.Coordinates([[0, 1, 2, 3, 4]], dims=["lat"])) + b = ArrayRaw(source=np.arange(5) + 200, coordinates=podpac.Coordinates([[5, 6, 7, 8, 9]], dims=["lat"])) + c = ArrayRaw(source=np.arange(5) + 300, coordinates=podpac.Coordinates([[10, 11, 12, 13, 14]], dims=["lat"])) + + node = TileCompositorRaw(sources=[a, b, c]) + + data = node.get_source_data() + np.testing.assert_array_equal(data["lat"], np.arange(15)) + np.testing.assert_array_equal(data, np.hstack([source.source for source in node.sources])) + + # with bounds + data = node.get_source_data({"lat": (2.5, 6.5)}) + np.testing.assert_array_equal(data["lat"], [3, 4, 5, 6]) + np.testing.assert_array_equal(data, [103, 104, 200, 201]) + + # error + with podpac.settings: + podpac.settings.set_unsafe_eval(True) + d = podpac.algorithm.Arithmetic(eqn="a+2", a=a) + node = TileCompositorRaw(sources=[a, b, c, d]) + with pytest.raises(ValueError, match="Cannot get composited source data"): + node.get_source_data() diff --git a/podpac/core/compositor/tile_compositor.py b/podpac/core/compositor/tile_compositor.py index 25462c9cb..141b61695 100644 --- a/podpac/core/compositor/tile_compositor.py +++ b/podpac/core/compositor/tile_compositor.py @@ -51,9 +51,10 @@ def composite(self, coordinates, data_arrays, result=None): res = res.combine_first(arr) # combine_first overrides MultiIndex names, even if they match. Reset them here: - for dim, index in res.indexes.items(): - if isinstance(index, pd.MultiIndex): - res = res.reindex({dim: pd.MultiIndex.from_tuples(index.values, names=arr.indexes[dim].names)}) + if int(xr.__version__.split(".")[0]) < 2022: # no longer necessary in newer versions of xarray + for dim, index in res.indexes.items(): + if isinstance(index, pd.MultiIndex): + res = res.reindex({dim: pd.MultiIndex.from_tuples(index.values, names=arr.indexes[dim].names)}) obounds = arr.attrs.get("bounds", {}) bounds = { @@ -62,11 +63,38 @@ def composite(self, coordinates, data_arrays, result=None): res = UnitsDataArray(res) if bounds: res.attrs["bounds"] = bounds + if "geotransform" in res.attrs: # Really hard to get the geotransform right, handle it in Coordinates + del res.attrs["geotransform"] if result is not None: result.data[:] = res.transpose(*result.dims).data return result return res + def get_source_data(self, bounds={}): + """ + Get composited source data, without interpolation. + + Arguments + --------- + bounds : dict + Dictionary of bounds by dimension, optional. + Keys must be dimension names, and values are (min, max) tuples, e.g. ``{'lat': (10, 20)}``. + + Returns + ------- + data : UnitsDataArray + Source data + """ + + if any(not hasattr(source, "get_source_data") for source in self.sources): + raise ValueError( + "Cannot get composited source data; all sources must have `get_source_data` implemented (such as nodes derived from a DataSource or TileCompositor node)." + ) + + coords = None # n/a + source_data_arrays = (source.get_source_data(bounds) for source in self.sources) # generator + return self.composite(coords, source_data_arrays) + class TileCompositor(InterpolationMixin, TileCompositorRaw): pass diff --git a/podpac/core/coordinates/__init__.py b/podpac/core/coordinates/__init__.py index 67e13ffff..b232ffc61 100644 --- a/podpac/core/coordinates/__init__.py +++ b/podpac/core/coordinates/__init__.py @@ -10,7 +10,7 @@ from podpac.core.coordinates.array_coordinates1d import ArrayCoordinates1d from podpac.core.coordinates.uniform_coordinates1d import UniformCoordinates1d from podpac.core.coordinates.stacked_coordinates import StackedCoordinates -from podpac.core.coordinates.rotated_coordinates import RotatedCoordinates +from podpac.core.coordinates.affine_coordinates import AffineCoordinates from podpac.core.coordinates.coordinates import Coordinates from podpac.core.coordinates.coordinates import merge_dims, concat, union from podpac.core.coordinates.group_coordinates import GroupCoordinates diff --git a/podpac/core/coordinates/affine_coordinates.py b/podpac/core/coordinates/affine_coordinates.py new file mode 100644 index 000000000..339cd88bb --- /dev/null +++ b/podpac/core/coordinates/affine_coordinates.py @@ -0,0 +1,365 @@ +from __future__ import division, unicode_literals, print_function, absolute_import + +from collections import OrderedDict + +import numpy as np +import traitlets as tl +import lazy_import +import warnings + +from podpac.core.coordinates.uniform_coordinates1d import UniformCoordinates1d + +affine = lazy_import.lazy_module("affine") + +from podpac.core.coordinates.array_coordinates1d import ArrayCoordinates1d +from podpac.core.coordinates.stacked_coordinates import StackedCoordinates +from podpac.core.coordinates.cfunctions import clinspace + + +class AffineCoordinates(StackedCoordinates): + """ + A grid of latitude and longitude coordinates, defined by an affine transformation. + + Parameters + ---------- + geotransform : tuple + GDAL geotransform + shape : tuple + shape (m, n) of the grid. + dims : tuple + Tuple of dimension names. + coords : dict-like + xarray coordinates (container of coordinate arrays) + coordinates : tuple + Tuple of 2d coordinate values in each dimension. + + Notes + ----- + + https://gdal.org/tutorials/geotransforms_tut.html + + GT(0) x-coordinate of the upper-left corner of the upper-left pixel. + GT(1) w-e pixel resolution / pixel width. + GT(2) row rotation (typically zero). + GT(3) y-coordinate of the upper-left corner of the upper-left pixel. + GT(4) column rotation (typically zero). + GT(5) n-s pixel resolution / pixel height (negative value for a north-up image). + + """ + + geotransform = tl.Tuple(tl.Float(), tl.Float(), tl.Float(), tl.Float(), tl.Float(), tl.Float(), read_only=True) + shape = tl.Tuple(tl.Integer(), tl.Integer(), read_only=True) + + def __init__(self, geotransform=None, shape=None): + """ + Create a grid of coordinates from a `geotransform` and `shape`. + + Parameters + ---------- + geotransform : tuple + GDAL geotransform + shape : tuple + shape (m, n) of the grid. + """ + if isinstance(geotransform, np.ndarray): + geotransform = tuple(geotransform.tolist()) + self.set_trait("geotransform", geotransform) + self.set_trait("shape", shape) + + # private traits + self._affine = affine.Affine.from_gdal(*self.geotransform) + + @tl.validate("shape") + def _validate_shape(self, d): + val = d["value"] + if val[0] <= 0 or val[1] <= 0: + raise ValueError("Invalid shape %s, shape must be positive" % (val,)) + return val + + # ------------------------------------------------------------------------------------------------------------------ + # Alternate Constructors + # ------------------------------------------------------------------------------------------------------------------ + + @classmethod + def from_definition(cls, d): + """ + Create AffineCoordinates from an affine coordinates definition. + + Arguments + --------- + d : dict + affine coordinates definition + + Returns + ------- + :class:`AffineCoordinates` + affine coordinates object + + See Also + -------- + definition + """ + + if "geotransform" not in d: + raise ValueError('AffineCoordinates definition requires "geotransform" property') + if "shape" not in d: + raise ValueError('AffineCoordinates definition requires "shape" property') + return AffineCoordinates(geotransform=d["geotransform"], shape=d["shape"]) + + # ------------------------------------------------------------------------------------------------------------------ + # standard methods + # ------------------------------------------------------------------------------------------------------------------ + + def __repr__(self): + return "%s(%s): Bounds(lat, lon)([%g, %g], [%g, %g]), Shape%s" % ( + self.__class__.__name__, + self.dims, + self.bounds["lat"][0], + self.bounds["lat"][1], + self.bounds["lon"][0], + self.bounds["lon"][1], + self.shape, + ) + + def __eq__(self, other): + if not self._eq_base(other): + return False + + if not other.is_affine: + return False + + if not np.allclose(self.geotransform, other.geotransform): + return False + + return True + + def _getsubset(self, index): + if isinstance(index, tuple) and isinstance(index[0], slice) and isinstance(index[1], slice): + lat = self["lat"].coordinates[index] + lon = self["lon"].coordinates[index] + + # We don't have to check every point in lat/lon for the same step + # since the self.is_affine call did that already + dlati = (lat[-1, 0] - lat[0, 0]) / (lat.shape[0] - 1) + dlatj = (lat[0, -1] - lat[0, 0]) / (lat.shape[1] - 1) + dloni = (lon[-1, 0] - lon[0, 0]) / (lon.shape[0] - 1) + dlonj = (lon[0, -1] - lon[0, 0]) / (lon.shape[1] - 1) + + # origin point + p0 = np.array([lat[0, 0], lon[0, 0]]) - np.array([[dlati, dlatj], [dloni, dlonj]]) @ np.ones(2) / 2 + + # This is defined as x ulc, x width, x height, y ulc, y width, y height + # x and y are defined by the CRS. Here we are assuming that it's always + # lon and lat == x and y + geotransform = [p0[1], dlonj, dloni, p0[0], dlatj, dlati] + + # get shape from indexed coordinates + shape = lat.shape + + return AffineCoordinates(geotransform=geotransform, shape=lat.shape) + + else: + return super(AffineCoordinates, self)._getsubset(index).simplify() + + # ------------------------------------------------------------------------------------------------------------------ + # Properties + # ------------------------------------------------------------------------------------------------------------------ + + @property + def _coords(self): + if not hasattr(self, "_coords_"): + self._coords_ = [ + ArrayCoordinates1d(c, name=dim) for c, dim in zip(self.coordinates.transpose(2, 0, 1), self.dims) + ] + return self._coords_ + + @property + def ndim(self): + return 2 + + @property + def affine(self): + """:affine.Affine: affine transformation for computing the coordinates from indexing values.""" + return self._affine + + @property + def dims(self): + return ("lat", "lon") + + @property + def is_affine(self): + return True + + @property + def origin(self): + origin = self.affine * [0, 0] + if self.dims == ("lat", "lon"): + origin = origin[::-1] + return origin + + @property + def coordinates(self): + """:tuple: computed coordinate values for each dimension.""" + + I = np.arange(self.shape[1]) + 0.5 + J = np.arange(self.shape[0]) + 0.5 + x, y = self.affine * np.meshgrid(I, J) + if self.dims == ("lat", "lon"): + c = np.stack([y, x]) + else: + c = np.stack([x, y]) + return c.transpose(1, 2, 0) + + @property + def definition(self): + d = OrderedDict() + d["geotransform"] = self.geotransform + d["shape"] = self.shape + return d + + @property + def full_definition(self): + return self.definition + + # ------------------------------------------------------------------------------------------------------------------ + # Methods + # ------------------------------------------------------------------------------------------------------------------ + + def copy(self): + """ + Make a copy of the affine coordinates. + + Returns + ------- + :class:`AffineCoordinates` + Copy of the affine coordinates. + """ + return AffineCoordinates(self.geotransform, self.shape) + + def get_area_bounds(self, boundary): + """Get coordinate area bounds, including boundary information, for each unstacked dimension. + + Arguments + --------- + boundary : dict + dictionary of boundary offsets for each unstacked dimension. Point dimensions can be omitted. + + Returns + ------- + area_bounds : dict + Dictionary of (low, high) coordinates area_bounds in each unstacked dimension + """ + + # TODO the boundary offsets need to be transformed + warnings.warn("AffineCoordinates area_bounds are not yet correctly implemented.") + return super(AffineCoordinates, self).get_area_bounds(boundary) + + def select(self, bounds, outer=False, return_index=False): + """ + Get the coordinate values that are within the given bounds in all dimensions. + + *Note: you should not generally need to call this method directly.* + + Parameters + ---------- + bounds : dict + dictionary of dim -> (low, high) selection bounds + outer : bool, optional + If True, do *outer* selections. Default False. + return_index : bool, optional + If True, return index for the selections in addition to coordinates. Default False. + + Returns + ------- + selection : :class:`StackedCoordinates`, :class:`AffineCoordinates` + coordinates consisting of the selection in all dimensions. + selection_index : list + index for the selected coordinates, only if ``return_index`` is True. + """ + + if not outer: + # if the geotransform is rotated, the inner selection is not a grid + # returning the general stacked coordinates is a general solution + return super(AffineCoordinates, self).select(bounds, outer=outer, return_index=return_index) + + # same rotation and step, new origin and shape + lat = self.coordinates[:, :, 0] + lon = self.coordinates[:, :, 1] + b = ( + (lat >= bounds["lat"][0]) + & (lat <= bounds["lat"][1]) + & (lon >= bounds["lon"][0]) + & (lon <= bounds["lon"][1]) + ) + + I, J = np.where(b) + imin = max(0, np.min(I) - 1) + jmin = max(0, np.min(J) - 1) + imax = min(self.shape[0] - 1, np.max(I) + 1) + jmax = min(self.shape[1] - 1, np.max(J) + 1) + + origin = np.array([lat[imin, jmin], lon[imin, jmin]]) + origin -= np.array([lat[0, 0], lon[0, 0]]) - self.origin + + shape = int(imax - imin + 1), int(jmax - jmin + 1) + + geotransform = ( + origin[1], + self.geotransform[1], + self.geotransform[2], + origin[0], + self.geotransform[4], + self.geotransform[5], + ) + + selected = AffineCoordinates(geotransform=geotransform, shape=shape) + + if return_index: + return selected, (slice(imin, imax + 1), slice(jmin, jmax + 1)) + else: + return selected + + def simplify(self): + # NOTE: podpac prefers unstacked UniformCoordinates to AffineCoordinates + # if that changes, just return self.copy() + if self.affine.is_rectilinear: + tol = 1e-15 # tolerance for deciding when a number is zero + a = self.affine + shape = self.shape + + if np.abs(a.e) <= tol and np.abs(a.a) <= tol: + order = -1 + step = np.array([a.d, a.b]) + else: + order = 1 + step = np.array([a.e, a.a]) + + origin = a.f + step[0] / 2, a.c + step[1] / 2 + end = origin[0] + step[0] * (shape[::order][0] - 1), origin[1] + step[1] * (shape[::order][1] - 1) + # when the shape == 1, UniformCoordinates1d cannot infer the step from the size + # we have have to create the UniformCoordinates1d manually + if shape[::order][0] == 1: + lat = UniformCoordinates1d(origin[0], end[0], step=step[0], name="lat") + else: + lat = clinspace(origin[0], end[0], shape[::order][0], "lat") + if shape[::order][1] == 1: + lon = UniformCoordinates1d(origin[1], end[1], step=step[1], name="lon") + else: + lon = clinspace(origin[1], end[1], shape[::order][1], "lon") + return [lat, lon][::order] + + return self.copy() + + # ------------------------------------------------------------------------------------------------------------------ + # Debug + # ------------------------------------------------------------------------------------------------------------------ + + def plot(self, marker="b.", origin_marker="bo", corner_marker="bx"): + from matplotlib import pyplot + + x = self.coordinates[:, :, 0] + y = self.coordinates[:, :, 1] + pyplot.plot(x.flatten(), y.flatten(), marker) + ox, oy = self.origin + pyplot.plot(ox, oy, origin_marker) + pyplot.gca().set_aspect("equal") diff --git a/podpac/core/coordinates/array_coordinates1d.py b/podpac/core/coordinates/array_coordinates1d.py index 28c5f254c..2d4c2f4a2 100644 --- a/podpac/core/coordinates/array_coordinates1d.py +++ b/podpac/core/coordinates/array_coordinates1d.py @@ -263,7 +263,12 @@ def __getitem__(self, index): # The following 3 lines are copied by UniformCoordinates1d.__getitem__ if self.ndim == 1 and np.ndim(index) > 1 and np.array(index).dtype == int: index = np.array(index).flatten().tolist() - return ArrayCoordinates1d(self.coordinates[index], **self.properties) + try: + return ArrayCoordinates1d(self.coordinates[index], **self.properties) + except IndexError as e: # This happens when index is a list, but should be a tuple + if isinstance(index, list): + return ArrayCoordinates1d(self.coordinates[tuple(index)], **self.properties) + raise (e) # ------------------------------------------------------------------------------------------------------------------ # Properties @@ -281,7 +286,7 @@ def ndim(self): @property def size(self): - """ Number of coordinates. """ + """Number of coordinates.""" return self.coordinates.size @property @@ -328,7 +333,7 @@ def step(self): @property def bounds(self): - """ Low and high coordinate bounds. """ + """Low and high coordinate bounds.""" if self.size == 0: lo, hi = np.nan, np.nan diff --git a/podpac/core/coordinates/base_coordinates.py b/podpac/core/coordinates/base_coordinates.py index 0da5f9d2c..479764032 100644 --- a/podpac/core/coordinates/base_coordinates.py +++ b/podpac/core/coordinates/base_coordinates.py @@ -79,11 +79,11 @@ def copy(self): raise NotImplementedError def unique(self, return_index=False): - """ Remove duplicate coordinate values.""" + """Remove duplicate coordinate values.""" raise NotImplementedError def get_area_bounds(self, boundary): - """Get coordinate area bounds, including boundary information, for each unstacked dimension. """ + """Get coordinate area bounds, including boundary information, for each unstacked dimension.""" raise NotImplementedError def select(self, bounds, outer=False, return_index=False): @@ -91,15 +91,15 @@ def select(self, bounds, outer=False, return_index=False): raise NotImplementedError def simplify(self): - """ Get the simplified/optimized representation of these coordinates. """ + """Get the simplified/optimized representation of these coordinates.""" raise NotImplementedError def flatten(self): - """ Get a copy of the coordinates with a flattened array. """ + """Get a copy of the coordinates with a flattened array.""" raise NotImplementedError def reshape(self, newshape): - """ Get a copy of the coordinates with a reshaped array (wraps numpy.reshape). """ + """Get a copy of the coordinates with a reshaped array (wraps numpy.reshape).""" raise NotImplementedError def issubset(self, other): diff --git a/podpac/core/coordinates/coordinates.py b/podpac/core/coordinates/coordinates.py index 1aafa1d9f..75653fd93 100644 --- a/podpac/core/coordinates/coordinates.py +++ b/podpac/core/coordinates/coordinates.py @@ -10,7 +10,6 @@ import itertools import json from collections import OrderedDict -from hashlib import md5 as hash_alg import numpy as np import traitlets as tl @@ -30,15 +29,14 @@ from podpac.core.coordinates.array_coordinates1d import ArrayCoordinates1d from podpac.core.coordinates.uniform_coordinates1d import UniformCoordinates1d from podpac.core.coordinates.stacked_coordinates import StackedCoordinates -from podpac.core.coordinates.rotated_coordinates import RotatedCoordinates +from podpac.core.coordinates.affine_coordinates import AffineCoordinates from podpac.core.coordinates.cfunctions import clinspace +from podpac.core.utils import hash_alg # Optional dependencies from lazy_import import lazy_module, lazy_class rasterio = lazy_module("rasterio") -affine_module = lazy_module("affine") - # Set up logging _logger = logging.getLogger(__name__) @@ -325,12 +323,34 @@ def from_xarray(cls, x, crs=None, validate_crs=False): coords : :class:`Coordinates` podpac Coordinates """ - + d = OrderedDict() if isinstance(x, (xr.DataArray, xr.Dataset)): - xcoords = x.coords # only pull crs from the DataArray attrs if the crs is not specified if crs is None: crs = x.attrs.get("crs") + + xcoords = x.coords + if "geotransform" in x.attrs: + other = cls.from_xarray(xcoords, crs=crs, validate_crs=validate_crs).udrop(["lat", "lon"]) + latshape = xcoords["lat"].shape + lonshape = xcoords["lon"].shape + if latshape == lonshape and len(latshape) == 2: + shape = latshape + else: + shape = [latshape[0], lonshape[0]] + xdims = list(xcoords.keys()) + if xdims.index("lat") > xdims.index("lon"): + shape = shape[::-1] + lat_lon = cls.from_geotransform(x.geotransform, shape=shape, crs=crs, validate_crs=validate_crs) + coords = merge_dims([other, lat_lon]) + + # These dims might have something like lat_lon-1, lat_lon-2, so eliminate the '-' ... + dims = [d.split("-")[0] for d in xcoords.dims if d != "output"] + # ... and make sure it's all unique without changing order (np.unique would change order...) + dims = [d for i, d in enumerate(dims) if d not in dims[:i]] + coords = coords.transpose(*dims) + return coords + elif isinstance(x, (xarray.core.coordinates.DataArrayCoordinates, xarray.core.coordinates.DatasetCoordinates)): xcoords = x else: @@ -342,7 +362,6 @@ def from_xarray(cls, x, crs=None, validate_crs=False): if crs is None: warnings.warn("using default crs for podpac coordinates loaded from xarray because no crs was provided") - d = OrderedDict() for dim in xcoords.dims: if dim in d: continue @@ -476,39 +495,11 @@ def from_url(cls, url): @classmethod def from_geotransform(cls, geotransform, shape, crs=None, validate_crs=True): """Creates Coordinates from GDAL Geotransform.""" - tol = 1e-15 # tolerance for deciding when a number is zero - # Handle the case of rotated coordinates - try: - rcoords = RotatedCoordinates.from_geotransform(geotransform, shape, dims=["lat", "lon"]) - except affine_module.UndefinedRotationError: - rcoords = None - _logger.debug("Rasterio source dataset does not have Rotated Coordinates") - - if rcoords is not None and np.abs(rcoords.theta % (np.pi / 2)) > tol: - # These are Rotated coordinates and we can return - coords = Coordinates([rcoords], dims=["lat,lon"], crs=crs) - return coords - - # Handle the case of uniform coordinates (not rotated, but N-S E-W aligned) - affine = rasterio.Affine.from_gdal(*geotransform) - if affine.e <= tol and affine.a <= tol: - order = -1 - step = np.array([affine.d, affine.b]) - else: - order = 1 - step = np.array([affine.e, affine.a]) - origin = affine.f + step[0] / 2, affine.c + step[1] / 2 - end = origin[0] + step[0] * (shape[::order][0] - 1), origin[1] + step[1] * (shape[::order][1] - 1) - coords = Coordinates( - [ - podpac.clinspace(origin[0], end[0], shape[::order][0], "lat"), - podpac.clinspace(origin[1], end[1], shape[::order][1], "lon"), - ][::order], - crs=crs, - validate_crs=validate_crs, - ) - return coords + cs = AffineCoordinates(geotransform, shape).simplify() + if isinstance(cs, AffineCoordinates): + cs = [cs] + return Coordinates(cs, crs=crs, validate_crs=validate_crs) @classmethod def from_definition(cls, d): @@ -550,8 +541,8 @@ def from_definition(cls, d): c = UniformCoordinates1d.from_definition(e) elif "name" in e and "values" in e: c = ArrayCoordinates1d.from_definition(e) - elif "dims" in e and "shape" in e and "theta" in e and "origin" in e and ("step" in e or "corner" in e): - c = RotatedCoordinates.from_definition(e) + elif "geotransform" in e and "shape" in e: + c = AffineCoordinates.from_definition(e) else: raise ValueError("Could not parse coordinates definition item with keys %s" % e.keys()) @@ -565,22 +556,22 @@ def from_definition(cls, d): # ------------------------------------------------------------------------------------------------------------------ def keys(self): - """ dict-like keys: dims """ + """dict-like keys: dims""" return self._coords.keys() def values(self): - """ dict-like values: coordinates for each key/dimension """ + """dict-like values: coordinates for each key/dimension""" return self._coords.values() def items(self): - """ dict-like items: (dim, coordinates) pairs """ + """dict-like items: (dim, coordinates) pairs""" return self._coords.items() def __iter__(self): return iter(self._coords) def get(self, dim, default=None): - """ dict-like get: get coordinates by dimension name with an optional """ + """dict-like get: get coordinates by dimension name with an optional""" try: return self[dim] except KeyError: @@ -653,7 +644,7 @@ def __len__(self): return len(self._coords) def update(self, other): - """ dict-like update: add/replace coordinates using another Coordinates object """ + """dict-like update: add/replace coordinates using another Coordinates object""" if not isinstance(other, Coordinates): raise TypeError("Cannot update Coordinates with object of type '%s'" % type(other)) @@ -762,7 +753,7 @@ def ushape(self): @property def ndim(self): - """:int: Number of dimensions. """ + """:int: Number of dimensions.""" return len(self.shape) @@ -815,7 +806,7 @@ def alt_units(self): @property def properties(self): - """:dict: Dictionary of the coordinate properties. """ + """:dict: Dictionary of the coordinate properties.""" d = OrderedDict() d["crs"] = self.crs @@ -869,7 +860,7 @@ def hash(self): @property def geotransform(self): - """ :tuple: GDAL geotransform. """ + """:tuple: GDAL geotransform.""" # Make sure we only have 1 time and alt dimension if "time" in self.udims and self["time"].size > 1: raise TypeError( @@ -899,11 +890,10 @@ def geotransform(self): self[first].start - self[first].step / 2, self[second].start - self[second].step / 2 ) * rasterio.transform.Affine.scale(self[first].step, self[second].step) transform = transform.to_gdal() - # Do the rotated coordinates cases - elif "lat,lon" in self.dims and isinstance(self._coords["lat,lon"], RotatedCoordinates): - transform = self._coords["lat,lon"].geotransform - elif "lon,lat" in self.dims and isinstance(self._coords["lon,lat"], RotatedCoordinates): - transform = self._coords["lon,lat"].geotransform + elif "lat_lon" in self.dims and isinstance(self._coords["lat_lon"], AffineCoordinates): + transform = self._coords["lat_lon"].geotransform + elif "lon_lat" in self.dims and isinstance(self._coords["lon_lat"], AffineCoordinates): + transform = self._coords["lon_lat"].geotransform else: raise TypeError( "Only 2-D coordinates that are uniform or rotated have a GDAL transform. These coordinates " @@ -933,7 +923,13 @@ def get_area_bounds(self, boundary): Dictionary of (low, high) coordinates area_bounds in each unstacked dimension """ - return {dim: self[dim].get_area_bounds(boundary.get(dim)) for dim in self.udims} + area_bounds = {} + for dim, c in self._coords.items(): + if isinstance(c, StackedCoordinates): + area_bounds.update(c.get_area_bounds(boundary)) + else: + area_bounds[dim] = c.get_area_bounds(boundary.get(dim)) + return area_bounds def drop(self, dims, ignore_missing=False): """ @@ -1033,7 +1029,10 @@ def udrop(self, dims, ignore_missing=False): cs.append(c) elif isinstance(c, StackedCoordinates): stacked = [s for s in c if s.name not in dims] - if len(stacked) > 1: + if len(stacked) == len(c): + # preserves parameterized stacked coordinates such as AffineCoordinates + cs.append(c) + elif len(stacked) > 1: cs.append(StackedCoordinates(stacked)) elif len(stacked) == 1: cs.append(stacked[0]) @@ -1414,7 +1413,7 @@ def transform(self, crs): cs.pop(i) cs.insert(i, c) - # transform the altitude if needed + # transform remaining altitude or stacked spatial dimensions if needed ts = [] for c in cs: tc = c._transform(transformer) @@ -1452,10 +1451,12 @@ def _simplified_transform(self, transformer, cs): # Transform all of the points for this dimension (either lat or lon) and record result this = self[t[i].name] that = self[t[j].name] - if this.size > 1: + if this.size > 1 and that.size > 1: other = clinspace(that.bounds[0], that.bounds[1], this.size) - else: + elif this.size == that.size: other = that.coordinates + else: + other = np.zeros(this.size) + that.coordinates.mean() diagonal = StackedCoordinates([this.coordinates, other], dims=[this.name, that.name]) t_diagonal = diagonal._transform(transformer) cs[self.dims.index(this.name)] = t_diagonal[this.name] @@ -1470,7 +1471,13 @@ def simplify(self): Simplified coordinates. """ - cs = [c.simplify() for c in self._coords.values()] + cs = [] + for c in self._coords.values(): + c2 = c.simplify() + if isinstance(c2, list): + cs += c2 + else: + cs.append(c2) return Coordinates(cs, **self.properties) def issubset(self, other): @@ -1512,9 +1519,8 @@ def __repr__(self): for c in self._coords.values(): if isinstance(c, Coordinates1d): rep += "\n\t%s: %s" % (c.name, c) - elif isinstance(c, RotatedCoordinates): - for dim in c.dims: - rep += "\n\t%s[%s]: Rotated(TODO)" % (c.name, dim) + elif isinstance(c, AffineCoordinates): + rep += "\n\t%s: %s" % (c.name, c) elif isinstance(c, StackedCoordinates): for dim in c.dims: rep += "\n\t%s[%s]: %s" % (c.name, dim, c[dim]) diff --git a/podpac/core/coordinates/coordinates1d.py b/podpac/core/coordinates/coordinates1d.py index bb65c9f03..e855255ed 100644 --- a/podpac/core/coordinates/coordinates1d.py +++ b/podpac/core/coordinates/coordinates1d.py @@ -41,12 +41,12 @@ class Coordinates1d(BaseCoordinates): @tl.observe("name") def _set_property(self, d): - if d["name"] is not None: + if d["new"] is not None: self._properties.add(d["name"]) def _set_name(self, value): # set name if it is not set already, otherwise check that it matches - if "name" not in self._properties: + if "name" not in self._properties and value is not None: self.name = value elif self.name != value: raise ValueError("Dimension mismatch, %s != %s" % (value, self.name)) @@ -69,7 +69,7 @@ def __repr__(self): return "%s: %s" % (name, desc) def _eq_base(self, other): - """ used by child __eq__ methods for common checks """ + """used by child __eq__ methods for common checks""" if not isinstance(other, Coordinates1d): return False @@ -160,13 +160,13 @@ def step(self): @property def bounds(self): - """ Low and high coordinate bounds. """ + """Low and high coordinate bounds.""" raise NotImplementedError @property def properties(self): - """:dict: Dictionary of the coordinate properties. """ + """:dict: Dictionary of the coordinate properties.""" return {key: getattr(self, key) for key in self._properties} diff --git a/podpac/core/coordinates/group_coordinates.py b/podpac/core/coordinates/group_coordinates.py index b949eeadf..3c11d39c9 100644 --- a/podpac/core/coordinates/group_coordinates.py +++ b/podpac/core/coordinates/group_coordinates.py @@ -1,8 +1,8 @@ from __future__ import division, unicode_literals, print_function, absolute_import import json -from hashlib import md5 as hash_alg import traitlets as tl +from podpac.core.utils import hash_alg from podpac.core.coordinates.coordinates import Coordinates from podpac.core.utils import JSONEncoder diff --git a/podpac/core/coordinates/rotated_coordinates.py b/podpac/core/coordinates/rotated_coordinates.py deleted file mode 100644 index 0151fb76c..000000000 --- a/podpac/core/coordinates/rotated_coordinates.py +++ /dev/null @@ -1,344 +0,0 @@ -from __future__ import division, unicode_literals, print_function, absolute_import - -from collections import OrderedDict - -import numpy as np -import traitlets as tl -import lazy_import - -rasterio = lazy_import.lazy_module("rasterio") - -from podpac.core.utils import ArrayTrait -from podpac.core.coordinates.array_coordinates1d import ArrayCoordinates1d -from podpac.core.coordinates.stacked_coordinates import StackedCoordinates - - -class RotatedCoordinates(StackedCoordinates): - """ - A grid of rotated latitude and longitude coordinates. - - RotatedCoordinates are parameterized spatial coordinates defined by a shape, rotation angle, upper left corner, and - step size. The lower right corner can be specified instead of the step. RotatedCoordinates can also be converted - to/from GDAL geotransform. - - Parameters - ---------- - shape : tuple - shape (m, n) of the grid. - theta : float - rotation angle, in radians - origin : np.ndarray(shape=(2,), dtype=float) - origin coordinates (position [0, 0]) - corner : np.ndarray(shape=(2,), dtype=float) - opposing corner coordinates (position [m-1, n-1]) - step : np.ndarray(shape=(2,), dtype=float) - Rotated distance between points in the grid, in each dimension. This is equivalent to the scaling of the - affine transformation used to calculate the coordinates. - dims : tuple - Tuple of dimension names. - coords : dict-like - xarray coordinates (container of coordinate arrays) - coordinates : tuple - Tuple of 2d coordinate values in each dimension. - """ - - shape = tl.Tuple(tl.Integer(), tl.Integer(), read_only=True) - theta = tl.Float(read_only=True) - origin = ArrayTrait(shape=(2,), dtype=float, read_only=True) - step = ArrayTrait(shape=(2,), dtype=float, read_only=True) - dims = tl.Tuple(tl.Unicode(), tl.Unicode(), read_only=True) - - def __init__(self, shape=None, theta=None, origin=None, step=None, corner=None, dims=None): - """ - Create a grid of rotated coordinates from a `shape`, `theta`, `origin`, and `step` or `corner`. - - Parameters - ---------- - shape : tuple - shape (m, n) of the grid. - theta : float - rotation angle, in radians - origin : np.ndarray(shape=(2,), dtype=float) - origin coordinates - corner : np.ndarray(shape=(2,), dtype=float) - opposing corner coordinates (corner or step required) - step : np.ndarray(shape=(2,), dtype=float) - Scaling, ie rotated distance between points in the grid, in each dimension. (corner or step required) - dims : tuple (required) - tuple of dimension names ('lat', 'lon', 'time', or 'alt'). - """ - - self.set_trait("shape", shape) - self.set_trait("theta", theta) - self.set_trait("origin", origin) - if step is None: - deg = np.rad2deg(theta) - a = ~rasterio.Affine.rotation(deg) * ~rasterio.Affine.translation(*origin) - d = np.array(a * corner) - np.array(a * origin) - step = d / np.array([shape[0] - 1, shape[1] - 1]) - self.set_trait("step", step) - if dims is not None: - self.set_trait("dims", dims) - - @tl.validate("dims") - def _validate_dims(self, d): - val = d["value"] - for dim in val: - if dim not in ["lat", "lon"]: - raise ValueError("RotatedCoordinates dims must be 'lat' or 'lon', not '%s'" % dim) - if val[0] == val[1]: - raise ValueError("Duplicate dimension '%s'" % val[0]) - return val - - @tl.validate("shape") - def _validate_shape(self, d): - val = d["value"] - if val[0] <= 0 or val[1] <= 0: - raise ValueError("Invalid shape %s, shape must be positive" % (val,)) - return val - - @tl.validate("step") - def _validate_step(self, d): - val = d["value"] - if val[0] == 0 or val[1] == 0: - raise ValueError("Invalid step %s, step cannot be 0" % val) - return val - - def _set_name(self, value): - self._set_dims(value.split("_")) - - def _set_dims(self, dims): - self.set_trait("dims", dims) - - # ------------------------------------------------------------------------------------------------------------------ - # Alternate Constructors - # ------------------------------------------------------------------------------------------------------------------ - - @classmethod - def from_geotransform(cls, geotransform, shape, dims=None): - affine = rasterio.Affine.from_gdal(*geotransform) - origin = affine.f, affine.c - deg = affine.rotation_angle - scale = ~affine.rotation(deg) * ~affine.translation(*origin) * affine - step = np.array([scale.e, scale.a]) - origin = affine.f + step[0] / 2, affine.c + step[1] / 2 - return cls(shape, np.deg2rad(deg), origin, step, dims=dims) - - @classmethod - def from_definition(cls, d): - """ - Create RotatedCoordinates from a rotated coordinates definition. - - Arguments - --------- - d : dict - rotated coordinates definition - - Returns - ------- - :class:`RotatedCoordinates` - rotated coordinates object - - See Also - -------- - definition - """ - - if "shape" not in d: - raise ValueError('RotatedCoordinates definition requires "shape" property') - if "theta" not in d: - raise ValueError('RotatedCoordinates definition requires "theta" property') - if "origin" not in d: - raise ValueError('RotatedCoordinates definition requires "origin" property') - if "step" not in d and "corner" not in d: - raise ValueError('RotatedCoordinates definition requires "step" or "corner" property') - if "dims" not in d: - raise ValueError('RotatedCoordinates definition requires "dims" property') - - shape = d["shape"] - theta = d["theta"] - origin = d["origin"] - kwargs = {k: v for k, v in d.items() if k not in ["shape", "theta", "origin"]} - return RotatedCoordinates(shape, theta, origin, **kwargs) - - # ------------------------------------------------------------------------------------------------------------------ - # standard methods - # ------------------------------------------------------------------------------------------------------------------ - - def __repr__(self): - return "%s(%s): Origin%s, Corner%s, rad[%.4f], shape%s" % ( - self.__class__.__name__, - self.dims, - self.origin, - self.corner, - self.theta, - self.shape, - ) - - def __eq__(self, other): - if not isinstance(other, RotatedCoordinates): - return False - - if self.dims != other.dims: - return False - - if self.shape != other.shape: - return False - - if self.affine != other.affine: - return False - - return True - - def __getitem__(self, index): - if isinstance(index, slice): - index = index, slice(None) - - if isinstance(index, tuple) and isinstance(index[0], slice) and isinstance(index[1], slice): - I = np.arange(self.shape[0])[index[0]] - J = np.arange(self.shape[1])[index[1]] - origin = self.affine * [I[0], J[0]] - step = self.step * [index[0].step or 1, index[1].step or 1] - shape = I.size, J.size - return RotatedCoordinates(shape, self.theta, origin, step, dims=self.dims) - - else: - # convert to raw StackedCoordinates (which creates the _coords attribute that the indexing requires) - return StackedCoordinates(self.coordinates, dims=self.dims).__getitem__(index) - - # ------------------------------------------------------------------------------------------------------------------ - # Properties - # ------------------------------------------------------------------------------------------------------------------ - - @property - def _coords(self): - raise RuntimeError("RotatedCoordinates do not have a _coords attribute.") - - @property - def ndim(self): - return 2 - - @property - def deg(self): - """ :float: rotation angle in degrees. """ - return np.rad2deg(self.theta) - - @property - def affine(self): - """:rasterio.Affine: affine transformation for computing the coordinates from indexing values. Contains the - tranlation, rotation, and scaling. - """ - t = rasterio.Affine.translation(*self.origin) - r = rasterio.Affine.rotation(self.deg) - s = rasterio.Affine.scale(*self.step) - return t * r * s - - @property - def corner(self): - """ :array: lower right corner. """ - return np.array(self.affine * np.array([self.shape[0] - 1, self.shape[1] - 1])) - - @property - def geotransform(self): - """:tuple: GDAL geotransform. - Note: This property may not provide the correct order of lat/lon in the geotransform as this class does not - always have knowledge of the dimension order of the specified dataset. As such it always supplies - geotransforms assuming that dims = ['lat', 'lon'] - """ - t = rasterio.Affine.translation(self.origin[1] - self.step[1] / 2, self.origin[0] - self.step[0] / 2) - r = rasterio.Affine.rotation(self.deg) - s = rasterio.Affine.scale(*self.step[::-1]) - return (t * r * s).to_gdal() - - @property - def coordinates(self): - """ :tuple: computed coordinave values for each dimension. """ - I = np.arange(self.shape[0]) - J = np.arange(self.shape[1]) - c1, c2 = self.affine * np.meshgrid(I, J) - return c1.T, c2.T - - @property - def definition(self): - d = OrderedDict() - d["dims"] = self.dims - d["shape"] = self.shape - d["theta"] = self.theta - d["origin"] = self.origin - d["step"] = self.step - return d - - @property - def full_definition(self): - return self.definition - - # ------------------------------------------------------------------------------------------------------------------ - # Methods - # ------------------------------------------------------------------------------------------------------------------ - - def copy(self): - """ - Make a copy of the rotated coordinates. - - Returns - ------- - :class:`RotatedCoordinates` - Copy of the rotated coordinates. - """ - return RotatedCoordinates(self.shape, self.theta, self.origin, self.step, dims=self.dims) - - def get_area_bounds(self, boundary): - """Get coordinate area bounds, including boundary information, for each unstacked dimension. - - Arguments - --------- - boundary : dict - dictionary of boundary offsets for each unstacked dimension. Point dimensions can be omitted. - - Returns - ------- - area_bounds : dict - Dictionary of (low, high) coordinates area_bounds in each unstacked dimension - """ - - # TODO the boundary offsets need to be rotated - warnings.warning("RotatedCoordinates area_bounds are not yet correctly implemented.") - return super(RotatedCoordinates, self).get_area_bounds(boundary) - - def select(self, bounds, outer=False, return_index=False): - """ - Get the coordinate values that are within the given bounds in all dimensions. - - *Note: you should not generally need to call this method directly.* - - Parameters - ---------- - bounds : dict - dictionary of dim -> (low, high) selection bounds - outer : bool, optional - If True, do *outer* selections. Default False. - return_index : bool, optional - If True, return index for the selections in addition to coordinates. Default False. - - Returns - ------- - selection : :class:`RotatedCoordinates`, :class:`DependentCoordinates`, :class:`StackedCoordinates` - rotated, dependent, or stacked coordinates consisting of the selection in all dimensions. - selection_index : list - index for the selected coordinates, only if ``return_index`` is True. - """ - - # TODO return RotatedCoordinates when possible - return super(RotatedCoordinates, self).select(bounds, outer=outer, return_index=return_index) - - # ------------------------------------------------------------------------------------------------------------------ - # Debug - # ------------------------------------------------------------------------------------------------------------------ - - # def plot(self, marker='b.', origin_marker='bo', corner_marker='bx'): - # from matplotlib import pyplot - # super(RotatedCoordinates, self).plot(marker=marker) - # ox, oy = self.origin - # cx, cy = self.corner - # pyplot.plot(ox, oy, origin_marker) - # pyplot.plot(cx, cy, corner_marker) diff --git a/podpac/core/coordinates/stacked_coordinates.py b/podpac/core/coordinates/stacked_coordinates.py index bdc0c631c..9ffd45b4f 100644 --- a/podpac/core/coordinates/stacked_coordinates.py +++ b/podpac/core/coordinates/stacked_coordinates.py @@ -8,6 +8,7 @@ import pandas as pd import traitlets as tl from six import string_types +import lazy_import from podpac.core.coordinates.base_coordinates import BaseCoordinates from podpac.core.coordinates.coordinates1d import Coordinates1d @@ -222,7 +223,10 @@ def __getitem__(self, index): return self._coords[self.dims.index(index)] else: - return StackedCoordinates([c[index] for c in self._coords]) + return self._getsubset(index) + + def _getsubset(self, index): + return StackedCoordinates([c[index] for c in self._coords]) def __setitem__(self, dim, c): if not dim in self.dims: @@ -257,7 +261,7 @@ def __contains__(self, item): return (self.flatten().coordinates == item).all(axis=1).any() - def __eq__(self, other): + def _eq_base(self, other): if not isinstance(other, StackedCoordinates): return False @@ -268,6 +272,12 @@ def __eq__(self, other): if self.shape != other.shape: return False + return True + + def __eq__(self, other): + if not self._eq_base(other): + return False + # full check of underlying coordinates if self._coords != other._coords: return False @@ -297,7 +307,7 @@ def name(self): @property def size(self): - """:int: Number of stacked coordinates. """ + """:int: Number of stacked coordinates.""" return self._coords[0].size @property @@ -338,7 +348,7 @@ def xcoords(self): @property def definition(self): - """:list: Serializable stacked coordinates definition. """ + """:list: Serializable stacked coordinates definition.""" return [c.definition for c in self._coords] @@ -460,9 +470,12 @@ def _index_len(index): index = slice(max(index.start or 0 for index in indices), min(index.stop or self.size for index in indices)) # for consistency if index.start == 0 and index.stop == self.size: - index = slice(None, None) + if self.ndim > 1: + index = [slice(None, None) for dim in self.dims] + else: + index = slice(None, None) elif any(_index_len(index) == 0 for index in indices): - return slice(0, 0) + index = slice(0, 0) else: # convert any slices to boolean array for i, index in enumerate(indices): @@ -475,11 +488,17 @@ def _index_len(index): # for consistency if np.all(index): - index = slice(None, None) + if self.ndim > 1: + index = [slice(None, None) for dim in self.dims] + else: + index = slice(None, None) return index def _transform(self, transformer): + if self.size == 0: + return self.copy() + coords = [c.copy() for c in self._coords] if "lat" in self.dims and "lon" in self.dims and "alt" in self.dims: @@ -524,7 +543,7 @@ def _transform(self, transformer): coords[ialt] = ArrayCoordinates1d(talt, "alt").simplify() - return StackedCoordinates(coords) + return StackedCoordinates(coords).simplify() def transpose(self, *dims, **kwargs): """ @@ -622,3 +641,62 @@ def issubset(self, other): ocs.append(StackedCoordinates([coords[dim] for dim in dims])) return all(a.issubset(o) for a, o in zip(acs, ocs)) + + def simplify(self): + if self.is_affine: + from podpac.core.coordinates.affine_coordinates import AffineCoordinates + + # build the geotransform directly + lat = self["lat"].coordinates + lon = self["lon"].coordinates + + # We don't have to check every point in lat/lon for the same step + # since the self.is_affine call did that already + dlati = (lat[-1, 0] - lat[0, 0]) / (lat.shape[0] - 1) + dlatj = (lat[0, -1] - lat[0, 0]) / (lat.shape[1] - 1) + dloni = (lon[-1, 0] - lon[0, 0]) / (lon.shape[0] - 1) + dlonj = (lon[0, -1] - lon[0, 0]) / (lon.shape[1] - 1) + + # origin point + p0 = [lat[0, 0], lon[0, 0]] - np.array([[dlati, dlatj], [dloni, dlonj]]) @ np.ones(2) / 2 + + # This is defined as x ulc, x width, x height, y ulc, y width, y height + # x and y are defined by the CRS. Here we are assuming that it's always + # lon and lat == x and y + geotransform = [p0[1], dlonj, dloni, p0[0], dlatj, dlati] + + a = AffineCoordinates(geotransform=geotransform, shape=self.shape) + + # simplify in order to convert to UniformCoordinates if appropriate + return a.simplify() + + return StackedCoordinates([c.simplify() for c in self._coords]) + + @property + def is_affine(self): + if set(self.dims) != {"lat", "lon"}: + return False + + if not (self.ndim == 2 and self.shape[0] > 1 and self.shape[1] > 1): + return False + + lat = self["lat"].coordinates + lon = self["lon"].coordinates + + d = lat[1:] - lat[:-1] + if not np.allclose(d, d[0, 0]): + return False + + d = lat[:, 1:] - lat[:, :-1] + if not np.allclose(d, d[0, 0]): + return False + + d = lon[1:] - lon[:-1] + if not np.allclose(d, d[0, 0]): + return False + + d = lon[:, 1:] - lon[:, :-1] + if not np.allclose(d, d[0, 0]): + return False + + return True diff --git a/podpac/core/coordinates/test/test_affine_coordinates.py b/podpac/core/coordinates/test/test_affine_coordinates.py new file mode 100644 index 000000000..bae9e1dc5 --- /dev/null +++ b/podpac/core/coordinates/test/test_affine_coordinates.py @@ -0,0 +1,430 @@ +from datetime import datetime +import json + +import pytest +import traitlets as tl +import numpy as np +import pandas as pd +import xarray as xr +import rasterio + +import podpac +from podpac.core.coordinates.stacked_coordinates import StackedCoordinates +from podpac.core.coordinates.array_coordinates1d import ArrayCoordinates1d +from podpac.core.coordinates.affine_coordinates import AffineCoordinates +from podpac.core.coordinates.uniform_coordinates1d import UniformCoordinates1d + +# origin [10, 20], pixel size [3, 2], north up +GEOTRANSFORM_NORTHUP = (10.0, 2.0, 0.0, 20.0, 0.0, -3.0) + +# origin [10, 20], step [2, 3], rotated 20 degrees +GEOTRANSFORM_ROTATED = (10.0, 1.879, -1.026, 20.0, 0.684, 2.819) + +from podpac import Coordinates + +UNIFORM = Coordinates.from_geotransform(geotransform=GEOTRANSFORM_NORTHUP, shape=(3, 4)) + + +class TestAffineCoordinatesCreation(object): + def test_init(self): + c = AffineCoordinates(geotransform=GEOTRANSFORM_NORTHUP, shape=(3, 4)) + + assert c.geotransform == GEOTRANSFORM_NORTHUP + assert c.shape == (3, 4) + assert c.is_affine + assert c.dims == ("lat", "lon") + assert c.udims == ("lat", "lon") + assert len(set(c.xdims)) == 2 + assert c.name == "lat_lon" + repr(c) + + def test_rotated(self): + c = AffineCoordinates(geotransform=GEOTRANSFORM_ROTATED, shape=(3, 4)) + + assert c.geotransform == GEOTRANSFORM_ROTATED + assert c.shape == (3, 4) + assert c.is_affine + assert c.dims == ("lat", "lon") + assert c.udims == ("lat", "lon") + assert len(set(c.xdims)) == 2 + assert c.name == "lat_lon" + repr(c) + + def test_invalid(self): + with pytest.raises(ValueError, match="Invalid shape"): + AffineCoordinates(geotransform=GEOTRANSFORM_NORTHUP, shape=(-3, 4)) + + with pytest.raises(ValueError, match="Invalid shape"): + AffineCoordinates(geotransform=GEOTRANSFORM_NORTHUP, shape=(3, 0)) + + def test_size_one_simplify(self): + c = AffineCoordinates(geotransform=GEOTRANSFORM_NORTHUP, shape=(1, 1)) + c2 = c.simplify() + assert isinstance(c2[0], UniformCoordinates1d) + assert isinstance(c2[1], UniformCoordinates1d) + assert c2[0].shape == (1,) + assert c2[1].shape == (1,) + assert c2[0].step == GEOTRANSFORM_NORTHUP[-1] + assert c2[1].step == GEOTRANSFORM_NORTHUP[1] + assert c2[1].name == "lon" + assert c2[0].name == "lat" + + # def test_copy(self): + # c = AffineCoordinates(geotransform=GEOTRANSFORM_NORTHUP, shape=(3, 4)) + # c2 = c.copy() + # assert c2 is not c + # assert c2 == c + + +# class TestAffineCoordinatesStandardMethods(object): +# def test_eq_type(self): +# c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) +# assert c != [] + +# def test_eq_shape(self): +# c1 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) +# c2 = RotatedCoordinates(shape=(4, 3), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) +# assert c1 != c2 + +# def test_eq_affine(self): +# c1 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) +# c2 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) +# c3 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 3, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) +# c4 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[11, 20], step=[1.0, 2.0], dims=["lat", "lon"]) +# c5 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.1, 2.0], dims=["lat", "lon"]) + +# assert c1 == c2 +# assert c1 != c3 +# assert c1 != c4 +# assert c1 != c5 + +# def test_eq_dims(self): +# c1 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) +# c2 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lon", "lat"]) +# assert c1 != c2 + + +# class TestRotatedCoordinatesSerialization(object): +# def test_definition(self): +# c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) +# d = c.definition + +# assert isinstance(d, dict) +# json.dumps(d, cls=podpac.core.utils.JSONEncoder) # test serializable +# c2 = RotatedCoordinates.from_definition(d) +# assert c2 == c + +# def test_from_definition_corner(self): +# c1 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], corner=[15, 17], dims=["lat", "lon"]) + +# d = {"shape": (3, 4), "theta": np.pi / 4, "origin": [10, 20], "corner": [15, 17], "dims": ["lat", "lon"]} +# c2 = RotatedCoordinates.from_definition(d) + +# assert c1 == c2 + +# def test_invalid_definition(self): +# d = {"theta": np.pi / 4, "origin": [10, 20], "step": [1.0, 2.0], "dims": ["lat", "lon"]} +# with pytest.raises(ValueError, match='RotatedCoordinates definition requires "shape"'): +# RotatedCoordinates.from_definition(d) + +# d = {"shape": (3, 4), "origin": [10, 20], "step": [1.0, 2.0], "dims": ["lat", "lon"]} +# with pytest.raises(ValueError, match='RotatedCoordinates definition requires "theta"'): +# RotatedCoordinates.from_definition(d) + +# d = {"shape": (3, 4), "theta": np.pi / 4, "step": [1.0, 2.0], "dims": ["lat", "lon"]} +# with pytest.raises(ValueError, match='RotatedCoordinates definition requires "origin"'): +# RotatedCoordinates.from_definition(d) + +# d = {"shape": (3, 4), "theta": np.pi / 4, "origin": [10, 20], "dims": ["lat", "lon"]} +# with pytest.raises(ValueError, match='RotatedCoordinates definition requires "step" or "corner"'): +# RotatedCoordinates.from_definition(d) + +# d = {"shape": (3, 4), "theta": np.pi / 4, "origin": [10, 20], "step": [1.0, 2.0]} +# with pytest.raises(ValueError, match='RotatedCoordinates definition requires "dims"'): +# RotatedCoordinates.from_definition(d) + +# def test_full_definition(self): +# c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) +# d = c.full_definition + +# assert isinstance(d, dict) +# assert set(d.keys()) == {"dims", "shape", "theta", "origin", "step"} +# json.dumps(d, cls=podpac.core.utils.JSONEncoder) # test serializable + + +class TestAffineCoordinatesProperties(object): + def test_origin(self): + c = AffineCoordinates(geotransform=GEOTRANSFORM_NORTHUP, shape=(3, 4)) + np.testing.assert_array_equal(c.origin, [20.0, 10.0]) + + def test_origin_rotated(self): + # lat, lon + c = AffineCoordinates(geotransform=GEOTRANSFORM_ROTATED, shape=(3, 4)) + np.testing.assert_array_equal(c.origin, [20.0, 10.0]) + + def test_coordinates(self): + c = AffineCoordinates(geotransform=GEOTRANSFORM_NORTHUP, shape=(3, 4)) + + assert c.coordinates.shape == (3, 4, 2) + + lat = c.coordinates[:, :, 0] + lon = c.coordinates[:, :, 1] + + np.testing.assert_allclose( + lat, + [ + [18.5, 18.5, 18.5, 18.5], + [15.5, 15.5, 15.5, 15.5], + [12.5, 12.5, 12.5, 12.5], + ], + ) + + np.testing.assert_allclose( + lon, + [ + [11.0, 13.0, 15.0, 17.0], + [11.0, 13.0, 15.0, 17.0], + [11.0, 13.0, 15.0, 17.0], + ], + ) + + # def test_coordinates_rotated(self): + # c = AffineCoordinates(geotransform=GEOTRANSFORM_ROTATED, shape=(3, 4)) + + # assert c.coordinates.shape == (3, 4, 2) + + # # lat + # np.testing.assert_allclose( + # c.coordinates[:, :, 0], + # [ + # [20.000, 22.819, 25.638, 28.457], + # [20.684, 23.503, 26.322, 29.141], + # [21.368, 24.187, 27.006, 29.825] + # ], + # ) + + # # lon + # np.testing.assert_allclose( + # c.coordinates[:, :, 1], + # [ + # [10.000, 8.974, 7.948, 6.922], + # [11.879, 10.853, 9.827, 8.801], + # [13.758, 12.732, 11.706, 10.680], + # ], + # ) + + +# class TestRotatedCoordinatesIndexing(object): +# def test_get_dim(self): +# c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) + +# lat = c["lat"] +# lon = c["lon"] +# assert isinstance(lat, ArrayCoordinates1d) +# assert isinstance(lon, ArrayCoordinates1d) +# assert lat.name == "lat" +# assert lon.name == "lon" +# assert_equal(lat.coordinates, c.coordinates[0]) +# assert_equal(lon.coordinates, c.coordinates[1]) + +# with pytest.raises(KeyError, match="Dimension .* not found"): +# c["other"] + +# def test_get_index_slices(self): +# c = RotatedCoordinates(shape=(5, 7), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) + +# # full +# c2 = c[1:4, 2:4] +# assert isinstance(c2, RotatedCoordinates) +# assert c2.shape == (3, 2) +# assert c2.theta == c.theta +# np.testing.assert_allclose(c2.origin, [c.coordinates[0][1, 2], c.coordinates[1][1, 2]]) +# np.testing.assert_allclose(c2.corner, [c.coordinates[0][3, 3], c.coordinates[1][3, 3]]) +# assert c2.dims == c.dims +# np.testing.assert_allclose(c2.coordinates[0], c.coordinates[0][1:4, 2:4]) +# np.testing.assert_allclose(c2.coordinates[1], c.coordinates[1][1:4, 2:4]) + +# # partial/implicit +# c2 = c[1:4] +# assert isinstance(c2, RotatedCoordinates) +# assert c2.shape == (3, 7) +# assert c2.theta == c.theta +# np.testing.assert_allclose(c2.origin, c.coordinates[0][1, 0], c.coordinates[1][1, 0]) +# np.testing.assert_allclose(c2.corner, c.coordinates[0][3, -1], c.coordinates[1][3, -1]) +# assert c2.dims == c.dims +# np.testing.assert_allclose(c2.coordinates[0], c.coordinates[0][1:4]) +# np.testing.assert_allclose(c2.coordinates[1], c.coordinates[1][1:4]) + +# # stepped +# c2 = c[1:4:2, 2:4] +# assert isinstance(c2, RotatedCoordinates) +# assert c2.shape == (2, 2) +# assert c2.theta == c.theta +# np.testing.assert_allclose(c2.origin, [c.coordinates[0][1, 2], c.coordinates[1][1, 2]]) +# np.testing.assert_allclose(c2.corner, [c.coordinates[0][3, 3], c.coordinates[1][3, 3]]) +# assert c2.dims == c.dims +# np.testing.assert_allclose(c2.coordinates[0], c.coordinates[0][1:4:2, 2:4]) +# np.testing.assert_allclose(c2.coordinates[1], c.coordinates[1][1:4:2, 2:4]) + +# # reversed +# c2 = c[4:1:-1, 2:4] +# assert isinstance(c2, RotatedCoordinates) +# assert c2.shape == (3, 2) +# assert c2.theta == c.theta +# np.testing.assert_allclose(c2.origin, c.coordinates[0][1, 2], c.coordinates[0][1, 2]) +# np.testing.assert_allclose(c2.corner, c.coordinates[0][3, 3], c.coordinates[0][3, 3]) +# assert c2.dims == c.dims +# np.testing.assert_allclose(c2.coordinates[0], c.coordinates[0][4:1:-1, 2:4]) +# np.testing.assert_allclose(c2.coordinates[1], c.coordinates[1][4:1:-1, 2:4]) + +# def test_get_index_fallback(self): +# c = RotatedCoordinates(shape=(5, 7), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) +# lat, lon = c.coordinates + +# I = [3, 1] +# J = slice(1, 4) +# B = lat > 6 + +# # int/slice/indices +# c2 = c[I, J] +# assert isinstance(c2, StackedCoordinates) +# assert c2.shape == (2, 3) +# assert c2.dims == c.dims +# assert_equal(c2["lat"].coordinates, lat[I, J]) +# assert_equal(c2["lon"].coordinates, lon[I, J]) + +# # boolean +# c2 = c[B] +# assert isinstance(c2, StackedCoordinates) +# assert c2.shape == (21,) +# assert c2.dims == c.dims +# assert_equal(c2["lat"].coordinates, lat[B]) +# assert_equal(c2["lon"].coordinates, lon[B]) + + +# class TestRotatedCoordinatesSelection(object): +# def test_select_single(self): +# c = RotatedCoordinates([LAT, LON], dims=['lat', 'lon']) + +# # single dimension +# bounds = {'lat': [0.25, .55]} +# E0, E1 = [0, 1, 1, 1], [3, 0, 1, 2] # expected + +# s = c.select(bounds) +# assert isinstance(s, StackedCoordinates) +# assert s == c[E0, E1] + +# s, I = c.select(bounds, return_index=True) +# assert isinstance(s, StackedCoordinates) +# assert s == c[I] +# assert_equal(I[0], E0) +# assert_equal(I[1], E1) + +# # a different single dimension +# bounds = {'lon': [12.5, 17.5]} +# E0, E1 = [0, 1, 1, 1, 1, 2], [3, 0, 1, 2, 3, 0] + +# s = c.select(bounds) +# assert isinstance(s, StackedCoordinates) +# assert s == c[E0, E1] + +# s, I = c.select(bounds, return_index=True) +# assert isinstance(s, StackedCoordinates) +# assert s == c[I] +# assert_equal(I[0], E0) +# assert_equal(I[1], E1) + +# # outer +# bounds = {'lat': [0.25, .75]} +# E0, E1 = [0, 0, 1, 1, 1, 1, 2, 2], [2, 3, 0, 1, 2, 3, 0, 1] + +# s = c.select(bounds, outer=True) +# assert isinstance(s, StackedCoordinates) +# assert s == c[E0, E1] + +# s, I = c.select(bounds, outer=True, return_index=True) +# assert isinstance(s, StackedCoordinates) +# assert s == c[E0, E1] +# assert_equal(I[0], E0) +# assert_equal(I[1], E1) + +# # no matching dimension +# bounds = {'alt': [0, 10]} +# s = c.select(bounds) +# assert s == c + +# s, I = c.select(bounds, return_index=True) +# assert s == c[I] +# assert s == c + +# def test_select_multiple(self): +# c = RotatedCoordinates([LAT, LON], dims=['lat', 'lon']) + +# # this should be the AND of both intersections +# bounds = {'lat': [0.25, 0.95], 'lon': [10.5, 17.5]} +# E0, E1 = [0, 1, 1, 1, 1, 2], [3, 0, 1, 2, 3, 0] +# s = c.select(bounds) +# assert s == c[E0, E1] + +# s, I = c.select(bounds, return_index=True) +# assert s == c[E0, E1] +# assert_equal(I[0], E0) +# assert_equal(I[1], E1) + +# def test_intersect(self): +# c = RotatedCoordinates([LAT, LON], dims=['lat', 'lon']) + + +# other_lat = ArrayCoordinates1d([0.25, 0.5, .95], name='lat') +# other_lon = ArrayCoordinates1d([10.5, 15, 17.5], name='lon') + +# # single other +# E0, E1 = [0, 1, 1, 1, 1, 2, 2, 2], [3, 0, 1, 2, 3, 0, 1, 2] +# s = c.intersect(other_lat) +# assert s == c[E0, E1] + +# s, I = c.intersect(other_lat, return_index=True) +# assert s == c[E0, E1] +# assert s == c[I] + +# E0, E1 = [0, 0, 1, 1, 1, 1, 2, 2, 2, 2], [2, 3, 0, 1, 2, 3, 0, 1, 2, 3] +# s = c.intersect(other_lat, outer=True) +# assert s == c[E0, E1] + +# E0, E1 = [0, 0, 0, 1, 1, 1, 1, 2], [1, 2, 3, 0, 1, 2, 3, 0] +# s = c.intersect(other_lon) +# assert s == c[E0, E1] + +# # multiple, in various ways +# E0, E1 = [0, 1, 1, 1, 1, 2], [3, 0, 1, 2, 3, 0] + +# other = StackedCoordinates([other_lat, other_lon]) +# s = c.intersect(other) +# assert s == c[E0, E1] + +# other = StackedCoordinates([other_lon, other_lat]) +# s = c.intersect(other) +# assert s == c[E0, E1] + +# from podpac.coordinates import Coordinates +# other = Coordinates([other_lat, other_lon]) +# s = c.intersect(other) +# assert s == c[E0, E1] + +# # full +# other = Coordinates(['2018-01-01'], dims=['time']) +# s = c.intersect(other) +# assert s == c + +# s, I = c.intersect(other, return_index=True) +# assert s == c +# assert s == c[I] + +# def test_intersect_invalid(self): +# c = RotatedCoordinates([LAT, LON], dims=['lat', 'lon']) + +# with pytest.raises(TypeError, match="Cannot intersect with type"): +# c.intersect({}) + +# with pytest.raises(ValueError, match="Cannot intersect mismatched dtypes"): +# c.intersect(ArrayCoordinates1d(['2018-01-01'], name='lat')) diff --git a/podpac/core/coordinates/test/test_coordinates.py b/podpac/core/coordinates/test/test_coordinates.py index 964a42606..97db6e362 100644 --- a/podpac/core/coordinates/test/test_coordinates.py +++ b/podpac/core/coordinates/test/test_coordinates.py @@ -13,7 +13,7 @@ from podpac.core.coordinates.coordinates1d import Coordinates1d from podpac.core.coordinates.array_coordinates1d import ArrayCoordinates1d from podpac.core.coordinates.stacked_coordinates import StackedCoordinates -from podpac.core.coordinates.rotated_coordinates import RotatedCoordinates +from podpac.core.coordinates.affine_coordinates import AffineCoordinates from podpac.core.coordinates.uniform_coordinates1d import UniformCoordinates1d from podpac.core.coordinates.cfunctions import crange, clinspace from podpac.core.coordinates.coordinates import Coordinates @@ -180,7 +180,7 @@ def test_stacked_shaped(self): assert c.size == 12 def test_rotated(self): - latlon = RotatedCoordinates((3, 4), np.pi / 4, [10, 20], [1.0, 2.0], dims=["lat", "lon"]) + latlon = AffineCoordinates(geotransform=(10.0, 2.0, 0.0, 20.0, 0.0, -3.0), shape=(3, 4)) c = Coordinates([latlon]) assert c.dims == ("lat_lon",) assert c.udims == ("lat", "lon") @@ -227,8 +227,8 @@ def test_mixed_shaped(self): assert c.size == 72 repr(c) - def test_mixed_rotated(sesf): - latlon = RotatedCoordinates((3, 4), np.pi / 4, [10, 20], [1.0, 2.0], dims=["lat", "lon"]) + def test_mixed_affine(sesf): + latlon = AffineCoordinates(geotransform=(10.0, 2.0, 0.0, 20.0, 0.0, -3.0), shape=(3, 4)) dates = [["2018-01-01", "2018-01-02", "2018-01-03"], ["2019-01-01", "2019-01-02", "2019-01-03"]] c = Coordinates([latlon, dates], dims=["lat_lon", "time"]) assert c.dims == ("lat_lon", "time") @@ -640,8 +640,8 @@ def test_definition_shaped(self): c2 = Coordinates.from_definition(d) assert c2 == c - def test_definition_rotated(self): - latlon = RotatedCoordinates((3, 4), np.pi / 4, [10, 20], [1.0, 2.0], dims=["lat", "lon"]) + def test_definition_affine(self): + latlon = AffineCoordinates(geotransform=(10.0, 2.0, 0.0, 20.0, 0.0, -3.0), shape=(3, 4)) c = Coordinates([latlon]) d = c.definition json.dumps(d, cls=podpac.core.utils.JSONEncoder) @@ -1780,25 +1780,53 @@ def test_concat_crs(self): class TestCoordinatesGeoTransform(object): - def uniform_working(self): + def test_uniform_working(self): # order: -lat, lon c = Coordinates([clinspace(1.5, 0.5, 5, "lat"), clinspace(1, 2, 9, "lon")]) + c2 = Coordinates.from_geotransform(c.geotransform, c.shape) + c3 = Coordinates.from_xarray(podpac.Node().create_output_array(c)) + assert c == c2 + assert c == c3 tf = np.array(c.geotransform).reshape(2, 3) np.testing.assert_almost_equal( - tf, np.array([[c["lon"].bounds[0], c["lon"].step, 0], [c["lat"].bounds[1], 0, c["lat"].step]]) + tf, + np.array( + [ + [c["lon"].bounds[0] - c["lon"].step / 2, c["lon"].step, 0], + [c["lat"].bounds[1] - c["lat"].step / 2, 0, c["lat"].step], + ] + ), ) # order: lon, lat c = Coordinates([clinspace(0.5, 1.5, 5, "lon"), clinspace(1, 2, 9, "lat")]) + c2 = Coordinates.from_geotransform(c.geotransform, c.shape) + c3 = Coordinates.from_xarray(podpac.Node().create_output_array(c)) + assert c == c2 + assert c == c3 tf = np.array(c.geotransform).reshape(2, 3) np.testing.assert_almost_equal( - tf, np.array([[c["lon"].bounds[0], 0, c["lon"].step], [c["lat"].bounds[0], c["lat"].step, 0]]) + tf, + np.array( + [ + [c["lon"].bounds[0] - c["lon"].step / 2, 0, c["lon"].step], + [c["lat"].bounds[0] - c["lat"].step / 2, c["lat"].step, 0], + ] + ), ) # order: lon, -lat, time c = Coordinates([clinspace(0.5, 1.5, 5, "lon"), clinspace(2, 1, 9, "lat"), crange(10, 11, 2, "time")]) + c2 = Coordinates.from_geotransform(c.geotransform, c.drop("time").shape) + assert c.drop("time") == c2 tf = np.array(c.geotransform).reshape(2, 3) np.testing.assert_almost_equal( - tf, np.array([[c["lon"].bounds[0], 0, c["lon"].step], [c["lat"].bounds[1], c["lat"].step, 0]]) + tf, + np.array( + [ + [c["lon"].bounds[0] - c["lon"].step / 2, 0, c["lon"].step], + [c["lat"].bounds[1] - c["lat"].step / 2, c["lat"].step, 0], + ] + ), ) # order: -lon, -lat, time, alt c = Coordinates( @@ -1809,12 +1837,20 @@ def uniform_working(self): crange(10, 11, 2, "alt"), ] ) + c2 = Coordinates.from_geotransform(c.geotransform, c.drop(["time", "alt"]).shape) + assert c.drop(["time", "alt"]) == c2 tf = np.array(c.geotransform).reshape(2, 3) np.testing.assert_almost_equal( - tf, np.array([[c["lon"].bounds[1], 0, c["lon"].step], [c["lat"].bounds[1], c["lat"].step, 0]]) + tf, + np.array( + [ + [c["lon"].bounds[1] - c["lon"].step / 2, 0, c["lon"].step], + [c["lat"].bounds[1] - c["lat"].step / 2, c["lat"].step, 0], + ] + ), ) - def error_time_alt_too_big(self): + def test_error_time_alt_too_big(self): # time c = Coordinates( [ @@ -1835,6 +1871,7 @@ def error_time_alt_too_big(self): ): c.geotransform + @pytest.mark.skip(reason="obsolete") def rot_coords_working(self): # order -lat, lon rc = RotatedCoordinates(shape=(4, 3), theta=np.pi / 8, origin=[10, 20], step=[-2.0, 1.0], dims=["lat", "lon"]) @@ -2094,3 +2131,74 @@ def test_transform_same_crs_same_result(self): assert_array_equal(c2["lat"].coordinates, c1["lat"].coordinates) assert_array_equal(c2["lon"].coordinates, c1["lon"].coordinates) + + def test_transform_size_1_lat(self): + c1 = Coordinates([ArrayCoordinates1d([1], name="lat"), clinspace(0, 2, 5, "lon")], crs="EPSG:3857") + c2 = c1.transform("EPSG:4326") + assert c2.shape == c1.shape + c1 = Coordinates([clinspace(0, 2, 5, "lat"), ArrayCoordinates1d([1], name="lon")], crs="EPSG:3857") + c2 = c1.transform("EPSG:4326") + assert c2.shape == c1.shape + + +class TestCoordinatesMethodSimplify(object): + def test_simplify_array_to_uniform(self): + c1 = Coordinates([[1, 2, 3, 4], [4, 6, 8]], dims=["lat", "lon"]) + c2 = Coordinates([[1, 2, 3, 5], [4, 6, 8]], dims=["lat", "lon"]) + c3 = Coordinates([clinspace(1, 4, 4), clinspace(4, 8, 3)], dims=["lat", "lon"]) + + # array -> uniform + assert c1.simplify() == c3.simplify() + + # array -> array + assert c2.simplify() == c2.simplify() + + # uniform -> uniform + assert c3.simplify() == c3.simplify() + + @pytest.mark.skip(reason="not implemented, spec uncertain") + def test_simplify_stacked_to_unstacked_arrays(self): + stacked = Coordinates([np.meshgrid([1, 2, 3, 5], [4, 6, 8])], dims=["lat_lon"]) + unstacked = Coordinates([[1, 2, 3, 5], [4, 6, 8]], dims=["lat", "lon"]) + + assert stacked.simplify() == unstacked + assert unstacked.simplify() == unstacked + + def test_stacked_to_unstacked_uniform(self): + stacked = Coordinates([np.meshgrid([4, 6, 8], [1, 2, 3, 4])[::-1]], dims=["lat_lon"]) + unstacked_uniform = Coordinates([clinspace(1, 4, 4), clinspace(4, 8, 3)], dims=["lat", "lon"]) + + # stacked grid -> uniform + assert stacked.simplify() == unstacked_uniform + + # uniform -> uniform + assert unstacked_uniform.simplify() == unstacked_uniform + + def test_stacked_to_affine(self): + geotransform_rotated = (10.0, 1.879, -1.026, 20.0, 0.684, 2.819) + affine = Coordinates([AffineCoordinates(geotransform=geotransform_rotated, shape=(4, 6))]) + stacked = Coordinates([StackedCoordinates([affine["lat_lon"]["lat"], affine["lat_lon"]["lon"]])]) + + # stacked -> affine + assert stacked.simplify() == affine + + # affine -> affine + assert affine.simplify() == affine + + def test_affine_to_uniform(self): + # NOTE: this assumes that podpac prefers unstacked UniformCoordinates to AffineCoordinates + geotransform_northup = (10.0, 2.0, 0.0, 20.0, 0.0, -3.0) + geotransform_rotated = (10.0, 1.879, -1.026, 20.0, 0.684, 2.819) + + c1 = Coordinates([AffineCoordinates(geotransform=geotransform_northup, shape=(4, 6))]) + c2 = Coordinates([AffineCoordinates(geotransform=geotransform_rotated, shape=(4, 6))]) + c3 = Coordinates([clinspace(18.5, 9.5, 4, name="lat"), clinspace(11, 21, 6, name="lon")]) + + # unrotated affine -> unstacked uniform + assert c1.simplify() == c3 + + # rotated affine -> rotated affine + assert c2.simplify() == c2 + + # unstacked uniform -> unstacked uniform + assert c3.simplify() == c3 diff --git a/podpac/core/coordinates/test/test_rotated_coordinates.py b/podpac/core/coordinates/test/test_rotated_coordinates.py deleted file mode 100644 index ecacde1dc..000000000 --- a/podpac/core/coordinates/test/test_rotated_coordinates.py +++ /dev/null @@ -1,456 +0,0 @@ -from datetime import datetime -import json - -import pytest -import traitlets as tl -import numpy as np -import pandas as pd -import xarray as xr -from numpy.testing import assert_equal, assert_allclose - -import podpac -from podpac.coordinates import ArrayCoordinates1d -from podpac.core.coordinates.stacked_coordinates import StackedCoordinates -from podpac.core.coordinates.array_coordinates1d import ArrayCoordinates1d -from podpac.core.coordinates.rotated_coordinates import RotatedCoordinates - - -class TestRotatedCoordinatesCreation(object): - def test_init_step(self): - # positive steps - c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - - assert c.shape == (3, 4) - assert c.theta == np.pi / 4 - assert_equal(c.origin, [10, 20]) - assert_equal(c.step, [1.0, 2.0]) - assert_allclose(c.corner, [7.171573, 25.656854]) - assert c.dims == ("lat", "lon") - assert c.udims == ("lat", "lon") - assert len(set(c.xdims)) == 2 - assert c.name == "lat_lon" - repr(c) - - # negative steps - c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[-1.0, -2.0], dims=["lat", "lon"]) - assert c.shape == (3, 4) - assert c.theta == np.pi / 4 - assert_equal(c.origin, [10, 20]) - assert_equal(c.step, [-1.0, -2.0]) - assert_allclose(c.corner, [12.828427, 14.343146]) - assert c.dims == ("lat", "lon") - assert c.udims == ("lat", "lon") - assert len(set(c.xdims)) == 2 - assert c.name == "lat_lon" - repr(c) - - def test_dims(self): - # lon_lat - c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lon", "lat"]) - assert c.dims == ("lon", "lat") - assert c.udims == ("lon", "lat") - assert len(set(c.xdims)) == 2 - assert c.name == "lon_lat" - - # alt - with pytest.raises(ValueError, match="RotatedCoordinates dims must be 'lat' or 'lon'"): - c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "alt"]) - - def test_init_corner(self): - c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], corner=[15, 17], dims=["lat", "lon"]) - assert c.shape == (3, 4) - assert c.theta == np.pi / 4 - assert_equal(c.origin, [10, 20]) - assert_allclose(c.step, [0.70710678, -1.88561808]) - assert_allclose(c.corner, [15.0, 17.0]) - assert c.dims == ("lat", "lon") - assert c.udims == ("lat", "lon") - assert len(set(c.xdims)) == 2 - assert c.name == "lat_lon" - repr(c) - - def test_thetas(self): - c = RotatedCoordinates(shape=(3, 4), theta=0 * np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - assert_allclose(c.corner, [12.0, 26.0]) - - c = RotatedCoordinates(shape=(3, 4), theta=1 * np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - assert_allclose(c.corner, [7.171573, 25.656854]) - - c = RotatedCoordinates(shape=(3, 4), theta=2 * np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - assert_allclose(c.corner, [4.0, 22.0]) - - c = RotatedCoordinates(shape=(3, 4), theta=3 * np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - assert_allclose(c.corner, [4.343146, 17.171573]) - - c = RotatedCoordinates(shape=(3, 4), theta=4 * np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - assert_allclose(c.corner, [8.0, 14.0]) - - c = RotatedCoordinates(shape=(3, 4), theta=5 * np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - assert_allclose(c.corner, [12.828427, 14.343146]) - - c = RotatedCoordinates(shape=(3, 4), theta=6 * np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - assert_allclose(c.corner, [16.0, 18.0]) - - c = RotatedCoordinates(shape=(3, 4), theta=7 * np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - assert_allclose(c.corner, [15.656854, 22.828427]) - - c = RotatedCoordinates(shape=(3, 4), theta=8 * np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - assert_allclose(c.corner, [12.0, 26.0]) - - c = RotatedCoordinates(shape=(3, 4), theta=-np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - assert_allclose(c.corner, [15.656854, 22.828427]) - - def test_invalid(self): - with pytest.raises(ValueError, match="Invalid shape"): - RotatedCoordinates(shape=(-3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - - with pytest.raises(ValueError, match="Invalid shape"): - RotatedCoordinates(shape=(3, 0), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - - with pytest.raises(ValueError, match="Invalid step"): - RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[0, 2.0], dims=["lat", "lon"]) - - with pytest.raises(ValueError, match="Duplicate dimension"): - RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lat"]) - - def test_copy(self): - c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - c2 = c.copy() - assert c2 is not c - assert c2 == c - - -class TestRotatedCoordinatesGeotransform(object): - def test_geotransform(self): - c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - assert_allclose(c.geotransform, (19.0, 1.4142136, -0.7071068, 9.5, 1.4142136, 0.7071068)) - - c2 = RotatedCoordinates.from_geotransform(c.geotransform, c.shape, dims=["lat", "lon"]) - assert c == c2 - - c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lon", "lat"]) - assert_allclose(c.geotransform, (19.0, 1.4142136, -0.7071068, 9.5, 1.4142136, 0.7071068)) - - c2 = RotatedCoordinates.from_geotransform(c.geotransform, c.shape, dims=["lon", "lat"]) - assert c == c2 - - -class TestRotatedCoordinatesStandardMethods(object): - def test_eq_type(self): - c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - assert c != [] - - def test_eq_shape(self): - c1 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - c2 = RotatedCoordinates(shape=(4, 3), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - assert c1 != c2 - - def test_eq_affine(self): - c1 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - c2 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - c3 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 3, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - c4 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[11, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - c5 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.1, 2.0], dims=["lat", "lon"]) - - assert c1 == c2 - assert c1 != c3 - assert c1 != c4 - assert c1 != c5 - - def test_eq_dims(self): - c1 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - c2 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lon", "lat"]) - assert c1 != c2 - - -class TestRotatedCoordinatesSerialization(object): - def test_definition(self): - c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - d = c.definition - - assert isinstance(d, dict) - json.dumps(d, cls=podpac.core.utils.JSONEncoder) # test serializable - c2 = RotatedCoordinates.from_definition(d) - assert c2 == c - - def test_from_definition_corner(self): - c1 = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], corner=[15, 17], dims=["lat", "lon"]) - - d = {"shape": (3, 4), "theta": np.pi / 4, "origin": [10, 20], "corner": [15, 17], "dims": ["lat", "lon"]} - c2 = RotatedCoordinates.from_definition(d) - - assert c1 == c2 - - def test_invalid_definition(self): - d = {"theta": np.pi / 4, "origin": [10, 20], "step": [1.0, 2.0], "dims": ["lat", "lon"]} - with pytest.raises(ValueError, match='RotatedCoordinates definition requires "shape"'): - RotatedCoordinates.from_definition(d) - - d = {"shape": (3, 4), "origin": [10, 20], "step": [1.0, 2.0], "dims": ["lat", "lon"]} - with pytest.raises(ValueError, match='RotatedCoordinates definition requires "theta"'): - RotatedCoordinates.from_definition(d) - - d = {"shape": (3, 4), "theta": np.pi / 4, "step": [1.0, 2.0], "dims": ["lat", "lon"]} - with pytest.raises(ValueError, match='RotatedCoordinates definition requires "origin"'): - RotatedCoordinates.from_definition(d) - - d = {"shape": (3, 4), "theta": np.pi / 4, "origin": [10, 20], "dims": ["lat", "lon"]} - with pytest.raises(ValueError, match='RotatedCoordinates definition requires "step" or "corner"'): - RotatedCoordinates.from_definition(d) - - d = {"shape": (3, 4), "theta": np.pi / 4, "origin": [10, 20], "step": [1.0, 2.0]} - with pytest.raises(ValueError, match='RotatedCoordinates definition requires "dims"'): - RotatedCoordinates.from_definition(d) - - def test_full_definition(self): - c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - d = c.full_definition - - assert isinstance(d, dict) - assert set(d.keys()) == {"dims", "shape", "theta", "origin", "step"} - json.dumps(d, cls=podpac.core.utils.JSONEncoder) # test serializable - - -class TestRotatedCoordinatesProperties(object): - def test_affine(self): - c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - R = c.affine - assert_allclose([R.a, R.b, R.c, R.d, R.e, R.f], [0.70710678, -1.41421356, 10.0, 0.70710678, 1.41421356, 20.0]) - - def test_coordinates(self): - c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - lat, lon = c.coordinates - - assert_allclose( - lat, - [ - [10.0, 8.58578644, 7.17157288, 5.75735931], - [10.70710678, 9.29289322, 7.87867966, 6.46446609], - [11.41421356, 10.0, 8.58578644, 7.17157288], - ], - ) - - assert_allclose( - lon, - [ - [20.0, 21.41421356, 22.82842712, 24.24264069], - [20.70710678, 22.12132034, 23.53553391, 24.94974747], - [21.41421356, 22.82842712, 24.24264069, 25.65685425], - ], - ) - - -class TestRotatedCoordinatesIndexing(object): - def test_get_dim(self): - c = RotatedCoordinates(shape=(3, 4), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - - lat = c["lat"] - lon = c["lon"] - assert isinstance(lat, ArrayCoordinates1d) - assert isinstance(lon, ArrayCoordinates1d) - assert lat.name == "lat" - assert lon.name == "lon" - assert_equal(lat.coordinates, c.coordinates[0]) - assert_equal(lon.coordinates, c.coordinates[1]) - - with pytest.raises(KeyError, match="Dimension .* not found"): - c["other"] - - def test_get_index_slices(self): - c = RotatedCoordinates(shape=(5, 7), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - - # full - c2 = c[1:4, 2:4] - assert isinstance(c2, RotatedCoordinates) - assert c2.shape == (3, 2) - assert c2.theta == c.theta - assert_allclose(c2.origin, [c.coordinates[0][1, 2], c.coordinates[1][1, 2]]) - assert_allclose(c2.corner, [c.coordinates[0][3, 3], c.coordinates[1][3, 3]]) - assert c2.dims == c.dims - assert_allclose(c2.coordinates[0], c.coordinates[0][1:4, 2:4]) - assert_allclose(c2.coordinates[1], c.coordinates[1][1:4, 2:4]) - - # partial/implicit - c2 = c[1:4] - assert isinstance(c2, RotatedCoordinates) - assert c2.shape == (3, 7) - assert c2.theta == c.theta - assert_allclose(c2.origin, c.coordinates[0][1, 0], c.coordinates[1][1, 0]) - assert_allclose(c2.corner, c.coordinates[0][3, -1], c.coordinates[1][3, -1]) - assert c2.dims == c.dims - assert_allclose(c2.coordinates[0], c.coordinates[0][1:4]) - assert_allclose(c2.coordinates[1], c.coordinates[1][1:4]) - - # stepped - c2 = c[1:4:2, 2:4] - assert isinstance(c2, RotatedCoordinates) - assert c2.shape == (2, 2) - assert c2.theta == c.theta - assert_allclose(c2.origin, [c.coordinates[0][1, 2], c.coordinates[1][1, 2]]) - assert_allclose(c2.corner, [c.coordinates[0][3, 3], c.coordinates[1][3, 3]]) - assert c2.dims == c.dims - assert_allclose(c2.coordinates[0], c.coordinates[0][1:4:2, 2:4]) - assert_allclose(c2.coordinates[1], c.coordinates[1][1:4:2, 2:4]) - - # reversed - c2 = c[4:1:-1, 2:4] - assert isinstance(c2, RotatedCoordinates) - assert c2.shape == (3, 2) - assert c2.theta == c.theta - assert_allclose(c2.origin, c.coordinates[0][1, 2], c.coordinates[0][1, 2]) - assert_allclose(c2.corner, c.coordinates[0][3, 3], c.coordinates[0][3, 3]) - assert c2.dims == c.dims - assert_allclose(c2.coordinates[0], c.coordinates[0][4:1:-1, 2:4]) - assert_allclose(c2.coordinates[1], c.coordinates[1][4:1:-1, 2:4]) - - def test_get_index_fallback(self): - c = RotatedCoordinates(shape=(5, 7), theta=np.pi / 4, origin=[10, 20], step=[1.0, 2.0], dims=["lat", "lon"]) - lat, lon = c.coordinates - - I = [3, 1] - J = slice(1, 4) - B = lat > 6 - - # int/slice/indices - c2 = c[I, J] - assert isinstance(c2, StackedCoordinates) - assert c2.shape == (2, 3) - assert c2.dims == c.dims - assert_equal(c2["lat"].coordinates, lat[I, J]) - assert_equal(c2["lon"].coordinates, lon[I, J]) - - # boolean - c2 = c[B] - assert isinstance(c2, StackedCoordinates) - assert c2.shape == (21,) - assert c2.dims == c.dims - assert_equal(c2["lat"].coordinates, lat[B]) - assert_equal(c2["lon"].coordinates, lon[B]) - - -# class TestRotatedCoordinatesSelection(object): -# def test_select_single(self): -# c = RotatedCoordinates([LAT, LON], dims=['lat', 'lon']) - -# # single dimension -# bounds = {'lat': [0.25, .55]} -# E0, E1 = [0, 1, 1, 1], [3, 0, 1, 2] # expected - -# s = c.select(bounds) -# assert isinstance(s, StackedCoordinates) -# assert s == c[E0, E1] - -# s, I = c.select(bounds, return_index=True) -# assert isinstance(s, StackedCoordinates) -# assert s == c[I] -# assert_equal(I[0], E0) -# assert_equal(I[1], E1) - -# # a different single dimension -# bounds = {'lon': [12.5, 17.5]} -# E0, E1 = [0, 1, 1, 1, 1, 2], [3, 0, 1, 2, 3, 0] - -# s = c.select(bounds) -# assert isinstance(s, StackedCoordinates) -# assert s == c[E0, E1] - -# s, I = c.select(bounds, return_index=True) -# assert isinstance(s, StackedCoordinates) -# assert s == c[I] -# assert_equal(I[0], E0) -# assert_equal(I[1], E1) - -# # outer -# bounds = {'lat': [0.25, .75]} -# E0, E1 = [0, 0, 1, 1, 1, 1, 2, 2], [2, 3, 0, 1, 2, 3, 0, 1] - -# s = c.select(bounds, outer=True) -# assert isinstance(s, StackedCoordinates) -# assert s == c[E0, E1] - -# s, I = c.select(bounds, outer=True, return_index=True) -# assert isinstance(s, StackedCoordinates) -# assert s == c[E0, E1] -# assert_equal(I[0], E0) -# assert_equal(I[1], E1) - -# # no matching dimension -# bounds = {'alt': [0, 10]} -# s = c.select(bounds) -# assert s == c - -# s, I = c.select(bounds, return_index=True) -# assert s == c[I] -# assert s == c - -# def test_select_multiple(self): -# c = RotatedCoordinates([LAT, LON], dims=['lat', 'lon']) - -# # this should be the AND of both intersections -# bounds = {'lat': [0.25, 0.95], 'lon': [10.5, 17.5]} -# E0, E1 = [0, 1, 1, 1, 1, 2], [3, 0, 1, 2, 3, 0] -# s = c.select(bounds) -# assert s == c[E0, E1] - -# s, I = c.select(bounds, return_index=True) -# assert s == c[E0, E1] -# assert_equal(I[0], E0) -# assert_equal(I[1], E1) - -# def test_intersect(self): -# c = RotatedCoordinates([LAT, LON], dims=['lat', 'lon']) - - -# other_lat = ArrayCoordinates1d([0.25, 0.5, .95], name='lat') -# other_lon = ArrayCoordinates1d([10.5, 15, 17.5], name='lon') - -# # single other -# E0, E1 = [0, 1, 1, 1, 1, 2, 2, 2], [3, 0, 1, 2, 3, 0, 1, 2] -# s = c.intersect(other_lat) -# assert s == c[E0, E1] - -# s, I = c.intersect(other_lat, return_index=True) -# assert s == c[E0, E1] -# assert s == c[I] - -# E0, E1 = [0, 0, 1, 1, 1, 1, 2, 2, 2, 2], [2, 3, 0, 1, 2, 3, 0, 1, 2, 3] -# s = c.intersect(other_lat, outer=True) -# assert s == c[E0, E1] - -# E0, E1 = [0, 0, 0, 1, 1, 1, 1, 2], [1, 2, 3, 0, 1, 2, 3, 0] -# s = c.intersect(other_lon) -# assert s == c[E0, E1] - -# # multiple, in various ways -# E0, E1 = [0, 1, 1, 1, 1, 2], [3, 0, 1, 2, 3, 0] - -# other = StackedCoordinates([other_lat, other_lon]) -# s = c.intersect(other) -# assert s == c[E0, E1] - -# other = StackedCoordinates([other_lon, other_lat]) -# s = c.intersect(other) -# assert s == c[E0, E1] - -# from podpac.coordinates import Coordinates -# other = Coordinates([other_lat, other_lon]) -# s = c.intersect(other) -# assert s == c[E0, E1] - -# # full -# other = Coordinates(['2018-01-01'], dims=['time']) -# s = c.intersect(other) -# assert s == c - -# s, I = c.intersect(other, return_index=True) -# assert s == c -# assert s == c[I] - -# def test_intersect_invalid(self): -# c = RotatedCoordinates([LAT, LON], dims=['lat', 'lon']) - -# with pytest.raises(TypeError, match="Cannot intersect with type"): -# c.intersect({}) - -# with pytest.raises(ValueError, match="Cannot intersect mismatched dtypes"): -# c.intersect(ArrayCoordinates1d(['2018-01-01'], name='lat')) diff --git a/podpac/core/coordinates/test/test_uniform_coordinates1d.py b/podpac/core/coordinates/test/test_uniform_coordinates1d.py index f7273d39a..51f1924ae 100644 --- a/podpac/core/coordinates/test/test_uniform_coordinates1d.py +++ b/podpac/core/coordinates/test/test_uniform_coordinates1d.py @@ -53,7 +53,7 @@ def test_numerical_inexact(self): c = UniformCoordinates1d(0, 49, 10) a = np.array([0, 10, 20, 30, 40], dtype=float) assert c.start == 0 - assert c.stop == 49 + assert c.stop == 40 assert c.step == 10 assert_equal(c.coordinates, a) assert_equal(c.bounds, [0, 40]) @@ -69,7 +69,7 @@ def test_numerical_inexact(self): c = UniformCoordinates1d(50, 1, -10) a = np.array([50, 40, 30, 20, 10], dtype=float) assert c.start == 50 - assert c.stop == 1 + assert c.stop == 10 assert c.step == -10 assert_equal(c.coordinates, a) assert_equal(c.bounds, [10, 50]) @@ -119,7 +119,7 @@ def test_datetime_inexact(self): c = UniformCoordinates1d("2018-01-01", "2018-01-06", "2,D") a = np.array(["2018-01-01", "2018-01-03", "2018-01-05"]).astype(np.datetime64) assert c.start == np.datetime64("2018-01-01") - assert c.stop == np.datetime64("2018-01-06") + assert c.stop == np.datetime64("2018-01-05") assert c.step == np.timedelta64(2, "D") assert_equal(c.coordinates, a) assert_equal(c.bounds, a[[0, -1]]) @@ -135,7 +135,7 @@ def test_datetime_inexact(self): c = UniformCoordinates1d("2018-01-06", "2018-01-01", "-2,D") a = np.array(["2018-01-06", "2018-01-04", "2018-01-02"]).astype(np.datetime64) assert c.start == np.datetime64("2018-01-06") - assert c.stop == np.datetime64("2018-01-01") + assert c.stop == np.datetime64("2018-01-02") assert c.step == np.timedelta64(-2, "D") assert_equal(c.coordinates, a) assert_equal(c.bounds, a[[-1, 0]]) @@ -217,7 +217,7 @@ def test_datetime_year_step(self): c = UniformCoordinates1d("2018-01-01", "2021-04-01", "1,Y") a = np.array(["2018-01-01", "2019-01-01", "2020-01-01", "2021-01-01"]).astype(np.datetime64) assert c.start == np.datetime64("2018-01-01") - assert c.stop == np.datetime64("2021-04-01") + assert c.stop == np.datetime64("2021-01-01") assert c.step == np.timedelta64(1, "Y") assert_equal(c.coordinates, a) assert_equal(c.bounds, a[[0, -1]]) @@ -232,7 +232,7 @@ def test_datetime_year_step(self): c = UniformCoordinates1d("2018-04-01", "2021-01-01", "1,Y") a = np.array(["2018-04-01", "2019-04-01", "2020-04-01"]).astype(np.datetime64) assert c.start == np.datetime64("2018-04-01") - assert c.stop == np.datetime64("2021-01-01") + assert c.stop == np.datetime64("2020-04-01") assert c.step == np.timedelta64(1, "Y") assert_equal(c.coordinates, a) assert_equal(c.bounds, a[[0, -1]]) @@ -248,7 +248,7 @@ def test_datetime_year_step(self): c = UniformCoordinates1d("2021-01-01", "2018-04-01", "-1,Y") a = np.array(["2021-01-01", "2020-01-01", "2019-01-01", "2018-01-01"]).astype(np.datetime64) assert c.start == np.datetime64("2021-01-01") - assert c.stop == np.datetime64("2018-04-01") + assert c.stop == np.datetime64("2018-01-01") assert c.step == np.timedelta64(-1, "Y") assert_equal(c.coordinates, a) assert_equal(c.bounds, a[[-1, 0]]) @@ -263,7 +263,7 @@ def test_datetime_year_step(self): c = UniformCoordinates1d("2021-04-01", "2018-01-01", "-1,Y") a = np.array(["2021-04-01", "2020-04-01", "2019-04-01", "2018-04-01"]).astype(np.datetime64) assert c.start == np.datetime64("2021-04-01") - assert c.stop == np.datetime64("2018-01-01") + assert c.stop == np.datetime64("2018-04-01") assert c.step == np.timedelta64(-1, "Y") assert_equal(c.coordinates, a) assert_equal(c.bounds, a[[-1, 0]]) @@ -611,7 +611,7 @@ def test_index(self): assert c2.name == c.name assert c2.properties == c.properties assert c2.start == 0 - assert c2.stop == 50 + assert c2.stop == 40 assert c2.step == 20 c2 = c[1:-1] @@ -727,7 +727,7 @@ def test_index_descending(self): assert c2.name == c.name assert c2.properties == c.properties assert c2.start == 50 - assert c2.stop == 0 + assert c2.stop == 10 assert c2.step == -20 c2 = c[1:-1] @@ -1237,3 +1237,67 @@ def test_issubset_coordinates(self): assert u.issubset(c1) assert not u.issubset(c2) assert not u.issubset(c3) + + def test_coordinates_floating_point_consistency(self): + c = podpac.Coordinates.from_url( + "?VERSION=1.3.0&HEIGHT=512&WIDTH=512&CRS=EPSG%3A3857&BBOX=-8061966.247294108,5322463.153553393,-7983694.730330088,5400734.670517412" + ).transform("EPSG:4326") + u = c["lon"] + u2 = podpac.coordinates.UniformCoordinates1d( + u.start - 2 * u.step, u.stop + 2 * u.step + 1e-14, step=u.step, fix_stop_val=True + ) + u3 = podpac.coordinates.UniformCoordinates1d( + u.start - 2 * u.step, u.stop + 2 * u.step + 1e-14, size=u.size + 4, fix_stop_val=True + ) + + assert u2.start == u3.start + assert u2.stop == u3.stop + step = (u2.stop - u2.start) / (u2.size - 1) + assert u2.step == step + assert u3.step == step + assert_equal(u2.coordinates, u3.coordinates) + + # Lat has a different order (i.e negative step) + u = c["lat"] + step = (u.coordinates[-1] - u.coordinates[0]) / (u.size - 1) + start = u.coordinates[0] + stop = u.coordinates[-1] + u2 = podpac.coordinates.UniformCoordinates1d( + start - 2 * step, stop + 2 * step + 1e-14, step=step, fix_stop_val=True + ) + u3 = podpac.coordinates.UniformCoordinates1d( + start - 2 * step, stop + 2 * step + 1e-14, size=u.size + 4, fix_stop_val=True + ) + + assert u2.start == u3.start + assert u2.stop == u3.stop + step = (u2.stop - u2.start) / (u2.size - 1) + assert u2.step == step + assert u3.step == step + assert_equal(u2.coordinates, u3.coordinates) + + # Need to make sure time data still works + u2 = podpac.coordinates.UniformCoordinates1d("2000-01-01", "2000-01-31", step="23,h") + u3 = podpac.coordinates.UniformCoordinates1d( + "2000-01-01T00", "2000-01-30T17", size=u2.size + ) # Won't allow me to specify something inconsistent... + + assert u2.start == u3.start + assert u2.stop == u3.stop + step = (u2.stop - u2.start) / (u2.size - 1) + assert u2.step == step + assert u3.step == step + assert_equal(u2.coordinates, u3.coordinates) + + # Now check consistency without the `fix_stop_val` flag + u = c["lon"] + u2 = podpac.coordinates.UniformCoordinates1d(u.start - 2 * u.step, u.stop + 2 * u.step + 1e-14, step=u.step) + c3 = podpac.Coordinates([u2], ["lon"]) + n = podpac.Node().create_output_array(podpac.Coordinates([u2], ["lon"])) + u2b = podpac.Coordinates.from_xarray(n) + assert_equal(u2b["lon"].coordinates, u2.coordinates) + + u2 = podpac.coordinates.UniformCoordinates1d(u.start - 2 * u.step, u.stop + 2 * u.step + 1e-14, size=u.size + 4) + n = podpac.Node().create_output_array(podpac.Coordinates([u2], ["lon"])) + u2b = podpac.Coordinates.from_xarray(n) + assert_equal(u2b["lon"].coordinates, u2.coordinates) diff --git a/podpac/core/coordinates/uniform_coordinates1d.py b/podpac/core/coordinates/uniform_coordinates1d.py index 3898c2487..f972c2d1c 100644 --- a/podpac/core/coordinates/uniform_coordinates1d.py +++ b/podpac/core/coordinates/uniform_coordinates1d.py @@ -37,9 +37,11 @@ class UniformCoordinates1d(Coordinates1d): start : float or datetime64 Start coordinate. stop : float or datetime64 - Stop coordinate. + Stop coordinate. Unless fix_stop_val == True at creation, this may not always be + exactly equal to what the user specified. Internally we ensure that stop = start + step * (size - 1) step : float or timedelta64 - Signed, non-zero step between coordinates. + Signed, non-zero step between coordinates. Note, the specified step my be changed internally to satisfy floating point consistency. + That is, the consistent step will ensure that step = (stop - start) / (size - 1) name : str Dimension name, one of 'lat', 'lon', 'time', 'alt'. coordinates : array, read-only @@ -59,7 +61,7 @@ class UniformCoordinates1d(Coordinates1d): step = tl.Union([tl.Float(), tl.Instance(np.timedelta64)], read_only=True) step.__doc__ = ":float, timedelta64: Signed, non-zero step between coordinates." - def __init__(self, start, stop, step=None, size=None, name=None): + def __init__(self, start, stop, step=None, size=None, name=None, fix_stop_val=False): """ Create uniformly-spaced 1d coordinates from a `start`, `stop`, and `step` or `size`. @@ -75,6 +77,17 @@ def __init__(self, start, stop, step=None, size=None, name=None): Number of coordinates (either step or size required). name : str, optional Dimension name, one of 'lat', 'lon', 'time', or 'alt'. + fix_stop_val : bool, optional + Default is False. If True, the constructor will modify the step to be consistent + instead of the stop value. Otherwise, the stop value *may* be modified to ensure that + stop = start + step * size + + Notes + ------ + When the user specifies fix_stop_val, then `stop` will always be exact as specified by the user. + + For floating point coordinates, the specified `step` my be changed internally to satisfy floating point consistency. + That is, for consistency `step = (stop - start) / (size - 1)` """ if step is not None and size is not None: @@ -116,6 +129,15 @@ def __init__(self, start, stop, step=None, size=None, name=None): self.set_trait("stop", stop) self.set_trait("step", step) + if not fix_stop_val: # Need to make sure that 'stop' is consistent with self.coordinates[-1] + self.set_trait("stop", add_coord(self.start, (self.size - 1) * self.step)) + + # Make sure step is floating-point error consistent in all cases + # This is only needed when the type is float + if fstep == step and self.size > 1: + step = divide_delta(self.stop - self.start, self.size - 1) + self.set_trait("step", step) + # set common properties super(UniformCoordinates1d, self).__init__(name=name) @@ -275,7 +297,7 @@ def __contains__(self, item): @cached_property def coordinates(self): - """:array, read-only: Coordinate values. """ + """:array, read-only: Coordinate values.""" coordinates = add_coord(self.start, np.arange(0, self.size) * self.step) # coordinates.setflags(write=False) # This breaks the 002-open-point-file example @@ -291,7 +313,7 @@ def shape(self): @property def size(self): - """ Number of coordinates. """ + """Number of coordinates.""" dname = np.array(self.step).dtype.name @@ -340,7 +362,7 @@ def is_uniform(self): @property def bounds(self): - """ Low and high coordinate bounds. """ + """Low and high coordinate bounds.""" lo = self.start hi = add_coord(self.start, self.step * (self.size - 1)) diff --git a/podpac/core/data/array_source.py b/podpac/core/data/array_source.py index f27736455..e6fc48f3c 100644 --- a/podpac/core/data/array_source.py +++ b/podpac/core/data/array_source.py @@ -54,8 +54,8 @@ class ArrayRaw(NoCacheMixin, DataSource): >>> output = node.eval(coords) """ - source = ArrayTrait().tag(attr=True) - coordinates = tl.Instance(Coordinates).tag(attr=True) + source = ArrayTrait().tag(attr=True, required=True) + coordinates = tl.Instance(Coordinates).tag(attr=True, required=True) _repr_keys = ["shape"] @@ -78,12 +78,12 @@ def _first_init(self, **kwargs): @property def shape(self): - """Returns the shape of :attr:`self.source` + """Returns the shape of :attr:`self.array` Returns ------- tuple - Shape of :attr:`self.source` + Shape of :attr:`self.array` """ return self.source.shape @@ -94,11 +94,11 @@ def get_data(self, coordinates, coordinates_index): return d def set_coordinates(self, value): - """ Not needed. """ + """Not needed.""" pass class Array(InterpolationMixin, ArrayRaw): - """ Array datasource with interpolation. """ + """Array datasource with interpolation.""" pass diff --git a/podpac/core/data/csv_source.py b/podpac/core/data/csv_source.py index 89b4b0e1e..5289dbd75 100644 --- a/podpac/core/data/csv_source.py +++ b/podpac/core/data/csv_source.py @@ -89,7 +89,7 @@ def open_dataset(self, f): @cached_property def dims(self): - """ list of dataset coordinate dimensions """ + """list of dataset coordinate dimensions""" lookup = { self._get_key(self.lat_key): "lat", self._get_key(self.lon_key): "lon", @@ -157,6 +157,6 @@ def _get_col(self, key): class CSV(InterpolationMixin, CSVRaw): - """ CSV datasource with interpolation. """ + """CSV datasource with interpolation.""" pass diff --git a/podpac/core/data/dataset_source.py b/podpac/core/data/dataset_source.py index d593c4894..2efcf789c 100644 --- a/podpac/core/data/dataset_source.py +++ b/podpac/core/data/dataset_source.py @@ -96,6 +96,6 @@ def get_coordinates(self): class Dataset(InterpolationMixin, DatasetRaw): - """ xarray dataset source with interpolation. """ + """xarray dataset source with interpolation.""" pass diff --git a/podpac/core/data/datasource.py b/podpac/core/data/datasource.py index 4a5c20da0..326b67f28 100644 --- a/podpac/core/data/datasource.py +++ b/podpac/core/data/datasource.py @@ -21,7 +21,7 @@ from podpac.core.coordinates import Coordinates, Coordinates1d, StackedCoordinates from podpac.core.coordinates.utils import VALID_DIMENSION_NAMES, make_coord_delta, make_coord_delta_array from podpac.core.node import Node -from podpac.core.utils import common_doc +from podpac.core.utils import common_doc, cached_property from podpac.core.node import COMMON_NODE_DOC log = logging.getLogger(__name__) @@ -155,7 +155,10 @@ class DataSource(Node): nan_val = tl.Any(np.nan).tag(attr=True) boundary = tl.Dict().tag(attr=True) - coordinate_index_type = tl.Enum(["slice", "numpy", "xarray"], default_value="numpy") + coordinate_index_type = tl.Enum( + ["slice", "numpy", "xarray"], + default_value="numpy", + ).tag(attr=True) cache_coordinates = tl.Bool(False) cache_output = tl.Bool() @@ -222,6 +225,21 @@ def coordinates(self): self.put_cache(nc, "coordinates") return nc + @property + def dims(self): + """datasource dims.""" + return self.coordinates.dims + + @property + def udims(self): + """datasource udims.""" + return self.coordinates.udims + + @property + def _crs(self): + """datasource crs.""" + return self.coordinates.crs + # ------------------------------------------------------------------------------------------------------------------ # Private Methods # ------------------------------------------------------------------------------------------------------------------ @@ -275,6 +293,89 @@ def _get_data(self, rc, rci): # Methods # ------------------------------------------------------------------------------------------------------------------ + def get_source_data(self, bounds={}): + """ + Get source data, without interpolation. + + Arguments + --------- + bounds : dict + Dictionary of bounds by dimension, optional. + Keys must be dimension names, and values are (min, max) tuples, e.g. ``{'lat': (10, 20)}``. + + Returns + ------- + data : UnitsDataArray + Source data + """ + + coords, I = self.coordinates.select(bounds, return_index=True) + return self._get_data(coords, I) + + def eval(self, coordinates, **kwargs): + """ + Wraps the super Node.eval method in order to cache with the correct coordinates. + + The output is independent of the crs or any extra dimensions, so this transforms and removes extra dimensions + before caching in the super eval method. + """ + + # check for missing dimensions + for c in self.coordinates.values(): + if isinstance(c, Coordinates1d): + if c.name not in coordinates.udims: + raise ValueError("Cannot evaluate these coordinates, missing dim '%s'" % c.name) + elif isinstance(c, StackedCoordinates): + if all(dim not in coordinates.udims for dim in c.udims): + raise ValueError("Cannot evaluate these coordinates, missing at least one dim in '%s'" % c.name) + + # store original requested coordinates + requested_coordinates = coordinates + # This is needed for the interpolation mixin to avoid floating-point discrepancies + # between the requested coordinates and the evaluated coordinates + self._requested_coordinates = requested_coordinates + + # remove extra dimensions + extra = [ + c.name + for c in coordinates.values() + if (isinstance(c, Coordinates1d) and c.name not in self.udims) + or (isinstance(c, StackedCoordinates) and all(dim not in self.udims for dim in c.dims)) + ] + coordinates = coordinates.drop(extra) + + # transform coordinates into native crs if different + if coordinates.crs.lower() != self._crs.lower(): + coordinates = coordinates.transform(self._crs) + + # note: super().eval (not self._eval) + # This call already sub-selects an 'output' if specified + output = super().eval(coordinates, **kwargs) + + # transform back to requested coordinates, if necessary + if coordinates.crs.lower() != requested_coordinates.crs.lower(): + # need to use the already-selected output, if it exists + try: + outputs = output["output"].data.tolist() + if isinstance(outputs, str): + # this will pass outputs=None to the create function, which is what we want in this case + # which is when it is a single output (not a dim) + outputs = [] + except KeyError: + # 'output' does not exist in the data, so outputs should be empty + outputs = [] + except Exception as e: + outputs = self.outputs + coords = Coordinates.from_xarray(output, crs=output.attrs.get("crs", None)) + # the coords.transform in the next line can cause floating point discrepancies between + # the requested coordinates and the output coordinates. This is handled in the + # InterpolationMixin using self._requested_coordinates + output = self.create_output_array( + coords.transform(requested_coordinates.crs), data=output.data, outputs=outputs + ) + + return output + @common_doc(COMMON_DATA_DOC) def _eval(self, coordinates, output=None, _selector=None): """Evaluates this node using the supplied coordinates. @@ -310,35 +411,6 @@ def _eval(self, coordinates, output=None, _selector=None): log.debug("Evaluating {} data source".format(self.__class__.__name__)) - # store requested coordinates for debugging - if settings["DEBUG"]: - self._requested_coordinates = coordinates - - # check for missing dimensions - for c in self.coordinates.values(): - if isinstance(c, Coordinates1d): - if c.name not in coordinates.udims: - raise ValueError("Cannot evaluate these coordinates, missing dim '%s'" % c.name) - elif isinstance(c, StackedCoordinates): - if all(s.name not in coordinates.udims for s in c): - raise ValueError("Cannot evaluate these coordinates, missing at least one dim in '%s'" % c.name) - - # remove extra dimensions - extra = [ - c.name - for c in coordinates.values() - if (isinstance(c, Coordinates1d) and c.name not in self.coordinates.udims) - or (isinstance(c, StackedCoordinates) and all(dim not in self.coordinates.udims for dim in c.dims)) - ] - coordinates = coordinates.drop(extra) - - # save before transforming - requested_coordinates = coordinates - - # transform coordinates into native crs if different - if self.coordinates.crs.lower() != coordinates.crs.lower(): - coordinates = coordinates.transform(self.coordinates.crs) - # Use the selector if _selector is not None: (rsc, rsci) = _selector(self.coordinates, coordinates, index_type=self.coordinate_index_type) @@ -368,18 +440,16 @@ def _eval(self, coordinates, output=None, _selector=None): # get data from data source rsd = self._get_data(rsc, rsci) - # data = rsd.part_transpose(requested_coordinates.dims) # this does not appear to be necessary anymore - data = rsd if output is None: - if requested_coordinates.crs.lower() != coordinates.crs.lower(): - if rsc.shape == data.shape: - data = self.create_output_array(rsc, data=data.data) - else: - crds = Coordinates.from_xarray(data, crs=data.attrs.get("crs", None)) - data = self.create_output_array(crds.transform(rsc.crs), data=data.data) - output = data + # if requested_coordinates.crs.lower() != coordinates.crs.lower(): + # if rsc.shape == rsd.shape: + # rsd = self.create_output_array(rsc, data=rsd.data) + # else: + # crds = Coordinates.from_xarray(rsd, crs=data.attrs.get("crs", None)) + # rsd = self.create_output_array(crds.transform(rsc.crs), data=rsd.data) + output = rsd else: - output.data[:] = data.data + output.data[:] = rsd.data # get indexed boundary rsb = self._get_boundary(rsci) @@ -409,6 +479,30 @@ def find_coordinates(self): return [self.coordinates] + def get_bounds(self, crs="default"): + """Get the full available coordinate bounds for the Node. + + Arguments + --------- + crs : str + Desired CRS for the bounds. Use 'source' to use the native source crs. + If not specified, podpac.settings["DEFAULT_CRS"] is used. Optional. + + Returns + ------- + bounds : dict + Bounds for each dimension. Keys are dimension names and values are tuples (min, max). + crs : str + The crs for the bounds. + """ + + if crs == "default": + crs = settings["DEFAULT_CRS"] + elif crs == "source": + crs = self.coordinates.crs + + return self.coordinates.transform(crs).bounds, crs + @common_doc(COMMON_DATA_DOC) def get_data(self, coordinates, coordinates_index): """{get_data} diff --git a/podpac/core/data/file_source.py b/podpac/core/data/file_source.py index d0acf0cc7..7433094b4 100644 --- a/podpac/core/data/file_source.py +++ b/podpac/core/data/file_source.py @@ -47,7 +47,7 @@ class BaseFileSource(DataSource): dataset object """ - source = tl.Unicode().tag(attr=True) + source = tl.Unicode().tag(attr=True, required=True) # list of attribute names, used by __repr__ and __str__ to display minimal info about the node _repr_keys = ["source"] @@ -65,7 +65,7 @@ def dataset(self): raise NotImplementedError() def close_dataset(self): - """ Close opened resources. Subclasses should implement if appropriate. """ + """Close opened resources. Subclasses should implement if appropriate.""" pass @@ -129,7 +129,7 @@ def _open(self, f, cache=True): return self.open_dataset(f) def open_dataset(self, f): - """ TODO """ + """TODO""" raise NotImplementedError() def close_dataset(self): @@ -177,7 +177,7 @@ class FileKeysMixin(tl.HasTraits): @property def _repr_keys(self): - """ list of attribute names, used by __repr__ and __str__ to display minimal info about the node""" + """list of attribute names, used by __repr__ and __str__ to display minimal info about the node""" keys = ["source"] if len(self.available_data_keys) > 1 and not isinstance(self.data_key, list): keys.append("data_key") diff --git a/podpac/core/data/h5py_source.py b/podpac/core/data/h5py_source.py index 52b471f75..8a7300baa 100644 --- a/podpac/core/data/h5py_source.py +++ b/podpac/core/data/h5py_source.py @@ -58,7 +58,7 @@ def dataset(self): return h5py.File(self.source, self.file_mode) def close_dataset(self): - """Closes the file. """ + """Closes the file.""" super(H5PYRaw, self).close_dataset() self.dataset.close() @@ -68,7 +68,7 @@ def close_dataset(self): @cached_property def dims(self): - """ dataset coordinate dims """ + """dataset coordinate dims""" try: if not isinstance(self.data_key, list): key = self.data_key @@ -123,6 +123,6 @@ def _find_h5py_keys(obj, keys=[]): class H5PY(InterpolationMixin, H5PYRaw): - """ h5py datasource with interpolation. """ + """h5py datasource with interpolation.""" pass diff --git a/podpac/core/data/ogc.py b/podpac/core/data/ogc.py index 7f0ddff73..986882529 100644 --- a/podpac/core/data/ogc.py +++ b/podpac/core/data/ogc.py @@ -13,7 +13,7 @@ from podpac.core.utils import common_doc, cached_property, resolve_bbox_order from podpac.core.data.datasource import DataSource -from podpac.core.interpolation.interpolation import InterpolationMixin +from podpac.core.interpolation.interpolation import InterpolationMixin, InterpolationTrait from podpac.core.node import NodeException from podpac.core.coordinates import Coordinates from podpac.core.coordinates import UniformCoordinates1d, ArrayCoordinates1d, Coordinates1d, StackedCoordinates @@ -156,10 +156,10 @@ class WCSRaw(DataSource): WCS : WCS datasource with podpac interpolation. """ - source = tl.Unicode().tag(attr=True) - layer = tl.Unicode().tag(attr=True) + source = tl.Unicode().tag(attr=True, required=True) + layer = tl.Unicode().tag(attr=True, required=True) version = tl.Unicode(default_value="1.0.0").tag(attr=True) - interpolation = tl.Unicode(default_value=None, allow_none=True).tag(attr=True) + interpolation = InterpolationTrait(default_value=None, allow_none=True).tag(attr=True) allow_mock_client = tl.Bool(False).tag(attr=True) username = tl.Unicode(allow_none=True) password = tl.Unicode(allow_none=True) @@ -167,6 +167,7 @@ class WCSRaw(DataSource): format = tl.CaselessStrEnum(["geotiff", "geotiff_byte"], default_value="geotiff") crs = tl.Unicode(default_value="EPSG:4326") max_size = tl.Long(default_value=None, allow_none=True) + wcs_kwargs = tl.Dict(help="Additional query parameters sent to the WCS server") _repr_keys = ["source", "layer"] @@ -372,7 +373,7 @@ def _get_chunk(self, coordinates): width = coordinates["lon"].size height = coordinates["lat"].size - kwargs = {} + kwargs = self.wcs_kwargs.copy() if "time" in coordinates: kwargs["time"] = coordinates["time"].coordinates.astype(str).tolist() @@ -381,8 +382,8 @@ def _get_chunk(self, coordinates): kwargs["interpolation"] = self.interpolation logger.info( - "WCS GetCoverage (source=%s, layer=%s, bbox=%s, shape=%s)" - % (self.source, self.layer, (w, n, e, s), (width, height)) + "WCS GetCoverage (source=%s, layer=%s, bbox=%s, shape=%s, time=%s)" + % (self.source, self.layer, (w, n, e, s), (width, height), kwargs.get("time")) ) crs = pyproj.CRS(coordinates.crs) @@ -413,14 +414,21 @@ def _get_chunk(self, coordinates): # get data using rasterio with rasterio.MemoryFile() as mf: mf.write(content) - dataset = mf.open(driver="GTiff") + try: + dataset = mf.open(driver="GTiff") + except rasterio.RasterioIOError: + raise WCSError("Could not read file with contents:", content) if "time" in coordinates and coordinates["time"].size > 1: # this should be easy to do, I'm just not sure how the data comes back. # is each time in a different band? raise NotImplementedError("TODO") - data = dataset.read(1).astype(float) + data = dataset.read().astype(float).squeeze() + + # Need to fix the order of the data in the case of multiple bands + if len(data.shape) == 3: + data = data.transpose((1, 2, 0)) # Need to fix the data order. The request and response order is always the same in WCS, but not in PODPAC if n > s: # By default it returns the data upside down, so this is backwards @@ -439,6 +447,6 @@ def get_layers(cls, source=None): class WCS(InterpolationMixin, WCSRaw): - """ WCS datasource with podpac interpolation. """ + """WCS datasource with podpac interpolation.""" coordinate_index_type = tl.Unicode("slice", read_only=True) diff --git a/podpac/core/data/ogr.py b/podpac/core/data/ogr.py index d41a8e5c5..76147179f 100644 --- a/podpac/core/data/ogr.py +++ b/podpac/core/data/ogr.py @@ -17,8 +17,8 @@ class OGRRaw(Node): """ """ - source = tl.Unicode().tag(attr=True) - layer = tl.Unicode().tag(attr=True) + source = tl.Unicode().tag(attr=True, required=True) + layer = tl.Unicode().tag(attr=True, required=True) attribute = tl.Unicode().tag(attr=True) nan_vals = tl.List().tag(attr=True) nan_val = tl.Any(np.nan).tag(attr=True) @@ -51,6 +51,41 @@ def extents(self): layer = self.datasource.GetLayerByName(self.layer) return layer.GetExtent() + def get_source_data(self, bounds={}): + """ + Not available for OGR nodes. + + Arguments + --------- + bounds : dict + Dictionary of bounds by dimension, optional. + Keys must be dimension names, and values are (min, max) tuples, e.g. ``{'lat': (10, 20)}``. + + raises + ------ + AttributeError : Cannot get source data for OGR datasources + """ + + raise AttributeError( + "Cannot get source data for OGR datasources. " + "The source data is a vector-based shapefile without a native resolution." + ) + + def find_coordinates(self): + """ + Not available for OGR nodes. + + raises + ------ + coord_list : list + list of available coordinates (Coordinates objects) + """ + + raise AttributeError( + "Cannot get available coordinates for OGR datasources. " + "The source data is a vector-based shapefile without native coordinates." + ) + @common_doc(COMMON_NODE_DOC) def _eval(self, coordinates, output=None, _selector=None): if "lat" not in coordinates.udims or "lon" not in coordinates.udims: diff --git a/podpac/core/data/pydap_source.py b/podpac/core/data/pydap_source.py index b1dafa1c9..415ff6d45 100644 --- a/podpac/core/data/pydap_source.py +++ b/podpac/core/data/pydap_source.py @@ -52,8 +52,8 @@ class PyDAPRaw(authentication.RequestsSessionMixin, DataSource): PyDAP : Interpolated OpenDAP datasource for general use. """ - source = tl.Unicode().tag(attr=True) - data_key = tl.Unicode().tag(attr=True) + source = tl.Unicode().tag(attr=True, required=True) + data_key = tl.Unicode().tag(attr=True, required=True) server_throttle_sleep_time = tl.Float( default_value=0.001, help="Some server have a throttling time for requests per period. " ).tag(attr=True) @@ -137,6 +137,6 @@ def keys(self): class PyDAP(InterpolationMixin, PyDAPRaw): - """ OpenDAP datasource with interpolation. """ + """OpenDAP datasource with interpolation.""" pass diff --git a/podpac/core/data/rasterio_source.py b/podpac/core/data/rasterio_source.py index 478db71a2..f52c03e36 100644 --- a/podpac/core/data/rasterio_source.py +++ b/podpac/core/data/rasterio_source.py @@ -80,7 +80,7 @@ def open_dataset(self, source, overview_level=None): envargs = {"AWS_HTTPS": self.aws_https} kwargs = {} if overview_level is not None: - kwargs = {'overview_level': overview_level} + kwargs = {"overview_level": overview_level} if source.startswith("s3://"): envargs["session"] = rasterio.session.AWSSession( aws_access_key_id=self.aws_access_key_id, @@ -191,7 +191,7 @@ def get_data_overviews(self, coordinates, coordinates_index): reduction_factor, np.abs(min_delta / self.coordinates[c].step) # self.coordinates is always uniform ) # Find the overview that's closest to this reduction factor - if reduction_factor < 2: # Then we shouldn't use an overview + if (reduction_factor < 2) or (len(self.overviews) == 0): # Then we shouldn't use an overview overview = 1 overview_level = None else: @@ -217,9 +217,11 @@ def get_data_overviews(self, coordinates, coordinates_index): ((inds[1].min() // overview), int(np.ceil(inds[1].max() / overview) + 1)), ) slc = (slice(window[0][0], window[0][1], 1), slice(window[1][0], window[1][1], 1)) - new_coords = Coordinates.from_geotransform(dataset.transform.to_gdal(), dataset.shape, crs=self.coordinates.crs) + new_coords = Coordinates.from_geotransform( + dataset.transform.to_gdal(), dataset.shape, crs=self.coordinates.crs + ) new_coords = new_coords[slc] - missing_coords = self.coordinates.drop(['lat', 'lon']) + missing_coords = self.coordinates.drop(["lat", "lon"]) new_coords = merge_dims([new_coords, missing_coords]) new_coords = new_coords.transpose(*self.coordinates.dims) coordinates_shape = new_coords.shape[:2] @@ -321,6 +323,6 @@ def get_band_numbers(self, key, value): class Rasterio(InterpolationMixin, RasterioRaw): - """ Rasterio datasource with interpolation. """ + """Rasterio datasource with interpolation.""" pass diff --git a/podpac/core/data/reprojection.py b/podpac/core/data/reprojection.py index 1a62542bb..53f5b53a9 100644 --- a/podpac/core/data/reprojection.py +++ b/podpac/core/data/reprojection.py @@ -30,9 +30,9 @@ class ReprojectedSource(DataSource): Coordinates where the source node should be evaluated. """ - source = NodeTrait().tag(attr=True) + source = NodeTrait().tag(attr=True, required=True) source_interpolation = InterpolationTrait().tag(attr=True) - reprojected_coordinates = tl.Instance(Coordinates).tag(attr=True) + reprojected_coordinates = tl.Instance(Coordinates).tag(attr=True, required=True) # list of attribute names, used by __repr__ and __str__ to display minimal info about the node _repr_keys = ["source", "interpolation"] diff --git a/podpac/core/data/test/test_array.py b/podpac/core/data/test/test_array.py index 52a320578..880a64162 100644 --- a/podpac/core/data/test/test_array.py +++ b/podpac/core/data/test/test_array.py @@ -26,7 +26,7 @@ def test_invalid_data(self): node = Array(source=["a", "b"], coordinates=self.coordinates) def test_get_data(self): - """ defined get_data function""" + """defined get_data function""" node = Array(source=self.data, coordinates=self.coordinates) output = node.eval(self.coordinates) diff --git a/podpac/core/data/test/test_dataset.py b/podpac/core/data/test/test_dataset.py index aafafb706..ce5de4721 100644 --- a/podpac/core/data/test/test_dataset.py +++ b/podpac/core/data/test/test_dataset.py @@ -22,7 +22,7 @@ def test_init_and_close(self): def test_dims(self): node = Dataset(source=self.source, time_key="day") - assert node.dims == ["time", "lat", "lon"] + assert np.all([d in ["time", "lat", "lon"] for d in node.dims]) # un-mapped keys # node = Dataset(source=self.source) @@ -37,7 +37,7 @@ def test_coordinates(self): # specify dimension keys node = Dataset(source=self.source, time_key="day") nc = node.coordinates - assert nc.dims == ("time", "lat", "lon") + # assert nc.dims == ("time", "lat", "lon") # Xarray is free to change this order -- we don't care np.testing.assert_array_equal(nc["lat"].coordinates, self.lat) np.testing.assert_array_equal(nc["lon"].coordinates, self.lon) np.testing.assert_array_equal(nc["time"].coordinates, self.time) @@ -47,23 +47,23 @@ def test_get_data(self): # specify data key node = Dataset(source=self.source, time_key="day", data_key="data") out = node.eval(node.coordinates) - np.testing.assert_array_equal(out, self.data) + np.testing.assert_array_equal(out.transpose("time", "lat", "lon"), self.data) node.close_dataset() node = Dataset(source=self.source, time_key="day", data_key="other") out = node.eval(node.coordinates) - np.testing.assert_array_equal(out, self.other) + np.testing.assert_array_equal(out.transpose("time", "lat", "lon"), self.other) node.close_dataset() def test_get_data_array_indexing(self): node = Dataset(source=self.source, time_key="day", data_key="data") - out = node.eval(node.coordinates[:, [0, 2]]) + out = node.eval(node.coordinates.transpose("time", "lat", "lon")[:, [0, 2]]) np.testing.assert_array_equal(out, self.data[:, [0, 2]]) node.close_dataset() def test_get_data_multiple(self): node = Dataset(source=self.source, time_key="day", data_key=["data", "other"]) - out = node.eval(node.coordinates) + out = node.eval(node.coordinates.transpose("time", "lat", "lon")) assert out.dims == ("time", "lat", "lon", "output") np.testing.assert_array_equal(out["output"], ["data", "other"]) np.testing.assert_array_equal(out.sel(output="data"), self.data) @@ -72,7 +72,7 @@ def test_get_data_multiple(self): # single node = Dataset(source=self.source, time_key="day", data_key=["other"]) - out = node.eval(node.coordinates) + out = node.eval(node.coordinates.transpose("time", "lat", "lon")) assert out.dims == ("time", "lat", "lon", "output") np.testing.assert_array_equal(out["output"], ["other"]) np.testing.assert_array_equal(out.sel(output="other"), self.other) @@ -80,7 +80,7 @@ def test_get_data_multiple(self): # alternate output names node = Dataset(source=self.source, time_key="day", data_key=["data", "other"], outputs=["a", "b"]) - out = node.eval(node.coordinates) + out = node.eval(node.coordinates.transpose("time", "lat", "lon")) assert out.dims == ("time", "lat", "lon", "output") np.testing.assert_array_equal(out["output"], ["a", "b"]) np.testing.assert_array_equal(out.sel(output="a"), self.data) @@ -89,7 +89,7 @@ def test_get_data_multiple(self): # default node = Dataset(source=self.source, time_key="day") - out = node.eval(node.coordinates) + out = node.eval(node.coordinates.transpose("time", "lat", "lon")) assert out.dims == ("time", "lat", "lon", "output") np.testing.assert_array_equal(out["output"], ["data", "other"]) np.testing.assert_array_equal(out.sel(output="data"), self.data) @@ -103,6 +103,6 @@ def test_extra_dimension_selection(self): # treat day as an "extra" dimension, and select the second day node = Dataset(source=self.source, data_key="data", selection={"day": 1}) - assert node.coordinates.dims == ("lat", "lon") + assert np.all([d in ["lat", "lon"] for d in node.dims]) out = node.eval(node.coordinates) - np.testing.assert_array_equal(out, self.data[1]) + np.testing.assert_array_equal(out, self.data[1].T) diff --git a/podpac/core/data/test/test_datasource.py b/podpac/core/data/test/test_datasource.py index 497bf00fc..ddd8ec1fd 100644 --- a/podpac/core/data/test/test_datasource.py +++ b/podpac/core/data/test/test_datasource.py @@ -112,6 +112,14 @@ def get_coordinates(self): with pytest.raises(AttributeError, match="can't set attribute"): node.coordinates = Coordinates([]) + def test_dims(self): + class MyDataSource(DataSource): + def get_coordinates(self): + return Coordinates([0, 0], dims=["lat", "lon"]) + + node = MyDataSource() + assert node.dims == ("lat", "lon") + def test_cache_coordinates(self): class MyDataSource(DataSource): get_coordinates_called = 0 @@ -337,7 +345,7 @@ def test_evaluate_crs_transform(self): def test_evaluate_selector(self): def selector(rsc, coordinates, index_type=None): - """ mock selector that just strides by 2 """ + """mock selector that just strides by 2""" new_rsci = tuple(slice(None, None, 2) for dim in rsc.dims) new_rsc = rsc[new_rsci] return new_rsc, new_rsci @@ -349,7 +357,7 @@ def selector(rsc, coordinates, index_type=None): np.testing.assert_array_equal(output["lon"].data, node.coordinates["lon"][::2].coordinates) def test_nan_vals(self): - """ evaluate note with nan_vals """ + """evaluate note with nan_vals""" # none node = MockDataSource() @@ -512,6 +520,100 @@ def _validate_boundary(self, d): np.testing.assert_array_equal(boundary["lat"], lat_boundary[index]) np.testing.assert_array_equal(boundary["lon"], lon_boundary[index]) + def test_eval_get_cache_extra_dims(self): + with podpac.settings: + podpac.settings["DEFAULT_CACHE"] = ["ram"] + + node = podpac.data.Array( + source=np.ones((3, 4)), + coordinates=podpac.Coordinates([range(3), range(4)], ["lat", "lon"]), + cache_ctrl=["ram"], + ) + coords1 = podpac.Coordinates([range(3), range(4), "2012-05-19"], ["lat", "lon", "time"]) + coords2 = podpac.Coordinates([range(3), range(4), "2019-09-10"], ["lat", "lon", "time"]) + + # retrieve from cache on the second evaluation + node.eval(coords1) + assert not node._from_cache + + node.eval(coords1) + assert node._from_cache + + # also retrieve from cache with different time coordinates + node.eval(coords2) + assert node._from_cache + + def test_eval_get_cache_transform_crs(self): + with podpac.settings: + podpac.settings["DEFAULT_CACHE"] = ["ram"] + + node = podpac.core.data.array_source.Array( + source=np.ones((3, 4)), + coordinates=podpac.Coordinates([range(3), range(4)], ["lat", "lon"], crs="EPSG:4326"), + cache_ctrl=["ram"], + ) + + # retrieve from cache on the second evaluation + node.eval(node.coordinates) + assert not node._from_cache + + node.eval(node.coordinates) + assert node._from_cache + + # also retrieve from cache with different crs + node.eval(node.coordinates.transform("EPSG:4326")) + assert node._from_cache + + def test_get_source_data(self): + node = podpac.data.Array( + source=np.ones((3, 4)), + coordinates=podpac.Coordinates([range(3), range(4)], ["lat", "lon"]), + ) + + data = node.get_source_data() + np.testing.assert_array_equal(data, node.source) + + def test_get_source_data_with_bounds(self): + node = podpac.data.Array( + source=np.ones((3, 4)), + coordinates=podpac.Coordinates([range(3), range(4)], ["lat", "lon"]), + ) + + data = node.get_source_data({"lon": (1.5, 4.5)}) + np.testing.assert_array_equal(data, node.source[:, 2:]) + + def test_get_bounds(self): + node = podpac.data.Array( + source=np.ones((3, 4)), + coordinates=podpac.Coordinates([range(3), range(4)], ["lat", "lon"], crs="EPSG:2193"), + ) + + with podpac.settings: + podpac.settings["DEFAULT_CRS"] = "EPSG:4326" + + # specify crs + bounds, crs = node.get_bounds(crs="EPSG:3857") + expected = { + "lat": (-13291827.558247397, -13291815.707967814), + "lon": (9231489.26794932, 9231497.142754894), + } + for k in expected: + np.testing.assert_almost_equal(bounds[k], expected[k]) + assert crs == "EPSG:3857" + + # native/source crs + bounds, crs = node.get_bounds(crs="source") + assert bounds == {"lat": (0, 2), "lon": (0, 3)} + assert crs == "EPSG:2193" + + # default crs + bounds, crs = node.get_bounds() + assert bounds == { + "lat": (-75.81365382984804, -75.81362774074242), + "lon": (82.92787904584206, 82.92794978642414), + } + assert crs == "EPSG:4326" + class TestDataSourceWithMultipleOutputs(object): def test_evaluate_no_overlap_with_output_extract_output(self): diff --git a/podpac/core/data/test/test_pydap.py b/podpac/core/data/test/test_pydap.py index 9794708dd..72210a4f6 100644 --- a/podpac/core/data/test/test_pydap.py +++ b/podpac/core/data/test/test_pydap.py @@ -12,7 +12,7 @@ class MockPyDAP(PyDAP): - """mock pydap data source """ + """mock pydap data source""" source = "http://demo.opendap.org" data_key = "key" @@ -50,7 +50,7 @@ def test_keys(self): assert "key" in keys def test_session(self): - """test session attribute and traitlet default """ + """test session attribute and traitlet default""" # hostname should be the same as the source, parsed by request node = PyDAP(source=self.source, data_key=self.data_key) diff --git a/podpac/core/data/test/test_rasterio.py b/podpac/core/data/test/test_rasterio.py index 4aec3da83..e770e132f 100644 --- a/podpac/core/data/test/test_rasterio.py +++ b/podpac/core/data/test/test_rasterio.py @@ -23,7 +23,7 @@ def test_init(self): node = Rasterio(source=self.source, band=self.band) def test_dataset(self): - """test dataset attribute and trait default """ + """test dataset attribute and trait default""" node = Rasterio(source=self.source, band=self.band) try: diff --git a/podpac/core/data/test/test_wcs.py b/podpac/core/data/test/test_wcs.py index 973e67296..71052330d 100644 --- a/podpac/core/data/test/test_wcs.py +++ b/podpac/core/data/test/test_wcs.py @@ -8,7 +8,7 @@ COORDS = podpac.Coordinates( [podpac.clinspace(-132.9023, -53.6051, 100, name="lon"), podpac.clinspace(23.6293, 53.7588, 100, name="lat")], - # crs="EPSG:4326", + crs="+proj=longlat +datum=WGS84 +no_defs +vunits=m", ) @@ -35,7 +35,7 @@ def getCoverage(self, **kwargs): class MockWCSRaw(WCSRaw): - """ Test node that uses the MockClient above. """ + """Test node that uses the MockClient above.""" @property def client(self): @@ -46,7 +46,7 @@ def get_coordinates(self): class MockWCS(WCS): - """ Test node that uses the MockClient above, and injects podpac interpolation. """ + """Test node that uses the MockClient above, and injects podpac interpolation.""" @property def client(self): @@ -107,7 +107,11 @@ def test_eval_extra_unstacked_dim(self): assert output.data.sum() == 1256581.0 def test_eval_extra_stacked_dim(self): - c = podpac.Coordinates([[COORDS["lat"][50], COORDS["lon"][50], 10]], dims=["lat_lon_alt"]) + c = podpac.Coordinates( + [[COORDS["lat"][50], COORDS["lon"][50], 10]], + dims=["lat_lon_alt"], + crs="+proj=longlat +datum=WGS84 +no_defs +vunits=m", + ) node = MockWCSRaw(source="mock", layer="mock", max_size=1000) output = node.eval(c) diff --git a/podpac/core/data/test/test_zarr.py b/podpac/core/data/test/test_zarr.py index 672b54cf8..4e21aa1b8 100644 --- a/podpac/core/data/test/test_zarr.py +++ b/podpac/core/data/test/test_zarr.py @@ -74,8 +74,9 @@ def test_eval_multiple(self): assert out.sel(output="a")[0, 0] == 0.0 assert out.sel(output="b")[0, 0] == 1.0 - @pytest.mark.aws + @pytest.mark.skip def test_s3(self): + # This file no longer exists path = "s3://podpac-internal-test/drought_parameters.zarr" node = Zarr(source=path, data_key="d0") node.close_dataset() diff --git a/podpac/core/data/zarr_source.py b/podpac/core/data/zarr_source.py index f03f0fce3..eb0785b95 100644 --- a/podpac/core/data/zarr_source.py +++ b/podpac/core/data/zarr_source.py @@ -202,6 +202,6 @@ def get_data(self, coordinates, coordinates_index): class Zarr(InterpolationMixin, ZarrRaw): - """ Zarr Datasource with Interpolation. """ + """Zarr Datasource with Interpolation.""" pass diff --git a/podpac/core/interpolation/interpolation.py b/podpac/core/interpolation/interpolation.py index 313813b9e..ddf70ef0a 100644 --- a/podpac/core/interpolation/interpolation.py +++ b/podpac/core/interpolation/interpolation.py @@ -12,17 +12,20 @@ from podpac.core.settings import settings from podpac.core.node import Node -from podpac.core.utils import NodeTrait, common_doc +from podpac.core.utils import NodeTrait, common_doc, cached_property from podpac.core.units import UnitsDataArray from podpac.core.coordinates import merge_dims, Coordinates from podpac.core.interpolation.interpolation_manager import InterpolationManager, InterpolationTrait from podpac.core.cache.cache_ctrl import CacheCtrl +from podpac.core.data.datasource import DataSource _logger = logging.getLogger(__name__) class InterpolationMixin(tl.HasTraits): + # interpolation = InterpolationTrait().tag(attr=True, required=False, default = "nearesttt") interpolation = InterpolationTrait().tag(attr=True) + _interp_node = None @property @@ -31,13 +34,22 @@ def _repr_keys(self): def _eval(self, coordinates, output=None, _selector=None): node = Interpolate( - interpolation=self.interpolation, source_id=self.hash, force_eval=True, cache_ctrl=CacheCtrl([]) + interpolation=self.interpolation, + source_id=self.hash, + force_eval=True, + cache_ctrl=CacheCtrl([]), + style=self.style, ) node._set_interpolation() selector = node._interpolation.select_coordinates node._source_xr = super()._eval(coordinates, _selector=selector) self._interp_node = node - r = node.eval(coordinates, output=output) + if isinstance(self, DataSource): + # This is required to ensure that the output coordinates + # match the requested coordinates to floating point precision + r = node.eval(self._requested_coordinates, output=output) + else: + r = node.eval(coordinates, output=output) # Helpful for debugging self._from_cache = node._from_cache return r @@ -101,7 +113,7 @@ class will be used without modification. """ - source = NodeTrait(allow_none=True).tag(attr=True) + source = NodeTrait(allow_none=True).tag(attr=True, required=True) source_id = tl.Unicode(allow_none=True).tag(attr=True) _source_xr = tl.Instance(UnitsDataArray, allow_none=True) # This is needed for the Interpolation Mixin @@ -117,6 +129,13 @@ class will be used without modification. _requested_source_data = tl.Instance(UnitsDataArray) _evaluated_coordinates = tl.Instance(Coordinates) + @tl.default("style") + def _default_style(self): # Pass through source style by default + if self.source is not None: + return self.source.style + else: + return super()._default_style() + # this adds a more helpful error message if user happens to try an inspect _interpolation before evaluate @tl.default("_interpolation") def _default_interpolation(self): @@ -227,12 +246,15 @@ def _eval(self, coordinates, output=None, _selector=None): # Drop extra coordinates extra_dims = [d for d in coordinates.udims if d not in source_coords.udims] - coordinates = coordinates.drop(extra_dims) + coordinates = coordinates.udrop(extra_dims) # Transform so that interpolation happens on the source data coordinate system if source_coords.crs.lower() != coordinates.crs.lower(): coordinates = coordinates.transform(source_coords.crs) + # Fix source coordinates in the case where some dimension are not being interpolated + coordinates = self._interpolation._fix_coordinates_for_none_interp(coordinates, source_coords) + if output is None: if "output" in source_out.dims: self.set_trait("outputs", source_out.coords["output"].data.tolist()) @@ -273,3 +295,22 @@ def find_coordinates(self): """ return self.source.find_coordinates() + + def get_bounds(self, crs="default"): + """Get the full available coordinate bounds for the Node. + + Arguments + --------- + crs : str + Desired CRS for the bounds. Use 'source' to use the native source crs. + If not specified, the default CRS in the podpac settings is used. Optional. + + Returns + ------- + bounds : dict + Bounds for each dimension. Keys are dimension names and values are tuples (hi, lo). + crs : str + The crs for the bounds. + """ + + return self.source.get_bounds(crs=crs) diff --git a/podpac/core/interpolation/interpolation_manager.py b/podpac/core/interpolation/interpolation_manager.py index 25ec66e92..daaa3f235 100644 --- a/podpac/core/interpolation/interpolation_manager.py +++ b/podpac/core/interpolation/interpolation_manager.py @@ -1,21 +1,24 @@ from __future__ import division, unicode_literals, print_function, absolute_import import logging + import warnings from copy import deepcopy from collections import OrderedDict from six import string_types import numpy as np +import xarray as xr import traitlets as tl from podpac.core import settings from podpac.core.units import UnitsDataArray -from podpac.core.coordinates import merge_dims, Coordinates +from podpac.core.coordinates import merge_dims, Coordinates, StackedCoordinates from podpac.core.coordinates.utils import VALID_DIMENSION_NAMES from podpac.core.interpolation.interpolator import Interpolator from podpac.core.interpolation.nearest_neighbor_interpolator import NearestNeighbor, NearestPreview from podpac.core.interpolation.rasterio_interpolator import RasterioInterpolator from podpac.core.interpolation.scipy_interpolator import ScipyPoint, ScipyGrid from podpac.core.interpolation.xarray_interpolator import XarrayInterpolator +from podpac.core.interpolation.none_interpolator import NoneInterpolator _logger = logging.getLogger(__name__) @@ -23,13 +26,22 @@ INTERPOLATION_DEFAULT = settings.settings.get("DEFAULT_INTERPOLATION", "nearest") """str : Default interpolation method used when creating a new :class:`Interpolation` class """ -INTERPOLATORS = [NearestNeighbor, XarrayInterpolator, RasterioInterpolator, ScipyPoint, ScipyGrid, NearestPreview] +INTERPOLATORS = [ + NoneInterpolator, + NearestNeighbor, + XarrayInterpolator, + RasterioInterpolator, + ScipyPoint, + ScipyGrid, + NearestPreview, +] """list : list of available interpolator classes""" INTERPOLATORS_DICT = {} """dict : Dictionary of a string interpolator name and associated interpolator class""" INTERPOLATION_METHODS = [ + "none", "nearest_preview", "nearest", "linear", @@ -466,7 +478,7 @@ def select_coordinates(self, source_coordinates, eval_coordinates, index_type="n for udims in interpolator_queue: interpolator = interpolator_queue[udims] extra_dims = [d for d in source_coordinates.udims if d not in udims] - sc = source_coordinates.drop(extra_dims) + sc = source_coordinates.udrop(extra_dims) # run interpolation. mutates selected coordinates and selected coordinates index sel_coords, sel_coords_idx = interpolator.select_coordinates( udims, sc, eval_coordinates, index_type=index_type @@ -491,9 +503,42 @@ def select_coordinates(self, source_coordinates, eval_coordinates, index_type="n validate_crs=False, ) if index_type == "numpy": - selected_coords_idx2 = np.ix_(*[np.ravel(selected_coords_idx[k]) for k in source_coordinates.dims]) - elif index_type in ["slice", "xarray"]: - selected_coords_idx2 = tuple([selected_coords_idx[d] for d in source_coordinates.dims]) + npcoords = [] + has_stacked = False + for k in source_coordinates.dims: + # Deal with nD stacked source coords (marked by coords being in tuple) + if isinstance(selected_coords_idx[k], tuple): + has_stacked = True + npcoords.extend([sci for sci in selected_coords_idx[k]]) + else: + npcoords.append(selected_coords_idx[k]) + if has_stacked: + # When stacked coordinates are nD we cannot use the catchall of the next branch + selected_coords_idx2 = npcoords + else: + # This would not be needed if everything went as planned in + # interpolator.select_coordinates, but this is a catchall that works + # for 90% of the cases + selected_coords_idx2 = np.ix_(*[np.ravel(npc) for npc in npcoords]) + elif index_type == "xarray": + selected_coords_idx2 = [] + for i in selected_coords.dims: + # Deal with nD stacked source coords (marked by coords being in tuple) + if isinstance(selected_coords_idx[i], tuple): + selected_coords_idx2.extend([xr.DataArray(sci, dims=[i]) for sci in selected_coords_idx[i]]) + else: + selected_coords_idx2.append(selected_coords_idx[i]) + selected_coords_idx2 = tuple(selected_coords_idx2) + elif index_type == "slice": + selected_coords_idx2 = [] + for i in selected_coords.dims: + # Deal with nD stacked source coords (marked by coords being in tuple) + if isinstance(selected_coords_idx[i], tuple): + selected_coords_idx2.extend(selected_coords_idx[i]) + else: + selected_coords_idx2.append(selected_coords_idx[i]) + + selected_coords_idx2 = tuple(selected_coords_idx2) else: raise ValueError("Unknown index_type '%s'" % index_type) return selected_coords, tuple(selected_coords_idx2) @@ -549,13 +594,25 @@ def interpolate(self, source_coordinates, source_data, eval_coordinates, output_ # TODO handle stacked issubset of unstacked case # this case is currently skipped because of the set(eval_coordinates) == set(source_coordinates))) if eval_coordinates.issubset(source_coordinates) and set(eval_coordinates) == set(source_coordinates): - try: - output_data.data[:] = source_data.interp(output_data.coords, method="nearest").transpose( - *output_data.dims - ) - except (NotImplementedError, ValueError): - output_data.data[:] = source_data.sel(output_data.coords).transpose(*output_data.dims) - return output_data + if any(isinstance(c, StackedCoordinates) and c.ndim > 1 for c in eval_coordinates.values()): + # TODO AFFINE + # currently this is bypassing the short-circuit in the shaped stacked coordinates case + pass + else: + try: + data = source_data.interp(output_data.coords, method="nearest") + except (NotImplementedError, ValueError): + try: + data = source_data.sel(output_data.coords[output_data.dims]) + except KeyError: + # Since the output is a subset of the original data, + # we can just rely on xarray's broadcasting capability + # to subselect data, as the final fallback + output_data[:] = 0 + data = source_data + output_data + + output_data.data[:] = data.transpose(*output_data.dims) + return output_data interpolator_queue = self._select_interpolator_queue( source_coordinates, eval_coordinates, "can_interpolate", strict=True @@ -606,10 +663,47 @@ def interpolate(self, source_coordinates, source_data, eval_coordinates, output_ return output_data + def _fix_coordinates_for_none_interp(self, eval_coordinates, source_coordinates): + interpolator_queue = self._select_interpolator_queue( + source_coordinates, eval_coordinates, "can_interpolate", strict=True + ) + if not any([isinstance(interpolator_queue[k], NoneInterpolator) for k in interpolator_queue]): + # Nothing to do, just return eval_coordinates + return eval_coordinates + + # Likely need to fix the output, since the shape of output will + # not match the eval coordinates in most cases + new_dims = [] + new_coords = [] + covered_udims = [] + for k in interpolator_queue: + if not isinstance(interpolator_queue[k], NoneInterpolator): + # Keep the eval_coordinates for these dimensions + for d in eval_coordinates.dims: + ud = d.split("_") + for u in ud: + if u in k: + new_dims.append(d) + new_coords.append(eval_coordinates[d]) + covered_udims.extend(ud) + break + else: + for d in source_coordinates.dims: + ud = d.split("_") + for u in ud: + if u in k: + new_dims.append(d) + new_coords.append(source_coordinates[d]) + covered_udims.extend(ud) + break + new_coordinates = Coordinates(new_coords, new_dims) + return new_coordinates + class InterpolationTrait(tl.Union): default_value = INTERPOLATION_DEFAULT + # .tag(attr=True, required=True, default = "linear") def __init__( self, trait_types=[tl.Dict(), tl.List(), tl.Enum(INTERPOLATION_METHODS), tl.Instance(InterpolationManager)], diff --git a/podpac/core/interpolation/nearest_neighbor_interpolator.py b/podpac/core/interpolation/nearest_neighbor_interpolator.py index e2fcf6010..ae39d038d 100644 --- a/podpac/core/interpolation/nearest_neighbor_interpolator.py +++ b/podpac/core/interpolation/nearest_neighbor_interpolator.py @@ -95,7 +95,7 @@ def is_stacked(d): ] else: - bounds = {d: None for d in source_coordinates.udims} + bounds = None if self.remove_nan: # Eliminate nans from the source data. Note, this could turn a uniform griddted dataset into a stacked one @@ -110,16 +110,38 @@ def is_stacked(d): continue source = source_coordinates[d] if is_stacked(d): - bound = np.stack([bounds[dd] for dd in d.split("_")], axis=1) + if bounds is not None: + bound = np.stack([bounds[dd] for dd in d.split("_")], axis=1) + else: + bound = None index = self._get_stacked_index(d, source, eval_coordinates, bound) + + if len(source.shape) == 2: # Handle case of 2D-stacked coordinates + ncols = source.shape[1] + index1 = index // ncols + index1 = self._resize_stacked_index(index1, d, eval_coordinates) + # With nD stacked coordinates, there are 'n' indices in the tuple + # All of these need to get into the data_index, and in the right order + data_index.append(index1) # This is a hack + index = index % ncols # The second half can go through the usual machinery + elif len(source.shape) > 2: # Handle case of nD-stacked coordinates + raise NotImplementedError index = self._resize_stacked_index(index, d, eval_coordinates) elif source_coordinates[d].is_uniform: request = eval_coordinates[d] - index = self._get_uniform_index(d, source, request, bounds[d]) + if bounds is not None: + bound = bounds[d] + else: + bound = None + index = self._get_uniform_index(d, source, request, bound) index = self._resize_unstacked_index(index, d, eval_coordinates) else: # non-uniform coordinates... probably an optimization here request = eval_coordinates[d] - index = self._get_nonuniform_index(d, source, request, bounds[d]) + if bounds is not None: + bound = bounds[d] + else: + bound = None + index = self._get_nonuniform_index(d, source, request, bound) index = self._resize_unstacked_index(index, d, eval_coordinates) data_index.append(index) @@ -219,7 +241,8 @@ def _get_stacked_index(self, dim, source, request, bounds=None): scales = np.array([self._get_scale(d, time_source, time_request) for d in udims])[None, :] tol = np.linalg.norm((tols * scales).squeeze()) src_coords, req_coords_diag = _higher_precision_time_stack(source, request, udims) - ckdtree_source = cKDTree(src_coords.T * scales) + # We need to unwravel the nD stacked coordinates + ckdtree_source = cKDTree(src_coords.reshape(src_coords.shape[0], -1).T * scales) # if the udims are all stacked in the same stack as part of the request coordinates, then we're done. # Otherwise we have to evaluate each unstacked set of dimensions independently @@ -229,7 +252,8 @@ def _get_stacked_index(self, dim, source, request, bounds=None): stacked = {d for d in request.dims for ud in udims if ud in d and request.is_stacked(ud)} if (len(indep_evals) + len(stacked)) <= 1: # output is stacked in the same way - req_coords = req_coords_diag.T + # The ckdtree call below needs the lat/lon pairs in the last axis position + req_coords = np.moveaxis(req_coords_diag, 0, -1) elif (len(stacked) == 0) | (len(indep_evals) == 0 and len(stacked) == len(udims)): req_coords = np.stack([i.ravel() for i in np.meshgrid(*req_coords_diag, indexing="ij")], axis=1) else: @@ -252,10 +276,16 @@ def _get_stacked_index(self, dim, source, request, bounds=None): if self.respect_bounds: if bounds is None: - bounds = [src_coords.min(0), src_coords.max(0)] + bounds = np.stack( + [ + src_coords.reshape(src_coords.shape[0], -1).T.min(0), + src_coords.reshape(src_coords.shape[0], -1).T.max(0), + ], + axis=1, + ) # Fix order of bounds bounds = bounds[:, [source.udims.index(dim) for dim in udims]] - index[np.any((req_coords > bounds[1]), axis=1) | np.any((req_coords < bounds[0]), axis=1)] = -1 + index[np.any((req_coords > bounds[1]), axis=-1) | np.any((req_coords < bounds[0]), axis=-1)] = -1 if tol and tol != np.inf: index[dist > tol] = -1 diff --git a/podpac/core/interpolation/none_interpolator.py b/podpac/core/interpolation/none_interpolator.py new file mode 100644 index 000000000..28f84f44b --- /dev/null +++ b/podpac/core/interpolation/none_interpolator.py @@ -0,0 +1,68 @@ +""" +Interpolator implementations +""" + +from __future__ import division, unicode_literals, print_function, absolute_import +from six import string_types + +import numpy as np +import xarray as xr +import traitlets as tl +from scipy.spatial import cKDTree + +# Optional dependencies + + +# podac imports +from podpac.core.interpolation.interpolator import COMMON_INTERPOLATOR_DOCS, Interpolator, InterpolatorException +from podpac.core.coordinates import Coordinates, UniformCoordinates1d, StackedCoordinates +from podpac.core.coordinates.utils import make_coord_delta, make_coord_value +from podpac.core.utils import common_doc +from podpac.core.coordinates.utils import get_timedelta +from podpac.core.interpolation.selector import Selector, _higher_precision_time_coords1d, _higher_precision_time_stack + + +@common_doc(COMMON_INTERPOLATOR_DOCS) +class NoneInterpolator(Interpolator): + """None Interpolation""" + + dims_supported = ["lat", "lon", "alt", "time"] + methods_supported = ["none"] + method = tl.Unicode(default_value="none") + + @common_doc(COMMON_INTERPOLATOR_DOCS) + def can_interpolate(self, udims, source_coordinates, eval_coordinates): + """ + {interpolator_interpolate} + """ + udims_subset = self._filter_udims_supported(udims) + + return udims_subset + + @common_doc(COMMON_INTERPOLATOR_DOCS) + def interpolate(self, udims, source_coordinates, source_data, eval_coordinates, output_data): + """ + {interpolator_interpolate} + """ + # Note, some of the following code duplicates code in the Selector class. + # This duplication is for the sake of optimization + + return source_data + + @common_doc(COMMON_INTERPOLATOR_DOCS) + def can_select(self, udims, source_coordinates, eval_coordinates): + """ + {interpolator_can_select} + """ + if not (self.method == "none"): + return tuple() + + udims_subset = self._filter_udims_supported(udims) + return udims_subset + + @common_doc(COMMON_INTERPOLATOR_DOCS) + def select_coordinates(self, udims, source_coordinates, eval_coordinates, index_type="numpy"): + """ + {interpolator_select} + """ + return source_coordinates.intersect(eval_coordinates, outer=False, return_index=True) diff --git a/podpac/core/interpolation/selector.py b/podpac/core/interpolation/selector.py index 70ffb7824..7cf3b484f 100644 --- a/podpac/core/interpolation/selector.py +++ b/podpac/core/interpolation/selector.py @@ -100,8 +100,15 @@ def select(self, source_coords, request_coords, index_type="numpy"): for coord1d in source_coords._coords.values(): ci = self._select1d(coord1d, request_coords, index_type) ci = np.sort(np.unique(ci)) - if index_type == "slice": + if len(coord1d.shape) == 2: # Handle case of 2D-stacked coordinates + ncols = coord1d.shape[1] + ci = (ci // ncols, ci % ncols) + if index_type == "slice": + ci = tuple([_index2slice(cii) for cii in ci]) + elif index_type == "slice": ci = _index2slice(ci) + if len(coord1d.shape) == 3: # Handle case of 3D-stacked coordinates + raise NotImplementedError c = coord1d[ci] coords.append(c) coords_inds.append(ci) @@ -109,7 +116,9 @@ def select(self, source_coords, request_coords, index_type="numpy"): if index_type == "numpy": coords_inds = self._merge_indices(coords_inds, source_coords.dims, request_coords.dims) elif index_type == "xarray": - pass # unlike numpy, xarray assumes indexes are orthogonal by default, so the 1d coordinates are already correct + # unlike numpy, xarray assumes indexes are orthogonal by default, so the 1d coordinates are already correct + # unless there are tuple coordinates (nD stacked coords) but those are handled in interpolation_manager + pass return coords, tuple(coords_inds) def _select1d(self, source, request, index_type): @@ -127,11 +136,18 @@ def _select1d(self, source, request, index_type): def _merge_indices(self, indices, source_dims, request_dims): # For numpy to broadcast correctly, we have to reshape each of the indices reshape = np.ones(len(indices), int) + new_indices = [] for i in range(len(indices)): reshape[:] = 1 reshape[i] = -1 - indices[i] = indices[i].reshape(*reshape) - return tuple(indices) + if isinstance(indices[i], tuple): + # nD stacked coordinates + # This means the source has shape (N, M, ...) + # But the coordinates are stacked (i.e. lat_lon with shape N, M for the lon and lat parts) + new_indices.append(tuple([ind.reshape(*reshape) for ind in indices[i]])) + else: + new_indices.append(indices[i].reshape(*reshape)) + return tuple(new_indices) def _select_uniform(self, source, request, index_type): crds = request[source.name] @@ -186,7 +202,8 @@ def _select_stacked(self, source, request, index_type): inds = np.array([]) # Parts of the below code is duplicated in NearestNeighborInterpolotor src_coords, req_coords_diag = _higher_precision_time_stack(source, request, udims) - ckdtree_source = cKDTree(src_coords.T) + # For nD stacked coordinates we need to unravel the stacked dimension + ckdtree_source = cKDTree(src_coords.reshape(src_coords.shape[0], -1).T) if (len(indep_evals) + len(stacked)) <= 1: req_coords = req_coords_diag.T elif (len(stacked) == 0) | (len(indep_evals) == 0 and len(stacked) == len(udims)): diff --git a/podpac/core/interpolation/test/test_interpolation.py b/podpac/core/interpolation/test/test_interpolation.py index e49ee3405..3579b4afb 100644 --- a/podpac/core/interpolation/test/test_interpolation.py +++ b/podpac/core/interpolation/test/test_interpolation.py @@ -81,6 +81,9 @@ def test_compositor_chain(self): np.testing.assert_array_equal(o.data, np.concatenate([self.s1.source, self.s2.source], axis=0)) + def test_get_bounds(self): + assert self.interp.get_bounds() == self.s1.get_bounds() + class TestInterpolationBehavior(object): def test_linear_1D_issue411and413(self): @@ -89,9 +92,12 @@ def test_linear_1D_issue411and413(self): raw_e_coords = [0, 0.5, 1, 1.5, 2] for dim in ["lat", "lon", "alt", "time"]: - ec = Coordinates([raw_e_coords], [dim]) + ec = Coordinates([raw_e_coords], [dim], crs="+proj=longlat +datum=WGS84 +no_defs +vunits=m") - arrb = ArrayRaw(source=data, coordinates=Coordinates([raw_coords], [dim])) + arrb = ArrayRaw( + source=data, + coordinates=Coordinates([raw_coords], [dim], crs="+proj=longlat +datum=WGS84 +no_defs +vunits=m"), + ) node = Interpolate(source=arrb, interpolation="linear") o = node.eval(ec) @@ -172,3 +178,32 @@ def test_selection_crs(self): o = node.eval(tocrds) assert o.crs == tocrds.crs assert_array_equal(o.data, np.linspace(0, 2, 9)) + + def test_floating_point_crs_disagreement(self): + tocrds = podpac.Coordinates([[39.1, 39.0, 38.9], [-77.1, -77, -77.2]], dims=["lat", "lon"], crs="EPSG:4326") + base = podpac.core.data.array_source.ArrayRaw( + source=np.random.rand(3, 3), coordinates=tocrds.transform("EPSG:32618") + ) + node = podpac.interpolators.Interpolate(source=base, interpolation="nearest") + o = node.eval(tocrds) + assert np.all((o.lat.data - tocrds["lat"].coordinates) == 0) + + # now check the Mixin + node2 = podpac.core.data.array_source.Array( + source=np.random.rand(3, 3), coordinates=tocrds.transform("EPSG:32618") + ) + o = node2.eval(tocrds) + assert np.all((o.lat.data - tocrds["lat"].coordinates) == 0) + + # now check the reverse operation + tocrds = podpac.Coordinates( + [podpac.clinspace(4307580, 4330177, 7), podpac.clinspace(309220, 327053, 8)], + dims=["lat", "lon"], + crs="EPSG:32618", + ) + srccrds = podpac.Coordinates( + [podpac.clinspace(39.2, 38.8, 9), podpac.clinspace(-77.3, -77.0, 9)], dims=["lat", "lon"], crs="EPSG:4326" + ) + node3 = podpac.core.data.array_source.Array(source=np.random.rand(9, 9), coordinates=srccrds) + o = node3.eval(tocrds) + assert np.all((o.lat.data - tocrds["lat"].coordinates) == 0) diff --git a/podpac/core/interpolation/test/test_interpolation_manager.py b/podpac/core/interpolation/test/test_interpolation_manager.py index eea45d8ed..540145929 100644 --- a/podpac/core/interpolation/test/test_interpolation_manager.py +++ b/podpac/core/interpolation/test/test_interpolation_manager.py @@ -26,7 +26,7 @@ class TestInterpolation(object): - """ Test interpolation class and support methods""" + """Test interpolation class and support methods""" def test_allow_missing_modules(self): """TODO: Allow user to be missing rasterio and scipy""" diff --git a/podpac/core/interpolation/test/test_interpolators.py b/podpac/core/interpolation/test/test_interpolators.py index 204d5bd57..2f6b7895f 100644 --- a/podpac/core/interpolation/test/test_interpolators.py +++ b/podpac/core/interpolation/test/test_interpolators.py @@ -31,6 +31,115 @@ def get_data(self, coordinates, coordinates_index): return self.create_output_array(coordinates, data=self.data[coordinates_index]) +class MockArrayDataSourceXR(InterpolationMixin, DataSource): + data = ArrayTrait().tag(attr=True) + coordinates = tl.Instance(Coordinates).tag(attr=True) + + def get_data(self, coordinates, coordinates_index): + dataxr = self.create_output_array(self.coordinates, data=self.data) + return self.create_output_array(coordinates, data=dataxr[coordinates_index].data) + + +class TestNone(object): + def test_none_select(self): + reqcoords = Coordinates([[-0.5, 1.5, 3.5], [0.5, 2.5, 4.5]], dims=["lat", "lon"]) + srccoords = Coordinates([[-1, 0, 1, 2, 3, 4, 5], [0, 1, 2, 3, 4, 5]], dims=["lat", "lon"]) + + # test straight ahead functionality + interp = InterpolationManager("none") + coords, cidx = interp.select_coordinates(srccoords, reqcoords) + assert coords == srccoords[1:5, 1:-1] + assert srccoords[cidx] == coords + + # test when selection is applied serially + interp = InterpolationManager([{"method": "none", "dims": ["lat"]}, {"method": "none", "dims": ["lon"]}]) + + coords, cidx = interp.select_coordinates(srccoords, reqcoords) + assert coords == srccoords[1:5, 1:-1] + assert srccoords[cidx] == coords + + # Test Case where rounding issues causes problem with endpoint + reqcoords = Coordinates([[0, 2, 4], [0, 2, 4]], dims=["lat", "lon"]) + lat = np.arange(0, 6.1, 1.3333333333333334) + lon = np.arange(0, 6.1, 1.333333333333334) # Notice one decimal less on this number + srccoords = Coordinates([lat, lon], dims=["lat", "lon"]) + + # test straight ahead functionality + interp = InterpolationManager("none") + coords, cidx = interp.select_coordinates(srccoords, reqcoords) + srccoords = Coordinates([lat, lon], dims=["lat", "lon"]) + assert srccoords[cidx] == coords + + def test_none_interpolation(self): + node = podpac.data.Array( + source=[0, 1, 2], + coordinates=podpac.Coordinates([[1, 5, 9]], dims=["lat"]), + interpolation="none", + ) + o = node.eval(podpac.Coordinates([podpac.crange(1, 9, 1)], dims=["lat"])) + np.testing.assert_array_equal(o.data, node.source) + + def test_none_heterogeneous(self): + # Heterogeneous + node = podpac.data.Array( + source=[[0, 1, 2], [0, 1, 2], [0, 1, 2], [0, 1, 2]], + coordinates=podpac.Coordinates([[1, 5, 9, 13], [0, 1, 2]], dims=["lat", "lon"]), + interpolation=[{"method": "none", "dims": ["lat"]}, {"method": "linear", "dims": ["lon"]}], + ) + o = node.eval(podpac.Coordinates([podpac.crange(1, 9, 2), [0.5, 1.5]], dims=["lat", "lon"])) + np.testing.assert_array_equal( + o.data, + [ + [0.5, 1.5], + [ + 0.5, + 1.5, + ], + [0.5, 1.5], + ], + ) + + # Heterogeneous _flipped + node = podpac.data.Array( + source=[[0, 1, 2], [0, 1, 2], [0, 1, 2], [0, 1, 2]], + coordinates=podpac.Coordinates([[1, 5, 9, 13], [0, 1, 2]], dims=["lat", "lon"]), + interpolation=[{"method": "linear", "dims": ["lon"]}, {"method": "none", "dims": ["lat"]}], + ) + o = node.eval(podpac.Coordinates([podpac.crange(1, 9, 2), [0.5, 1.5]], dims=["lat", "lon"])) + np.testing.assert_array_equal( + o.data, + [ + [0.5, 1.5], + [ + 0.5, + 1.5, + ], + [0.5, 1.5], + ], + ) + + # Examples + # source eval + # lat_lon lat, lon + node = podpac.data.Array( + source=[0, 1, 2], + coordinates=podpac.Coordinates([[[1, 5, 9], [1, 5, 9]]], dims=[["lat", "lon"]]), + interpolation=[{"method": "none", "dims": ["lon", "lat"]}], + ) + o = node.eval(podpac.Coordinates([podpac.crange(1, 9, 1), podpac.crange(1, 9, 1)], dims=["lon", "lat"])) + np.testing.assert_array_equal(o.data, node.source) + + # source eval + # lat, lon lat_lon + node = podpac.data.Array( + source=[[0, 1, 2], [0, 1, 2], [0, 1, 2], [0, 1, 2]], + coordinates=podpac.Coordinates([[1, 5, 9, 13], [0, 1, 2]], dims=["lat", "lon"]), + interpolation=[{"method": "none", "dims": ["lat", "lon"]}], + ) + o = node.eval(podpac.Coordinates([[podpac.crange(1, 9, 2), podpac.crange(1, 9, 2)]], dims=[["lat", "lon"]])) + np.testing.assert_array_equal(o.data, node.source[:-1, 1:]) + + class TestNearest(object): def test_nearest_preview_select(self): reqcoords = Coordinates([[-0.5, 1.5, 3.5], [0.5, 2.5, 4.5]], dims=["lat", "lon"]) @@ -464,12 +573,134 @@ def test_respect_bounds(self): np.testing.assert_array_equal(output.data[1:], source[[0, 2]]) assert np.isnan(output.data[0]) + def test_2Dstacked(self): + # With Time + source = np.random.rand(5, 4, 2) + coords_src = Coordinates( + [ + [ + np.arange(5)[:, None] + 0.1 * np.ones((5, 4)), + np.arange(4)[None, :] + 0.1 * np.ones((5, 4)), + ], + [0.4, 0.7], + ], + ["lat_lon", "time"], + ) + coords_dst = Coordinates([np.arange(4) + 0.2, np.arange(1, 4) - 0.2, [0.5]], ["lat", "lon", "time"]) + node = MockArrayDataSource( + data=source, + coordinates=coords_src, + interpolation={ + "method": "nearest", + "interpolators": [NearestNeighbor], + }, + ) + output = node.eval(coords_dst) + np.testing.assert_array_equal(output, source[:4, 1:, :1]) + + # Using 'xarray' coordinates type + node = MockArrayDataSourceXR( + data=source, + coordinates=coords_src, + coordinate_index_type="xarray", + interpolation={ + "method": "nearest", + "interpolators": [NearestNeighbor], + }, + ) + output = node.eval(coords_dst) + np.testing.assert_array_equal(output, source[:4, 1:, :1]) + + # Using 'slice' coordinates type + node = MockArrayDataSource( + data=source, + coordinates=coords_src, + coordinate_index_type="slice", + interpolation={ + "method": "nearest", + "interpolators": [NearestNeighbor], + }, + ) + output = node.eval(coords_dst) + np.testing.assert_array_equal(output, source[:4, 1:, :1]) + + # Without Time + source = np.random.rand(5, 4) + node = MockArrayDataSource( + data=source, + coordinates=coords_src.drop("time"), + interpolation={ + "method": "nearest", + "interpolators": [NearestNeighbor], + }, + ) + output = node.eval(coords_dst) + np.testing.assert_array_equal(output, source[:4, 1:]) + + # def test_3Dstacked(self): + # # With Time + # source = np.random.rand(5, 4, 2) + # coords_src = Coordinates([[ + # np.arange(5)[:, None, None] + 0.1 * np.ones((5, 4, 2)), + # np.arange(4)[None, :, None] + 0.1 * np.ones((5, 4, 2)), + # np.arange(2)[None, None, :] + 0.1 * np.ones((5, 4, 2))]], ["lat_lon_time"]) + # coords_dst = Coordinates([np.arange(4)+0.2, np.arange(1, 4)-0.2, [0.5]], ["lat", "lon", "time"]) + # node = MockArrayDataSource( + # data=source, + # coordinates=coords_src, + # interpolation={ + # "method": "nearest", + # "interpolators": [NearestNeighbor], + # }, + # ) + # output = node.eval(coords_dst) + # np.testing.assert_array_equal(output, source[:4, 1:, :1]) + + # # Using 'xarray' coordinates type + # node = MockArrayDataSourceXR( + # data=source, + # coordinates=coords_src, + # coordinate_index_type='xarray', + # interpolation={ + # "method": "nearest", + # "interpolators": [NearestNeighbor], + # }, + # ) + # output = node.eval(coords_dst) + # np.testing.assert_array_equal(output, source[:4, 1:, :1]) + + # # Using 'slice' coordinates type + # node = MockArrayDataSource( + # data=source, + # coordinates=coords_src, + # coordinate_index_type='slice', + # interpolation={ + # "method": "nearest", + # "interpolators": [NearestNeighbor], + # }, + # ) + # output = node.eval(coords_dst) + # np.testing.assert_array_equal(output, source[:4, 1:, :1]) + + # # Without Time + # source = np.random.rand(5, 4) + # node = MockArrayDataSource( + # data=source, + # coordinates=coords_src.drop('time'), + # interpolation={ + # "method": "nearest", + # "interpolators": [NearestNeighbor], + # }, + # ) + # output = node.eval(coords_dst) + # np.testing.assert_array_equal(output, source[:4, 1:]) + class TestInterpolateRasterioInterpolator(object): """test interpolation functions""" def test_interpolate_rasterio(self): - """ regular interpolation using rasterio""" + """regular interpolation using rasterio""" assert rasterio is not None @@ -581,7 +812,7 @@ def test_interpolate_scipy_grid(self): assert np.isnan(output.data[4, 4]) # TODO: how to handle outside bounds def test_interpolate_irregular_arbitrary_2dims(self): - """ irregular interpolation """ + """irregular interpolation""" # Note, this test also tests the looper helper @@ -605,7 +836,7 @@ def test_interpolate_irregular_arbitrary_2dims(self): # assert output.data[0, 0] == source[] def test_interpolate_looper_helper(self): - """ irregular interpolation """ + """irregular interpolation""" # Note, this test also tests the looper helper @@ -675,7 +906,7 @@ def test_interpolate_irregular_arbitrary_swap(self): np.testing.assert_array_equal(output.lon.values, coords_dst["lon"].coordinates) def test_interpolate_irregular_lat_lon(self): - """ irregular interpolation """ + """irregular interpolation""" source = np.random.rand(5, 5) coords_src = Coordinates([clinspace(0, 10, 5), clinspace(0, 10, 5)], dims=["lat", "lon"]) @@ -697,7 +928,7 @@ def test_interpolate_irregular_lat_lon(self): class TestInterpolateScipyPoint(object): def test_interpolate_scipy_point(self): - """ interpolate point data to nearest neighbor with various coords_dst""" + """interpolate point data to nearest neighbor with various coords_dst""" source = np.random.rand(6) coords_src = Coordinates([[[0, 2, 4, 6, 8, 10], [0, 2, 4, 5, 6, 10]]], dims=["lat_lon"]) @@ -863,7 +1094,7 @@ def test_interpolate_xarray_grid(self): assert np.all(~np.isnan(output.data)) def test_interpolate_irregular_arbitrary_2dims(self): - """ irregular interpolation """ + """irregular interpolation""" # try >2 dims source = np.random.rand(5, 5, 3) @@ -921,7 +1152,7 @@ def test_interpolate_irregular_arbitrary_swap(self): assert np.all(output.lon.values == coords_dst["lon"].coordinates) def test_interpolate_irregular_lat_lon(self): - """ irregular interpolation """ + """irregular interpolation""" source = np.random.rand(5, 5) coords_src = Coordinates([clinspace(0, 10, 5), clinspace(0, 10, 5)], dims=["lat", "lon"]) diff --git a/podpac/core/interpolation/xarray_interpolator.py b/podpac/core/interpolation/xarray_interpolator.py index a3b1ce3da..99c25a09f 100644 --- a/podpac/core/interpolation/xarray_interpolator.py +++ b/podpac/core/interpolation/xarray_interpolator.py @@ -98,11 +98,26 @@ def interpolate(self, udims, source_coordinates, source_data, eval_coordinates, coords=[eval_coordinates.xcoords[new_dim]], ) continue - if not source_coordinates.is_stacked(d) and eval_coordinates.is_stacked(d): + if ( + not source_coordinates.is_stacked(d) + and eval_coordinates.is_stacked(d) + and len(eval_coordinates[d].shape) == 1 + ): + # Handle case for stacked coordinates (i.e. along a curve) new_dim = [dd for dd in eval_coordinates.dims if d in dd][0] coords[d] = xr.DataArray( eval_coordinates[d].coordinates, dims=[new_dim], coords=[eval_coordinates.xcoords[new_dim]] ) + elif ( + not source_coordinates.is_stacked(d) + and eval_coordinates.is_stacked(d) + and len(eval_coordinates[d].shape) > 1 + ): + # Dependent coordinates (i.e. a warped coordinate system) + keep_coords = {k: v for k, v in eval_coordinates.xcoords.items() if k in eval_coordinates.xcoords[d][0]} + coords[d] = xr.DataArray( + eval_coordinates[d].coordinates, dims=eval_coordinates.xcoords[d][0], coords=keep_coords + ) else: # TODO: Check dependent coordinates coords[d] = eval_coordinates[d].coordinates @@ -127,4 +142,4 @@ def interpolate(self, udims, source_coordinates, source_data, eval_coordinates, output_data = source_data.interp(method=self.method, **coords) - return output_data.transpose(*eval_coordinates.dims) + return output_data.transpose(*eval_coordinates.xdims) diff --git a/podpac/core/managers/aws.py b/podpac/core/managers/aws.py index bf3836f47..953e2f9e2 100644 --- a/podpac/core/managers/aws.py +++ b/podpac/core/managers/aws.py @@ -13,8 +13,6 @@ from six import string_types -import boto3 -import botocore import traitlets as tl import numpy as np @@ -24,6 +22,22 @@ from podpac.core.utils import common_doc, JSONEncoder from podpac import version +# Optional imports +from lazy_import import lazy_module, lazy_class + +try: + import boto3 + import botocore +except: + + class err: + def __init__(self, *args, **kwargs): + raise ImportError("boto3 is not installed, please install to use this functionality.") + + class boto3: + Session = err + + # Set up logging _log = logging.getLogger(__name__) @@ -31,7 +45,7 @@ class LambdaException(Exception): - """ Exception during execution of a Lambda node""" + """Exception during execution of a Lambda node""" pass diff --git a/podpac/core/managers/parallel.py b/podpac/core/managers/parallel.py index c667411e0..940e316f0 100644 --- a/podpac/core/managers/parallel.py +++ b/podpac/core/managers/parallel.py @@ -22,7 +22,20 @@ zarr = lazy_module("zarr") zarrGroup = lazy_class("zarr.Group") -botocore = lazy_module("botocore") +try: + import botocore +except: + + class dum: + pass + + class mod: + ClientError = dum + ReadTimeoutError = dum + + class botocore: + exceptions = mod + # Set up logging _log = logging.getLogger(__name__) diff --git a/podpac/core/node.py b/podpac/core/node.py index 8929efd5b..d04a85dd5 100644 --- a/podpac/core/node.py +++ b/podpac/core/node.py @@ -12,7 +12,7 @@ import warnings from collections import OrderedDict from copy import deepcopy -from hashlib import md5 as hash_alg +import logging import numpy as np import traitlets as tl @@ -27,11 +27,14 @@ from podpac.core.utils import trait_is_defined from podpac.core.utils import _get_query_params_from_url, _get_from_url, _get_param from podpac.core.utils import probe_node +from podpac.core.utils import NodeTrait +from podpac.core.utils import hash_alg from podpac.core.coordinates import Coordinates from podpac.core.style import Style from podpac.core.cache import CacheCtrl, get_default_cache_ctrl, make_cache_ctrl, S3CacheStore, DiskCacheStore from podpac.core.managers.multi_threading import thread_manager +_logger = logging.getLogger(__name__) COMMON_NODE_DOC = { "requested_coordinates": """The set of coordinates requested by a user. The Node will be evaluated using these coordinates.""", @@ -79,13 +82,13 @@ class NodeException(Exception): - """ Base class for exceptions when using podpac nodes """ + """Base class for exceptions when using podpac nodes""" pass class NodeDefinitionError(NodeException): - """ Raised node definition errors, such as when the definition is circular or is not yet unavailable. """ + """Raised node definition errors, such as when the definition is circular or is not yet unavailable.""" pass @@ -129,7 +132,7 @@ class Node(tl.HasTraits): outputs = tl.List(trait=tl.Unicode(), allow_none=True).tag(attr=True) output = tl.Unicode(default_value=None, allow_none=True).tag(attr=True) - units = tl.Unicode(default_value=None, allow_none=True).tag(attr=True) + units = tl.Unicode(default_value=None, allow_none=True).tag(attr=True, hidden=True) style = tl.Instance(Style) dtype = tl.Enum([float], default_value=float) @@ -183,7 +186,7 @@ def _cache_ctrl_default(self): _traits_initialized_guard = False def __init__(self, **kwargs): - """ Do not overwrite me """ + """Do not overwrite me""" # Shortcut for users to make setting the cache_ctrl simpler: if "cache_ctrl" in kwargs and isinstance(kwargs["cache_ctrl"], list): @@ -278,11 +281,13 @@ def eval(self, coordinates, **kwargs): if settings["DEBUG"]: self._requested_coordinates = coordinates - key = "output" - cache_coordinates = coordinates.transpose(*sorted(coordinates.dims)) # order agnostic caching + item = "output" - if not self.force_eval and self.cache_output and self.has_cache(key, cache_coordinates): - data = self.get_cache(key, cache_coordinates) + # get standardized coordinates for caching + cache_coordinates = coordinates.transpose(*sorted(coordinates.dims)).simplify() + + if not self.force_eval and self.cache_output and self.has_cache(item, cache_coordinates): + data = self.get_cache(item, cache_coordinates) if output is not None: order = [dim for dim in output.dims if dim not in data.dims] + list(data.dims) output.transpose(*order)[:] = data @@ -290,7 +295,7 @@ def eval(self, coordinates, **kwargs): else: data = self._eval(coordinates, **kwargs) if self.cache_output: - self.put_cache(data, key, cache_coordinates) + self.put_cache(data, item, cache_coordinates) self._from_cache = False # extract single output, if necessary @@ -311,7 +316,7 @@ def eval(self, coordinates, **kwargs): data.attrs["layer_style"] = self.style if self.units is not None: - data.attrs["units"] + data.attrs["units"] = self.units # Add crs if it is missing if "crs" not in data.attrs: @@ -351,8 +356,39 @@ def find_coordinates(self): raise NotImplementedError + def get_bounds(self, crs="default"): + """Get the full available coordinate bounds for the Node. + + Arguments + --------- + crs : str + Desired CRS for the bounds. + If not specified, the default CRS in the podpac settings is used. Optional. + + Returns + ------- + bounds : dict + Bounds for each dimension. Keys are dimension names and values are tuples (min, max). + crs : str + The CRS for the bounds. + """ + + if crs == "default": + crs = podpac.settings["DEFAULT_CRS"] + + bounds = {} + for coords in self.find_coordinates(): + ct = coords.transform(crs) + for dim, (lo, hi) in ct.bounds.items(): + if dim not in bounds: + bounds[dim] = (lo, hi) + else: + bounds[dim] = (min(lo, bounds[dim][0]), max(hi, bounds[dim][1])) + + return bounds, crs + @common_doc(COMMON_DOC) - def create_output_array(self, coords, data=np.nan, attrs=None, **kwargs): + def create_output_array(self, coords, data=np.nan, attrs=None, outputs=None, **kwargs): """ Initialize an output data array @@ -362,6 +398,10 @@ def create_output_array(self, coords, data=np.nan, attrs=None, **kwargs): {arr_coords} data : None, number, or array-like (optional) {arr_init_type} + attrs : dict + Attributes to add to output -- UnitsDataArray.create uses the 'crs' portion contained in here + outputs : list[string], optional + Default is self.outputs. List of strings listing the outputs **kwargs {arr_kwargs} @@ -384,8 +424,12 @@ def create_output_array(self, coords, data=np.nan, attrs=None, **kwargs): attrs["geotransform"] = coords.geotransform except (TypeError, AttributeError): pass + if outputs is None: + outputs = self.outputs + if outputs == []: + outputs = None - return UnitsDataArray.create(coords, data=data, outputs=self.outputs, dtype=self.dtype, attrs=attrs, **kwargs) + return UnitsDataArray.create(coords, data=data, outputs=outputs, dtype=self.dtype, attrs=attrs, **kwargs) def trait_is_defined(self, name): return trait_is_defined(self, name) @@ -589,7 +633,7 @@ def json_pretty(self): @cached_property def hash(self): - """ hash for this node, used in caching and to determine equality. """ + """hash for this node, used in caching and to determine equality.""" # deepcopy so that the cached definition property is not modified by the deletes below d = deepcopy(self.definition) @@ -762,7 +806,7 @@ def rem_cache(self, key, coordinates=None, mode="all"): if self.cache_ctrl is None: return - self.cache_ctrl.rem(self, key=key, coordinates=coordinates, mode=mode) + self.cache_ctrl.rem(self, item=key, coordinates=coordinates, mode=mode) # --------------------------------------------------------# # Class Methods (Deserialization) @@ -837,7 +881,28 @@ def from_definition(cls, definition): kwargs[k] = _lookup_attr(nodes, name, v) if "style" in d: - kwargs["style"] = Style.from_definition(d["style"]) + style_class = getattr(node_class, "style", Style) + if isinstance(style_class, tl.TraitType): + # Now we actually have to look through the class to see + # if there is a custom initializer for style + for attr in dir(node_class): + atr = getattr(node_class, attr) + if not isinstance(atr, tl.traitlets.DefaultHandler) or atr.trait_name != "style": + continue + try: + style_class = atr(node_class) + except Exception as e: + # print ("couldn't make style from class", e) + try: + style_class = atr(node_class()) + except: + # print ("couldn't make style from class instance", e) + style_class = style_class.klass + try: + kwargs["style"] = style_class.from_definition(d["style"]) + except Exception as e: + kwargs["style"] = Style.from_definition(d["style"]) + # print ("couldn't make style from inferred style class", e) for k in d: if k not in ["node", "inputs", "attrs", "lookup_attrs", "plugin", "style"]: @@ -990,6 +1055,158 @@ def from_name_params(cls, name, params=None): return cls.from_definition(d) + @classmethod + def get_ui_spec(cls, help_as_html=False): + """Get spec of node attributes for building a ui + + Parameters + ---------- + help_as_html : bool, optional + Default is False. If True, the docstrings will be converted to html before storing in the spec. + + Returns + ------- + dict + Spec for this node that is readily json-serializable + """ + filter = [] + spec = {"help": cls.__doc__, "module": cls.__module__ + "." + cls.__name__, "attrs": {}, "style": {}} + # Strip out starting spaces in the help text so that markdown parsing works correctly + if spec["help"] is None: + spec["help"] = "No help text to display." + spec["help"] = spec["help"].replace("\n ", "\n") + + if help_as_html: + from numpydoc.docscrape_sphinx import SphinxDocString + from docutils.core import publish_string + + tmp = SphinxDocString(spec["help"]) + tmp2 = publish_string(str(tmp), writer_name="html") + slc = slice(tmp2.index(b'
'), tmp2.index(b"")) + spec["help"] = tmp2[slc].decode() + + # find any default values that are defined by function with decorators + # e.g. using @tl.default("trait_name") + # def _default_trait_name(self): ... + function_defaults = {} + for attr in dir(cls): + atr = getattr(cls, attr) + if not isinstance(atr, tl.traitlets.DefaultHandler): + continue + try: + try: + def_val = atr(cls()) + except: + def_val = atr(cls) + if isinstance(def_val, NodeTrait): + def_val = def_val.name + print("Changing Nodetrait to string") + # if "NodeTrait" not in str(atr(cls)): + function_defaults[atr.trait_name] = def_val + except Exception: + _logger.warning( + "For node {}: Failed to generate default from function for trait {}".format( + cls.__name__, atr.trait_name + ) + ) + + for attr in dir(cls): + if attr in filter: + continue + attrt = getattr(cls, attr) + if not isinstance(attrt, tl.TraitType): + continue + if not attrt.metadata.get("attr", False): + continue + type_ = attrt.__class__.__name__ + + try: + schema = getattr(attrt, "_schema") + except: + schema = None + + type_extra = str(attrt) + if type_ == "Union": + type_ = [t.__class__.__name__ for t in attrt.trait_types] + type_extra = "Union" + elif type_ == "Instance": + type_ = attrt.klass.__name__ + if type_ == "Node": + type_ = "NodeTrait" + type_extra = attrt.klass + elif type_ == "Dict" and schema is None: + try: + schema = { + "key": getattr(attrt, "_key_trait").__class__.__name__, + "value": getattr(attrt, "_value_trait").__class__.__name__, + } + except Exception as e: + print("Could not find schema for", attrt, " of type", type_) + schema = None + + required = attrt.metadata.get("required", False) + hidden = attrt.metadata.get("hidden", False) + if attr in function_defaults: + default_val = function_defaults[attr] + else: + default_val = attrt.default() + if not isinstance(type_extra, str): + type_extra = str(type_extra) + try: + if np.isnan(default_val): + default_val = "nan" + except: + pass + + if default_val == tl.Undefined: + default_val = None + + spec["attrs"][attr] = { + "type": type_, + "type_str": type_extra, # May remove this if not needed + "values": getattr(attrt, "values", None), + "default": default_val, + "help": attrt.help, + "required": required, + "hidden": hidden, + "schema": schema, + } + + try: + # This returns the + style_json = json.loads(cls().style.json) # load the style from the cls + except: + style_json = {} + + spec["style"] = style_json # this does not work, because node not created yet? + + """ + I will manually define generic defaults here. Eventually we may want to + dig into this and create node specific styling. This will have to be done under each + node. But may be difficult to add style to each node? + + Example: podpac.core.algorithm.utility.SinCoords.Style ----> returns a tl.Instance + BUT if I do: + podpac.core.algorithm.utility.SinCoords().style.json ---> outputs style + + ERROR if no parenthesis are given. So how can this be done without instantiating the class? + + Will need to ask @MPU how to define a node specific style. + """ + # spec["style"] = { + # "name": "?", + # "units": "m", + # "clim": [-1.0, 1.0], + # "colormap": "jet", + # "enumeration_legend": "?", + # "enumeration_colors": "?", + # "default_enumeration_legend": "unknown", + # "default_enumeration_color": (0.2, 0.2, 0.2), + # } + + spec.update(getattr(cls, "_ui_spec", {})) + return spec + def _lookup_input(nodes, name, value): # containers @@ -1064,7 +1281,7 @@ def _lookup_attr(nodes, name, value): class NoCacheMixin(tl.HasTraits): - """ Mixin to use no cache by default. """ + """Mixin to use no cache by default.""" cache_ctrl = tl.Instance(CacheCtrl, allow_none=True) @@ -1074,7 +1291,7 @@ def _cache_ctrl_default(self): class DiskCacheMixin(tl.HasTraits): - """ Mixin to add disk caching to the Node by default. """ + """Mixin to add disk caching to the Node by default.""" cache_ctrl = tl.Instance(CacheCtrl, allow_none=True) diff --git a/podpac/core/style.py b/podpac/core/style.py index b93d8388d..69c9857cd 100644 --- a/podpac/core/style.py +++ b/podpac/core/style.py @@ -44,7 +44,7 @@ def __init__(self, node=None, *args, **kwargs): super(Style, self).__init__(*args, **kwargs) name = tl.Unicode() - units = tl.Unicode(allow_none=True) + units = tl.Unicode(allow_none=True, default_value="") clim = tl.List(default_value=[None, None]) colormap = tl.Unicode(allow_none=True, default_value=None) enumeration_legend = tl.Dict(key_trait=tl.Int(), value_trait=tl.Unicode(), default_value=None, allow_none=True) @@ -79,7 +79,7 @@ def _validate_enumeration_legend(self, d): @property def full_enumeration_colors(self): - """ Convert enumeration_colors into a tuple suitable for matplotlib ListedColormap. """ + """Convert enumeration_colors into a tuple suitable for matplotlib ListedColormap.""" return tuple( [ self.enumeration_colors.get(value, self.default_enumeration_color) @@ -89,7 +89,7 @@ def full_enumeration_colors(self): @property def full_enumeration_legend(self): - """ Convert enumeration_legend into a tuple suitable for matplotlib. """ + """Convert enumeration_legend into a tuple suitable for matplotlib.""" return tuple( [ self.enumeration_legend.get(value, self.default_enumeration_legend) @@ -119,6 +119,31 @@ def json(self): return json.dumps(self.definition, separators=(",", ":"), cls=JSONEncoder) + @classmethod + def get_style_ui(self): + """ + Attempting to expose style units to get_ui_spec(). This will grab defaults in general. + BUT this will not set defaults for each particular node. + """ + d = OrderedDict() + if self.name: + d["name"] = self.name + if self.units: + d["units"] = self.units + if self.colormap: + d["colormap"] = self.colormap + if self.enumeration_legend: + d["enumeration_legend"] = self.enumeration_legend + if self.enumeration_colors: + d["enumeration_colors"] = self.enumeration_colors + if self.default_enumeration_legend != DEFAULT_ENUMERATION_LEGEND: + d["default_enumeration_legend"] = self.default_enumeration_legend + if self.default_enumeration_color != DEFAULT_ENUMERATION_COLOR: + d["default_enumeration_color"] = self.default_enumeration_color + if self.clim != [None, None]: + d["clim"] = self.clim + return d + @property def definition(self): d = OrderedDict() diff --git a/podpac/core/test/test_authentication.py b/podpac/core/test/test_authentication.py index 1f8fd1902..966be77a8 100644 --- a/podpac/core/test/test_authentication.py +++ b/podpac/core/test/test_authentication.py @@ -2,6 +2,7 @@ import requests import traitlets as tl import s3fs +from numpy.testing import assert_equal from podpac import settings, Node from podpac.core.authentication import RequestsSessionMixin, S3Mixin, set_credentials @@ -22,10 +23,10 @@ def test_set_credentials(self): set_credentials() with pytest.raises(ValueError): - set_credentials(None, username="test", password="test") + set_credentials(None, uname="test", password="test") with pytest.raises(ValueError): - set_credentials("", username="test", password="test") + set_credentials("", uname="test", password="test") # make sure these are empty at first assert not settings["username@test.com"] @@ -34,17 +35,17 @@ def test_set_credentials(self): # test input/getpass # TODO: how do you test this? - # set both username and password - set_credentials(hostname="test.com", username="testuser", password="testpass") + # set both username and pw + set_credentials(hostname="test.com", uname="testuser", password="testpass") assert settings["username@test.com"] == "testuser" assert settings["password@test.com"] == "testpass" # set username only - set_credentials(hostname="test.com", username="testuser2") + set_credentials(hostname="test.com", uname="testuser2") assert settings["username@test.com"] == "testuser2" assert settings["password@test.com"] == "testpass" - # set password only + # set pw only set_credentials(hostname="test.com", password="testpass3") assert settings["username@test.com"] == "testuser2" assert settings["password@test.com"] == "testpass3" @@ -94,8 +95,8 @@ def test_property_values(self): node = SomeNode(hostname="propertyvalues.com") node.set_credentials(username="testuser2", password="testpass2") - assert node.username == "testuser2" - assert node.password == "testpass2" + assert_equal(node.username, "testuser2") + assert_equal(node.password, "testpass2") def test_session(self): with settings: diff --git a/podpac/core/test/test_node.py b/podpac/core/test/test_node.py index 53ba91597..038539273 100644 --- a/podpac/core/test/test_node.py +++ b/podpac/core/test/test_node.py @@ -212,6 +212,32 @@ def test_find_coordinates_not_implemented(self): with pytest.raises(NotImplementedError): node.find_coordinates() + def test_get_bounds(self): + class MyNode(Node): + def find_coordinates(self): + return [ + podpac.Coordinates([[0, 1, 2], [0, 10, 20]], dims=["lat", "lon"], crs="EPSG:2193"), + podpac.Coordinates([[3, 4], [30, 40]], dims=["lat", "lon"], crs="EPSG:2193"), + ] + + node = MyNode() + + with podpac.settings: + podpac.settings["DEFAULT_CRS"] = "EPSG:4326" + + # specify crs + bounds, crs = node.get_bounds(crs="EPSG:2193") + assert bounds == {"lat": (0, 4), "lon": (0, 40)} + assert crs == "EPSG:2193" + + # default crs + bounds, crs = node.get_bounds() + assert bounds == { + "lat": (-75.81397534013118, -75.81362774074242), + "lon": (82.92787904584206, 82.9280189659297), + } + assert crs == "EPSG:4326" + class TestCreateOutputArray(object): def test_create_output_array_default(self): diff --git a/podpac/core/test/test_units.py b/podpac/core/test/test_units.py index 54414d13c..d0f27db73 100644 --- a/podpac/core/test/test_units.py +++ b/podpac/core/test/test_units.py @@ -8,7 +8,7 @@ import xarray as xr from pint.errors import DimensionalityError -from podpac.core.coordinates import Coordinates, clinspace, RotatedCoordinates +from podpac.core.coordinates import Coordinates, clinspace, AffineCoordinates from podpac.core.style import Style from podpac.core.units import ureg @@ -83,7 +83,7 @@ def test_pow(self): dims=["lat", "lon", "alt"], attrs={"units": ureg.meter}, ) - assert (a ** 2).attrs["units"] == ureg.meter ** 2 + assert (a**2).attrs["units"] == ureg.meter**2 def test_set_to_value_using_UnitsDataArray_as_mask_does_nothing_if_mask_has_dim_not_in_array(self): a = UnitsDataArray( @@ -273,7 +273,7 @@ def test_units_allpass(self): assert a6[0, 0].data[()] == False a7 = a1 * a2 - assert a7[0, 0].to(ureg.m ** 2).data[()] == (1 * ureg.meter * ureg.inch).to(ureg.meter ** 2).magnitude + assert a7[0, 0].to(ureg.m**2).data[()] == (1 * ureg.meter * ureg.inch).to(ureg.meter**2).magnitude a8 = a2 / a1 assert a8[0, 0].to_base_units().data[()] == (1 * ureg.inch / ureg.meter).to_base_units().magnitude @@ -328,7 +328,7 @@ def test_ufuncs(self): np.mean(a1) np.min(a1) np.max(a1) - a1 ** 2 + a1**2 # These don't have units! np.dot(a2.T, a1) @@ -532,8 +532,6 @@ def test_to_image_vmin_vmax(self): class TestToGeoTiff(object): def make_square_array(self, order=1, bands=1): - # order = -1 - # bands = 3 node = Array( source=np.arange(8 * bands).reshape(3 - order, 3 + order, bands), coordinates=Coordinates([clinspace(4, 0, 2, "lat"), clinspace(1, 4, 4, "lon")][::order], crs="EPSG:4326"), @@ -542,12 +540,14 @@ def make_square_array(self, order=1, bands=1): return node def make_rot_array(self, order=1, bands=1): - # order = -1 - # bands = 3 - rc = RotatedCoordinates( - shape=(2, 4), theta=np.pi / 8, origin=[10, 20], step=[-2.0, 1.0], dims=["lat", "lon"][::order] - ) - c = Coordinates([rc]) + if order == 1: + geotransform = (10.0, 1.879, -1.026, 20.0, 0.684, 2.819) + else: + # I think this requires changing the geotransform? Not yet supported + raise NotImplementedError("TODO") + + rc = AffineCoordinates(geotransform=geotransform, shape=(2, 4)) + c = Coordinates([rc], crs="EPSG:4326") node = Array( source=np.arange(8 * bands).reshape(3 - order, 3 + order, bands), coordinates=c, @@ -555,7 +555,7 @@ def make_rot_array(self, order=1, bands=1): ) return node - def test_to_geotiff_rountrip_1band(self): + def test_to_geotiff_roundtrip_1band(self): # lat/lon order, usual node = self.make_square_array() out = node.eval(node.coordinates) @@ -584,7 +584,7 @@ def test_to_geotiff_rountrip_1band(self): rout = rnode.eval(rnode.coordinates) np.testing.assert_almost_equal(rout.data, out.data) - def test_to_geotiff_rountrip_2band(self): + def test_to_geotiff_roundtrip_2band(self): # lat/lon order, usual node = self.make_square_array(bands=2) out = node.eval(node.coordinates) @@ -631,11 +631,12 @@ def test_to_geotiff_rountrip_2band(self): rout = rnode.eval(rnode.coordinates) np.testing.assert_almost_equal(out.data[..., 1], rout.data) - @pytest.mark.skip("TODO: We can remove this skipped test after solving #363") - def test_to_geotiff_rountrip_rotcoords(self): + def test_to_geotiff_roundtrip_rotcoords(self): # lat/lon order, usual node = self.make_rot_array() + out = node.eval(node.coordinates) + with tempfile.NamedTemporaryFile("wb") as fp: out.to_geotiff(fp) fp.write(b"a") # for some reason needed to get good comparison @@ -647,16 +648,16 @@ def test_to_geotiff_rountrip_rotcoords(self): rout = rnode.eval(rnode.coordinates) np.testing.assert_almost_equal(out.data, rout.data) - # lon/lat order, unsual - node = self.make_square_array(order=-1) - out = node.eval(node.coordinates) - with tempfile.NamedTemporaryFile("wb") as fp: - out.to_geotiff(fp) - fp.write(b"a") # for some reason needed to get good comparison + # # lon/lat order, unsual + # node = self.make_square_array(order=-1) + # out = node.eval(node.coordinates) + # with tempfile.NamedTemporaryFile("wb") as fp: + # out.to_geotiff(fp) + # fp.write(b"a") # for some reason needed to get good comparison - fp.seek(0) - rnode = Rasterio(source=fp.name, outputs=node.outputs) - assert node.coordinates == rnode.coordinates + # fp.seek(0) + # rnode = Rasterio(source=fp.name, outputs=node.outputs) + # assert node.coordinates == rnode.coordinates - rout = rnode.eval(rnode.coordinates) - np.testing.assert_almost_equal(out.data, rout.data) + # rout = rnode.eval(rnode.coordinates) + # np.testing.assert_almost_equal(out.data, rout.data) diff --git a/podpac/core/units.py b/podpac/core/units.py index 99dbca72f..e61aa4c49 100644 --- a/podpac/core/units.py +++ b/podpac/core/units.py @@ -39,6 +39,7 @@ from lazy_import import lazy_module, lazy_class rasterio = lazy_module("rasterio") +affine = lazy_module("affine") # Set up logging _logger = logging.getLogger(__name__) @@ -136,7 +137,10 @@ def to_netcdf(self, *args, **kwargs): o = self for d in self.dims: if "_" in d and "dim" not in d: # This it is stacked - o = o.reset_index(d) + try: + o = o.reset_index(d) + except KeyError: + pass # This is fine, actually didn't need to reset because not a real dim o._pp_serialize() r = super(UnitsDataArray, o).to_netcdf(*args, **kwargs) self._pp_deserialize() @@ -656,7 +660,7 @@ def to_geotiff(fp, data, geotransform=None, crs=None, **kwargs): """ # This only works for data that essentially has lat/lon only - dims = data.coords.dims + dims = list(data.coords.keys()) if "lat" not in dims or "lon" not in dims: raise NotImplementedError("Cannot export GeoTIFF for dataset with lat/lon coordinates.") if "time" in dims and len(data.coords["time"]) > 1: @@ -681,6 +685,7 @@ def to_geotiff(fp, data, geotransform=None, crs=None, **kwargs): # Geotransform should ALWAYS be defined as (lon_origin, lon_dj, lon_di, lat_origin, lat_dj, lat_di) # if isinstance(data, xr.DataArray) and data.dims.index('lat') > data.dims.index('lon'): # geotransform = geotransform[3:] + geotransform[:3] + if geotransform is None: try: geotransform = Coordinates.from_xarray(data).geotransform @@ -703,7 +708,7 @@ def to_geotiff(fp, data, geotransform=None, crs=None, **kwargs): if len(data.shape) == 2: data = data[:, :, None] - geotransform = rasterio.Affine.from_gdal(*geotransform) + geotransform = affine.Affine.from_gdal(*geotransform) # Update the kwargs that rasterio will use. Anything added by the user will take priority. kwargs2 = dict( diff --git a/podpac/core/utils.py b/podpac/core/utils.py index c87ea13f0..7949845bd 100644 --- a/podpac/core/utils.py +++ b/podpac/core/utils.py @@ -8,12 +8,11 @@ import sys import json import datetime -import functools -import importlib import logging -import time +import inspect from collections import OrderedDict from copy import deepcopy +from hashlib import sha256 as hash_alg try: import urllib.parse as urllib @@ -130,7 +129,7 @@ def create_logfile( if sys.version < "3.6": # for Python 2 and Python < 3.6 compatibility class OrderedDictTrait(tl.Dict): - """ OrderedDict trait """ + """OrderedDict trait""" default_value = OrderedDict() @@ -145,13 +144,12 @@ def validate(self, obj, value): super(OrderedDictTrait, self).validate(obj, value) return value - else: OrderedDictTrait = tl.Dict class ArrayTrait(tl.TraitType): - """ A coercing numpy array trait. """ + """A coercing numpy array trait.""" def __init__(self, ndim=None, shape=None, dtype=None, dtypes=None, default_value=None, *args, **kwargs): if ndim is not None and shape is not None and len(shape) != ndim: @@ -198,7 +196,7 @@ def validate(self, obj, value): class TupleTrait(tl.List): - """ An instance of a Python tuple that accepts the 'trait' argument (like Set, List, and Dict). """ + """An instance of a Python tuple that accepts the 'trait' argument (like Set, List, and Dict).""" def validate(self, obj, value): value = super(TupleTrait, self).validate(obj, value) @@ -206,6 +204,8 @@ def validate(self, obj, value): class NodeTrait(tl.Instance): + _schema = {"test": "info"} + def __init__(self, *args, **kwargs): from podpac import Node as _Node @@ -218,6 +218,19 @@ def validate(self, obj, value): return value +class DimsTrait(tl.List): + _schema = {"test": "info"} + + def __init__(self, *args, **kwargs): + super().__init__(tl.Enum(["lat", "lon", "time", "alt"]), *args, minlen=1, maxlen=4, **kwargs) + + # def validate(self, obj, value): + # super().validate(obj, value) + # if podpac.core.settings.settings["DEBUG"]: + # value = deepcopy(value) + # return value + + class JSONEncoder(json.JSONEncoder): def default(self, obj): # podpac objects with definitions @@ -514,7 +527,7 @@ def probe_node(node, lat=None, lon=None, time=None, alt=None, crs=None, nested=F """ def partial_definition(key, definition): - """ Needed to build partial node definitions """ + """Needed to build partial node definitions""" new_def = OrderedDict() for k in definition: new_def[k] = definition[k] @@ -522,7 +535,7 @@ def partial_definition(key, definition): return new_def def flatten_list(l): - """ Needed to flatten the inputs list for all the dependencies """ + """Needed to flatten the inputs list for all the dependencies""" nl = [] for ll in l: if isinstance(ll, list): @@ -533,7 +546,7 @@ def flatten_list(l): return nl def get_entry(key, out, definition): - """ Needed for the nested version of the pipeline """ + """Needed for the nested version of the pipeline""" # We have to rearrange the outputs entry = OrderedDict() entry["name"] = out[key]["name"] @@ -553,20 +566,24 @@ def get_entry(key, out, definition): v = float(node.eval(coords)) definition = node.definition out = OrderedDict() - for key in definition: - if key == "podpac_version": + for item in definition: + if item == "podpac_version": continue - d = partial_definition(key, definition) + d = partial_definition(item, definition) n = podpac.Node.from_definition(d) - value = float(n.eval(coords)) - inputs = flatten_list(list(d[key].get("inputs", {}).values())) + o = n.eval(coords) + if o.size == 1: + value = float(o) + else: + value = o.data.tolist() + inputs = flatten_list(list(d[item].get("inputs", {}).values())) active = True - out[key] = { + out[item] = { "active": active, "value": value, "units": n.style.units, "inputs": inputs, - "name": n.style.name if n.style.name else key, + "name": n.style.name if n.style.name else item, "node_hash": n.hash, } # Fix sources for Compositors @@ -574,10 +591,64 @@ def get_entry(key, out, definition): searching_for_active = True for inp in inputs: out[inp]["active"] = False - if out[inp]["value"] == out[key]["value"] and np.isfinite(out[inp]["value"]) and searching_for_active: + if out[inp]["value"] == out[item]["value"] and np.isfinite(out[inp]["value"]) and searching_for_active: out[inp]["active"] = True searching_for_active = False if nested: out = get_entry(list(out.keys())[-1], out, definition) return out + + +def get_ui_node_spec(module=None, category="default", help_as_html=False): + """ + Returns a dictionary describing the specifications for each Node in a module. + + Parameters + ----------- + module: module + The Python module for which the ui specs should be summarized. Only the top-level + classes will be included in the spec. (i.e. no recursive search through submodules) + category: str, optional + Default is "default". Top-level category name for the group of Nodes. + help_as_html: bool, optional + Default is False. If True, the docstrings will be converted to html before storing in the spec. + + Returns + -------- + dict + Dictionary of {category: {Node1: spec_1, Node2: spec2, ...}} describing the specs for each Node. + """ + import podpac + import podpac.datalib # May not be imported by default + + spec = {} + + if module is None: + modcat = zip( + [podpac.data, podpac.algorithm, podpac.compositor, podpac.datalib], + ["data", "algorithm", "compositor", "datalib"], + ) + for mod, cat in modcat: + spec.update(get_ui_node_spec(mod, cat, help_as_html=help_as_html)) + return spec + + spec[category] = {} + disabled_categories = ["Algorithm", "DataSource", "DroughtMonitorCategory", "DroughtCategory", "IntakeCatalog"] + for obj in dir(module): + # print(obj) + if obj in disabled_categories: + ob = getattr(module, obj) + # print(ob) + # print(ob.get_ui_spec()) + # would be fairly annoying to have to check all of the attrs for abstract + # still need a better solution + continue + ob = getattr(module, obj) + if not inspect.isclass(ob): + continue + if not issubclass(ob, podpac.Node): + continue + spec[category][obj] = ob.get_ui_spec(help_as_html=help_as_html) + + return spec diff --git a/podpac/datalib/cosmos_stations.py b/podpac/datalib/cosmos_stations.py index 68ca06ea7..ebcd45fb5 100644 --- a/podpac/datalib/cosmos_stations.py +++ b/podpac/datalib/cosmos_stations.py @@ -157,9 +157,9 @@ class COSMOSStationsRaw(TileCompositorRaw): stations_url = tl.Unicode("sitesNoLegend.js") dims = ["lat", "lon", "time"] - @tl.default("interpolation") - def _interpolation_default(self): - return {"method": "nearest", "params": {"use_selector": False, "remove_nan": False, "time_scale": "1,M"}} + from podpac.style import Style + + style = Style(colormap="jet") ## PROPERTIES @cached_property(use_cache_ctrl=True) @@ -283,7 +283,7 @@ def latlon_from_label(self, label): return c def _get_label_inds(self, label): - """ Helper function to get source indices for partially matched labels """ + """Helper function to get source indices for partially matched labels""" ind = [] for lab in label: ind.extend([i for i, l in enumerate(self.stations_label) if lab.lower() in l.lower()]) @@ -386,7 +386,9 @@ def get_station_data(self, label=None, lat_lon=None): class COSMOSStations(InterpolationMixin, COSMOSStationsRaw): - pass + @tl.default("interpolation") + def _interpolation_default(self): + return {"method": "nearest", "params": {"use_selector": False, "remove_nan": False, "time_scale": "1,M"}} if __name__ == "__main__": diff --git a/podpac/datalib/drought_monitor.py b/podpac/datalib/drought_monitor.py index 3c7011efa..54bc74ba4 100644 --- a/podpac/datalib/drought_monitor.py +++ b/podpac/datalib/drought_monitor.py @@ -9,12 +9,19 @@ class DroughtMonitorCategory(Zarr): class DroughtCategory(Algorithm): + # soil_moisture = NodeTrait().tag(attr=True, required=True) + # d0 = NodeTrait().tag(attr=True, required=True) + # d1 = NodeTrait().tag(attr=True, required=True) + # d2 = NodeTrait().tag(attr=True, required=True) + # d3 = NodeTrait().tag(attr=True, required=True) + # d4 = NodeTrait().tag(attr=True, required=True) soil_moisture = NodeTrait().tag(attr=True) d0 = NodeTrait().tag(attr=True) d1 = NodeTrait().tag(attr=True) d2 = NodeTrait().tag(attr=True) d3 = NodeTrait().tag(attr=True) d4 = NodeTrait().tag(attr=True) + style = Style( clim=[0, 6], enumeration_colors={ diff --git a/podpac/datalib/egi.py b/podpac/datalib/egi.py index f50e79b05..5725584bd 100644 --- a/podpac/datalib/egi.py +++ b/podpac/datalib/egi.py @@ -15,6 +15,7 @@ import requests from six import string_types +from traitlets.traitlets import default import numpy as np import xarray as xr import traitlets as tl @@ -97,17 +98,27 @@ class EGI(InterpolationMixin, DataSource): The data array compiled from downloaded EGI data """ - base_url = tl.Unicode(default_value=BASE_URL).tag(attr=True) + base_url = tl.Unicode(default_value=BASE_URL).tag( + attr=True, required=False, default="https://n5eil01u.ecs.nsidc.org/egi/request" + ) # required - short_name = tl.Unicode().tag(attr=True) - data_key = tl.Unicode().tag(attr=True) - lat_key = tl.Unicode(allow_none=True).tag(attr=True) - lon_key = tl.Unicode(allow_none=True).tag(attr=True) - time_key = tl.Unicode(allow_none=True).tag(attr=True) + short_name = tl.Unicode().tag(attr=True, required=True) + data_key = tl.Unicode().tag(attr=True, required=True) + lat_key = tl.Unicode(allow_none=True).tag(attr=True, required=True) + lon_key = tl.Unicode(allow_none=True).tag(attr=True, required=True) + time_key = tl.Unicode(allow_none=True).tag(attr=True, required=True) + udims_overwrite = tl.List() min_bounds_span = tl.Dict(allow_none=True).tag(attr=True) + @property + def udims(self): + if self.udims_overwrite: + return self.udims_overwrite + """ This needs to be implemented so this node will cache properly. See Datasource.eval.""" + raise NotImplementedError + # optional # full list of supported formats ["GeoTIFF", "HDF-EOS5", "NetCDF4-CF", "NetCDF-3", "ASCII", "HDF-EOS", "KML"] @@ -464,9 +475,25 @@ def _read_zips(self, zip_files): return all_data - ################ - # Token Handling - ################ + ###################################### + # Token and Authentication Handling # + ###################################### + def set_credentials(self, username=None, password=None): + """Shortcut to :func:`podpac.authentication.set_crendentials` using class member :attr:`self.hostname` for the hostname + + Parameters + ---------- + username : str, optional + Username to store in settings for `self.hostname`. + If no username is provided and the username does not already exist in the settings, + the user will be prompted to enter one. + password : str, optional + Password to store in settings for `self.hostname` + If no password is provided and the password does not already exist in the settings, + the user will be prompted to enter one. + """ + return authentication.set_credentials("urs.earthdata.nasa.gov", username=username, password=password) + def _authenticate(self): if self.token is None: self.get_token() @@ -568,3 +595,10 @@ def _get_ip(self): s.close() return ip + + @classmethod + def get_ui_spec(cls, help_as_html=False): + spec = super().get_ui_spec(help_as_html=help_as_html) + spec["attrs"]["username"] = {} + spec["attrs"]["password"] = {} + return spec diff --git a/podpac/datalib/gfs.py b/podpac/datalib/gfs.py index d71716c9d..2de2d4f31 100644 --- a/podpac/datalib/gfs.py +++ b/podpac/datalib/gfs.py @@ -62,10 +62,10 @@ class GFS(S3Mixin, DiskCacheMixin, TileCompositor): source hour, e.g. '1200' """ - parameter = tl.Unicode().tag(attr=True) - level = tl.Unicode().tag(attr=True) - date = tl.Unicode().tag(attr=True) - hour = tl.Unicode().tag(attr=True) + parameter = tl.Unicode().tag(attr=True, required=True) + level = tl.Unicode().tag(attr=True, required=True) + date = tl.Unicode().tag(attr=True, required=True) + hour = tl.Unicode().tag(attr=True, required=True) @property def _repr_keys(self): diff --git a/podpac/datalib/intake_catalog.py b/podpac/datalib/intake_catalog.py index 75aade506..de56fee6e 100644 --- a/podpac/datalib/intake_catalog.py +++ b/podpac/datalib/intake_catalog.py @@ -60,8 +60,8 @@ class IntakeCatalog(podpac.data.DataSource): """ # input parameters - source = tl.Unicode().tag(attr=True) - uri = tl.Unicode().tag(attr=True) + source = tl.Unicode().tag(attr=True, required=True) + uri = tl.Unicode().tag(attr=True, required=True) # optional input parameters field = tl.Unicode(default_value=None, allow_none=True).tag(attr=True) diff --git a/podpac/datalib/modis_pds.py b/podpac/datalib/modis_pds.py index c38f40515..9b57c168c 100644 --- a/podpac/datalib/modis_pds.py +++ b/podpac/datalib/modis_pds.py @@ -94,7 +94,7 @@ def _available(s3, *l): def get_tile_coordinates(h, v): - """ use pre-fetched lat and lon bounds to get coordinates for a single tile """ + """use pre-fetched lat and lon bounds to get coordinates for a single tile""" lat_start, lat_stop = SINUSOIDAL_VERTICAL[v] lon_start, lon_stop = SINUSOIDAL_HORIZONTAL[h] lat = podpac.clinspace(lat_start, lat_stop, 2400, name="lat") @@ -131,7 +131,7 @@ class MODISSource(RasterioRaw): _repr_keys = ["prefix", "data_key"] def init(self): - """ validation """ + """validation""" for key in ["horizontal", "vertical", "date", "data_key"]: if not getattr(self, key): raise ValueError("MODISSource '%s' required" % key) @@ -192,8 +192,8 @@ class MODISComposite(S3Mixin, TileCompositorRaw): individual object (varies by product) """ - product = tl.Enum(values=PRODUCTS, help="MODIS product ID").tag(attr=True) - data_key = tl.Unicode(help="data to retrieve (varies by product)").tag(attr=True) + product = tl.Enum(values=PRODUCTS, help="MODIS product ID").tag(attr=True, required=True) + data_key = tl.Unicode(help="data to retrieve (varies by product)").tag(attr=True, required=True) tile_width = (1, 2400, 2400) start_date = "2013-01-01" @@ -214,7 +214,7 @@ def available_tiles(self): return [(h, v) for h in _available(self.s3, self.product) for v in _available(self.s3, self.product, h)] def select_sources(self, coordinates, _selector=None): - """ 2d select sources filtering """ + """2d select sources filtering""" # filter tiles spatially ct = coordinates.transform(CRS) diff --git a/podpac/datalib/satutils.py b/podpac/datalib/satutils.py index 8e8358eb0..756876974 100644 --- a/podpac/datalib/satutils.py +++ b/podpac/datalib/satutils.py @@ -29,7 +29,7 @@ def _get_asset_info(item, name): - """ for forwards/backwards compatibility, convert B0x to/from Bx as needed """ + """for forwards/backwards compatibility, convert B0x to/from Bx as needed""" if name in item.assets: return item.assets[name] diff --git a/podpac/datalib/smap_egi.py b/podpac/datalib/smap_egi.py index 67cf398f5..53fe8475f 100644 --- a/podpac/datalib/smap_egi.py +++ b/podpac/datalib/smap_egi.py @@ -139,7 +139,7 @@ class SMAP(EGI): product = tl.Enum(SMAP_PRODUCTS, default_value="SPL4SMAU").tag(attr=True) nan_vals = [-9999.0] min_bounds_span = tl.Dict(default_value={"lon": 0.3, "lat": 0.3, "time": "3,h"}).tag(attr=True) - check_quality_flags = tl.Bool(True).tag(attr=True) + check_quality_flags = tl.Bool(True).tag(attr=True, default=True) quality_flag_key = tl.Unicode(allow_none=True).tag(attr=True) data_key = tl.Unicode(allow_none=True, default_value=None).tag(attr=True) base_url = tl.Unicode(default_value=BASE_URL).tag(attr=True) @@ -156,6 +156,10 @@ def short_name(self): def _product_data(self): return SMAP_PRODUCT_DICT[self.product] + @property + def udims(self): + return ["lat", "lon", "time"] + @property def lat_key(self): return self._product_data[0] @@ -289,14 +293,14 @@ def append_file(self, all_data, data): lat = all_data.lat.sel(lat=data.lat, method="nearest") # When the difference between old and new coordintaes are large, it means there are new coordinates - Ilat = np.abs(lat.data - data.lat) > 1e-3 + Ilat = (np.abs(lat.data - data.lat) > 1e-3).data # Use the new data's coordinates for the new coordinates - lat.data[Ilat] = data.lat[Ilat] + lat.data[Ilat] = data.lat.data[Ilat] # Repeat for lon lon = all_data.lon.sel(lon=data.lon, method="nearest") - Ilon = np.abs(lon.data - data.lon) > 1e-3 - lon.data[Ilon] = data.lon[Ilon] + Ilon = (np.abs(lon.data - data.lon) > 1e-3).data + lon.data[Ilon] = data.lon.data[Ilon] # Assign to data data.lon.data[:] = lon.data diff --git a/podpac/datalib/soilgrids.py b/podpac/datalib/soilgrids.py index 3b099f39c..2b12659aa 100644 --- a/podpac/datalib/soilgrids.py +++ b/podpac/datalib/soilgrids.py @@ -9,7 +9,7 @@ class SoilGridsBase(WCS): - """ Base SoilGrids WCS datasource. """ + """Base SoilGrids WCS datasource.""" format = "geotiff_byte" max_size = 16384 @@ -17,73 +17,73 @@ class SoilGridsBase(WCS): class SoilGridsWRB(SoilGridsBase): - """ SoilGrids: WRB classes and probabilities (WCS) """ + """SoilGrids: WRB classes and probabilities (WCS)""" source = "https://maps.isric.org/mapserv?map=/map/wrb.map" class SoilGridsBDOD(SoilGridsBase): - """ SoilGrids: Bulk density (WCS) """ + """SoilGrids: Bulk density (WCS)""" source = "https://maps.isric.org/mapserv?map=/map/bdod.map" class SoilGridsCEC(SoilGridsBase): - """ SoilGrids: Cation exchange capacity and ph 7 (WCS) """ + """SoilGrids: Cation exchange capacity and ph 7 (WCS)""" source = "https://maps.isric.org/mapserv?map=/map/cec.map" class SoilGridsCFVO(SoilGridsBase): - """ SoilGrids: Coarse fragments volumetric (WCS)""" + """SoilGrids: Coarse fragments volumetric (WCS)""" source = "https://maps.isric.org/mapserv?map=/map/cfvo.map" class SoilGridsClay(SoilGridsBase): - """ SoilGrids: Clay content (WCS) """ + """SoilGrids: Clay content (WCS)""" source = "https://maps.isric.org/mapserv?map=/map/clay.map" class SoilGridsNitrogen(SoilGridsBase): - """ SoilGrids: Nitrogen (WCS) """ + """SoilGrids: Nitrogen (WCS)""" source = "https://maps.isric.org/mapserv?map=/map/nitrogen.map" class SoilGridsPHH2O(SoilGridsBase): - """ SoilGrids: Soil pH in H2O (WCS) """ + """SoilGrids: Soil pH in H2O (WCS)""" source = "https://maps.isric.org/mapserv?map=/map/phh2o.map" class SoilGridsSand(SoilGridsBase): - """ SoilGrids: Sand content (WCS) """ + """SoilGrids: Sand content (WCS)""" source = "https://maps.isric.org/mapserv?map=/map/sand.map" class SoilGridsSilt(SoilGridsBase): - """ SoilGrids: Silt content (WCS) """ + """SoilGrids: Silt content (WCS)""" source = "https://maps.isric.org/mapserv?map=/map/silt.map" class SoilGridsSOC(SoilGridsBase): - """ SoilGrids: Soil organic carbon content (WCS) """ + """SoilGrids: Soil organic carbon content (WCS)""" source = "https://maps.isric.org/mapserv?map=/map/soc.map" class SoilGridsOCS(SoilGridsBase): - """ SoilGrids: Soil organic carbon stock (WCS) """ + """SoilGrids: Soil organic carbon stock (WCS)""" source = "https://maps.isric.org/mapserv?map=/map/ocs.map" class SoilGridsOCD(SoilGridsBase): - """ SoilGrids: Organic carbon densities (WCS) """ + """SoilGrids: Organic carbon densities (WCS)""" source = "https://maps.isric.org/mapserv?map=/map/ocd.map" diff --git a/podpac/datalib/terraintiles.py b/podpac/datalib/terraintiles.py index 15e7c5b5b..5e35d6884 100644 --- a/podpac/datalib/terraintiles.py +++ b/podpac/datalib/terraintiles.py @@ -519,7 +519,7 @@ def _mercator_to_tilespace(xm, ym, zoom): (x, y) int tile coordinates """ - tiles = 2 ** zoom + tiles = 2**zoom diameter = 2 * np.pi x = int(tiles * (xm + np.pi) / diameter) y = int(tiles * (np.pi - ym) / diameter) diff --git a/podpac/datalib/weathercitizen.py b/podpac/datalib/weathercitizen.py index 78ed745d3..391de7225 100644 --- a/podpac/datalib/weathercitizen.py +++ b/podpac/datalib/weathercitizen.py @@ -62,7 +62,7 @@ class WeatherCitizen(InterpolationMixin, DataSource): Display log messages or progress """ - source = tl.Unicode(allow_none=True, default_value="geosensors") + source = tl.Unicode(allow_none=True, default_value="geosensors").tag(attr=True, required=True) data_key = tl.Unicode(allow_none=True, default_value="properties.pressure").tag(attr=True) uuid = tl.Unicode(allow_none=True, default_value=None).tag(attr=True) device = tl.Unicode(allow_none=True, default_value=None).tag(attr=True) diff --git a/podpac/version.py b/podpac/version.py index d7f318b18..d10985b3b 100644 --- a/podpac/version.py +++ b/podpac/version.py @@ -16,8 +16,8 @@ ## UPDATE VERSION HERE ############## MAJOR = 3 -MINOR = 1 -HOTFIX = 2 +MINOR = 2 +HOTFIX = 0 ############## diff --git a/setup.py b/setup.py index 75481dd77..d58fb150e 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ "requests>=2.18", "lazy-import>=0.2.2", "psutil", + "affine", ] if sys.version_info.major == 2: @@ -58,6 +59,9 @@ "sat-search>=0.2", "sat-stac>=0.3", ], + "node_ui": [ + "numpydoc" + ], "dev": [ "pylint>=1.8.2", "pytest-cov>=2.5.1", @@ -74,7 +78,7 @@ if sys.version_info.major == 2: extras_require["dev"] += ["pytest>=3.3.2"] else: - extras_require["dev"] += ["sphinx>=2.3, <3.0", "sphinx-rtd-theme>=0.4", "sphinx-autobuild>=0.7", "pytest>=5.0"] + extras_require["dev"] += ["sphinx>=2.3, <3.0", "sphinx-rtd-theme>=0.4", "sphinx-autobuild>=0.7", "pytest>=5.0", "numpydoc"] if sys.version >= "3.6": extras_require["dev"] += [