Adds a validate cli command

CHANGELOG.md

@@ -9,6 +9,7 @@
 - Optional dependency on s3fs ([#178](https://github.com/stac-utils/stactools/pull/178)), enabling:
   - Using s3 files as external data for testing
   - Using s3 hrefs with stactools functionality by installing with `pip install stactools[s3]` (or `pip install stactools[all]`)
+- `stac validate` command for validating JSON and checking links ([#151](https://github.com/stac-utils/stactools/pull/151))
 
 ## stactools 0.2.1a2
 

src/stactools/cli/__init__.py

@@ -9,14 +9,15 @@ def register_plugin(registry):
     # Register subcommands
 
     from stactools.cli.commands import (copy, info, layout, merge, migrate,
-                                        version)
+                                        version, validate)
 
     registry.register_subcommand(copy.create_copy_command)
     registry.register_subcommand(copy.create_move_assets_command)
     registry.register_subcommand(info.create_info_command)
     registry.register_subcommand(info.create_describe_command)
     registry.register_subcommand(layout.create_layout_command)
     registry.register_subcommand(merge.create_merge_command)
+    registry.register_subcommand(validate.create_validate_command)
     registry.register_subcommand(version.create_version_command)
 
     # TODO

src/stactools/cli/commands/validate.py

@@ -0,0 +1,80 @@
+import sys
+from typing import Optional, List
+
+import click
+import pystac
+from pystac import Item, Catalog, STACValidationError, STACObject
+
+from stactools.core.utils import href_exists
+
+
+def create_validate_command(cli):
+    @cli.command("validate", short_help="Validate a stac object.")
+    @click.argument("href")
+    @click.option("--recurse/--no-recurse",
+                  default=True,
+                  help=("If false, do not validate any children "
+                        "(only useful for Catalogs and Collections"))
+    @click.option("--links/--no-links",
+                  default=True,
+                  help=("If false, do not check any of the objects's links."))
+    @click.option(
+        "--assets/--no-assets",
+        default=True,
+        help=("If false, do not check any of the collection's/item's assets."))
+    def validate_command(href, recurse, links, assets):
+        """Validates a STAC object.
+
+        Prints any validation errors to stdout.
+        """
+        object = pystac.read_file(href)
+
+        if isinstance(object, Item):
+            errors = validate(object, None, False, links, assets)
+        else:
+            errors = validate(object, object, recurse, links, assets)
+
+        if not errors:
+            click.secho("OK", fg="green", nl=False)
+            click.echo(f" STAC object at {href} is valid!")
+        else:
+            for error in errors:
+                click.secho("ERROR", fg="red", nl=False)
+                click.echo(f" {error}")
+            sys.exit(1)
+
+    return validate_command
+
+
+def validate(object: STACObject, root: Optional[STACObject], recurse: bool,
+             links: bool, assets: bool) -> List[str]:
+    errors: List[str] = []
+
+    try:
+        object.validate()
+    except FileNotFoundError as e:
+        errors.append(f"File not found: {e}")
+    except STACValidationError as e:
+        errors.append(f"{e}\n{e.source}")
+
+    if links:
+        for link in object.get_links():
+            if not href_exists(link.get_absolute_href()):
+                errors.append(
+                    f"Missing link in {object.self_href}: \"{link.rel}\" -> {link.href}"
+                )
+
+    if assets and not isinstance(object, Catalog):
+        for name, asset in object.get_assets().items():
+            if not href_exists(asset.get_absolute_href()):
+                errors.append(
+                    f"Asset '{name}' does not exist: {asset.get_absolute_href()}"
+                )
+
+    if recurse:
+        for child in object.get_children():
+            errors.extend(validate(child, root, recurse, links, assets))
+        for item in object.get_items():
+            errors.extend(validate(item, root, False, links, assets))
+
+    return errors

src/stactools/core/utils/__init__.py

@@ -1,5 +1,7 @@
 from typing import Callable, Optional, TypeVar
 
+import fsspec
+
 T = TypeVar('T')
 U = TypeVar('U')
 
@@ -9,3 +11,13 @@ def map_opt(fn: Callable[[T], U], v: Optional[T]) -> Optional[U]:
     None if the input option is None.
     """
     return v if v is None else fn(v)
+
+
+def href_exists(href: str) -> bool:
+    """Returns true if the asset exists.
+
+    Uses fssepc and its `exists` method:
+    https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.spec.AbstractFileSystem.exists.
+    """
+    fs, _, paths = fsspec.get_fs_token_paths(href)
+    return paths and fs.exists(paths[0])

tests/cli/commands/test_validate.py

@@ -0,0 +1,46 @@
+from typing import List, Callable
+from stactools.cli.commands.validate import create_validate_command
+
+from stactools.testing import CliTestCase
+
+from tests import test_data
+
+
+class ValidatateTest(CliTestCase):
+    def create_subcommand_functions(self) -> List[Callable]:
+        return [create_validate_command]
+
+    def test_valid_item(self):
+        path = test_data.get_path(
+            "data-files/catalogs/test-case-1/country-1/area-1-1/"
+            "area-1-1-imagery/area-1-1-imagery.json")
+        result = self.run_command(["validate", path, "--no-assets"])
+        self.assertEqual(0, result.exit_code)
+
+    def test_invalid_item(self):
+        path = test_data.get_path(
+            "data-files/catalogs/test-case-1/country-1/area-1-1/"
+            "area-1-1-imagery/area-1-1-imagery-invalid.json")
+        result = self.run_command(["validate", path])
+        self.assertEqual(1, result.exit_code)
+
+    def test_collection_with_invalid_item(self):
+        path = test_data.get_path(
+            "data-files/catalogs/test-case-1/country-1/area-1-1/collection-invalid.json"
+        )
+        result = self.run_command(["validate", path])
+        self.assertEqual(1, result.exit_code)
+
+    def test_collection_with_invalid_item_no_validate_all(self):
+        path = test_data.get_path(
+            "data-files/catalogs/test-case-1/country-1/area-1-1/collection-invalid.json"
+        )
+        result = self.run_command(["validate", path, "--no-recurse"])
+        self.assertEqual(0, result.exit_code)
+
+    def test_collection_invalid_asset(self):
+        path = test_data.get_path(
+            "data-files/catalogs/test-case-1/country-1"
+            "/area-1-1/area-1-1-imagery/area-1-1-imagery.json")
+        result = self.run_command(["validate", path])
+        self.assertEqual(1, result.exit_code)

tests/data-files/catalogs/missing-stuff/catalog.json

@@ -0,0 +1,22 @@
+{
+  "id": "test",
+  "stac_version": "1.0.0-beta.2",
+  "description": "test catalog",
+  "links": [
+    {
+      "rel": "child",
+      "href": "./country-1/catalog.json",
+      "type": "application/json"
+    },
+    {
+      "rel": "child",
+      "href": "./country-2/catalog.json",
+      "type": "application/json"
+    },
+    {
+      "rel": "root",
+      "href": "./catalog.json",
+      "type": "application/json"
+    }
+  ]
+}

tests/data-files/catalogs/missing-stuff/country-1/area-1-1/area-1-1-imagery/area-1-1-imagery.json

@@ -0,0 +1,69 @@
+{
+  "type": "Feature",
+  "stac_version": "1.0.0-beta.2",
+  "id": "area-1-1-imagery",
+  "properties": {
+    "datetime": "2019-10-04 18:55:37Z"
+  },
+  "geometry": {
+    "type": "Polygon",
+    "coordinates": [
+      [
+        [
+          -2.5048828125,
+          3.8916575492899987
+        ],
+        [
+          -1.9610595703125,
+          3.8916575492899987
+        ],
+        [
+          -1.9610595703125,
+          4.275202171119132
+        ],
+        [
+          -2.5048828125,
+          4.275202171119132
+        ],
+        [
+          -2.5048828125,
+          3.8916575492899987
+        ]
+      ]
+    ]
+  },
+  "bbox": [
+    -2.5048828125,
+    3.8916575492899987,
+    -1.9610595703125,
+    3.8916575492899987
+  ],
+  "collection": "area-1-1",
+  "links": [
+    {
+      "rel": "collection",
+      "href": "../collection.json",
+      "type": "application/json"
+    },
+    {
+      "rel": "root",
+      "href": "../../../catalog.json",
+      "type": "application/json"
+    },
+    {
+      "rel": "parent",
+      "href": "../collection.json",
+      "type": "application/json"
+    }
+  ],
+  "assets": {
+    "ortho": {
+      "href": "http://example.com/area-1-1_ortho.tif",
+      "type": "image/vnd.stac.geotiff"
+    },
+    "dsm": {
+      "href": "http://example.com/area-1-1_dsm.tif",
+      "type": "image/vnd.stac.geotiff"
+    }
+  }
+}

tests/data-files/catalogs/missing-stuff/country-1/area-1-1/area-1-1-labels/area-1-1-labels.json

@@ -0,0 +1,93 @@
+{
+  "type": "Feature",
+  "stac_version": "1.0.0-beta.2",
+  "id": "area-1-1-labels",
+  "properties": {
+    "datetime": "2019-10-04 18:55:37Z",
+    "label:description": "labels for area-1-1",
+    "label:type": "vector",
+    "label:properties": [
+      "label"
+    ],
+    "label:classes": [
+      {
+        "name": "label",
+        "classes": [
+          "one",
+          "two"
+        ]
+      }
+    ],
+    "label:task": [
+      "classification"
+    ],
+    "label:method": [
+      "manual"
+    ]
+  },
+  "geometry": {
+    "type": "Polygon",
+    "coordinates": [
+      [
+        [
+          -2.5048828125,
+          3.8916575492899987
+        ],
+        [
+          -1.9610595703125,
+          3.8916575492899987
+        ],
+        [
+          -1.9610595703125,
+          4.275202171119132
+        ],
+        [
+          -2.5048828125,
+          4.275202171119132
+        ],
+        [
+          -2.5048828125,
+          3.8916575492899987
+        ]
+      ]
+    ]
+  },
+  "bbox": [
+    -2.5048828125,
+    3.8916575492899987,
+    -1.9610595703125,
+    3.8916575492899987
+  ],
+  "collection": "area-1-1",
+  "links": [
+    {
+      "rel": "source",
+      "href": "../area-1-1-imagery/area-1-1-imagery.json",
+      "type": "application/json"
+    },
+    {
+      "rel": "collection",
+      "href": "../collection.json",
+      "type": "application/json"
+    },
+    {
+      "rel": "root",
+      "href": "../../../catalog.json",
+      "type": "application/json"
+    },
+    {
+      "rel": "parent",
+      "href": "../collection.json",
+      "type": "application/json"
+    }
+  ],
+  "assets": {
+    "labels": {
+      "href": "http://example.com/area-1-1-labels.geojson",
+      "type": "application/geo+json"
+    }
+  },
+  "stac_extensions": [
+    "label"
+  ]
+}