Skip to content

Commit

Permalink
projsync: make it filter out files not intended for the current version
Browse files Browse the repository at this point in the history 
* Add a PROJ_DATA.VERSION in proj.db to indicate the target PROJ-data
  package version
* Make projsync use that information and the version_added and
  version_removed properties added in OSGeo/PROJ-data#67
  to filter out files that are not relevant
* Add --no-version-filtering and --verbose switches
  • Loading branch information
@rouault
rouault committed May 17, 2021
1 parent 5fe51fa commit aee1709
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 2 deletions.
4 changes: 4 additions & 0 deletions data/sql/metadata.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,7 @@ INSERT INTO "metadata" VALUES('EPSG.DATE', '2021-05-16');
-- The value of ${PROJ_VERSION} is substituted at build time by the actual
-- value.
INSERT INTO "metadata" VALUES('PROJ.VERSION', '${PROJ_VERSION}');

-- Version of the PROJ-data package with which this database is the most
-- compatible.
INSERT INTO "metadata" VALUES('PROJ_DATA.VERSION', '1.7');
19 changes: 18 additions & 1 deletion docs/source/apps/projsync.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ Synopsis
| [--source-id ID] [--area-of-use NAME]
| [--file NAME]
| [--all] [--exclude-world-coverage]
| [--quiet] [--dry-run] [--list-files]
| [--quiet | --verbose] [--dry-run] [--list-files]
| [--no-version-filtering]
Description
***********
Expand Down Expand Up @@ -108,6 +109,12 @@ The following control parameters can appear in any order:

Quiet mode

.. option:: --verbose

.. versionadded:: 8.1

Verbose mode (more than default)

.. option:: --dry-run

Simulate the behavior of the tool without downloading resource files.
Expand All @@ -116,6 +123,16 @@ The following control parameters can appear in any order:

List file names, with the source_id and area_of_use properties.

.. option:: --no-version-filtering

.. versionadded:: 8.1

By default, projsync only downloads files that are compatible of
the PROJ_DATA.VERSION metadata of :file:`proj.db`, taking into account the
``version_added`` and ``version_removed`` properties of entries in :file:`files.geojson`.
When specifying this switch, all files referenced in :file:`files.geojson`
will be candidate (combined with other filters).


At least one of :option:`--list-files`, :option:`--file`, :option:`--source-id`,
:option:`--area-of-use`, :option:`--bbox` or :option:`--all` must be specified.
Expand Down
84 changes: 83 additions & 1 deletion src/apps/projsync.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,9 @@ static void usage() {
std::cerr << " [--source-id ID] [--area-of-use NAME]" << std::endl;
std::cerr << " [--file NAME]" << std::endl;
std::cerr << " [--all] [--exclude-world-coverage]" << std::endl;
std::cerr << " [--quiet] [--dry-run] [--list-files]" << std::endl;
std::cerr << " [--quiet | --verbose] [--dry-run] [--list-files]"
<< std::endl;
std::cerr << " [--no-version-filtering]" << std::endl;
std::exit(1);
}

Expand Down Expand Up @@ -124,10 +126,12 @@ int main(int argc, char *argv[]) {
double queried_north = 0.0;
bool intersects = true;
bool quiet = false;
bool verbose = false;
bool includeWorldCoverage = true;
bool queryAll = false;
std::string queriedFilename;
std::string files_geojson_local;
bool versionFiltering = true;

for (int i = 1; i < argc; i++) {
std::string arg(argv[i]);
Expand Down Expand Up @@ -208,8 +212,12 @@ int main(int argc, char *argv[]) {
includeWorldCoverage = false;
} else if (arg == "--all") {
queryAll = true;
} else if (arg == "--no-version-filtering") {
versionFiltering = false;
} else if (arg == "-q" || arg == "--quiet") {
quiet = true;
} else if (arg == "--verbose") {
verbose = true;
} else {
usage();
}
Expand Down Expand Up @@ -281,6 +289,22 @@ int main(int argc, char *argv[]) {
std::cout << "filename,source_id,area_of_use,file_size" << std::endl;
}

std::string proj_data_version_str;
int proj_data_version_major = 0;
int proj_data_version_minor = 0;
{
const char *proj_data_version =
proj_context_get_database_metadata(ctx, "PROJ_DATA.VERSION");
if (proj_data_version) {
proj_data_version_str = proj_data_version;
const auto tokens = split(proj_data_version, '.');
if (tokens.size() >= 2) {
proj_data_version_major = atoi(tokens[0].c_str());
proj_data_version_minor = atoi(tokens[1].c_str());
}
}
}

try {
const auto j = json::parse(text);
bool foundMatchSourceIdCriterion = false;
Expand Down Expand Up @@ -315,6 +339,64 @@ int main(int argc, char *argv[]) {
continue;
}
const auto name(j_name.get<std::string>());

if (versionFiltering && proj_data_version_major > 0 &&
properties.contains("version_added")) {
const auto j_version_added = properties["version_added"];
if (j_version_added.is_string()) {
const auto version_added(
j_version_added.get<std::string>());
const auto tokens = split(version_added, '.');
if (tokens.size() >= 2) {
int version_major = atoi(tokens[0].c_str());
int version_minor = atoi(tokens[1].c_str());
if (proj_data_version_major < version_major ||
(proj_data_version_major == version_major &&
proj_data_version_minor < version_minor)) {
// File only useful for a later PROJ version
if (verbose) {
std::cout << "Skipping " << name
<< " as it is only useful starting "
"with PROJ-data "
<< version_added
<< " and we are targetting "
<< proj_data_version_str << std::endl;
}
continue;
}
}
}
}

if (versionFiltering &&
proj_data_version_major > 0 &&
properties.contains("version_removed")) {
const auto j_version_removed = properties["version_removed"];
if (j_version_removed.is_string()) {
const auto version_removed(
j_version_removed.get<std::string>());
const auto tokens = split(version_removed, '.');
if (tokens.size() >= 2) {
int version_major = atoi(tokens[0].c_str());
int version_minor = atoi(tokens[1].c_str());
if (proj_data_version_major > version_major ||
(proj_data_version_major == version_major &&
proj_data_version_minor >= version_minor)) {
// File only useful for a previous PROJ version
if (verbose) {
std::cout << "Skipping " << name
<< " as it is no longer useful "
"starting with PROJ-data "
<< version_removed
<< " and we are targetting "
<< proj_data_version_str << std::endl;
}
continue;
}
}
}
}

files.insert(name);

if (!properties.contains("source_id")) {
Expand Down
22 changes: 22 additions & 0 deletions test/cli/test_projsync
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,28 @@ fi
cat ${TMP_OUT} | grep "nz_linz_nzgeoid2009.tif" >/dev/null || (cat ${TMP_OUT}; exit 100)
cat ${TMP_OUT} | grep "us_noaa_alaska.tif" >/dev/null || (cat ${TMP_OUT}; exit 100)

cat > dummy.geojson <<EOF
{
"type": "FeatureCollection",
"features": [
{ "type": "Feature", "properties": { "url": "https://example.com/removed_in_1.7.tif", "name": "removed_in_1.7", "area_of_use": "Australia", "type": "VERTICAL_OFFSET_GEOGRAPHIC_TO_VERTICAL", "source_crs_code": "EPSG:7843", "source_crs_name": "GDA2020", "target_crs_code": "EPSG:9458", "target_crs_name": "AVWS height", "source": "Geoscience Australia", "source_country": "Australia", "source_id": "au_ga", "source_url": "http://www.ga.gov.au", "description": "GDA2020 (EPSG:7843) to AVWS height (EPSG:9458)", "file_size": 60878523, "sha256sum": "c06f24dae030587cc2556a316782ade83b6d91f1b8d03922ddd0ce58f3868baa", "version_removed": "1.7" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 93.0083333, -61.0083333 ], [ 174.0083333, -61.0083333 ], [ 174.0083333, -8.0083333 ], [ 93.0083333, -8.0083333 ], [ 93.0083333, -61.0083333 ] ] ] } },
{ "type": "Feature", "properties": { "url": "https://example.com/added_in_99_99.tif", "name": "added_in_99_99.tif", "area_of_use": "Australia", "type": "VERTICAL_OFFSET_GEOGRAPHIC_TO_VERTICAL", "source_crs_code": "EPSG:7843", "source_crs_name": "GDA2020", "target_crs_code": "EPSG:9458", "target_crs_name": "AVWS height", "source": "Geoscience Australia", "source_country": "Australia", "source_id": "au_ga", "source_url": "http://www.ga.gov.au", "description": "GDA2020 (EPSG:7843) to AVWS height (EPSG:9458)", "file_size": 60878523, "sha256sum": "c06f24dae030587cc2556a316782ade83b6d91f1b8d03922ddd0ce58f3868baa", "version_added": "99.99" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 93.0083333, -61.0083333 ], [ 174.0083333, -61.0083333 ], [ 174.0083333, -8.0083333 ], [ 93.0083333, -8.0083333 ], [ 93.0083333, -61.0083333 ] ] ] } },
{ "type": "Feature", "properties": { "url": "https://example.com/without_version.tif", "name": "without_version.tif", "area_of_use": "Australia", "type": "VERTICAL_OFFSET_GEOGRAPHIC_TO_VERTICAL", "source_crs_code": "EPSG:7843", "source_crs_name": "GDA2020", "target_crs_code": "EPSG:9458", "target_crs_name": "AVWS height", "source": "Geoscience Australia", "source_country": "Australia", "source_id": "au_ga", "source_url": "http://www.ga.gov.au", "description": "GDA2020 (EPSG:7843) to AVWS height (EPSG:9458)", "file_size": 60878523, "sha256sum": "c06f24dae030587cc2556a316782ade83b6d91f1b8d03922ddd0ce58f3868baa" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 93.0083333, -61.0083333 ], [ 174.0083333, -61.0083333 ], [ 174.0083333, -8.0083333 ], [ 93.0083333, -8.0083333 ], [ 93.0083333, -61.0083333 ] ] ] } }
]
}
EOF

echo "Testing $EXE --source-id au_ga --verbose --dry-run --local-geojson-file dummy.geojson"
if ! $EXE --source-id au_ga --verbose --dry-run --local-geojson-file dummy.geojson > ${TMP_OUT}; then
echo "--source-id au_ga --verbose --dry-run --local-geojson-file dummy.geojson"
cat ${TMP_OUT}
exit 100
fi
rm dummy.geojson
cat ${TMP_OUT} | grep "Skipping removed_in_1.7 as it is no longer useful starting with PROJ-data 1.7" >/dev/null || (cat ${TMP_OUT}; exit 100)
cat ${TMP_OUT} | grep "Skipping added_in_99_99.tif as it is only useful starting with PROJ-data 99.99" >/dev/null || (cat ${TMP_OUT}; exit 100)
cat ${TMP_OUT} | grep "Would download https://cdn.proj.org/without_version.tif" >/dev/null || (cat ${TMP_OUT}; exit 100)

rm -rf ${PROJ_USER_WRITABLE_DIRECTORY}
rm ${TMP_OUT}

Expand Down

0 comments on commit aee1709

Please sign in to comment.