Refactor how we fetch external data.

Prior to this, a custom cmake script (girder.cmake) was used via a direct URL for each external data file. After this change, we use cmake's build-in ExternalData module. This continues to get all external data files listed and automatically untar any .tgz files. External data file references have been moved from testing/test-data to tests/external-data. Eventually I'd like there to be only one of the tests and testing directories, as having two makes things confusing. This also deletes some abandoned cmake scripts and removes some unused data files. As a benefit, data files are now obtained by sha512, which should guarantee that we have the correct file, even if it changes. Before, the download url was used. This makes it so that we only need one entry in the repo per data file instead of two. The only downside is that ExternalData emits a -hash-stamp file in the download directory.
OpenGeoscience · Jul 12, 2018 · 5803c2e · 5803c2e
1 parent a56a346
commit 5803c2e
Show file tree

Hide file tree

Showing 72 changed files with 51 additions and 206 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -41,13 +41,6 @@ function(add_geojs_test test_name)
   set_property(TEST "notes-report" APPEND PROPERTY DEPENDS "${test_name}")
 endfunction()
 
-# set variables for girder.cmake
-set(Girder_KEY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/testing/test-data")
-set(Girder_DATA_DIR "${GEOJS_DEPLOY_DIR}/data")
-include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/girder.cmake)
-
-add_download_target()
-
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/testing/test-runners/geojs_test_runner.py.in
                ${CMAKE_CURRENT_BINARY_DIR}/test/geojs_test_runner.py
 )
@@ -73,12 +66,37 @@ add_test(
 )
 set_property(TEST "notes-report" APPEND PROPERTY DEPENDS "notes-reset")
 
-add_custom_target(
-  data_files
-  ALL
-  DEPENDS ${Girder_DOWNLOAD_FILES}
-)
-add_test(NAME get_data_files COMMAND "${CMAKE_COMMAND}" --build ${CMAKE_BINARY_DIR} --target data_files)
+# Use ExternalData to download all referenced files in tests/external-data
+include(ExternalData)
+set(ExternalData_URL_TEMPLATES "https://data.kitware.com/api/v1/file/hashsum/%(algo)/%(hash)/download")
+set(ExternalData_BINARY_ROOT "${GEOJS_DEPLOY_DIR}/data")
+set(ExternalData_SOURCE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/tests/external-data")
+# Get a list of files to download based on the name *.sha512
+file(GLOB data_files_list "${ExternalData_SOURCE_ROOT}/*.sha512")
+# Obtain the files using ExternalData
+foreach(data_file ${data_files_list})
+  string(REGEX REPLACE "^(.*)\\.sha512$" "\\1" base_file "${data_file}")
+  ExternalData_Expand_Arguments(data_files tmp DATA{${base_file}})
+endforeach()
+# Add a target to get the files
+ExternalData_Add_Target(data_files)
+# For any file that is a .tgz file, expand it after we get it
+foreach(data_file ${data_files_list})
+  string(REGEX REPLACE "^(.*)\\.sha512$" "\\1" base_file "${data_file}")
+  get_filename_component(base_file "${base_file}" NAME)
+  get_filename_component(base_fileext "${base_file}" EXT)
+  if("${base_fileext}" STREQUAL ".tgz")
+    get_filename_component(base_filename "${base_file}" NAME_WE)
+    add_custom_command(
+      TARGET data_files POST_BUILD
+      DEPENDS "${ExternalData_BINARY_ROOT}/${base_file}"
+      COMMAND ${CMAKE_COMMAND} -E make_directory "${ExternalData_BINARY_ROOT}/${base_filename}"
+      COMMAND ${CMAKE_COMMAND} -E chdir "${ExternalData_BINARY_ROOT}/${base_filename}" tar zxf "${ExternalData_BINARY_ROOT}/${base_file}"
+    )
+  endif()
+endforeach()
+# Create a test that gets this target
+add_test(NAME get_data_files COMMAND "${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}" --target data_files)
 
 if(HEADLESS_TESTS)
   add_test(

diff --git a/cmake/configure-js-coverage-test.cmake b/cmake/configure-js-coverage-test.cmake
diff --git a/cmake/configure-js-unit-test.cmake b/cmake/configure-js-unit-test.cmake
diff --git a/cmake/girder.cmake b/cmake/girder.cmake
diff --git a/testing/test-data/AdderallCities2015.csv.md5 b/testing/test-data/AdderallCities2015.csv.md5
diff --git a/testing/test-data/AdderallCities2015.csv.url b/testing/test-data/AdderallCities2015.csv.url
diff --git a/testing/test-data/base-images.tgz.md5 b/testing/test-data/base-images.tgz.md5
diff --git a/testing/test-data/base-images.tgz.url b/testing/test-data/base-images.tgz.url
diff --git a/testing/test-data/blue.jpg.md5 b/testing/test-data/blue.jpg.md5
diff --git a/testing/test-data/blue.jpg.url b/testing/test-data/blue.jpg.url
diff --git a/testing/test-data/cities.csv.md5 b/testing/test-data/cities.csv.md5
diff --git a/testing/test-data/cities.csv.url b/testing/test-data/cities.csv.url
diff --git a/testing/test-data/countries.json.md5 b/testing/test-data/countries.json.md5
diff --git a/testing/test-data/countries.json.url b/testing/test-data/countries.json.url
diff --git a/testing/test-data/earthquakes-video.webm.md5 b/testing/test-data/earthquakes-video.webm.md5
diff --git a/testing/test-data/earthquakes-video.webm.url b/testing/test-data/earthquakes-video.webm.url
diff --git a/testing/test-data/earthquakes.json.md5 b/testing/test-data/earthquakes.json.md5
diff --git a/testing/test-data/earthquakes.json.url b/testing/test-data/earthquakes.json.url
diff --git a/testing/test-data/grid.jpg.md5 b/testing/test-data/grid.jpg.md5
diff --git a/testing/test-data/grid.jpg.url b/testing/test-data/grid.jpg.url
diff --git a/testing/test-data/hurricanes.json.md5 b/testing/test-data/hurricanes.json.md5
diff --git a/testing/test-data/hurricanes.json.url b/testing/test-data/hurricanes.json.url
diff --git a/testing/test-data/land_polygons.json.md5 b/testing/test-data/land_polygons.json.md5
diff --git a/testing/test-data/land_polygons.json.url b/testing/test-data/land_polygons.json.url
diff --git a/testing/test-data/land_shallow_topo_2048.png.md5 b/testing/test-data/land_shallow_topo_2048.png.md5
diff --git a/testing/test-data/land_shallow_topo_2048.png.url b/testing/test-data/land_shallow_topo_2048.png.url
diff --git a/testing/test-data/oahu-dense.json.md5 b/testing/test-data/oahu-dense.json.md5
diff --git a/testing/test-data/oahu-dense.json.url b/testing/test-data/oahu-dense.json.url
diff --git a/testing/test-data/oahu-medium.json.md5 b/testing/test-data/oahu-medium.json.md5
diff --git a/testing/test-data/oahu-medium.json.url b/testing/test-data/oahu-medium.json.url
diff --git a/testing/test-data/oahu.json.md5 b/testing/test-data/oahu.json.md5
diff --git a/testing/test-data/oahu.json.url b/testing/test-data/oahu.json.url
diff --git a/testing/test-data/red.jpg.md5 b/testing/test-data/red.jpg.md5
diff --git a/testing/test-data/red.jpg.url b/testing/test-data/red.jpg.url
diff --git a/testing/test-data/roads.json.md5 b/testing/test-data/roads.json.md5
diff --git a/testing/test-data/roads.json.url b/testing/test-data/roads.json.url
diff --git a/testing/test-data/sample.json.md5 b/testing/test-data/sample.json.md5
diff --git a/testing/test-data/sample.json.url b/testing/test-data/sample.json.url
diff --git a/testing/test-data/spark.png.md5 b/testing/test-data/spark.png.md5
diff --git a/testing/test-data/spark.png.url b/testing/test-data/spark.png.url
diff --git a/testing/test-data/temperature_data.tsv.md5 b/testing/test-data/temperature_data.tsv.md5
diff --git a/testing/test-data/temperature_data.tsv.url b/testing/test-data/temperature_data.tsv.url
diff --git a/testing/test-data/tilefancy.png.md5 b/testing/test-data/tilefancy.png.md5
diff --git a/testing/test-data/tilefancy.png.url b/testing/test-data/tilefancy.png.url
diff --git a/testing/test-data/tiles.tgz.md5 b/testing/test-data/tiles.tgz.md5
diff --git a/testing/test-data/tiles.tgz.url b/testing/test-data/tiles.tgz.url
diff --git a/testing/test-data/vtkCube.dat.md5 b/testing/test-data/vtkCube.dat.md5
diff --git a/testing/test-data/vtkCube.dat.url b/testing/test-data/vtkCube.dat.url
diff --git a/testing/test-data/vtkSceneMetadata.json.md5 b/testing/test-data/vtkSceneMetadata.json.md5
diff --git a/testing/test-data/vtkSceneMetadata.json.url b/testing/test-data/vtkSceneMetadata.json.url
diff --git a/testing/test-data/white.jpg.md5 b/testing/test-data/white.jpg.md5
diff --git a/testing/test-data/white.jpg.url b/testing/test-data/white.jpg.url
diff --git a/tests/external-data/AdderallCities2015.csv.sha512 b/tests/external-data/AdderallCities2015.csv.sha512
@@ -0,0 +1 @@
+c3e984482cc6db1193a6dca2a55396a2baad8541a5c8c679f33750b76f766f40a119ec3e63abbabcd095d752c3df8ce21bf24cbe629502121f24ba90b4b0674c
diff --git a/tests/external-data/base-images.tgz.sha512 b/tests/external-data/base-images.tgz.sha512
@@ -0,0 +1 @@
+d7c9ac0d2324d047ee687fa0c1f857b24b03beae3cb84060d7507c53aca2b8a184abdcb6161ceb2cb238219b8235cfab75f61d9a1a2ff9fcf521b3f96c5f6c84
diff --git a/tests/external-data/blue.jpg.sha512 b/tests/external-data/blue.jpg.sha512
@@ -0,0 +1 @@
+867b1f3c568289efc7d0dba97d827a2bc4d83a7465cebcb3b5aec7bac6a38cf70d037d1814402bc97ad1f2f6737cfb5ce97db0a4fb53a716e77fd3ba57a7ab3b
diff --git a/tests/external-data/cities.csv.sha512 b/tests/external-data/cities.csv.sha512
@@ -0,0 +1 @@
+5a665e5feda24f28e5cf4ed0801b67e73bbcf3ea781b2e50d11284214e67b25b68e6a1c48da46e5e4d4d0c54c2ec18f88d292224b4541fb279396cf7b94beac9
diff --git a/tests/external-data/earthquakes-video.webm.sha512 b/tests/external-data/earthquakes-video.webm.sha512
@@ -0,0 +1 @@
+834a9d05f5fb00145e529fa4b217398db0df188c69d751f3869128b6e9c92d3000f85378752c56d9d9b5fa0870437dd9bdfeb5d62f6c87c2c03a7f1a20ee8523
diff --git a/tests/external-data/earthquakes.json.sha512 b/tests/external-data/earthquakes.json.sha512
@@ -0,0 +1 @@
+f098b6437411384b552419b4a36264c1bb3fed816ccfe9545145175e0b92a0b7ad5ebdcb9dddd0a12a90499143ffa471c02f6e049be5b973db607ff066892500
diff --git a/tests/external-data/grid.jpg.sha512 b/tests/external-data/grid.jpg.sha512
@@ -0,0 +1 @@
+60d201a14c7d31e7881301e6784e0372ddf27f26e5e4eafba9ba39158dfd050e3683faaa660fcde47e6c994dd3ee64c5a5231474ca75090053ef9207fedd9029
diff --git a/tests/external-data/hurricanes.json.sha512 b/tests/external-data/hurricanes.json.sha512
@@ -0,0 +1 @@
+012f15036bfc9ac3abb81a2a61e2c7b602ef3d8f7bd3b3f0fb66972ee69034730655f69febb2df5d657f22cd2b1e69170f6568bcbb03d7ebdecdfbafb80cc3b7
diff --git a/tests/external-data/land_polygons.json.sha512 b/tests/external-data/land_polygons.json.sha512
@@ -0,0 +1 @@
+30a828392d58678599130e0dca6d7f27e7e07e4e5b5d7f7a37871eb395d53d97b76134c0a07e805fbdfac0f42e6d3ca6e287c9727815a9dc711d541b1c8f68a1
diff --git a/tests/external-data/land_shallow_topo_2048.png.sha512 b/tests/external-data/land_shallow_topo_2048.png.sha512
@@ -0,0 +1 @@
+8a8330dba5bacdb511038ad0f6ee5a764a40aa7a8868a445749f653ae5d85d8317684ac706e7a9f049590170df6bc3fefc2912d52124d1b3b17aa43c529ff2a8
diff --git a/tests/external-data/oahu-dense.json.sha512 b/tests/external-data/oahu-dense.json.sha512
@@ -0,0 +1 @@
+692a44ec4a18b16d1530403a9a2bf286ff2ef7d45fa58f555c278f91a0fa708b5626a3f38955e06a11cdc06a1009e859328687c1f32a2169e0c8ef1b518418c1
diff --git a/tests/external-data/oahu-medium.json.sha512 b/tests/external-data/oahu-medium.json.sha512
@@ -0,0 +1 @@
+83375c5c2678b11d8de7f59fd8e2f9b889d3ec20f4a81279d996711bfc0942bd9dcdce149cbca995930459c26b3bf58be60569687085adf361ff805436b75aab
diff --git a/tests/external-data/oahu.json.sha512 b/tests/external-data/oahu.json.sha512
@@ -0,0 +1 @@
+e44282c44fa95f0b40c2135ee94ff40a755771f3b4bf9acda0eef0048cd0fb29b3a71352e2eecdca6cfc35fafde96ffb1a9658c9fee2346071808a9123c26cb5
diff --git a/tests/external-data/red.jpg.sha512 b/tests/external-data/red.jpg.sha512
@@ -0,0 +1 @@
+70884f5c3d1747633412dcf64d4886d985b49f508b68eb9f3874b2fdcfad38e7623f4a7c7355baa1656503f46e2e091576a19ded9fad7c7c65387c313206d2e6
diff --git a/tests/external-data/roads.json.sha512 b/tests/external-data/roads.json.sha512
@@ -0,0 +1 @@
+ae8d8b99c3fab73798ddcc246ea53b1bc8d598414d00df4ce697373a343fbf7ffed4dbae5f07a997ac7731b5f86140686b8425b99bd5818b8c9ec68e7c4d3315
diff --git a/tests/external-data/sample.json.sha512 b/tests/external-data/sample.json.sha512
@@ -0,0 +1 @@
+5d4b00559f17fac607eafe4a9bb933386ebd572fbe545e6138b8eafbbf510074a8ae398c7df0420a017218af83b18cc322633b77007b63a27dfea7a50db70244
diff --git a/tests/external-data/temperature_data.tsv.sha512 b/tests/external-data/temperature_data.tsv.sha512
@@ -0,0 +1 @@
+bde5cdad7c4518694393de83da938bcf5d485b41116b51f0fce4b462737992d2973f076c577040fa816873b196c6397244aa03736fc7750a0eea0ba5cbcc48e4
diff --git a/tests/external-data/tilefancy.png.sha512 b/tests/external-data/tilefancy.png.sha512
@@ -0,0 +1 @@
+455effa59d421cbb73c6def815813063f1c649363de4007fa0de00cc2e0f24cba745c046266e5a83fc43b121a648017d509d7bf03c30fbee1448817e3f849683
diff --git a/tests/external-data/tiles.tgz.sha512 b/tests/external-data/tiles.tgz.sha512
@@ -0,0 +1 @@
+30defdfd28f6773c78f31b64b728d42630583b4e2b74c7098d75e6887bf5d8a6bea08a85669ad969058ac2c98dc45d5512684c62227403e7fcf57ea4dc1675f5
diff --git a/tests/external-data/white.jpg.sha512 b/tests/external-data/white.jpg.sha512
@@ -0,0 +1 @@
+ea7a9d7ff76775e742572f89e90cce1248ec99c33b2f486e0fa1d19ab461b87dff324533ecb186a3db14e40a3826da97b5d66566360a201228f60140b0e89942
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		c3e984482cc6db1193a6dca2a55396a2baad8541a5c8c679f33750b76f766f40a119ec3e63abbabcd095d752c3df8ce21bf24cbe629502121f24ba90b4b0674c
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		d7c9ac0d2324d047ee687fa0c1f857b24b03beae3cb84060d7507c53aca2b8a184abdcb6161ceb2cb238219b8235cfab75f61d9a1a2ff9fcf521b3f96c5f6c84
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		867b1f3c568289efc7d0dba97d827a2bc4d83a7465cebcb3b5aec7bac6a38cf70d037d1814402bc97ad1f2f6737cfb5ce97db0a4fb53a716e77fd3ba57a7ab3b
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		5a665e5feda24f28e5cf4ed0801b67e73bbcf3ea781b2e50d11284214e67b25b68e6a1c48da46e5e4d4d0c54c2ec18f88d292224b4541fb279396cf7b94beac9
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		834a9d05f5fb00145e529fa4b217398db0df188c69d751f3869128b6e9c92d3000f85378752c56d9d9b5fa0870437dd9bdfeb5d62f6c87c2c03a7f1a20ee8523
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		f098b6437411384b552419b4a36264c1bb3fed816ccfe9545145175e0b92a0b7ad5ebdcb9dddd0a12a90499143ffa471c02f6e049be5b973db607ff066892500
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		60d201a14c7d31e7881301e6784e0372ddf27f26e5e4eafba9ba39158dfd050e3683faaa660fcde47e6c994dd3ee64c5a5231474ca75090053ef9207fedd9029
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		012f15036bfc9ac3abb81a2a61e2c7b602ef3d8f7bd3b3f0fb66972ee69034730655f69febb2df5d657f22cd2b1e69170f6568bcbb03d7ebdecdfbafb80cc3b7
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		30a828392d58678599130e0dca6d7f27e7e07e4e5b5d7f7a37871eb395d53d97b76134c0a07e805fbdfac0f42e6d3ca6e287c9727815a9dc711d541b1c8f68a1
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		8a8330dba5bacdb511038ad0f6ee5a764a40aa7a8868a445749f653ae5d85d8317684ac706e7a9f049590170df6bc3fefc2912d52124d1b3b17aa43c529ff2a8