Merge RFC4 (#1865)

This commit is the result of the squashing of rfc4_dev branch in a single
commit. It implements mostly RFC 4 related work.

* Grid handling:
  - remove obsolete and presumably unfinished implementation of grid catalog functionality
  - all grid functionality is in grids.cpp/.hpp
  - vertical and horizontal grid shift: rework to no longer load whole grid into memory
  - remove hgrids and vgrids member from PJ structure, and store them in hgridshift/vgridshift/deformation structures
  - build systems: add optional libtiff dependency. Must be explicitly disabled if not desired
  - add support for horizontal and vertical grids in GeoTIFF, if libtiff is available
  - add GenericShiftGridSet and GenericShiftGrid classes, relying on TIFF grids, that can be used for generic purpose grid-based adjustment
  - add a +proj=xyzgridshift method to perform geocentric translation by grid. Used for French NTF to RGF93 transformation using gr3df97a.tif grid
  - deformation: add support for +grids= for GeoTIFF grids
  - horizontal grid shift: fix failures on points slightly outside a subgrid (fixes #209)

* File management:
  - add a filemanager.cpp/.hpp to deal with file related work
  - test for legacy proj_api.h fileapi
  - proj.h: add proj_context_set_fileapi() and proj_context_set_sqlite3_vfs_name() (fixes #866)
  - add capability to read resource files from the user writable directory

* Network access:
  - build systems: add optional curl dependency
  - add a curl-based default implementation for network related functionality
  - proj.h: add C API to control network functionality, and optionaly provide network callbacks
  - add data/proj.ini with default settings
  - add a SQLite3 local cache of downloaded chunks
  - add proj_is_download_needed() and proj_download_file()

* Use Win32 Unicode APIs and expect all strings to be UTF-8 (fixes #1765)
  For backward compatibility, if PROJ_LIB content is found to be not UTF-8 or
  pointing to a non existing directory, then an attempt at interpretating it
  in the ANSI page encoding is done.
  proj_context_set_search_paths() now assumes strings to be in UTF-8, and
  functions returning paths will also return values in UTF-8.

.cirrus.yml

@@ -16,7 +16,7 @@ task:
     folder: $HOME/.ccache
 
   pkginstall_script:
-    - pkg install -y autoconf automake libtool pkgconf sqlite3 wget unzip ccache gmake
+    - pkg install -y autoconf automake libtool pkgconf sqlite3 tiff wget unzip ccache gmake
   download_grid_script:
     - wget https://download.osgeo.org/proj/proj-datumgrid-1.8.zip
     - (cd data && unzip -o ../proj-datumgrid-1.8.zip)

CMakeLists.txt

@@ -83,6 +83,7 @@ elseif("${CMAKE_C_COMPILER_ID}" STREQUAL "Intel")
     set(PROJ_CXX_WARN_FLAGS -Wall)
   endif()
 endif()
+
 set(PROJ_C_WARN_FLAGS "${PROJ_C_WARN_FLAGS}"
   CACHE STRING "C flags used to compile PROJ targets")
 set(PROJ_CXX_WARN_FLAGS "${PROJ_CXX_WARN_FLAGS}"
@@ -133,6 +134,39 @@ IF("${SQLITE3_VERSION}" VERSION_LESS "3.11")
   message(SEND_ERROR "sqlite3 >= 3.11 required!")
 ENDIF()
 
+################################################################################
+# Check for libtiff
+################################################################################
+
+option(DISABLE_TIFF "Disable TIFF support" OFF)
+mark_as_advanced(DISABLE_TIFF)
+if(DISABLE_TIFF)
+  message(WARNING "TIFF support has been disabled and will result in the inability to read some grids")
+else()
+  find_package(TIFF REQUIRED)
+  if(TIFF_FOUND)
+    boost_report_value(TIFF_FOUND)
+  else()
+    message(SEND_ERROR "libtiff dependency not found!")
+  endif()
+  add_definitions(-DTIFF_ENABLED)
+endif()
+
+################################################################################
+# Check for curl
+################################################################################
+
+option(ENABLE_CURL "Enable Curl support" ON)
+if(ENABLE_CURL)
+  find_package(CURL REQUIRED)
+  if(CURL_FOUND)
+    boost_report_value(CURL_FOUND)
+  else()
+    message(SEND_ERROR "curl dependency not found!")
+  endif()
+  add_definitions(-DCURL_ENABLED)
+endif()
+
 ################################################################################
 # threading configuration
 ################################################################################

Doxyfile

@@ -772,7 +772,7 @@ WARN_LOGFILE           =
 # spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
 # Note: If this tag is empty the current directory is searched.
 
-INPUT                  = src/iso19111 include/proj src/proj.h src/proj_experimental.h src/general_doc.dox
+INPUT                  = src/iso19111 include/proj src/proj.h src/proj_experimental.h src/general_doc.dox src/filemanager.cpp src/networkfilemanager.cpp
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

appveyor.yml

@@ -27,9 +27,10 @@ build_script:
   - set PATH=%CD%;%PATH%
   - cd ..
   - vcpkg install sqlite3:"%platform%"-windows
+  - vcpkg install tiff:"%platform%"-windows
+  - vcpkg install curl:"%platform%"-windows
   - set SQLITE3_BIN=%APPVEYOR_BUILD_FOLDER%\sqlite3\bin
   - mkdir %SQLITE3_BIN%
-  - copy c:\projects\proj\vcpkg\installed\"%platform%"-windows\bin\sqlite3.dll %SQLITE3_BIN%
   - ps: |
         appveyor DownloadFile https://sqlite.org/2018/sqlite-tools-win32-x86-3250100.zip
         7z x sqlite-tools-win32-x86-3250100.zip
@@ -50,6 +51,8 @@ build_script:
   - set PROJ_DIR=%APPVEYOR_BUILD_FOLDER%\proj_dir
   - cmake -G "%VS_FULL%" .. -DCMAKE_BUILD_TYPE=Release -DBUILD_LIBPROJ_SHARED="%BUILD_LIBPROJ_SHARED%" -DCMAKE_C_FLAGS="/WX" -DCMAKE_CXX_FLAGS="/WX" -DCMAKE_TOOLCHAIN_FILE=C:/projects/proj/vcpkg/scripts/buildsystems/vcpkg.cmake -DCMAKE_INSTALL_PREFIX="%PROJ_DIR%"
   - cmake --build . --config Release --target install
+  - copy c:\projects\proj\vcpkg\installed\"%platform%"-windows\bin\*.dll %PROJ_DIR%\bin
+  - dir %PROJ_DIR%\bin
 
 test_script:
   - echo test_script

cmake/ProjTest.cmake

@@ -2,6 +2,11 @@
 # add test with sh script
 #
 
+function(proj_test_set_properties TESTNAME)
+    set_tests_properties( ${TESTNAME}
+        PROPERTIES ENVIRONMENT "PROJ_IGNORE_USER_WRITABLE_DIRECTORY=YES;PROJ_LIB=${PROJECT_BINARY_DIR}/data/for_tests")
+endfunction()
+
 function(proj_add_test_script_sh SH_NAME BIN_USE)
   if(UNIX)
     get_filename_component(testname ${SH_NAME} NAME_WE)
@@ -26,8 +31,8 @@ function(proj_add_test_script_sh SH_NAME BIN_USE)
         COMMAND ${PROJECT_SOURCE_DIR}/test/cli/${SH_NAME}
         ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${${BIN_USE}}
       )
-    set_tests_properties( ${testname}
-        PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data/for_tests")
+
+      proj_test_set_properties(${testname})
     endif()
 
   endif()
@@ -43,8 +48,6 @@ function(proj_add_gie_test TESTNAME TESTCASE)
       COMMAND ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${GIE_BIN}
       ${TESTFILE}
     )
-    set_tests_properties( ${TESTNAME}
-        PROPERTIES ENVIRONMENT "PROJ_LIB=${PROJECT_BINARY_DIR}/data/for_tests")
-
+    proj_test_set_properties(${TESTNAME})
 
 endfunction()

configure.ac

@@ -20,6 +20,8 @@ AC_PROG_LN_S
 AC_PROG_MAKE_SET
 AM_PROG_LIBTOOL
 
+PKG_PROG_PKG_CONFIG
+
 dnl Enable as much warnings as possible
 AX_CFLAGS_WARN_ALL(C_WFLAGS)
 AX_CXXFLAGS_WARN_ALL(CXX_WFLAGS)
@@ -169,6 +171,15 @@ AC_SUBST(NO_ZERO_AS_NULL_POINTER_CONSTANT_FLAG,$NO_ZERO_AS_NULL_POINTER_CONSTANT
 CFLAGS="${CFLAGS} -fvisibility=hidden"
 CXXFLAGS="${CXXFLAGS} -fvisibility=hidden"
 
+case "${host_os}" in
+    cygwin* | mingw32* | pw32* | beos* | darwin*)
+        CFLAGS="${CFLAGS} -DNOMINMAX"
+        CXXFLAGS="${CXXFLAGS} -DNOMINMAX"
+        ;;
+    *)
+        ;;
+esac
+
 dnl Checks for libraries.
 save_CFLAGS="$CFLAGS"
 CFLAGS=`echo "$CFLAGS" | sed "s/-Werror/ /"`
@@ -240,6 +251,78 @@ if test x"$SQLITE3_CHECK" != x"yes" ; then
     AC_MSG_ERROR([Please install sqlite3 binary.])
 fi
 
+dnl ---------------------------------------------------------------------------
+dnl Check for libtiff
+dnl ---------------------------------------------------------------------------
+
+AC_ARG_ENABLE([tiff-is-strongly-discouraged],
+              AS_HELP_STRING([--disable-tiff-is-strongly-discouraged],
+                    [Disable TIFF support. Strongly discouraged !]),
+              [enable_tiff=no],
+              [enable_tiff=yes])
+
+if test "x$enable_tiff" = "xyes" -o "x$enable_tiff" = ""; then
+    if test "x$TIFF_CFLAGS$TIFF_LIBS" = "x" ; then
+        if $PKG_CONFIG libtiff; then
+            PKG_CHECK_MODULES([TIFF], [libtiff])
+        else
+            PKG_CHECK_MODULES([TIFF], [libtiff-4])
+        fi
+    fi
+    TIFF_ENABLED_FLAGS=-DTIFF_ENABLED
+fi
+AC_SUBST(TIFF_CFLAGS,$TIFF_CFLAGS)
+AC_SUBST(TIFF_LIBS,$TIFF_LIBS)
+AC_SUBST(TIFF_ENABLED_FLAGS,$TIFF_ENABLED_FLAGS)
+
+dnl ---------------------------------------------------------------------------
+dnl Check for curl
+dnl ---------------------------------------------------------------------------
+
+FOUND_CURL=no
+CURL_CFLAGS=
+CURL_LIB=
+
+AC_ARG_WITH(curl,
+    [  --with-curl[=ARG]       Enable curl support (ARG=path to curl-config.)],,,)
+
+dnl Clear some cache variables
+unset ac_cv_path_LIBCURL
+
+if test "`basename xx/$with_curl`" = "curl-config" ; then
+  LIBCURL_CONFIG="$with_curl"
+elif test "$with_curl" = "no" ; then
+  LIBCURL_CONFIG=no
+else
+  AC_PATH_PROG(LIBCURL_CONFIG, curl-config, not-found)
+fi
+
+if test "$LIBCURL_CONFIG" = "not-found" ; then
+  AC_MSG_ERROR([curl not found. If wanting to do a build without curl support (and thus without built-in networking capability), explictly disable it with --without-curl])
+elif test "$LIBCURL_CONFIG" != "no" ; then
+
+  CURL_VERNUM=`$LIBCURL_CONFIG --vernum`
+  CURL_VER=`$LIBCURL_CONFIG --version | awk '{print $2}'`
+
+  AC_MSG_RESULT([        found libcurl version $CURL_VER])
+
+  AC_CHECK_LIB(curl,curl_global_init,FOUND_CURL=yes,FOUND_CURL=no,`$LIBCURL_CONFIG --libs`)
+
+  if test "$FOUND_CURL" = "no" ; then
+    AC_MSG_ERROR([curl not found. If wanting to do a build without curl support (and thus without built-in networking capability), explictly disable it with --without-curl])
+  fi
+  CURL_ENABLED_FLAGS=-DCURL_ENABLED
+fi
+
+if test "$FOUND_CURL" = "yes" ; then
+  CURL_CFLAGS=`$LIBCURL_CONFIG --cflags`
+  CURL_LIBS=`$LIBCURL_CONFIG --libs`
+fi
+
+AC_SUBST(CURL_CFLAGS,$CURL_CFLAGS)
+AC_SUBST(CURL_LIBS,$CURL_LIBS)
+AC_SUBST(CURL_ENABLED_FLAGS,$CURL_ENABLED_FLAGS)
+
 dnl ---------------------------------------------------------------------------
 dnl Check for external Google Test
 dnl ---------------------------------------------------------------------------

data/CMakeLists.txt

@@ -2,6 +2,10 @@
 # files containing dictionary of useful projection
 #
 
+set(CONFIG_FILES
+  proj.ini
+)
+
 set(PROJ_DICTIONARY
   null
   world
@@ -54,7 +58,7 @@ add_custom_command(
 add_custom_target(generate_proj_db ALL DEPENDS ${PROJ_DB})
 
 if(NOT "${CMAKE_CURRENT_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_BINARY_DIR}")
-  foreach(FILE ${PROJ_DICTIONARY} ${GRIDSHIFT_FILES})
+  foreach(FILE ${CONFIG_FILES} ${PROJ_DICTIONARY} ${GRIDSHIFT_FILES})
     configure_file(${FILE} ${FILE} COPYONLY)
   endforeach()
 endif()
@@ -78,18 +82,25 @@ set(DATA_FOR_TESTS
     nzgd2kgrid0005.gsb
     ITRF2000
     BETA2007.gsb)
+file(GLOB DATA_TESTS tests/*)
 
 execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/for_tests)
-foreach(FILE ${DATA_FOR_TESTS})
+execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_CURRENT_BINARY_DIR}/for_tests/tests)
+foreach(FILE ${DATA_FOR_TESTS} ${CONFIG_FILES})
     if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${FILE})
         configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${FILE} ${CMAKE_CURRENT_BINARY_DIR}/for_tests/${FILE} COPYONLY)
     endif()
 endforeach()
+foreach(FILE ${DATA_TESTS})
+    get_filename_component(FILENAME ${FILE} NAME)
+    configure_file(${FILE} ${CMAKE_CURRENT_BINARY_DIR}/for_tests/tests/${FILENAME} COPYONLY)
+endforeach()
 
 #
 #install
 #
 set(ALL_DATA_FILE
+  ${CONFIG_FILES}
   ${PROJ_DICTIONARY}
   ${GRIDSHIFT_FILES}
   ${PROJ_DB}

data/Makefile.am

@@ -1,6 +1,6 @@
 DATAPATH = $(top_srcdir)/data
 
-pkgdata_DATA = GL27 nad.lst nad27 nad83 world other.extra \
+pkgdata_DATA = proj.ini GL27 nad.lst nad27 nad83 world other.extra \
 		CH null \
 		ITRF2000 ITRF2008 ITRF2014 proj.db \
 		projjson.schema.json
@@ -38,12 +38,55 @@ SQL_ORDERED_LIST = sql/begin.sql \
 		sql/customizations.sql \
 		sql/commit.sql
 
-EXTRA_DIST = GL27 nad.lst nad27 nad83 \
+EXTRA_DIST = proj.ini GL27 nad.lst nad27 nad83 \
 		world other.extra \
 		CH \
 		ITRF2000 ITRF2008 ITRF2014 \
 		projjson.schema.json \
-		CMakeLists.txt tests/test_nodata.gtx null \
+		CMakeLists.txt \
+		tests/test_nodata.gtx \
+		tests/test_vgrid_bigendian_bigtiff.tif \
+		tests/test_vgrid_bigendian.tif \
+		tests/test_vgrid_bigtiff.tif \
+		tests/test_vgrid_bottomup_with_matrix.tif \
+		tests/test_vgrid_bottomup_with_scale.tif \
+		tests/test_vgrid_deflate_floatingpointpredictor.tif \
+		tests/test_vgrid_deflate.tif \
+		tests/test_vgrid_float64.tif \
+		tests/test_vgrid_in_second_channel.tif \
+		tests/test_vgrid_int16.tif \
+		tests/test_vgrid_int32.tif \
+		tests/test_vgrid_uint32.tif \
+		tests/test_vgrid_invalid_channel_type.tif \
+		tests/test_vgrid_nodata.tif \
+		tests/test_vgrid_pixelisarea.tif \
+		tests/test_vgrid_pixelispoint.tif \
+		tests/test_vgrid_uint16.tif \
+		tests/test_vgrid_uint16_with_scale_offset.tif \
+		tests/test_vgrid_unsupported_byte.tif \
+		tests/test_vgrid_with_overview.tif \
+		tests/test_vgrid_with_subgrid.tif \
+		tests/test_hgrid.tif \
+		tests/test_hgrid_separate.tif \
+		tests/test_hgrid_tiled.tif \
+		tests/test_hgrid_tiled_separate.tif \
+		tests/test_hgrid_strip.tif \
+		tests/test_hgrid_positive_west.tif \
+		tests/test_hgrid_lon_shift_first.tif \
+		tests/test_hgrid_radian.tif \
+		tests/test_hgrid_degree.tif \
+		tests/test_hgrid_with_overview.tif \
+		tests/test_hgrid_extra_ifd_with_other_info.tif \
+		tests/test_hgrid_with_subgrid.tif \
+		tests/test_hgrid_with_subgrid_no_grid_name.tif \
+		tests/subset_of_gr3df97a.tif \
+		tests/egm96_15_uncompressed_truncated.tif \
+		tests/test_vgrid_single_strip_truncated.tif \
+		tests/nkgrf03vel_realigned_extract.tif \
+		tests/nkgrf03vel_realigned_xy_extract.ct2 \
+		tests/nkgrf03vel_realigned_z_extract.gtx \
+		tests/test_hgrid_with_two_level_of_subgrids_no_grid_name.tif \
+		null \
 		generate_all_sql_in.cmake sql_filelist.cmake \
 		$(SQL_ORDERED_LIST)
 
@@ -127,6 +170,8 @@ check-local:
 			echo "WARNING: grid $$x missing: some tests will be skipped"; \
 		fi \
 	done; \
+	ln -sf ../$(DATAPATH)/tests for_tests; \
+	ln -sf ../$(DATAPATH)/proj.ini for_tests; \
 	ln -sf ../proj.db for_tests
 
 clean-local:

data/proj.ini

@@ -0,0 +1,16 @@
+[general]
+; Lines starting by ; are commented lines.
+;
+
+; Network capabilities disabled by default.
+; Can be overriden with the PROJ_NETWORK=ON environment variable.
+; network = on
+
+; Can be overriden with the PROJ_NETWORK_ENDPOINT environment variable.
+cdn_endpoint = https://cdn.proj.org
+
+cache_enabled = on
+
+cache_size_MB = 300
+
+cache_ttl_sec = 86400

docs/source/apps/cct.rst

@@ -91,6 +91,17 @@ cartesian coordinates) and
 *Coordinate Transformations*, which are coordinate operations where
 input and output datums differ (e.g. change of reference frame).
 
+Use of remote grids
+*******************
+
+.. versionadded:: 7.0.0
+
+If the :envvar:`PROJ_NETWORK` environment variable is set to ``ON``,
+:program:`cct` will attempt to use remote grids stored on CDN (Content
+Delivery Network) storage, when they are not available locally.
+
+More details are available in the :ref:`network` section.
+
 Examples
 ********
 

docs/source/apps/cs2cs.rst

@@ -171,6 +171,16 @@ normally be in DMS format (use ``-f %.12f`` for decimal degrees with 12 decimal
 places), while projected (cartesian) coordinates will be in linear
 (meter, feet) units.
 
+Use of remote grids
+-------------------
+
+.. versionadded:: 7.0.0
+
+If the :envvar:`PROJ_NETWORK` environment variable is set to ``ON``,
+:program:`cs2cs` will attempt to use remote grids stored on CDN (Content
+Delivery Network) storage, when they are not available locally.
+
+More details are available in the :ref:`network` section.
 
 Examples
 ********