From 9bc81810b4606596f1eefc7310e3361336c5951c Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Fri, 13 Dec 2024 02:45:23 +0100 Subject: [PATCH] Add a CMake EMBED_GRIDS_DIRECTORY option to embed .tif/.json files into libproj ``` .. option:: EMBED_GRIDS_DIRECTORY= .. versionadded:: 9.6 Embed files from ending with .tif or .json in the PROJ library itself. The pointed directory can potentially be the full PROJ-data package (uncompressed). In that case, about 6 GB of free disk and 16 GB of RAM are required to build PROJ. When using this parameter, EMBED_RESOURCE_FILES must be set to ON. If the content of the directory changes, you need to run CMake again to update the list of files. ``` --- .github/workflows/fedora_rawhide/start.sh | 12 ++- docs/source/install.rst | 20 ++++- src/embedded_resources.c | 7 ++ src/embedded_resources.h | 5 ++ src/filemanager.cpp | 98 ++++++++++++++++++++++- src/lib_proj.cmake | 60 +++++++++++++- 6 files changed, 196 insertions(+), 6 deletions(-) diff --git a/.github/workflows/fedora_rawhide/start.sh b/.github/workflows/fedora_rawhide/start.sh index ff219fb606..a31abede25 100755 --- a/.github/workflows/fedora_rawhide/start.sh +++ b/.github/workflows/fedora_rawhide/start.sh @@ -2,7 +2,7 @@ set -e -dnf install -y cmake clang ccache ninja-build sqlite-devel libtiff-devel libcurl-devel diffutils +dnf install -y cmake clang ccache ninja-build sqlite-devel libtiff-devel libcurl-devel diffutils wget cd "$WORK_DIR" @@ -22,6 +22,16 @@ CC=clang CXX=clang++ cmake .. \ -DEMBED_RESOURCE_FILES=ON -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON -DUSE_CCACHE=ON -DPROJ_DB_CACHE_DIR=$HOME/.ccache .. make -j$(nproc) ctest -j$(nproc) + +# Try EMBED_GRIDS_DIRECTORY option +wget https://raw.githubusercontent.com/OSGeo/PROJ-data/refs/heads/master/us_nga/us_nga_egm96_15.tif +mkdir grids +mv us_nga_egm96_15.tif grids +CC=clang CXX=clang++ cmake .. -DEMBED_GRIDS_DIRECTORY=$PWD/grids +make -j$(nproc) +echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979 +echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979 | grep 44.643 >/dev/null || (echo "Expected 49dN 2dE 44.643 as a result" && /bin/false) + cd .. ccache -s diff --git a/docs/source/install.rst b/docs/source/install.rst index beffee6f53..002f8dac87 100644 --- a/docs/source/install.rst +++ b/docs/source/install.rst @@ -450,9 +450,23 @@ All cached entries can be viewed using ``cmake -LAH`` from a build directory. .. versionadded:: 9.6 + When ON, :file:`proj.db` and :file:`proj.ini` will be embedded into the PROJ library. Default is OFF for shared library builds (BUILD_SHARED_LIBS=ON), and ON for static library builds (BUILD_SHARED_LIBS=OFF). - When ON, :file:`proj.db` and :file:`proj.ini` will be embedded into the PROJ library. + +.. option:: EMBED_GRIDS_DIRECTORY= + + .. versionadded:: 9.6 + + Embed files from ending with .tif or .json in the PROJ library itself. + + The pointed directory can potentially be the full PROJ-data package (uncompressed). + In that case, about 6 GB of free disk and 16 GB of RAM are required to build PROJ. + + When using this parameter, EMBED_RESOURCE_FILES must be set to ON. + + If the content of the directory changes, you need to run CMake again to + update the list of files. .. option:: USE_ONLY_EMBEDDED_RESOURCE_FILES=ON/OFF @@ -462,7 +476,9 @@ All cached entries can be viewed using ``cmake -LAH`` from a build directory. :file:`proj.db` and :file:`proj.ini` on the file system, and fallback to the embedded version if not found. By setting USE_ONLY_EMBEDDED_RESOURCE_FILES=ON, no attempt at locating - those files on the file system is made. Default is OFF. + those files on the file system is made. And if EMBED_GRIDS_DIRECTORY has also + been set, no attempt at locating grid files on the file system is made. + Default is OFF. Users will also typically want to set EMBED_PROJ_DATA_PATH=OFF if setting USE_ONLY_EMBEDDED_RESOURCE_FILES=OFF. diff --git a/src/embedded_resources.c b/src/embedded_resources.c index 04b721f11e..adae0b8a35 100644 --- a/src/embedded_resources.c +++ b/src/embedded_resources.c @@ -1,3 +1,6 @@ +#include +#include + #include "embedded_resources.h" #if USE_SHARP_EMBED @@ -32,3 +35,7 @@ const char *pj_get_embedded_proj_ini(unsigned int *pnSize) { } #endif + +#ifdef USE_EMBEDDED_GRIDS +#include "file_embed/embedded_grids.c" +#endif diff --git a/src/embedded_resources.h b/src/embedded_resources.h index 39a119b0a7..560d8ec794 100644 --- a/src/embedded_resources.h +++ b/src/embedded_resources.h @@ -8,6 +8,11 @@ extern "C" { const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize); const char *pj_get_embedded_proj_ini(unsigned int *pnSize); +#ifdef USE_EMBEDDED_GRIDS +const unsigned char *pj_get_embedded_grid(const char *filename, + unsigned int *pnSize); +#endif + #ifdef __cplusplus } #endif diff --git a/src/filemanager.cpp b/src/filemanager.cpp index 2e3f3d8a26..4a748ac4f1 100644 --- a/src/filemanager.cpp +++ b/src/filemanager.cpp @@ -893,6 +893,80 @@ std::unique_ptr FileApiAdapter::open(PJ_CONTEXT *ctx, // --------------------------------------------------------------------------- +#if USE_EMBEDDED_GRIDS + +class FileMemory : public File { + PJ_CONTEXT *m_ctx; + const unsigned char *const m_data; + const size_t m_size; + size_t m_pos = 0; + + FileMemory(const FileMemory &) = delete; + FileMemory &operator=(const FileMemory &) = delete; + + protected: + FileMemory(const std::string &filename, PJ_CONTEXT *ctx, + const unsigned char *data, size_t size) + : File(filename), m_ctx(ctx), m_data(data), m_size(size) {} + + public: + size_t read(void *buffer, size_t sizeBytes) override; + size_t write(const void *, size_t) override; + bool seek(unsigned long long offset, int whence = SEEK_SET) override; + unsigned long long tell() override { return m_pos; } + void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; } + + bool hasChanged() const override { return false; } + + static std::unique_ptr open(PJ_CONTEXT *ctx, const char *filename, + FileAccess access, + const unsigned char *data, size_t size) { + if (access != FileAccess::READ_ONLY) + return nullptr; + return std::unique_ptr(new FileMemory(filename, ctx, data, size)); + } +}; + +size_t FileMemory::read(void *buffer, size_t sizeBytes) { + if (m_pos >= m_size) + return 0; + if (sizeBytes >= m_size - m_pos) { + const size_t bytesToCopy = m_size - m_pos; + memcpy(buffer, m_data + m_pos, bytesToCopy); + m_pos = m_size; + return bytesToCopy; + } + memcpy(buffer, m_data + m_pos, sizeBytes); + m_pos += sizeBytes; + return sizeBytes; +} + +size_t FileMemory::write(const void *, size_t) { + // shouldn't happen given we have bailed out in open() in non read-only + // modes + return 0; +} + +bool FileMemory::seek(unsigned long long offset, int whence) { + if (whence == SEEK_SET) { + m_pos = static_cast(offset); + return m_pos == offset; + } else if (whence == SEEK_CUR) { + const unsigned long long newPos = m_pos + offset; + m_pos = static_cast(newPos); + return m_pos == newPos; + } else { + if (offset != 0) + return false; + m_pos = m_size; + return true; + } +} + +#endif + +// --------------------------------------------------------------------------- + std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename, FileAccess access) { if (starts_with(filename, "http://") || starts_with(filename, "https://")) { @@ -909,11 +983,31 @@ std::unique_ptr FileManager::open(PJ_CONTEXT *ctx, const char *filename, if (ctx->fileApi.open_cbk != nullptr) { return FileApiAdapter::open(ctx, filename, access); } + + std::unique_ptr ret; +#if !(USE_EMBEDDED_GRIDS && USE_ONLY_EMBEDDED_RESOURCE_FILES) #ifdef _WIN32 - return FileWin32::open(ctx, filename, access); + ret = FileWin32::open(ctx, filename, access); #else - return FileStdio::open(ctx, filename, access); + ret = FileStdio::open(ctx, filename, access); +#endif +#endif + +#if USE_EMBEDDED_GRIDS +#if USE_ONLY_EMBEDDED_RESOURCE_FILES + if (!ret) #endif + { + unsigned int size = 0; + const unsigned char *in_memory_data = + pj_get_embedded_grid(filename, &size); + if (in_memory_data) { + ret = FileMemory::open(ctx, filename, access, in_memory_data, size); + } + } +#endif + + return ret; } // --------------------------------------------------------------------------- diff --git a/src/lib_proj.cmake b/src/lib_proj.cmake index a36633926c..b841caa063 100644 --- a/src/lib_proj.cmake +++ b/src/lib_proj.cmake @@ -411,7 +411,6 @@ if (EMBED_RESOURCE_FILES AND NOT IS_SHARP_EMBED_AVAILABLE_RES) -P ${PROJECT_SOURCE_DIR}/cmake/FileEmbed.cmake DEPENDS generate_proj_db "${PROJECT_BINARY_DIR}/data/proj.db" ) - target_sources(proj PRIVATE embedded_resources.c "${EMBEDDED_PROJ_DB}") set(EMBEDDED_PROJ_INI "file_embed/proj_ini.c") add_custom_command( @@ -434,6 +433,65 @@ elseif(EMBED_RESOURCE_FILES AND IS_SHARP_EMBED_AVAILABLE_RES) set_target_properties(proj_resources PROPERTIES C_STANDARD 23) target_sources(proj PRIVATE $) endif() + +set(EMBED_GRIDS_DIRECTORY "" CACHE PATH "Directory that contains .tif and .json files to embed into libproj") +if (EMBED_GRIDS_DIRECTORY) + if (NOT EMBED_RESOURCE_FILES) + message(FATAL_ERROR "EMBED_RESOURCE_FILES should be set to ON when EMBED_GRIDS_DIRECTORY is set") + endif() + + if (NOT IS_DIRECTORY ${EMBED_GRIDS_DIRECTORY}) + message(FATAL_ERROR "${EMBED_GRIDS_DIRECTORY} is not a valid directory") + endif() + + file(GLOB FILES "${EMBED_GRIDS_DIRECTORY}/*.tif" "${EMBED_GRIDS_DIRECTORY}/*.json") + if (NOT FILES) + message(FATAL_ERROR "No .tif or .json files found in ${EMBED_GRIDS_DIRECTORY}") + endif() + set(EMBEDDED_GRIDS_C_PROLOG_CONTENT "") + set(EMBEDDED_GRIDS_C_CONTENT "") + string(APPEND EMBEDDED_GRIDS_C_CONTENT + "const unsigned char *pj_get_embedded_grid(const char* filename, unsigned int *pnSize)\n" + "{\n") + foreach(FILE ${FILES}) + get_filename_component(FILENAME ${FILE} NAME) + message(STATUS "Embedding ${FILENAME}") + set(C_IDENTIFIER_GRID_NAME "${FILENAME}") + string(REPLACE "." "_" C_IDENTIFIER_GRID_NAME "${C_IDENTIFIER_GRID_NAME}") + string(REPLACE "-" "_" C_IDENTIFIER_GRID_NAME "${C_IDENTIFIER_GRID_NAME}") + set(C_FILENAME "file_embed/${C_IDENTIFIER_GRID_NAME}.c") + add_custom_command( + OUTPUT "${C_FILENAME}" + COMMAND ${CMAKE_COMMAND} + -DRUN_FILE_EMBED_GENERATE=1 + "-DFILE_EMBED_GENERATE_PATH=${FILE}" + -P ${PROJECT_SOURCE_DIR}/cmake/FileEmbed.cmake + DEPENDS "${FILE}" + ) + + string(APPEND EMBEDDED_GRIDS_C_CONTENT + " if (strcmp(filename, \"${FILENAME}\") == 0)\n" + " {\n" + " *pnSize = ${C_IDENTIFIER_GRID_NAME}_size;\n" + " return ${C_IDENTIFIER_GRID_NAME}_data;\n" + " }\n") + + target_sources(proj PRIVATE "${C_FILENAME}") + + string(APPEND EMBEDDED_GRIDS_C_PROLOG_CONTENT "extern const uint8_t ${C_IDENTIFIER_GRID_NAME}_data[];\n") + string(APPEND EMBEDDED_GRIDS_C_PROLOG_CONTENT "extern const unsigned ${C_IDENTIFIER_GRID_NAME}_size;\n") + endforeach() + string(APPEND EMBEDDED_GRIDS_C_CONTENT + " *pnSize = 0;\n" + " return NULL;\n" + "}\n") + file(WRITE "file_embed/embedded_grids.c" "${EMBEDDED_GRIDS_C_PROLOG_CONTENT}\n${EMBEDDED_GRIDS_C_CONTENT}") + target_compile_definitions(proj PRIVATE USE_EMBEDDED_GRIDS) + if (IS_SHARP_EMBED_AVAILABLE_RES) + target_compile_definitions(proj_resources PRIVATE USE_EMBEDDED_GRIDS) + endif() +endif() + if (EMBED_RESOURCE_FILES) target_sources(proj PRIVATE memvfs.c) target_compile_definitions(proj PRIVATE EMBED_RESOURCE_FILES)