Add a CMake EMBED_GRIDS_DIRECTORY option to embed .tif/.json files in…

…to libproj

```
.. option:: EMBED_GRIDS_DIRECTORY=<directory>

    .. versionadded:: 9.6

    Embed files from <directory> ending with .tif or .json in the PROJ library itself.

    The pointed directory can potentially be the full PROJ-data package (uncompressed).
    In that case, about 6 GB of free disk and 16 GB of RAM are required to build PROJ.

    When using this parameter, EMBED_RESOURCE_FILES must be set to ON.

    If the content of the directory changes, you need to run CMake again to
    update the list of files.
```

.github/workflows/fedora_rawhide/start.sh

@@ -2,7 +2,7 @@
 
 set -e
 
-dnf install -y cmake clang ccache ninja-build sqlite-devel libtiff-devel libcurl-devel diffutils
+dnf install -y cmake clang ccache ninja-build sqlite-devel libtiff-devel libcurl-devel diffutils wget
 
 cd "$WORK_DIR"
 
@@ -22,6 +22,16 @@ CC=clang CXX=clang++ cmake .. \
     -DEMBED_RESOURCE_FILES=ON -DUSE_ONLY_EMBEDDED_RESOURCE_FILES=ON -DUSE_CCACHE=ON -DPROJ_DB_CACHE_DIR=$HOME/.ccache ..
 make -j$(nproc)
 ctest -j$(nproc)
+
+# Try EMBED_GRIDS_DIRECTORY option
+wget https://raw.githubusercontent.com/OSGeo/PROJ-data/refs/heads/master/us_nga/us_nga_egm96_15.tif
+mkdir grids
+mv us_nga_egm96_15.tif grids
+CC=clang CXX=clang++ cmake .. -DEMBED_GRIDS_DIRECTORY=$PWD/grids
+make -j$(nproc)
+echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979
+echo 49 2 0 | bin/cs2cs "WGS84 + EGM96 height" EPSG:4979  | grep 44.643 >/dev/null || (echo "Expected 49dN 2dE 44.643 as a result" && /bin/false)
+
 cd ..
 
 ccache -s

docs/source/install.rst

@@ -450,9 +450,23 @@ All cached entries can be viewed using ``cmake -LAH`` from a build directory.
 
     .. versionadded:: 9.6
 
+    When ON, :file:`proj.db` and :file:`proj.ini` will be embedded into the PROJ library.
     Default is OFF for shared library builds (BUILD_SHARED_LIBS=ON), and ON
     for static library builds (BUILD_SHARED_LIBS=OFF).
-    When ON, :file:`proj.db` and :file:`proj.ini` will be embedded into the PROJ library.
+
+.. option:: EMBED_GRIDS_DIRECTORY=<directory>
+
+    .. versionadded:: 9.6
+
+    Embed files from <directory> ending with .tif or .json in the PROJ library itself.
+
+    The pointed directory can potentially be the full PROJ-data package (uncompressed).
+    In that case, about 6 GB of free disk and 16 GB of RAM are required to build PROJ.
+
+    When using this parameter, EMBED_RESOURCE_FILES must be set to ON.
+
+    If the content of the directory changes, you need to run CMake again to
+    update the list of files.
 
 .. option:: USE_ONLY_EMBEDDED_RESOURCE_FILES=ON/OFF
 
@@ -462,7 +476,9 @@ All cached entries can be viewed using ``cmake -LAH`` from a build directory.
     :file:`proj.db` and :file:`proj.ini` on the file system, and fallback to the
     embedded version if not found.
     By setting USE_ONLY_EMBEDDED_RESOURCE_FILES=ON, no attempt at locating
-    those files on the file system is made. Default is OFF.
+    those files on the file system is made. And if EMBED_GRIDS_DIRECTORY has also
+    been set, no attempt at locating grid files on the file system is made.
+    Default is OFF.
     Users will also typically want to set EMBED_PROJ_DATA_PATH=OFF if setting
     USE_ONLY_EMBEDDED_RESOURCE_FILES=OFF.
 

src/embedded_resources.c

@@ -1,3 +1,5 @@
+#include <string.h>
+
 #include "embedded_resources.h"
 
 #if USE_SHARP_EMBED
@@ -32,3 +34,7 @@ const char *pj_get_embedded_proj_ini(unsigned int *pnSize) {
 }
 
 #endif
+
+#ifdef USE_EMBEDDED_GRIDS
+#include "file_embed/embedded_grids.c"
+#endif

src/embedded_resources.h

@@ -8,6 +8,11 @@ extern "C" {
 const unsigned char *pj_get_embedded_proj_db(unsigned int *pnSize);
 const char *pj_get_embedded_proj_ini(unsigned int *pnSize);
 
+#ifdef USE_EMBEDDED_GRIDS
+const unsigned char *pj_get_embedded_grid(const char *filename,
+                                          unsigned int *pnSize);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

src/filemanager.cpp

@@ -893,6 +893,80 @@ std::unique_ptr<File> FileApiAdapter::open(PJ_CONTEXT *ctx,
 
 // ---------------------------------------------------------------------------
 
+#if USE_EMBEDDED_GRIDS
+
+class FileMemory : public File {
+    PJ_CONTEXT *m_ctx;
+    const unsigned char *const m_data;
+    const size_t m_size;
+    size_t m_pos = 0;
+
+    FileMemory(const FileMemory &) = delete;
+    FileMemory &operator=(const FileMemory &) = delete;
+
+  protected:
+    FileMemory(const std::string &filename, PJ_CONTEXT *ctx,
+               const unsigned char *data, size_t size)
+        : File(filename), m_ctx(ctx), m_data(data), m_size(size) {}
+
+  public:
+    size_t read(void *buffer, size_t sizeBytes) override;
+    size_t write(const void *, size_t) override;
+    bool seek(unsigned long long offset, int whence = SEEK_SET) override;
+    unsigned long long tell() override { return m_pos; }
+    void reassign_context(PJ_CONTEXT *ctx) override { m_ctx = ctx; }
+
+    bool hasChanged() const override { return false; }
+
+    static std::unique_ptr<File> open(PJ_CONTEXT *ctx, const char *filename,
+                                      FileAccess access,
+                                      const unsigned char *data, size_t size) {
+        if (access != FileAccess::READ_ONLY)
+            return nullptr;
+        return std::unique_ptr<File>(new FileMemory(filename, ctx, data, size));
+    }
+};
+
+size_t FileMemory::read(void *buffer, size_t sizeBytes) {
+    if (m_pos >= m_size)
+        return 0;
+    if (sizeBytes >= m_size - m_pos) {
+        const size_t bytesToCopy = m_size - m_pos;
+        memcpy(buffer, m_data + m_pos, bytesToCopy);
+        m_pos = m_size;
+        return bytesToCopy;
+    }
+    memcpy(buffer, m_data + m_pos, sizeBytes);
+    m_pos += sizeBytes;
+    return sizeBytes;
+}
+
+size_t FileMemory::write(const void *, size_t) {
+    // shouldn't happen given we have bailed out in open() in non read-only
+    // modes
+    return 0;
+}
+
+bool FileMemory::seek(unsigned long long offset, int whence) {
+    if (whence == SEEK_SET) {
+        m_pos = static_cast<size_t>(offset);
+        return m_pos == offset;
+    } else if (whence == SEEK_CUR) {
+        const unsigned long long newPos = m_pos + offset;
+        m_pos = static_cast<size_t>(newPos);
+        return m_pos == newPos;
+    } else {
+        if (offset != 0)
+            return false;
+        m_pos = m_size;
+        return true;
+    }
+}
+
+#endif
+
+// ---------------------------------------------------------------------------
+
 std::unique_ptr<File> FileManager::open(PJ_CONTEXT *ctx, const char *filename,
                                         FileAccess access) {
     if (starts_with(filename, "http://") || starts_with(filename, "https://")) {
@@ -909,11 +983,31 @@ std::unique_ptr<File> FileManager::open(PJ_CONTEXT *ctx, const char *filename,
     if (ctx->fileApi.open_cbk != nullptr) {
         return FileApiAdapter::open(ctx, filename, access);
     }
+
+    std::unique_ptr<File> ret;
+#if !(USE_EMBEDDED_GRIDS && USE_ONLY_EMBEDDED_RESOURCE_FILES)
 #ifdef _WIN32
-    return FileWin32::open(ctx, filename, access);
+    ret = FileWin32::open(ctx, filename, access);
 #else
-    return FileStdio::open(ctx, filename, access);
+    ret = FileStdio::open(ctx, filename, access);
+#endif
+#endif
+
+#if USE_EMBEDDED_GRIDS
+#if USE_ONLY_EMBEDDED_RESOURCE_FILES
+    if (!ret)
 #endif
+    {
+        unsigned int size = 0;
+        const unsigned char *in_memory_data =
+            pj_get_embedded_grid(filename, &size);
+        if (in_memory_data) {
+            ret = FileMemory::open(ctx, filename, access, in_memory_data, size);
+        }
+    }
+#endif
+
+    return ret;
 }
 
 // ---------------------------------------------------------------------------

src/lib_proj.cmake

@@ -411,7 +411,6 @@ if (EMBED_RESOURCE_FILES AND NOT IS_SHARP_EMBED_AVAILABLE_RES)
         -P ${PROJECT_SOURCE_DIR}/cmake/FileEmbed.cmake
         DEPENDS generate_proj_db "${PROJECT_BINARY_DIR}/data/proj.db"
     )
-    target_sources(proj PRIVATE embedded_resources.c "${EMBEDDED_PROJ_DB}")
 
     set(EMBEDDED_PROJ_INI "file_embed/proj_ini.c")
     add_custom_command(
@@ -434,6 +433,65 @@ elseif(EMBED_RESOURCE_FILES AND IS_SHARP_EMBED_AVAILABLE_RES)
     set_target_properties(proj_resources PROPERTIES C_STANDARD 23)
     target_sources(proj PRIVATE $<TARGET_OBJECTS:proj_resources>)
 endif()
+
+set(EMBED_GRIDS_DIRECTORY "" CACHE PATH "Directory that contains .tif and .json files to embed into libproj")
+if (EMBED_GRIDS_DIRECTORY)
+    if (NOT EMBED_RESOURCE_FILES)
+        message(FATAL_ERROR "EMBED_RESOURCE_FILES should be set to ON when EMBED_GRIDS_DIRECTORY is set")
+    endif()
+
+    if (NOT IS_DIRECTORY ${EMBED_GRIDS_DIRECTORY})
+        message(FATAL_ERROR "${EMBED_GRIDS_DIRECTORY} is not a valid directory")
+    endif()
+
+    file(GLOB FILES "${EMBED_GRIDS_DIRECTORY}/*.tif" "${EMBED_GRIDS_DIRECTORY}/*.json")
+    if (NOT FILES)
+        message(FATAL_ERROR "No .tif or .json files found in ${EMBED_GRIDS_DIRECTORY}")
+    endif()
+    set(EMBEDDED_GRIDS_C_PROLOG_CONTENT "")
+    set(EMBEDDED_GRIDS_C_CONTENT "")
+    string(APPEND EMBEDDED_GRIDS_C_CONTENT
+        "const unsigned char *pj_get_embedded_grid(const char* filename, unsigned int *pnSize)\n"
+        "{\n")
+    foreach(FILE ${FILES})
+        get_filename_component(FILENAME ${FILE} NAME)
+        message(STATUS "Embedding ${FILENAME}")
+        set(C_IDENTIFIER_GRID_NAME "${FILENAME}")
+        string(REPLACE "." "_" C_IDENTIFIER_GRID_NAME "${C_IDENTIFIER_GRID_NAME}")
+        string(REPLACE "-" "_" C_IDENTIFIER_GRID_NAME "${C_IDENTIFIER_GRID_NAME}")
+        set(C_FILENAME "file_embed/${C_IDENTIFIER_GRID_NAME}.c")
+        add_custom_command(
+            OUTPUT "${C_FILENAME}"
+            COMMAND ${CMAKE_COMMAND}
+            -DRUN_FILE_EMBED_GENERATE=1
+            "-DFILE_EMBED_GENERATE_PATH=${FILE}"
+            -P ${PROJECT_SOURCE_DIR}/cmake/FileEmbed.cmake
+            DEPENDS "${FILE}"
+        )
+
+        string(APPEND EMBEDDED_GRIDS_C_CONTENT
+               "    if (strcmp(filename, \"${FILENAME}\") == 0)\n"
+               "    {\n"
+               "        *pnSize = ${C_IDENTIFIER_GRID_NAME}_size;\n"
+               "        return ${C_IDENTIFIER_GRID_NAME}_data;\n"
+               "    }\n")
+
+        target_sources(proj PRIVATE "${C_FILENAME}")
+
+        string(APPEND EMBEDDED_GRIDS_C_PROLOG_CONTENT "extern const uint8_t ${C_IDENTIFIER_GRID_NAME}_data[];\n")
+        string(APPEND EMBEDDED_GRIDS_C_PROLOG_CONTENT "extern const unsigned ${C_IDENTIFIER_GRID_NAME}_size;\n")
+    endforeach()
+    string(APPEND EMBEDDED_GRIDS_C_CONTENT
+           "    *pnSize = 0;\n"
+           "    return NULL;\n"
+           "}\n")
+    file(WRITE "file_embed/embedded_grids.c" "${EMBEDDED_GRIDS_C_PROLOG_CONTENT}\n${EMBEDDED_GRIDS_C_CONTENT}")
+    target_compile_definitions(proj PRIVATE USE_EMBEDDED_GRIDS)
+    if (IS_SHARP_EMBED_AVAILABLE_RES)
+        target_compile_definitions(proj_resources PRIVATE USE_EMBEDDED_GRIDS)
+    endif()
+endif()
+
 if (EMBED_RESOURCE_FILES)
     target_sources(proj PRIVATE memvfs.c)
     target_compile_definitions(proj PRIVATE EMBED_RESOURCE_FILES)