Improve UTF8 Support On Windows

.github/workflows/run_tests_osx.yml

@@ -7,7 +7,7 @@
 name: Run macOS-based netCDF Tests
 
 
-on: [pull_request]
+on: [pull_request,push]
 
 jobs:
 

.github/workflows/run_tests_ubuntu.yml

@@ -4,7 +4,7 @@
 
 name: Run Ubuntu/Linux netCDF Tests
 
-on: [pull_request]
+on: [pull_request,push]
 
 jobs:
 

.github/workflows/run_tests_win_mingw.yml

@@ -7,7 +7,7 @@
 name: Run MSYS2, MinGW64-based Tests
 
 
-on: [ pull_request ]
+on: [pull_request,push]
 
 jobs:
 

CMakeLists.txt

@@ -55,6 +55,21 @@ IF(UNAME)
   set(TMP_BUILDNAME "${osname}-${osrel}-${cpu}")
 ENDIF()
 
+# Define some Platforms
+if(osname MATCHES "CYGWIN.*")
+SET(ISCYGWIN yes)
+endif()
+if(osname MATCHES "Darwin.*")
+SET(ISOSX yes)
+endif()
+if(MSVC)
+SET(ISMSVC yes)
+endif()
+if(osname MATCHES "MINGW.*")
+SET(ISMINGW yes)
+SET(MINGW yes)
+endif()
+
 ###
 # Allow for some customization of the buildname.
 # This will make it easier to identify different builds,
@@ -86,6 +101,7 @@ INCLUDE(GNUInstallDirs)
 
 IF(MSVC)
   SET(GLOBAL PROPERTY USE_FOLDERS ON)
+  ADD_COMPILE_OPTIONS("/utf-8")
 ENDIF()
 
 #Add custom CMake Module
@@ -412,7 +428,6 @@ IF(NOT MSVC)
   ENDIF(BUILD_FORTRAN)
 ENDIF()
 
-
 ###
 # Allow the user to specify libraries
 # to link against, similar to automakes 'LIBS' variable.
@@ -1794,6 +1809,26 @@ IF(ENABLE_MMAP)
 ENDIF(ENABLE_MMAP)
 
 #CHECK_FUNCTION_EXISTS(alloca HAVE_ALLOCA)
+
+# Used in the `configure_file` calls below
+SET(ISCMAKE "1")
+IF(MSVC)
+SET(ISMSVC ON CACHE BOOL "" FORCE)
+SET(REGEDIT ON CACHE BOOL "" FORCE)
+# Get windows major version and build number
+EXECUTE_PROCESS(COMMAND "systeminfo" OUTPUT_VARIABLE WININFO)
+IF(WININFO STREQUAL "")
+  SET(WVM 0)
+  SET(WVB 0)
+ELSE()
+  STRING(REGEX MATCH "\nOS Version:[ \t]+[0-9.]+" WINVERLINE "${WININFO}")
+  STRING(REGEX REPLACE "[^0-9]*([0-9]+)[.]([0-9])+[.]([0-9]+)" "\\1" WVM "${WINVERLINE}")
+  STRING(REGEX REPLACE "[^0-9]*([0-9]+)[.]([0-9])+[.]([0-9]+)" "\\3" WVB "${WINVERLINE}")
+ENDIF()
+SET(WINVERMAJOR ${WVM} CACHE STRING "" FORCE)
+SET(WINVERBUILD ${WVB} CACHE STRING "" FORCE)
+ENDIF()
+
 #####
 # End system inspection checks.
 #####
@@ -2418,13 +2453,6 @@ configure_file(
   ${netCDF_SOURCE_DIR}/include/netcdf_dispatch.h.in
   ${netCDF_BINARY_DIR}/include/netcdf_dispatch.h @ONLY NEWLINE_STYLE LF)
 
-# Used in the `configure_file` calls below
-SET(ISCMAKE "1")
-IF(MSVC)
-SET(ISMSVC "1")
-SET(REGEDIT 1)
-ENDIF()
-
 ####
 # Build test_common.sh
 #####

RELEASE_NOTES.md

@@ -7,8 +7,9 @@ This file contains a high-level description of this package's evolution. Release
 
 ## 4.8.2 - TBD
 
+* [Bug Fix] Improve UTF8 support on windows so that it can use utf8 natively. See [Github #2222](https://github.com/Unidata/netcdf-c/pull/2222).
 * [Enhancement] Add complete bitgroom support to NCZarr. See [Github #2197](https://github.com/Unidata/netcdf-c/pull/2197).
-* [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179(https://github.com/Unidata/netcdf-c/pull/2179).
+* [Bug Fix] Clean up the handling of deeply nested VLEN types. Marks nc_free_vlen() and nc_free_string as deprecated in favor of ncaux_reclaim_data(). See [Github #2179](https://github.com/Unidata/netcdf-c/pull/2179).
 * [Bug Fix] Make sure that netcdf.h accurately defines the flags in the open/create mode flags. See [Github #2183](https://github.com/Unidata/netcdf-c/pull/2183).
 * [Enhancement] Improve support for msys2+mingw platform. See [Github #2171](https://github.com/Unidata/netcdf-c/pull/2171).
 * [Bug Fix] Clean up the various inter-test dependencies in ncdump for CMake. See [Github #2168](https://github.com/Unidata/netcdf-c/pull/2168).

config.h.cmake.in

@@ -157,12 +157,6 @@ are set when opening a binary file on Windows. */
 /* if true, Allow dynamically loaded plugins */
 #cmakedefine ENABLE_PLUGINS 1
 
-/* Do we have access to the Windows Registry */
-#cmakedefine REGEDIT 1
-
-/* define the possible sources for remote test servers */
-#cmakedefine REMOTETESTSERVERS	"${REMOTETESTSERVERS}"
-
 /* if true, run extra tests which may not work yet */
 #cmakedefine EXTRA_TESTS 1
 
@@ -505,6 +499,12 @@ with zip */
 /* Define to the version of this package. */
 #cmakedefine PACKAGE_VERSION "${netCDF_VERSION}"
 
+/* Do we have access to the Windows Registry */
+#cmakedefine REGEDIT 1
+
+/* define the possible sources for remote test servers */
+#cmakedefine REMOTETESTSERVERS	"${REMOTETESTSERVERS}"
+
 /* The size of `ulonglong` as computed by sizeof. */
 #cmakedefine SIZEOF_ULONGLONG @SIZEOF_ULONGLONG@
 
@@ -626,6 +626,10 @@ with zip */
 /* Version number of package */
 #cmakedefine VERSION "${netCDF_VERSION}"
 
+/* Capture  Windows version and build */
+#cmakedefine WINVERMAJOR ${WINVERMAJOR}
+#cmakedefine WINVERBUILD ${WINVERBUILD}
+
 /* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
    significant byte first (like Motorola and SPARC, unlike Intel). */
 #if defined AC_APPLE_UNIVERSAL_BUILD

configure.ac

@@ -106,6 +106,15 @@ ISMINGW=yes
 ISMSYS=yes
 fi
 
+# Get windows version info
+WINVER=`systeminfo | sed -e '/^OS Version:/p' -ed | sed -e 's|[^0-9]*\([0-9.]*\).*|\1|'`
+WINVERMAJOR=`echo $WVER | sed -e 's|\([^.]*\)[.]\([^.]*\)[.]\(.*\)|\1|'`
+WINVERBUILD=`echo $WVER | sed -e 's|\([^.]*\)[.]\([^.]*\)[.]\(.*\)|\3|'`
+if test "x$WINVERMAJOR" = x ; then WINVERMAJOR=0; fi
+if test "x$WINVERBUILD" = x ; then WINVERBUILD=0; fi
+AC_DEFINE_UNQUOTED([WINVERMAJOR], [$WINVERMAJOR], [windows version major])
+AC_DEFINE_UNQUOTED([WINVERBUILD], [$WINVERBUILD], [windows version build])
+
 AC_MSG_NOTICE([checking supported formats])
 
 # An explicit disable of netcdf-4 | netcdf4 is treated as if it was disable-hdf5
@@ -1218,6 +1227,8 @@ AM_CONDITIONAL(ISMINGW, [test "x$ISMINGW" = xyes])
 AM_CONDITIONAL(ISMSYS, [test "x$ISMSYS" = xyes])
 
 AC_SUBST([ISMSVC], [${ISMSVC}])
+  AC_SUBST([WINVERMAJOR], [${WINVERMAJOR}])
+  AC_SUBST([WINVERBUILD], [${WINVERBUILD}])
 AC_SUBST([ISCYGWIN], [${ISCYGWIN}])
 AC_SUBST([ISOSX], [${ISOSX}])
 AC_SUBST([ISMINGW], [${ISMINGW}])

dap4_test/test_common.h

@@ -26,26 +26,6 @@ static char* outfile = NULL;
 static int ncid = 0;
 static int translatenc4 = 0;
 
-static int
-readfile(const char* filename, NCbytes* content)
-{
-    FILE* stream;
-    char part[1024];
-
-    stream = fopen(filename,"r");
-    if(stream == NULL) return errno;
-    for(;;) {
-	size_t count = fread(part, 1, sizeof(part), stream);
-	if(count <= 0) break;
-	ncbytesappendn(content,part,count);
-	if(ferror(stream)) {fclose(stream); return NC_EIO;}
-	if(feof(stream)) break;
-    }
-    ncbytesnull(content);
-    fclose(stream);
-    return NC_NOERR;
-}
-
 static void
 fail(int code)
 {
@@ -86,7 +66,7 @@ setup(int tdmr, int argc, char** argv)
     outfile = NULL;
     input = ncbytesnew();
     output = ncbytesnew();
-    if((ret = readfile(infile,input))) fail(ret);
+    if((ret = NC_readfile(infile,input))) fail(ret);
     {
 	const char* trans = getenv("translatenc4");
 	if(trans != NULL)

dap4_test/test_data.c

@@ -12,7 +12,7 @@ Test the netcdf-4 data building process.
 #include <stdio.h>
 #include "netcdf.h"
 
-#define DEBUG
+#undef DEBUG
 
 static void
 fail(int code)

include/nc_tests.h

@@ -20,6 +20,8 @@
 #include "netcdf.h"
 #include "netcdf_filter.h"
 #include "nc_logging.h"
+#include "ncpathmgr.h"
+#include "ncrc.h"
 #ifdef USE_PARALLEL
 #include "netcdf_par.h"
 #endif

include/ncrc.h

@@ -103,6 +103,8 @@ EXTERNL int NC__testurl(const char* path, char** basenamep);
 EXTERNL int NC_isLittleEndian(void);
 EXTERNL char* NC_entityescape(const char* s);
 EXTERNL int NC_readfile(const char* filename, NCbytes* content);
+EXTERNL int NC_readfilen(const char* filename, NCbytes* content, long long len);
+EXTERNL int NC_readfileF(FILE* fp, NCbytes* content, long long len);
 EXTERNL int NC_writefile(const char* filename, size_t size, void* content);
 EXTERNL char* NC_mktmp(const char* base);
 EXTERNL int NC_getmodelist(const char* modestr, NClist** modelistp);

libdap2/dceconstraints.c

@@ -13,7 +13,7 @@
 #include "dapincludes.h"
 #include "dceparselex.h"
 
-#define DEBUG
+#undef DEBUG
 
 #define LBRACE "{"
 #define RBRACE "}"

libdap4/d4file.c

@@ -395,17 +395,17 @@ set_curl_properties(NCD4INFO* d4info)
 	FILE* f = NULL;
 	char* fname = d4info->auth->curlflags.cookiejar;
 	/* See if the file exists already */
-        f = fopen(fname,"r");
+        f = NCfopen(fname,"r");
 	if(f == NULL) {
 	    /* Ok, create it */
-	    f = fopen(fname,"w+");
+	    f = NCfopen(fname,"w+");
 	    if(f == NULL) {
 	        fprintf(stderr,"Cookie file cannot be read and written: %s\n",fname);
 	        {ret= NC_EPERM; goto fail;}
 	    }
 	} else { /* test if file can be written */
 	    fclose(f);
-	    f = fopen(fname,"r+");
+	    f = NCfopen(fname,"r+");
 	    if(f == NULL) {
 	        fprintf(stderr,"Cookie file is cannot be written: %s\n",fname);
 	        {ret = NC_EPERM; goto fail;}

libdap4/d4includes.h

@@ -42,6 +42,7 @@
 #include "ncuri.h"
 #include "nclog.h"
 #include "ncdap.h"
+#include "ncpathmgr.h"
 
 #include "d4util.h"
 

libdap4/d4util.c

@@ -346,73 +346,6 @@ NCD4_elidenuls(char* s, size_t slen)
     return j;
 }
 
-int
-NCD4_readfile(const char* filename, NCbytes* content)
-{
-    int ret = NC_NOERR;
-    FILE* stream = NULL;
-    char part[1024];
-
-    stream = fopen(filename,"r");
-    if(stream == NULL) {ret=errno; goto done;}
-    for(;;) {
-	size_t count = fread(part, 1, sizeof(part), stream);
-	if(count <= 0) break;
-	ncbytesappendn(content,part,count);
-	if(ferror(stream)) {ret = NC_EIO; goto done;}
-	if(feof(stream)) break;
-    }
-    ncbytesnull(content);
-done:
-    if(stream) fclose(stream);
-    return ret;
-}
-
-/**
-Wrap mktmp and return the generated name
-*/
-
-int
-NCD4_mktmp(const char* base, char** tmpnamep)
-{
-    int fd;
-    char tmp[NC_MAX_PATH];
-#ifdef HAVE_MKSTEMP
-    mode_t mask;
-#endif
-
-    strncpy(tmp,base,sizeof(tmp));
-#ifdef HAVE_MKSTEMP
-    strlcat(tmp,"XXXXXX", sizeof(tmp));
-    /* Note Potential problem: old versions of this function
-       leave the file in mode 0666 instead of 0600 */
-    mask=umask(0077);
-    fd = mkstemp(tmp);
-    (void)umask(mask);
-#else /* !HAVE_MKSTEMP */
-    /* Need to simulate by using some kind of pseudo-random number */
-    {
-	int rno = rand();
-	char spid[7];
-	if(rno < 0) rno = -rno;
-        snprintf(spid,sizeof(spid),"%06d",rno);
-        strlcat(tmp,spid,sizeof(tmp));
-#if defined(_WIN32) || defined(_WIN64)
-        fd=open(tmp,O_RDWR|O_BINARY|O_CREAT, _S_IREAD|_S_IWRITE);
-#  else
-        fd=open(tmp,O_RDWR|O_CREAT|O_EXCL, S_IRWXU);
-#  endif
-    }
-#endif /* !HAVE_MKSTEMP */
-    if(fd < 0) {
-       nclog(NCLOGERR, "Could not create temp file: %s",tmp);
-       return THROW(NC_EPERM);
-    } else
-	close(fd);
-    if(tmpnamep) *tmpnamep = strdup(tmp);
-    return THROW(NC_NOERR);
-}
-
 void
 NCD4_hostport(NCURI* uri, char* space, size_t len)
 {