Merge pull request #2643 from philip-davis/dataspaces2

Update DataSpaces engine to support DataSpaces2

cmake/DetectOptions.cmake

@@ -216,9 +216,9 @@ endif()
 
 # DataSpaces
 if(ADIOS2_USE_DataSpaces STREQUAL AUTO)
-  find_package(DataSpaces 1.8)
+  find_package(DataSpaces 2.1.1)
 elseif(ADIOS2_USE_DataSpaces)
-  find_package(DataSpaces 1.8 REQUIRED)
+  find_package(DataSpaces 2.1.1 REQUIRED)
 endif()
 if(DATASPACES_FOUND)
   set(ADIOS2_HAVE_DataSpaces TRUE)

cmake/FindDataSpaces.cmake

@@ -34,7 +34,33 @@ if(NOT DATASPACES_FOUND)
       get_filename_component(DATASPACES_ROOT "${DSPACES_CONF}/../.." ABSOLUTE)
     endif()
   endif()
-  if(DATASPACES_ROOT)
+  find_package(PkgConfig)
+  if(PKG_CONFIG_FOUND)
+    set(_CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH})
+    if(DATASPACES_ROOT)
+      list(INSERT CMAKE_PREFIX_PATH 0 "${DATASPACES_ROOT}")
+    elseif(NOT ENV{DATASPACES_ROOT} STREQUAL "")
+      list(INSERT CMAKE_PREFIX_PATH 0 "$ENV{DATASPACES_ROOT}")
+    endif()
+    set(PKG_CONFIG_USE_CMAKE_PREFIX_PATH ON)
+    pkg_check_modules(PC_DSPACES dspaces)
+    set(CMAKE_PREFIX_PATH ${_CMAKE_PREFIX_PATH})
+    unset(_CMAKE_PREFIX_PATH)
+    if(PC_DSPACES_FOUND)
+      if(BUILD_SHARED_LIBS)
+        set(_PC_TYPE)
+      else()
+        set(_PC_TYPE _STATIC)
+      endif()
+      set(DATASPACES_LIBRARIES ${PC_DSPACES${_PC_TYPE}_LINK_LIBRARIES})
+      set(DATASPACES_LIBRARY_HINT ${PC_DSPACES${_PC_TYPE}_LIBRARY_DIRS})
+      set(DATASPACES_INCLUDE_DIR ${PC_DSPACES${_PC_TYPE}_INCLUDE_DIRS})
+      set(DATASPACES_VERSION ${PC_DSPACES_VERSION})
+      find_library(DATASPACES_LIBRARY dspaces HINTS ${DATASPACES_LIBRARY_HINT})
+      set(HAVE_DSPACES2 TRUE)
+    endif()
+  endif()
+  if(DATASPACES_ROOT AND NOT PC_DSPACES_FOUND)
     find_program(DSPACES_CONF dspaces_config ${DATASPACES_ROOT}/bin)
     if(DSPACES_CONF)
 	  execute_process(COMMAND ${DSPACES_CONF} -l
@@ -50,50 +76,47 @@ if(NOT DATASPACES_FOUND)
 	  foreach(LOOP_VAR ${LINK_LIBS})
 	    STRING(FIND ${LOOP_VAR} "-u" DEL_FLG)
 	    if(("${DEL_FLG}" EQUAL "-1"))
-	        STRING(FIND ${LOOP_VAR} "/" HINT_FLG)
-		if(NOT("${HINT_FLG}" EQUAL "-1"))
+	      STRING(FIND ${LOOP_VAR} "/" HINT_FLG)
+		  if(NOT("${HINT_FLG}" EQUAL "-1"))
 		    list(APPEND DATASPACES_LIBRARY_HINT ${LOOP_VAR})
-		else()
+		  else()
 		    unset(LOCAL_LIBRARY CACHE)
 		    unset(LOCAL_LIBRARY-NOTFOUND CACHE)
 		    STRING(FIND ${LOOP_VAR} "stdc++" CPP_FLG)
 		    if("${CPP_FLG}" EQUAL "-1")
-		        find_library(LOCAL_LIBRARY NAMES "${LOOP_VAR}" HINTS ${DATASPACES_LIBRARY_HINT})
-		        if(LOCAL_LIBRARY)
+		      find_library(LOCAL_LIBRARY NAMES "${LOOP_VAR}" HINTS ${DATASPACES_LIBRARY_HINT})
+		      if(LOCAL_LIBRARY)
 			    list(APPEND DATASPACES_LIBRARIES ${LOCAL_LIBRARY})
-		        else()
+		      else()
 			    list(APPEND DATASPACES_LIBRARIES ${LOOP_VAR})
-		        endif()
+		      endif()
 		    endif()
-		endif()
-	      endif()
-  	    endforeach()
-	execute_process(COMMAND ${DSPACES_CONF} -v
+		  endif()
+	    endif()
+  	  endforeach()
+	  execute_process(COMMAND ${DSPACES_CONF} -v
         RESULT_VARIABLE RESULT_VAR
         OUTPUT_VARIABLE DATASPACES_VERSION
         ERROR_QUIET
         OUTPUT_STRIP_TRAILING_WHITESPACE)	
-    endif ()
-
+    endif()
     set(DATASPACES_INCLUDE_OPTS HINTS ${DATASPACES_ROOT}/include)
-
+    find_path(DATASPACES_INCLUDE_DIR dataspaces.h ${DATASPACES_INCLUDE_OPTS})
+    find_library(DATASPACES_LIBRARY dspaces HINTS ${DATASPACES_LIBRARY_HINT}) 
   endif()
 
-  find_path(DATASPACES_INCLUDE_DIR dataspaces.h ${DATASPACES_INCLUDE_OPTS})
-  find_library(DSPACES_LIBRARY dspaces HINTS ${DATASPACES_LIBRARY_HINT}) 
-
   include(FindPackageHandleStandardArgs)
   find_package_handle_standard_args(DataSpaces
     FOUND_VAR DATASPACES_FOUND
     VERSION_VAR DATASPACES_VERSION
     REQUIRED_VARS DATASPACES_VERSION DATASPACES_INCLUDE_DIR 
-      DATASPACES_LIBRARIES DSPACES_LIBRARY
+      DATASPACES_LIBRARIES #DATASPACES_LIBRARY
   )
   if(DATASPACES_FOUND)
     if(DATASPACES_FOUND AND NOT TARGET DataSpaces::DataSpaces)
       add_library(DataSpaces::DataSpaces UNKNOWN IMPORTED)
       set_target_properties(DataSpaces::DataSpaces PROPERTIES
-       	IMPORTED_LOCATION             "${DSPACES_LIBRARY}"
+       	IMPORTED_LOCATION             "${DATASPACES_LIBRARY}"
         INTERFACE_LINK_LIBRARIES      "${DATASPACES_LIBRARIES}"
         INTERFACE_INCLUDE_DIRECTORIES "${DATASPACES_INCLUDE_DIR}"
       )

cmake/FindLIBFABRIC.cmake

@@ -30,11 +30,11 @@ if(NOT (PC_LIBFABRIC_FOUND STREQUAL "IGNORE"))
       if(BUILD_SHARED_LIBS)
         set(_PC_TYPE)
       else()
-        set(_PF_TYPE _STATIC)
+        set(_PC_TYPE _STATIC)
       endif()
       set(LIBFABRIC_INCLUDE_DIRS ${PC_LIBFABRIC${_PC_TYPE}_INCLUDE_DIRS})
       set(LIBFABRIC_LIBRARIES ${PC_LIBFABRIC${_PC_TYPE}_LINK_LIBRARIES})
-      set(LIBFABRIC_DEFINITIONS ${PC_LIBFABRIC${PC_TYPE}_CFLAGS_OTHER})
+      set(LIBFABRIC_DEFINITIONS ${PC_LIBFABRIC${_PC_TYPE}_CFLAGS_OTHER})
     endif()
   endif()
 endif()

source/adios2/CMakeLists.txt

@@ -213,6 +213,7 @@ if(ADIOS2_HAVE_SST)
 endif()
 
 if(ADIOS2_HAVE_DataSpaces)
+  add_subdirectory(toolkit/dataspaces)
   target_sources(adios2_core_mpi PRIVATE
      toolkit/dataspaces/ds_writer.c
      engine/dataspaces/DataSpacesWriter.cpp engine/dataspaces/DataSpacesWriter.tcc 

source/adios2/engine/dataspaces/DataSpacesReader.cpp

@@ -14,8 +14,13 @@
 #include "DataSpacesReader.tcc"
 #include "adios2/helper/adiosCommMPI.h"
 #include "adios2/helper/adiosFunctions.h" //CSVToVector
+#include "adios2/toolkit/dataspaces/DSpacesConfig.h"
 #include "adios2/toolkit/dataspaces/ds_data.h"
+#ifdef HAVE_DSPACES2
+#include "dspaces.h"
+#else
 #include "dataspaces.h"
+#endif /* HAVE_DSPACES2 */
 
 namespace adios2
 {
@@ -83,23 +88,47 @@ StepStatus DataSpacesReader::BeginStep(StepMode mode, const float timeout_sec)
     strcpy(meta_lk, lk_name.c_str());
 
     MPI_Comm lock_comm = m_data.mpi_comm;
+#ifndef HAVE_DSPACES2
     dspaces_lock_on_read(meta_lk, &lock_comm);
+#endif /* HAVE_DSPACES2 */
 
     int nVars = 0;
     if (!m_ProvideLatest)
     {
         if (rank == 0)
         {
+#ifdef HAVE_DSPACES2
+            dspaces_client_t *client = get_client_handle();
+            char meta_str[256];
+            unsigned int metalen;
+            sprintf(meta_str, "VARMETA@%s", fstr);
+            int err = dspaces_get_meta(*client, meta_str, META_MODE_NEXT,
+                                       m_CurrentStep, &bcast_array[1],
+                                       (void **)&buffer, &metalen);
+            bcast_array[0] = metalen;
+#else
             buffer = dspaces_get_next_meta(m_CurrentStep, fstr, &bcast_array[0],
                                            &bcast_array[1]);
+#endif /* HAVE_DSPACES2 */
         }
     }
     else
     {
         if (rank == 0)
         {
+#ifdef HAVE_DSPACES2
+            dspaces_client_t *client = get_client_handle();
+            char meta_str[256];
+            unsigned int metalen;
+            sprintf(meta_str, "VARMETA@%s", fstr);
+            int err = dspaces_get_meta(*client, meta_str, META_MODE_LAST,
+                                       m_CurrentStep, &bcast_array[1],
+                                       (void **)&buffer, &metalen);
+            bcast_array[0] = metalen;
+#else
             buffer = dspaces_get_latest_meta(m_CurrentStep, fstr,
                                              &bcast_array[0], &bcast_array[1]);
+#endif /* HAVE_DSPACES2 */
         }
     }
     MPI_Bcast(bcast_array, 2, MPI_INT, 0, m_data.mpi_comm);
@@ -228,12 +257,13 @@ void DataSpacesReader::EndStep()
     strcpy(meta_lk, lk_name.c_str());
 
     MPI_Comm lock_comm = m_data.mpi_comm;
+#ifndef HAVE_DSPACES2
     dspaces_unlock_on_read(meta_lk, &lock_comm);
+#endif /* HAVE_DSPACES2 */
 }
 
 void DataSpacesReader::DoClose(const int transportIndex)
 {
-
     globals_adios_set_dataspaces_disconnected_from_reader();
 }
 

source/adios2/engine/dataspaces/DataSpacesReader.tcc

@@ -15,7 +15,12 @@
 
 #include "DataSpacesReader.h"
 #include "adios2/helper/adiosFunctions.h" //CSVToVector
+#include "adios2/toolkit/dataspaces/DSpacesConfig.h"
+#ifdef HAVE_DSPACES2
+#include "dspaces.h"
+#else
 #include "dataspaces.h"
+#endif /* HAVE_DSPACES2 */
 
 namespace adios2
 {
@@ -67,7 +72,11 @@ void DataSpacesReader::ReadDsData(Variable<T> &variable, T *data, int version)
         */
     if (variable.m_Shape.size() == 0)
     {
+#ifndef HAVE_DSPACES2
         gdims_in[0] = dspaces_get_num_space_server();
+#else
+        gdims_in[0] = 1;
+#endif
         lb_in[0] = 0;
         ub_in[0] = 0;
         ndims = 1;
@@ -108,11 +117,16 @@ void DataSpacesReader::ReadDsData(Variable<T> &variable, T *data, int version)
     std::string l_Name = ds_in_name + std::to_string(version);
     char *cstr = new char[l_Name.length() + 1];
     strcpy(cstr, l_Name.c_str());
-
+#ifdef HAVE_DSPACES2
+    dspaces_client_t *client = get_client_handle();
+    dspaces_define_gdim(*client, var_str, ndims, gdims_in);
+    dspaces_get(*client, var_str, version, variable.m_ElementSize, ndims, lb_in,
+                ub_in, (void *)data, -1);
+#else
     dspaces_define_gdim(var_str, ndims, gdims_in);
     dspaces_get(var_str, version, variable.m_ElementSize, ndims, lb_in, ub_in,
                 (void *)data);
-
+#endif /* HAVE_DSPACES2 */
     delete[] cstr;
     delete[] var_str;
 }

source/adios2/engine/dataspaces/DataSpacesWriter.cpp

@@ -15,8 +15,13 @@
 #include "DataSpacesWriter.tcc"
 #include "adios2/helper/adiosCommMPI.h"
 #include "adios2/helper/adiosFunctions.h" //CSVToVector
+#include "adios2/toolkit/dataspaces/DSpacesConfig.h"
 #include "adios2/toolkit/dataspaces/ds_data.h"
+#ifdef HAVE_DSPACES2
+#include "dspaces.h"
+#else
 #include "dataspaces.h"
+#endif /* HAVE_DSPACES2 */
 
 namespace adios2
 {
@@ -73,10 +78,13 @@ void DataSpacesWriter::EndStep()
     char *meta_lk = new char[local_file_var.length() + 1];
     strcpy(meta_lk, local_file_var.c_str());
     MPI_Comm lock_comm = m_data.mpi_comm;
-
+#ifndef HAVE_DSPACES2
     dspaces_lock_on_write(meta_lk, &lock_comm);
+#endif /* HAVE_DSPACES2 */
     WriteVarInfo();
+#ifndef HAVE_DSPACES2
     dspaces_unlock_on_write(meta_lk, &lock_comm);
+#endif /* HAVE_DSPACES2 */
 }
 void DataSpacesWriter::Flush(const int transportIndex) {}
 
@@ -88,9 +96,23 @@ void DataSpacesWriter::DoClose(const int transportIndex)
     char *meta_lk = new char[local_file_var.length() + 1];
     strcpy(meta_lk, local_file_var.c_str());
 
+#ifdef HAVE_DSPACES2
+    int rank;
+    MPI_Comm_rank(m_data.mpi_comm, &rank);
+    if (rank == 0)
+    {
+        local_file_var = "VARMETA@" + f_Name;
+        char *local_str = new char[local_file_var.length() + 1];
+        strcpy(local_str, local_file_var.c_str());
+        dspaces_client_t *client = get_client_handle();
+        dspaces_put_meta(*client, local_str, m_CurrentStep + 1, NULL, 0);
+        delete[] local_str;
+    }
+    MPI_Barrier(m_data.mpi_comm);
+#else
     dspaces_lock_on_write(meta_lk, &(m_data.mpi_comm));
     dspaces_unlock_on_write(meta_lk, &(m_data.mpi_comm));
-
+#endif /* HAVE_DSPACES2 */
     globals_adios_set_dataspaces_disconnected_from_writer();
 }
 
@@ -122,13 +144,12 @@ void DataSpacesWriter::WriteVarInfo()
 
     if (rank == 0)
     {
-
         std::string ds_file_var;
         int var_num = ndim_vector.size();
         int var_name_max_length = 128;
-        int buf_len = var_num * sizeof(int) + var_num * sizeof(int) +
-                      MAX_DS_NDIM * var_num * sizeof(uint64_t) +
-                      var_num * var_name_max_length * sizeof(char);
+        int buf_len =
+            var_num * (2 * sizeof(int) + MAX_DS_NDIM * sizeof(uint64_t) +
+                       var_name_max_length * sizeof(char));
         int *dim_meta, *elemSize_meta;
         uint64_t *gdim_meta;
         dim_meta = (int *)malloc(var_num * sizeof(int));
@@ -174,12 +195,17 @@ void DataSpacesWriter::WriteVarInfo()
         ndim = 1;
         lb[0] = 0;
         ub[0] = buf_len - 1;
+#ifdef HAVE_DSPACES2
+        dspaces_client_t *client = get_client_handle();
+        dspaces_put_meta(*client, local_str, m_CurrentStep, buffer, buf_len);
+#else
         gdims[0] = (ub[0] - lb[0] + 1) * dspaces_get_num_space_server();
         dspaces_define_gdim(local_str, ndim, gdims);
 
         dspaces_put(local_str, m_CurrentStep, elemsize, ndim, lb, ub, buffer);
 
         dspaces_put_sync(); // wait on previous put to finish
+#endif /* HAVE_DSPACES2 */
         delete[] local_str;
         free(dim_meta);
         free(elemSize_meta);

source/adios2/engine/dataspaces/DataSpacesWriter.tcc

@@ -15,8 +15,14 @@
 
 #include "DataSpacesWriter.h"
 #include "adios2/helper/adiosFunctions.h"
+#include "adios2/toolkit/dataspaces/DSpacesConfig.h"
+#include "adios2/toolkit/dataspaces/ds.h"
 #include "adios2/toolkit/dataspaces/ds_data.h"
+#ifdef HAVE_DSPACES2
+#include "dspaces.h"
+#else
 #include "dataspaces.h"
+#endif /* HAVE_DSPACES2 */
 
 namespace adios2
 {
@@ -48,7 +54,11 @@ void DataSpacesWriter::DoPutSyncCommon(Variable<T> &variable, const T *values)
         */
     if (variable.m_SingleValue)
     {
+#ifdef HAVE_DSPACES2
+        gdims_in[0] = 1;
+#else
         gdims_in[0] = dspaces_get_num_space_server();
+#endif /* HAVE_DSPACES2 */
         lb_in[0] = 0;
         ub_in[0] = 0;
         ndims = 1;
@@ -110,11 +120,18 @@ void DataSpacesWriter::DoPutSyncCommon(Variable<T> &variable, const T *values)
     char *cstr = new char[l_Name.length() + 1];
     strcpy(cstr, l_Name.c_str());
 
+#ifdef HAVE_DSPACES2
+    dspaces_client_t *client = get_client_handle();
+    dspaces_define_gdim(*client, var_str, ndims, gdims_in);
+    dspaces_put(*client, var_str, version, variable.m_ElementSize, ndims, lb_in,
+                ub_in, values);
+#else
     dspaces_define_gdim(var_str, ndims, gdims_in);
     dspaces_put(var_str, version, variable.m_ElementSize, ndims, lb_in, ub_in,
                 values);
     dspaces_put_sync();
     dspaces_put_sync();
+#endif /* HAVE_DSPACES2 */
     delete[] cstr;
     delete[] var_str;
 }