Remote fdb

CMakeLists.txt

@@ -33,13 +33,6 @@ ecbuild_add_option( FEATURE GRIB
 
 ecbuild_find_package( NAME metkit VERSION 1.11.22 REQUIRED )
 
-### FDB backend in persistent memory, i.e. pmem (NVRAM)
-
-ecbuild_add_option( FEATURE PMEMFDB # option present in fdb5_config.h
-                    DEFAULT OFF
-                    DESCRIPTION "Persistent memory (NVRAM) support for FDB"
-                    REQUIRED_PACKAGES "NAME pmem" )
-
 ### FDB backend in CEPH object store (using Rados)
 
 find_package( RADOS QUIET )
@@ -166,6 +159,7 @@ add_subdirectory( tests )
 
 ### finalize
 
+
 ecbuild_install_project( NAME fdb )
 
 ecbuild_print_summary()

VERSION

@@ -1 +1 @@
-5.13.3
+5.13.105

src/fdb5/CMakeLists.txt

@@ -80,8 +80,6 @@ list( APPEND fdb5_srcs
     database/BaseArchiveVisitor.h
     database/Catalogue.cc
     database/Catalogue.h
-    database/DB.cc
-    database/DB.h
     database/DatabaseNotFoundException.cc
     database/DatabaseNotFoundException.h
     database/DataStats.cc
@@ -126,7 +124,6 @@ list( APPEND fdb5_srcs
     database/IndexFactory.h
     database/Key.cc
     database/Key.h
-    database/ReadVisitor.cc
     database/ReadVisitor.h
     database/Report.cc
     database/Report.h
@@ -153,6 +150,8 @@ list( APPEND fdb5_srcs
     io/LustreFileHandle.h
     io/HandleGatherer.cc
     io/HandleGatherer.h
+    io/FieldHandle.cc
+    io/FieldHandle.h
     rules/MatchAlways.cc
     rules/MatchAlways.h
     rules/MatchAny.cc
@@ -220,18 +219,37 @@ if(HAVE_FDB_REMOTE)
         api/RemoteFDB.cc
         api/RemoteFDB.h
 
-        remote/RemoteConfiguration.h
-        remote/RemoteConfiguration.cc
+        remote/Connection.h
+        remote/Connection.cc
         remote/RemoteFieldLocation.h
         remote/RemoteFieldLocation.cc
         remote/Messages.h
         remote/Messages.cc
-        remote/Handler.h
-        remote/Handler.cc
-        remote/AvailablePortList.cc
-        remote/AvailablePortList.h
         remote/FdbServer.h
         remote/FdbServer.cc
+
+        remote/client/Client.h
+        remote/client/Client.cc
+        remote/client/ClientConnection.h
+        remote/client/ClientConnection.cc
+        remote/client/ClientConnectionRouter.h
+        remote/client/ClientConnectionRouter.cc
+        remote/client/RemoteStore.cc
+        remote/client/RemoteStore.h
+        remote/client/RemoteCatalogue.h
+        remote/client/RemoteCatalogue.cc
+
+        remote/server/AvailablePortList.cc
+        remote/server/AvailablePortList.h
+        remote/server/CatalogueHandler.h
+        remote/server/CatalogueHandler.cc
+        remote/server/StoreHandler.h
+        remote/server/StoreHandler.cc
+        remote/server/ServerConnection.h
+        remote/server/ServerConnection.cc
+
+        # remote/RemoteConfiguration.h
+        # remote/RemoteConfiguration.cc
     )
 endif()
 
@@ -310,58 +328,6 @@ if( HAVE_TOCFDB )
         fdb-reconsolidate-toc )
 endif()
 
-if( HAVE_PMEMFDB )
-
-    list( APPEND fdb5_srcs
-        pmem/DataPool.cc
-        pmem/DataPool.h
-        pmem/DataPoolManager.cc
-        pmem/DataPoolManager.h
-        pmem/MemoryBufferStream.h
-        pmem/MemoryBufferStream.cc
-        pmem/PBaseNode.cc
-        pmem/PBaseNode.h
-        pmem/PBranchingNode.cc
-        pmem/PBranchingNode.h
-        pmem/PDataNode.cc
-        pmem/PDataNode.h
-        pmem/PDataRoot.cc
-        pmem/PDataRoot.h
-        pmem/PIndexRoot.cc
-        pmem/PIndexRoot.h
-        pmem/PMemDB.cc
-        pmem/PMemDB.h
-        pmem/PMemDBReader.cc
-        pmem/PMemDBReader.h
-        pmem/PMemDBWriter.cc
-        pmem/PMemDBWriter.h
-        pmem/PMemFieldLocation.cc
-        pmem/PMemFieldLocation.h
-        pmem/PMemIndex.cc
-        pmem/PMemIndex.h
-        pmem/PMemIndexLocation.cc
-        pmem/PMemIndexLocation.h
-        pmem/PRoot.cc
-        pmem/PRoot.h
-        pmem/Pool.cc
-        pmem/Pool.h
-        pmem/PoolEntry.cc
-        pmem/PoolEntry.h
-        pmem/PoolGroup.cc
-        pmem/PoolGroup.h
-        pmem/PoolManager.cc
-        pmem/PoolManager.h
-        pmem/PMemStats.cc
-        pmem/PMemStats.h
-        pmem/PMemEngine.cc
-        pmem/PMemEngine.h
-    )
-
-else()
-    set( PMEM_LIBRARIES "" )
-    set( PMEM_INCLUDE_DIRS "" )
-endif()
-
 if( HAVE_RADOSFDB )
     list( APPEND fdb5_srcs
         rados/RadosFieldLocation.cc

src/fdb5/LibFdb5.h

@@ -19,7 +19,8 @@
 
 #include "eckit/system/Library.h"
 
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
+#include "fdb5/types/TypesRegistry.h"
 #include "fdb5/api/helpers/Callback.h"
 
 namespace fdb5 {

src/fdb5/api/DistFDB.h

@@ -47,6 +47,8 @@ class DistFDB : public FDBBase {
 
     ListIterator list(const FDBToolRequest& request) override;
 
+    AxesIterator axesIterator(const FDBToolRequest& request, int level=3) override { NOTIMP; }
+
     DumpIterator dump(const FDBToolRequest& request, bool simple) override;
 
     StatusIterator status(const FDBToolRequest& request) override;

src/fdb5/api/FDB.cc

@@ -28,6 +28,7 @@
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/database/Key.h"
 #include "fdb5/io/HandleGatherer.h"
+#include "fdb5/io/FieldHandle.h"
 #include "fdb5/message/MessageDecoder.h"
 #include "fdb5/types/Type.h"
 
@@ -145,13 +146,6 @@ bool FDB::sorted(const metkit::mars::MarsRequest &request) {
     return sorted;
 }
 
-class ListElementDeduplicator : public metkit::hypercube::Deduplicator<ListElement> {
-public:
-    bool toReplace(const ListElement& existing, const ListElement& replacement) const override {
-        return existing.timestamp() < replacement.timestamp();
-    }
-};
-
 eckit::DataHandle* FDB::read(const eckit::URI& uri) {
     auto location = std::unique_ptr<FieldLocation>(FieldLocationFactory::instance().build(uri.scheme(), uri));
     return location->dataHandle();
@@ -168,7 +162,12 @@ eckit::DataHandle* FDB::read(const std::vector<eckit::URI>& uris, bool sorted) {
     return result.dataHandle();
 }
 
-eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) {
+eckit::DataHandle* FDB::read(bool seekable, ListIterator& it, bool sorted) {
+
+    if (seekable) {
+        return new FieldHandle(it);
+    }
+
     eckit::Timer timer;
     timer.start();
 
@@ -219,9 +218,9 @@ eckit::DataHandle* FDB::read(ListIterator& it, bool sorted) {
     return result.dataHandle();
 }
 
-eckit::DataHandle* FDB::retrieve(const metkit::mars::MarsRequest& request) {
+eckit::DataHandle* FDB::retrieve(const metkit::mars::MarsRequest& request, bool seekable) {
     ListIterator it = inspect(request);
-    return read(it, sorted(request));
+    return read(seekable, it, sorted(request));
 }
 
 ListIterator FDB::inspect(const metkit::mars::MarsRequest& request) {

src/fdb5/api/FDB.h

@@ -89,9 +89,9 @@ class FDB {
 
     eckit::DataHandle* read(const std::vector<eckit::URI>& uris, bool sorted = false);
 
-    eckit::DataHandle* read(ListIterator& it, bool sorted = false);
+    eckit::DataHandle* read(bool seekable, ListIterator& it, bool sorted = false);
 
-    eckit::DataHandle* retrieve(const metkit::mars::MarsRequest& request);
+    eckit::DataHandle* retrieve(const metkit::mars::MarsRequest& request, bool seekable = false);
 
     ListIterator inspect(const metkit::mars::MarsRequest& request);
 

src/fdb5/api/FDBFactory.h

@@ -25,7 +25,7 @@
 #include "eckit/utils/Regex.h"
 #include "eckit/memory/NonCopyable.h"
 
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/config/Config.h"
 #include "fdb5/api/FDBStats.h"
 #include "fdb5/api/helpers/AxesIterator.h"
@@ -39,13 +39,17 @@
 #include "fdb5/api/helpers/StatusIterator.h"
 #include "fdb5/api/helpers/Callback.h"
 
-namespace eckit {
-namespace message {
+namespace eckit::message {
+
 class Message;
-}
-}  // namespace eckit
 
-namespace metkit { class MarsRequest; }
+}  // namespace eckit::message
+
+namespace metkit {
+
+class MarsRequest;
+
+} // namespace metkit
 
 namespace fdb5 {
 
@@ -89,7 +93,7 @@ class FDBBase : private eckit::NonCopyable {
 
     virtual MoveIterator move(const FDBToolRequest& request, const eckit::URI& dest) = 0;
 
-    virtual AxesIterator axesIterator(const FDBToolRequest& request, int axes) { NOTIMP; }
+    virtual AxesIterator axesIterator(const FDBToolRequest& request, int axes) = 0;
 
     void registerArchiveCallback(ArchiveCallback callback) {callback_ = callback;}
 

src/fdb5/api/FDBStats.cc

@@ -29,6 +29,7 @@ namespace fdb5 {
 
 FDBStats::FDBStats() :
     numArchive_(0),
+    numLocation_(0),
     numFlush_(0),
     numRetrieve_(0),
     bytesArchive_(0),
@@ -48,6 +49,7 @@ FDBStats::~FDBStats() {}
 
 FDBStats& FDBStats::operator+=(const FDBStats& rhs) {
     numArchive_ += rhs.numArchive_;
+    numLocation_ += rhs.numLocation_;
     numFlush_ += rhs.numFlush_;
     numRetrieve_ += rhs.numRetrieve_;
     bytesArchive_ += rhs.bytesArchive_;
@@ -81,6 +83,9 @@ void FDBStats::addArchive(size_t length, eckit::Timer& timer, size_t nfields) {
                          << ", total: " << Seconds(elapsedArchive_) << std::endl;
 }
 
+void FDBStats::addLocation(size_t nfields) {
+    numLocation_ += nfields;
+}
 
 void FDBStats::addRetrieve(size_t length, eckit::Timer& timer) {
 

src/fdb5/api/FDBStats.h

@@ -36,9 +36,11 @@ class FDBStats : public eckit::Statistics {
     ~FDBStats();
 
     size_t numArchive() const { return numArchive_; }
+    size_t numLocation() const { return numLocation_; }
     size_t numFlush() const { return numFlush_; }
 
     void addArchive(size_t length, eckit::Timer& timer, size_t nfields=1);
+    void addLocation(size_t nfields=1);
     void addRetrieve(size_t length, eckit::Timer& timer);
     void addFlush(eckit::Timer& timer);
 
@@ -49,6 +51,7 @@ class FDBStats : public eckit::Statistics {
 private: // members
 
     size_t numArchive_;
+    size_t numLocation_;
     size_t numFlush_;
     size_t numRetrieve_;
 

src/fdb5/api/LocalFDB.cc

@@ -21,7 +21,7 @@
 #include "fdb5/api/helpers/FDBToolRequest.h"
 #include "fdb5/api/LocalFDB.h"
 #include "fdb5/database/Archiver.h"
-#include "fdb5/database/DB.h"
+#include "fdb5/database/Catalogue.h"
 #include "fdb5/database/EntryVisitMechanism.h"
 #include "fdb5/database/Index.h"
 #include "fdb5/database/Inspector.h"
@@ -143,7 +143,7 @@ void LocalFDB::print(std::ostream &s) const {
 
 
 static FDBBuilder<LocalFDB> localFdbBuilder("local");
-
+static FDBBuilder<LocalFDB> builder("catalogue"); // Enable type=catalogue to build localFDB (serverside).
 //----------------------------------------------------------------------------------------------------------------------
 
 } // namespace fdb5

src/fdb5/api/LocalFDB.h

@@ -63,14 +63,16 @@ class LocalFDB : public FDBBase {
 
     void flush() override;
 
-private: // methods
-
-    void print(std::ostream& s) const override;
+protected: // methods
 
     template <typename VisitorType, typename ... Ts>
     APIIterator<typename VisitorType::ValueType> queryInternal(const FDBToolRequest& request, Ts ... args);
 
-private: // members
+private: // methods
+
+    void print(std::ostream& s) const override;
+
+protected: // members
 
     std::string home_;