Refactor block file logic + util.h/cpp moved to util/system.h

CMakeLists.txt

@@ -186,6 +186,7 @@ set(SERVER_SOURCES
         ./src/chain.cpp
         ./src/checkpoints.cpp
         ./src/consensus/tx_verify.cpp
+        ./src/flatfile.cpp
         ./src/httprpc.cpp
         ./src/httpserver.cpp
         ./src/indirectmap.h
@@ -416,7 +417,7 @@ set(UTIL_SOURCES
         ./src/arith_uint256.cpp
         ./src/uint256.cpp
         ./src/util/threadnames.cpp
-        ./src/util.cpp
+        ./src/util/system.cpp
         ./src/utilstrencodings.cpp
         ./src/utilmoneystr.cpp
         ./src/utiltime.cpp

src/Makefile.am

@@ -190,6 +190,7 @@ BITCOIN_CORE_H = \
   pairresult.h \
   addressbook.h \
   wallet/db.h \
+  flatfile.h \
   fs.h \
   hash.h \
   httprpc.h \
@@ -283,7 +284,7 @@ BITCOIN_CORE_H = \
   uint256.h \
   undo.h \
   util/memory.h \
-  util.h \
+  util/system.h \
   util/macros.h \
   util/threadnames.h \
   utilstrencodings.h \
@@ -326,6 +327,7 @@ libbitcoin_server_a_SOURCES = \
   checkpoints.cpp \
   consensus/params.cpp \
   consensus/tx_verify.cpp \
+  flatfile.cpp \
   consensus/zerocoin_verify.cpp \
   evo/deterministicmns.cpp \
   evo/evodb.cpp \
@@ -546,7 +548,7 @@ libbitcoin_util_a_SOURCES = \
   sync.cpp \
   threadinterrupt.cpp \
   uint256.cpp \
-  util.cpp \
+  util/system.cpp \
   utilmoneystr.cpp \
   util/threadnames.cpp \
   utilstrencodings.cpp \

src/Makefile.test.include

@@ -64,6 +64,7 @@ BITCOIN_TESTS =\
   test/DoS_tests.cpp \
   test/evo_deterministicmns_tests.cpp \
   test/evo_specialtx_tests.cpp \
+  test/flatfile_tests.cpp \
   test/getarg_tests.cpp \
   test/hash_tests.cpp \
   test/key_tests.cpp \

src/addrdb.cpp

@@ -14,7 +14,7 @@
 #include "random.h"
 #include "streams.h"
 #include "tinyformat.h"
-#include "util.h"
+#include "util/system.h"
 
 namespace {
 

src/addrman.h

@@ -12,7 +12,7 @@
 #include "random.h"
 #include "sync.h"
 #include "timedata.h"
-#include "util.h"
+#include "util/system.h"
 
 #include <map>
 #include <set>

src/bench/bench_pivx.cpp

@@ -6,7 +6,7 @@
 #include "bench.h"
 
 #include "key.h"
-#include "util.h"
+#include "util/system.h"
 
 int
 main(int argc, char** argv)

src/bench/checkqueue.cpp

@@ -3,7 +3,7 @@
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
 #include "bench.h"
-#include "util.h"
+#include "util/system.h"
 #include "checkqueue.h"
 #include "prevector.h"
 #include "random.h"

src/bip38.cpp

@@ -8,7 +8,7 @@
 #include "crypto/aes.h"
 #include "hash.h"
 #include "pubkey.h"
-#include "util.h"
+#include "util/system.h"
 #include "utilstrencodings.h"
 #include "random.h"
 

src/blockassembler.cpp

@@ -21,7 +21,7 @@
 #include "spork.h"
 #include "timedata.h"
 #include "txmempool.h"
-#include "util.h"
+#include "util/system.h"
 #include "validation.h"
 #include "validationinterface.h"
 

src/chain.cpp

@@ -142,19 +142,19 @@ std::string CBlockIndex::ToString() const
         GetBlockHash().ToString());
 }
 
-CDiskBlockPos CBlockIndex::GetBlockPos() const
+FlatFilePos CBlockIndex::GetBlockPos() const
 {
-    CDiskBlockPos ret;
+    FlatFilePos ret;
     if (nStatus & BLOCK_HAVE_DATA) {
         ret.nFile = nFile;
         ret.nPos = nDataPos;
     }
     return ret;
 }
 
-CDiskBlockPos CBlockIndex::GetUndoPos() const
+FlatFilePos CBlockIndex::GetUndoPos() const
 {
-    CDiskBlockPos ret;
+    FlatFilePos ret;
     if (nStatus & BLOCK_HAVE_UNDO) {
         ret.nFile = nFile;
         ret.nPos = nUndoPos;

src/chain.h

@@ -11,13 +11,14 @@
 #define BITCOIN_CHAIN_H
 
 #include "chainparams.h"
+#include "flatfile.h"
 #include "optional.h"
 #include "pow.h"
 #include "primitives/block.h"
 #include "timedata.h"
 #include "tinyformat.h"
 #include "uint256.h"
-#include "util.h"
+#include "util/system.h"
 #include "libzerocoin/Denominations.h"
 
 #include <vector>
@@ -87,47 +88,6 @@ class CBlockFileInfo
     }
 };
 
-struct CDiskBlockPos {
-    int nFile;
-    unsigned int nPos;
-
-    ADD_SERIALIZE_METHODS;
-
-    template <typename Stream, typename Operation>
-    inline void SerializationOp(Stream& s, Operation ser_action) {
-        READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED));
-        READWRITE(VARINT(nPos));
-    }
-
-    CDiskBlockPos()
-    {
-        SetNull();
-    }
-
-    CDiskBlockPos(int nFileIn, unsigned int nPosIn)
-    {
-        nFile = nFileIn;
-        nPos = nPosIn;
-    }
-
-    friend bool operator==(const CDiskBlockPos& a, const CDiskBlockPos& b)
-    {
-        return (a.nFile == b.nFile && a.nPos == b.nPos);
-    }
-
-    friend bool operator!=(const CDiskBlockPos& a, const CDiskBlockPos& b)
-    {
-        return !(a == b);
-    }
-
-    void SetNull()
-    {
-        nFile = -1;
-        nPos = 0;
-    }
-    bool IsNull() const { return (nFile == -1); }
-};
-
 enum BlockStatus {
     //! Unused.
     BLOCK_VALID_UNKNOWN = 0,
@@ -253,8 +213,8 @@ class CBlockIndex
 
     std::string ToString() const;
 
-    CDiskBlockPos GetBlockPos() const;
-    CDiskBlockPos GetUndoPos() const;
+    FlatFilePos GetBlockPos() const;
+    FlatFilePos GetUndoPos() const;
     CBlockHeader GetBlockHeader() const;
     uint256 GetBlockHash() const { return *phashBlock; }
     int64_t GetBlockTime() const { return (int64_t)nTime; }

src/chainparamsbase.cpp

@@ -7,7 +7,7 @@
 #include "chainparamsbase.h"
 
 #include "tinyformat.h"
-#include "util.h"
+#include "util/system.h"
 
 #include <assert.h>
 

src/consensus/params.cpp

@@ -5,7 +5,7 @@
 
 #include "consensus/params.h"
 #include "consensus/upgrades.h"
-#include "util.h"
+#include "util/system.h"
 
 namespace Consensus {
 

src/core_read.cpp

@@ -10,7 +10,7 @@
 #include "script/script.h"
 #include "serialize.h"
 #include "streams.h"
-#include "util.h"
+#include "util/system.h"
 #include "utilstrencodings.h"
 #include "version.h"
 

src/core_write.cpp

@@ -13,7 +13,7 @@
 #include "serialize.h"
 #include "streams.h"
 #include <univalue.h>
-#include "util.h"
+#include "util/system.h"
 #include "utilmoneystr.h"
 #include "utilstrencodings.h"
 

src/crypter.cpp

@@ -9,7 +9,7 @@
 #include "crypto/sha512.h"
 #include "script/script.h"
 #include "script/standard.h"
-#include "util.h"
+#include "util/system.h"
 #include "init.h"
 #include "uint256.h"
 

src/dbwrapper.cpp

@@ -4,7 +4,7 @@
 
 #include "dbwrapper.h"
 
-#include "util.h"
+#include "util/system.h"
 
 #include <leveldb/cache.h>
 #include <leveldb/env.h>

src/dbwrapper.h

@@ -9,7 +9,7 @@
 #include "fs.h"
 #include "serialize.h"
 #include "streams.h"
-#include "util.h"
+#include "util/system.h"
 #include "version.h"
 
 #include <typeindex>

src/flatfile.cpp

@@ -0,0 +1,98 @@
+// Copyright (c) 2009-2010 Satoshi Nakamoto
+// Copyright (c) 2009-2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <stdexcept>
+
+#include "flatfile.h"
+#include "logging.h"
+#include "tinyformat.h"
+#include "util/system.h"
+
+FlatFileSeq::FlatFileSeq(fs::path dir, const char* prefix, size_t chunk_size) :
+    m_dir(std::move(dir)),
+    m_prefix(prefix),
+    m_chunk_size(chunk_size)
+{
+    if (chunk_size == 0) {
+        throw std::invalid_argument("chunk_size must be positive");
+    }
+}
+
+std::string FlatFilePos::ToString() const
+{
+    return strprintf("FlatFilePos(nFile=%i, nPos=%i)", nFile, nPos);
+}
+
+fs::path FlatFileSeq::FileName(const FlatFilePos& pos) const
+{
+    return m_dir / strprintf("%s%05u.dat", m_prefix, pos.nFile);
+}
+
+FILE* FlatFileSeq::Open(const FlatFilePos& pos, bool read_only)
+{
+    if (pos.IsNull()) {
+        return nullptr;
+    }
+    fs::path path = FileName(pos);
+    fs::create_directories(path.parent_path());
+    FILE* file = fsbridge::fopen(path, read_only ? "rb": "rb+");
+    if (!file && !read_only)
+        file = fsbridge::fopen(path, "wb+");
+    if (!file) {
+        LogPrintf("Unable to open file %s\n", path.string());
+        return nullptr;
+    }
+    if (pos.nPos && fseek(file, pos.nPos, SEEK_SET)) {
+        LogPrintf("Unable to seek to position %u of %s\n", pos.nPos, path.string());
+        fclose(file);
+        return nullptr;
+    }
+    return file;
+}
+
+size_t FlatFileSeq::Allocate(const FlatFilePos& pos, size_t add_size, bool& out_of_space)
+{
+    out_of_space = false;
+
+    unsigned int n_old_chunks = (pos.nPos + m_chunk_size - 1) / m_chunk_size;
+    unsigned int n_new_chunks = (pos.nPos + add_size + m_chunk_size - 1) / m_chunk_size;
+    if (n_new_chunks > n_old_chunks) {
+        size_t old_size = pos.nPos;
+        size_t new_size = n_new_chunks * m_chunk_size;
+        size_t inc_size = new_size - old_size;
+
+        if (CheckDiskSpace(m_dir, inc_size)) {
+            FILE *file = Open(pos);
+            if (file) {
+                LogPrintf("Pre-allocating up to position 0x%x in %s%05u.dat\n", new_size, m_prefix, pos.nFile);
+                AllocateFileRange(file, pos.nPos, inc_size);
+                fclose(file);
+                return inc_size;
+            }
+        } else {
+            out_of_space = true;
+        }
+    }
+    return 0;
+}
+
+bool FlatFileSeq::Flush(const FlatFilePos& pos, bool finalize)
+{
+    FILE* file = Open(FlatFilePos(pos.nFile, 0)); // Avoid fseek to nPos
+    if (!file) {
+        return error("%s: failed to open file %d", __func__, pos.nFile);
+    }
+    if (finalize && !TruncateFile(file, pos.nPos)) {
+        fclose(file);
+        return error("%s: failed to truncate file %d", __func__, pos.nFile);
+    }
+    if (!FileCommit(file)) {
+        fclose(file);
+        return error("%s: failed to commit file %d", __func__, pos.nFile);
+    }
+
+    fclose(file);
+    return true;
+}