From 40b447d3821e8bb84ee530dd6387d6b0c942bf26 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Mon, 30 Sep 2019 16:33:01 -0500
Subject: [PATCH 01/16] Fixed block reporting to work for trimmed block logs.

---
 programs/eosio-blocklog/main.cpp | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp
index ffef2283b57..27835c63ebd 100644
--- a/programs/eosio-blocklog/main.cpp
+++ b/programs/eosio-blocklog/main.cpp
@@ -73,7 +73,11 @@ void blocklog::read_log() {
    EOS_ASSERT( end->block_num() > 1, block_log_exception, "Only one block found in block log" );
 
    //fix message below, first block might not be 1, first_block_num is not set yet
-   ilog( "existing block log contains block num 1 through block num ${n}", ("n",end->block_num()) );
+   ilog( "existing block log contains block num ${first} through block num ${n}",
+         ("first",block_logger.first_block_num())("n",end->block_num()) );
+   if (first_block < block_logger.first_block_num()) {
+      first_block = block_logger.first_block_num();
+   }
 
    optional<chainbase::database> reversible_blocks;
    try {
@@ -129,10 +133,10 @@ void blocklog::read_log() {
                  ("ref_block_prefix", ref_block_prefix)
                  (pretty_output.get_object());
       fc::variant v(std::move(enhanced_object));
-       if (no_pretty_print)
-          fc::json::to_stream(*out, v, fc::json::stringify_large_ints_and_doubles);
-       else
-          *out << fc::json::to_pretty_string(v) << "\n";
+      if (no_pretty_print)
+         fc::json::to_stream(*out, v, fc::json::stringify_large_ints_and_doubles);
+      else
+         *out << fc::json::to_pretty_string(v) << "\n";
    };
    bool contains_obj = false;
    while((block_num <= last_block) && (next = block_logger.read_block_by_num( block_num ))) {
@@ -142,6 +146,7 @@ void blocklog::read_log() {
       ++block_num;
       contains_obj = true;
    }
+
    if (reversible_blocks) {
       const reversible_block_object* obj = nullptr;
       while( (block_num <= last_block) && (obj = reversible_blocks->find<reversible_block_object,by_num>(block_num)) ) {
@@ -153,6 +158,7 @@ void blocklog::read_log() {
          contains_obj = true;
       }
    }
+
    if (as_json_array)
       *out << "]";
    rt.report();

From b0052d74336f104e0e657ef2e574a451fff1463c Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Mon, 30 Sep 2019 16:44:21 -0500
Subject: [PATCH 02/16] Moved trim_data into block_log. GH #7939

---
 libraries/chain/block_log.cpp                 | 165 +++++++++++++++++-
 .../chain/include/eosio/chain/block_log.hpp   |  26 +++
 programs/eosio-blocklog/main.cpp              | 107 +-----------
 3 files changed, 190 insertions(+), 108 deletions(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index d8fe7443bef..29fd96468fc 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -102,6 +102,7 @@ namespace eosio { namespace chain {
          uint64_t previous();
          uint32_t version() const { return _version; }
          uint32_t first_block_num() const { return _first_block_num; }
+         constexpr static uint32_t      _buf_len                          = 1U << 24;
       private:
          void update_buffer();
 
@@ -118,9 +119,7 @@ namespace eosio { namespace chain {
          std::unique_ptr<char[]>        _buffer_ptr;
          std::string                    _block_file_name;
          constexpr static int64_t       _unset_position                   = -1;
-         constexpr static uint32_t      _buf_len                          = 1U << 24;
          constexpr static uint64_t      _position_size                    = sizeof(_current_position_in_file);
-         constexpr static int           _blknum_offset_from_pos           = 14; //offset from start of block to 4 byte block number, valid for the only allowed versions
       };
 
       constexpr uint64_t buffer_location_to_file_location(uint32_t buffer_location) { return buffer_location << 3; }
@@ -132,6 +131,7 @@ namespace eosio { namespace chain {
          void write(uint64_t pos);
          void complete();
          void update_buffer_position();
+         constexpr static uint64_t          _buffer_bytes             = 1U << 22;
       private:
          void prepare_buffer();
          bool shift_buffer();
@@ -144,9 +144,39 @@ namespace eosio { namespace chain {
          int64_t                            _current_position         = 0;
          int64_t                            _start_of_buffer_position = 0;
          int64_t                            _end_of_buffer_position   = 0;
-         constexpr static uint64_t          _buffer_bytes             = 1U << 22;
          constexpr static uint64_t          _max_buffer_length        = file_location_to_buffer_location(_buffer_bytes);
       };
+
+      /*
+       *  @brief datastream adapter that adapts FILE* for use with fc unpack
+       *
+       *  This class supports unpack functionality but not pack.
+       */
+      class fileptr_datastream {
+      public:
+         explicit fileptr_datastream( FILE* file, const std::string& filename ) : _file(file), _filename(filename) {}
+
+         void skip( size_t s ) {
+            std::vector<char> d( s );
+            read( &d[0], s );
+         }
+
+         bool read( char* d, size_t s ) {
+            size_t result = fread( d, 1, s, _file );
+            EOS_ASSERT( result == s, block_log_exception,
+                        "in file: ${file} only able to read ${act} of the expected ${exp}",
+                        ("file", _filename)("act",result)("exp",s) );
+            return true;
+         }
+
+         bool get( unsigned char& c ) { return get( *(char*)&c ); }
+
+         bool get( char& c ) { return read(&c, 1); }
+
+      private:
+         FILE* const _file;
+         const std::string _filename;
+      };
    }
 
    block_log::block_log(const fc::path& data_dir)
@@ -751,10 +781,10 @@ namespace eosio { namespace chain {
       uint32_t bnum = 0;
       if (block_pos >= _start_of_buffer_position) {
          const uint32_t index_of_block = block_pos - _start_of_buffer_position;
-         bnum = *reinterpret_cast<uint32_t*>(buf + index_of_block + _blknum_offset_from_pos);  //block number of previous block (is big endian)
+         bnum = *reinterpret_cast<uint32_t*>(buf + index_of_block + trim_data::blknum_offset);  //block number of previous block (is big endian)
       }
       else {
-         const auto blknum_offset_pos = block_pos + _blknum_offset_from_pos;
+         const auto blknum_offset_pos = block_pos + trim_data::blknum_offset;
          auto status = fseek(_file.get(), blknum_offset_pos, SEEK_SET);
          EOS_ASSERT( status == 0, block_log_exception, "Could not seek in '${blocks_log}' to position: ${pos}. Returned status: ${status}", ("blocks_log", _block_file_name)("pos", blknum_offset_pos)("status", status) );
          auto size = fread((void*)&bnum, sizeof(bnum), 1, _file.get());
@@ -906,4 +936,127 @@ namespace eosio { namespace chain {
    bool block_log::is_supported_version(uint32_t version) {
       return std::clamp(version, min_supported_version, max_supported_version) == version;
    }
-} } /// eosio::chain
+   trim_data::trim_data(fc::path block_dir) {
+
+      // code should follow logic in block_log::repair_log
+
+      using namespace std;
+      block_file_name = block_dir / "blocks.log";
+      index_file_name = block_dir / "blocks.index";
+      blk_in = FC_FOPEN(block_file_name.generic_string().c_str(), "rb");
+      EOS_ASSERT( blk_in != nullptr, block_log_not_found, "cannot read file ${file}", ("file",block_file_name.string()) );
+      ind_in = FC_FOPEN(index_file_name.generic_string().c_str(), "rb");
+      EOS_ASSERT( ind_in != nullptr, block_log_not_found, "cannot read file ${file}", ("file",index_file_name.string()) );
+      auto size = fread((void*)&version,sizeof(version), 1, blk_in);
+      EOS_ASSERT( size == 1, block_log_unsupported_version, "invalid format for file ${file}", ("file",block_file_name.string()));
+      cout << "block log version= " << version << '\n';
+      EOS_ASSERT( block_log::is_supported_version(version), block_log_unsupported_version, "block log version ${v} is not supported", ("v",version));
+
+      detail::fileptr_datastream ds(blk_in, block_file_name.string());
+      if (version == 1) {
+         first_block = 1;
+         genesis_state gs;
+         fc::raw::unpack(ds, gs);
+         chain_id = gs.compute_chain_id();
+      }
+      else {
+         size = fread((void *) &first_block, sizeof(first_block), 1, blk_in);
+         EOS_ASSERT(size == 1, block_log_exception, "invalid format for file ${file}",
+                    ("file", block_file_name.string()));
+         if (block_log::contains_genesis_state(version, first_block)) {
+            genesis_state gs;
+            fc::raw::unpack(ds, gs);
+            chain_id = gs.compute_chain_id();
+         }
+         else if (block_log::contains_chain_id(version, first_block)) {
+            chain_id = chain_id_type{};
+            ds >> *chain_id;
+         }
+         else {
+            EOS_ASSERT( false, block_log_exception,
+                        "Block log ${file} is not supported. version: ${ver} and first_block: ${first_block} does not contain "
+                        "a genesis_state nor a chain_id.",
+                        ("file", block_file_name.string())("ver", version)("first_block", first_block));
+         }
+
+         const auto expected_totem = block_log::npos;
+         std::decay_t<decltype(block_log::npos)> actual_totem;
+         size = fread ( (char*)&actual_totem, sizeof(actual_totem), 1, blk_in);
+
+         EOS_ASSERT(size == 1, block_log_exception,
+                    "Expected to read ${size} bytes, but did not read any bytes", ("size", sizeof(actual_totem)));
+         EOS_ASSERT(actual_totem == expected_totem, block_log_exception,
+                    "Expected separator between block log header and blocks was not found( expected: ${e}, actual: ${a} )",
+                    ("e", fc::to_hex((char*)&expected_totem, sizeof(expected_totem) ))("a", fc::to_hex((char*)&actual_totem, sizeof(actual_totem) )));
+      }
+
+      const uint64_t start_of_blocks = ftell(blk_in);
+      first_block_pos = block_pos(first_block);
+      EOS_ASSERT(start_of_blocks == first_block_pos, block_log_exception,
+                 "Block log ${file} was determined to have its first block at ${determined}, but the block index "
+                 "indicates the first block is at ${index}",
+                 ("file", block_file_name.string())("determined", start_of_blocks)("index",first_block_pos));
+
+      cout << "first block= " << first_block << '\n';
+      const auto status = fseek(ind_in, 0, SEEK_END);                //get length of blocks.index (gives number of blocks)
+      EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} end", ("file", index_file_name.string()) );
+      EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} end", ("file", index_file_name.string()) );
+      const uint64_t file_end = ftell(ind_in);                //get length of blocks.index (gives number of blocks)
+      last_block = first_block + file_end/sizeof(uint64_t) - 1;
+      cout << "last block=  " << last_block << '\n';
+   }
+
+   trim_data::~trim_data() {
+      if (blk_in != nullptr)
+         fclose(blk_in);
+      if (ind_in != nullptr)
+         fclose(ind_in);
+   }
+
+   uint64_t trim_data::block_pos(uint32_t n) {
+      using namespace std;
+      index_pos = sizeof(uint64_t) * (n - first_block);
+      auto status = fseek(ind_in, index_pos, SEEK_SET);
+      EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file for block ${b}", ("file", index_file_name.string())("pos", index_pos)("b",n) );
+      const uint64_t pos = ftell(ind_in);
+      EOS_ASSERT( pos == index_pos, block_log_exception, "cannot seek to ${file} entry for block ${b}", ("file", index_file_name.string())("b",n) );
+      uint64_t block_n_pos;
+      auto size = fread((void*)&block_n_pos, sizeof(block_n_pos), 1, ind_in);                   //filepos of block n
+      EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}", ("file", index_file_name.string())("b",n) );
+      return block_n_pos;
+   }
+
+   void trim_data::find_block_pos(uint32_t n) {
+      //get file position of block n from blocks.index then confirm block n is found in blocks.log at that position
+      //sets fpos0 and fpos1, throws exception if block at fpos0 is not block n
+      using namespace std;
+      auto temp = block_pos(first_block);
+      fpos0 = block_pos(n);
+      auto size = fread((void*)&fpos1, sizeof(fpos1), 1, ind_in);                   //filepos of block n+1
+      if (n != last_block)
+         EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}, size=${size}",
+                     ("file", index_file_name.string())("b",n + 1)("size",size) );
+
+      cout << "According to blocks.index:\n";
+      cout << "    block " << n << " starts at position " << fpos0 << '\n';
+      cout << "    block " << n + 1;
+
+      if (n != last_block)
+         cout << " starts at position " << fpos1 << '\n';
+      else
+         cout << " is past end\n";
+
+      //read blocks.log and verify block number n is found at file position fpos0
+      auto status = fseek(blk_in, fpos0 + blknum_offset, SEEK_SET);
+      EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", block_file_name.string())("pos", fpos0 + blknum_offset) );
+      const uint64_t block_offset_pos = ftell(blk_in);
+      EOS_ASSERT( block_offset_pos == fpos0 + blknum_offset, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", block_file_name.string())("pos", fpos0 + blknum_offset) );
+      uint32_t prior_blknum;
+      size = fread((void*)&prior_blknum, sizeof(prior_blknum), 1, blk_in);     //read bigendian block number of prior block
+      EOS_ASSERT( size == 1, block_log_exception, "cannot read prior block");
+      const uint32_t bnum = fc::endian_reverse_u32(prior_blknum) + 1;          //convert to little endian, add 1 since prior block
+      cout << "At position " << fpos0 << " in " << index_file_name.generic_string() << " find block " << bnum << (bnum == n ? " as expected\n": " - not good!\n");
+      EOS_ASSERT( bnum == n, block_log_exception, "${index} does not agree with ${blocks}", ("index", index_file_name.string())("blocks", block_file_name.string()) );
+   }
+
+   } } /// eosio::chain
diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp
index db2fe7f4cd5..739e50e6b2d 100644
--- a/libraries/chain/include/eosio/chain/block_log.hpp
+++ b/libraries/chain/include/eosio/chain/block_log.hpp
@@ -85,4 +85,30 @@ namespace eosio { namespace chain {
          std::unique_ptr<detail::block_log_impl> my;
    };
 
+//to derive blknum_offset==14 see block_header.hpp and note on disk struct is packed
+//   block_timestamp_type timestamp;                  //bytes 0:3
+//   account_name         producer;                   //bytes 4:11
+//   uint16_t             confirmed;                  //bytes 12:13
+//   block_id_type        previous;                   //bytes 14:45, low 4 bytes is big endian block number of previous block
+
+   struct trim_data {            //used by trim_blocklog_front(), trim_blocklog_end(), and smoke_test()
+      trim_data(fc::path block_dir);
+      ~trim_data();
+      void find_block_pos(uint32_t n);
+      uint64_t block_pos(uint32_t n);
+      fc::path block_file_name, index_file_name;        //full pathname for blocks.log and blocks.index
+      uint32_t version = 0;                              //blocklog version
+      uint32_t first_block = 0;                          //first block in blocks.log
+      uint32_t last_block = 0;                          //last block in blocks.log
+      FILE* blk_in = nullptr;                            //C style files for reading blocks.log and blocks.index
+      FILE* ind_in = nullptr;                            //C style files for reading blocks.log and blocks.index
+      //we use low level file IO because it is distinctly faster than C++ filebuf or iostream
+      uint64_t index_pos = 0;                            //filepos in blocks.index for block n, +8 for block n+1
+      uint64_t fpos0 = 0;                                //filepos in blocks.log for block n and block n+1
+      uint64_t fpos1 = 0;                                //filepos in blocks.log for block n and block n+1
+      uint64_t first_block_pos = 0;                      //file position in blocks.log for the first block in the log
+      fc::optional<chain_id_type> chain_id;
+
+      static constexpr int blknum_offset{14};            //offset from start of block to 4 byte block number, valid for the only allowed versions
+   };
 } }
diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp
index 27835c63ebd..ab5ac942b1b 100644
--- a/programs/eosio-blocklog/main.cpp
+++ b/programs/eosio-blocklog/main.cpp
@@ -208,109 +208,12 @@ void blocklog::initialize(const variables_map& options) {
 
 }
 
-constexpr int blknum_offset{14};                      //offset from start of block to 4 byte block number
-//this offset is valid for version 1 and 2 blocklogs, version is checked before using blknum_offset
-
-//to derive blknum_offset==14 see block_header.hpp and note on disk struct is packed
-//   block_timestamp_type timestamp;                  //bytes 0:3
-//   account_name         producer;                   //bytes 4:11
-//   uint16_t             confirmed;                  //bytes 12:13
-//   block_id_type        previous;                   //bytes 14:45, low 4 bytes is big endian block number of previous block
-
-struct trim_data {            //used by trim_blocklog_front(), trim_blocklog_end(), and smoke_test()
-   trim_data(bfs::path block_dir);
-   ~trim_data() {
-      fclose(blk_in);
-      fclose(ind_in);
-   }
-   void find_block_pos(uint32_t n);
-   bfs::path block_file_name, index_file_name;        //full pathname for blocks.log and blocks.index
-   uint32_t version = 0;                              //blocklog version
-   uint32_t first_block = 0;                          //first block in blocks.log
-   uint32_t last_block = 0;                          //last block in blocks.log
-   FILE* blk_in = nullptr;                            //C style files for reading blocks.log and blocks.index
-   FILE* ind_in = nullptr;                            //C style files for reading blocks.log and blocks.index
-   //we use low level file IO because it is distinctly faster than C++ filebuf or iostream
-   uint64_t index_pos = 0;                            //filepos in blocks.index for block n, +8 for block n+1
-   uint64_t fpos0 = 0;                                //filepos in blocks.log for block n and block n+1
-   uint64_t fpos1 = 0;                                //filepos in blocks.log for block n and block n+1
-};
-
-
-trim_data::trim_data(bfs::path block_dir) {
-   report_time rt("trimming log");
-   using namespace std;
-   block_file_name = block_dir / "blocks.log";
-   index_file_name = block_dir / "blocks.index";
-   blk_in = FOPEN(block_file_name.c_str(), "rb");
-   EOS_ASSERT( blk_in != nullptr, block_log_not_found, "cannot read file ${file}", ("file",block_file_name.string()) );
-   ind_in = FOPEN(index_file_name.c_str(), "rb");
-   EOS_ASSERT( ind_in != nullptr, block_log_not_found, "cannot read file ${file}", ("file",index_file_name.string()) );
-   auto size = fread((void*)&version,sizeof(version), 1, blk_in);
-   EOS_ASSERT( size == 1, block_log_unsupported_version, "invalid format for file ${file}", ("file",block_file_name.string()));
-   cout << "block log version= " << version << '\n';
-   EOS_ASSERT( block_log::is_supported_version(version), block_log_unsupported_version, "block log version ${v} is not supported", ("v",version));
-   if (version == 1) {
-      first_block = 1;
-   }
-   else {
-      size = fread((void *) &first_block, sizeof(first_block), 1, blk_in);
-      EOS_ASSERT(size == 1, block_log_exception, "invalid format for file ${file}",
-                 ("file", block_file_name.string()));
-   }
-   cout << "first block= " << first_block << '\n';
-   const auto status = fseek(ind_in, 0, SEEK_END);                //get length of blocks.index (gives number of blocks)
-   EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} end", ("file", index_file_name.string()) );
-   const uint64_t file_end = ftell(ind_in);                //get length of blocks.index (gives number of blocks)
-   last_block = first_block + file_end/sizeof(uint64_t) - 1;
-   cout << "last block=  " << last_block << '\n';
-   rt.report();
-}
-
-void trim_data::find_block_pos(uint32_t n) {
-   //get file position of block n from blocks.index then confirm block n is found in blocks.log at that position
-   //sets fpos0 and fpos1, throws exception if block at fpos0 is not block n
-   report_time rt("finding block position");
-   using namespace std;
-   index_pos = sizeof(uint64_t) * (n - first_block);
-   auto status = fseek(ind_in, index_pos, SEEK_SET);
-   EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file for block ${b}", ("file", index_file_name.string())("pos", index_pos)("b",n) );
-   const uint64_t pos = ftell(ind_in);
-   EOS_ASSERT( pos == index_pos, block_log_exception, "cannot seek to ${file} entry for block ${b}", ("file", index_file_name.string())("b",n) );
-   auto size = fread((void*)&fpos0, sizeof(fpos0), 1, ind_in);                   //filepos of block n
-   EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}", ("file", index_file_name.string())("b",n) );
-   size = fread((void*)&fpos1, sizeof(fpos1), 1, ind_in);                   //filepos of block n+1
-   if (n != last_block)
-      EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}, size=${size}", ("file", index_file_name.string())("b",n + 1)("size",size) );
-
-   cout << "According to blocks.index:\n";
-   cout << "    block " << n << " starts at position " << fpos0 << '\n';
-   cout << "    block " << n + 1;
-
-   if (n != last_block)
-      cout << " starts at position " << fpos1 << '\n';
-   else
-      cout << " is past end\n";
-
-   //read blocks.log and verify block number n is found at file position fpos0
-   status = fseek(blk_in, fpos0 + blknum_offset, SEEK_SET);
-   EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", block_file_name.string())("pos", fpos0 + blknum_offset) );
-   const uint64_t block_offset_pos = ftell(blk_in);
-   EOS_ASSERT( block_offset_pos == fpos0 + blknum_offset, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", block_file_name.string())("pos", fpos0 + blknum_offset) );
-   uint32_t prior_blknum;
-   size = fread((void*)&prior_blknum, sizeof(prior_blknum), 1, blk_in);     //read bigendian block number of prior block
-   EOS_ASSERT( size == 1, block_log_exception, "cannot read prior block");
-   const uint32_t bnum = endian_reverse_u32(prior_blknum) + 1;          //convert to little endian, add 1 since prior block
-   cout << "At position " << fpos0 << " in " << index_file_name << " find block " << bnum << (bnum == n ? " as expected\n": " - not good!\n");
-   EOS_ASSERT( bnum == n, block_log_exception, "${index} does not agree with ${blocks}", ("index", index_file_name.string())("blocks", block_file_name.string()) );
-   rt.report();
-}
-
 int trim_blocklog_end(bfs::path block_dir, uint32_t n) {       //n is last block to keep (remove later blocks)
    report_time rt("trimming blocklog end");
    using namespace std;
    trim_data td(block_dir);
-   cout << "\nIn directory " << block_dir << " will trim all blocks after block " << n << " from " << td.block_file_name << " and " << td.index_file_name << ".\n";
+   cout << "\nIn directory " << block_dir << " will trim all blocks after block " << n << " from "
+        << td.block_file_name.generic_string() << " and " << td.index_file_name.generic_string() << ".\n";
    if (n < td.first_block) {
       cerr << "All blocks are after block " << n << " so do nothing (trim_end would delete entire blocks.log)\n";
       return 1;
@@ -522,8 +425,8 @@ void smoke_test(bfs::path block_dir) {
    uint64_t file_pos;
    auto size = fread((void*)&file_pos, sizeof(uint64_t), 1, td.blk_in);
    EOS_ASSERT( size == 1, block_log_exception, "${file} read fails", ("file", td.block_file_name.string()) );
-   status = fseek(td.blk_in, file_pos + blknum_offset, SEEK_SET);
-   EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", td.block_file_name.string())("pos", file_pos + blknum_offset) );
+   status = fseek(td.blk_in, file_pos + trim_data::blknum_offset, SEEK_SET);
+   EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", td.block_file_name.string())("pos", file_pos + trim_data::blknum_offset) );
    uint32_t bnum;
    size = fread((void*)&bnum, sizeof(uint32_t), 1, td.blk_in);
    EOS_ASSERT( size == 1, block_log_exception, "${file} read fails", ("file", td.block_file_name.string()) );
@@ -570,7 +473,7 @@ int main(int argc, char** argv) {
                return -1;
          }
          if (blog.first_block != 0) {
-            if (trim_blocklog_front(vmap.at("blocks-dir").as<bfs::path>(), blog.first_block) != 0)
+            if (!trim_blocklog_front(vmap.at("blocks-dir").as<bfs::path>(), blog.first_block))
                return -1;
          }
          return 0;

From f52866f0a889029df1192d65321615a4c55a6a57 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Mon, 30 Sep 2019 16:45:19 -0500
Subject: [PATCH 03/16] Adding missed invarients to existing code. GH #7939

---
 libraries/chain/block_log.cpp | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index 29fd96468fc..ae6285f0fbc 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -372,6 +372,8 @@ namespace eosio { namespace chain {
    }
 
    void block_log::reset( const chain_id_type& chain_id, uint32_t first_block_num ) {
+      EOS_ASSERT( first_block_num > 1, block_log_exception,
+                  "Block log version ${ver} needs to be created with a genesis state if starting from block number 1." );
       my->reset(chain_id, signed_block_ptr(), first_block_num);
    }
 
@@ -561,6 +563,15 @@ namespace eosio { namespace chain {
       uint32_t first_block_num = 1;
       if (version != 1) {
          old_block_stream.read ( (char*)&first_block_num, sizeof(first_block_num) );
+
+         // this assert is only here since repair_log is only used for --hard-replay-blockchain, which removes any
+         // existing state, if another API needs to use it, this can be removed and the check for the first block's
+         // previous block id will need to accommodate this.
+         EOS_ASSERT( first_block_num == 1, block_log_exception,
+                     "Block log ${file} must contain a genesis state and start at block number 1.  This block log "
+                     "starts at block number ${first_block_num}.",
+                     ("file", (backup_dir / "blocks.log").generic_string())("first_block_num", first_block_num));
+
          new_block_stream.write( (char*)&first_block_num, sizeof(first_block_num) );
       }
 
@@ -577,6 +588,12 @@ namespace eosio { namespace chain {
 
          new_block_stream << chain_id;
       }
+      else {
+         EOS_ASSERT( false, block_log_exception,
+                     "Block log ${file} is not supported. version: ${ver} and first_block_num: ${fbn} does not contain "
+                     "a genesis_state nor a chain_id.",
+                     ("file", (backup_dir / "blocks.log").generic_string())("ver", version)("fbn", first_block_num));
+      }
 
       if (version != 1) {
          auto expected_totem = npos;

From d798600354ef67d3574e8549e7ab7a500a4bf3aa Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Mon, 30 Sep 2019 16:46:44 -0500
Subject: [PATCH 04/16] Movede trim_blocklog_front logic into block_log and
 added support for version 3. GH #7939

---
 libraries/chain/block_log.cpp                 |  88 +++++++++
 .../chain/include/eosio/chain/block_log.hpp   |   5 +-
 .../include/eosio/chain/chain_id_type.hpp     |   2 +-
 programs/eosio-blocklog/main.cpp              | 183 +-----------------
 4 files changed, 96 insertions(+), 182 deletions(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index ae6285f0fbc..0f037762f5e 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -953,6 +953,94 @@ namespace eosio { namespace chain {
    bool block_log::is_supported_version(uint32_t version) {
       return std::clamp(version, min_supported_version, max_supported_version) == version;
    }
+
+   bool block_log::trim_blocklog_front(const fc::path& block_dir, uint32_t truncate_at_block) {
+      using namespace std;
+      ilog("In directory ${dir} will trim all blocks before block ${n} from blocks.log and blocks.index.",
+           ("dir", block_dir.generic_string())("n", truncate_at_block));
+      trim_data original_block_log(block_dir);
+      if (truncate_at_block <= original_block_log.first_block) {
+         ilog("There are no blocks before block ${n} so do nothing.", ("n", truncate_at_block));
+         return false;
+      }
+      if (truncate_at_block > original_block_log.last_block) {
+         ilog("All blocks are before block ${n} so do nothing (trim front would delete entire blocks.log).", ("n", truncate_at_block));
+         return false;
+      }
+      original_block_log.find_block_pos(truncate_at_block);
+
+      fc::path new_block_filename = block_dir / "blocks.out";
+      if (fc::remove(new_block_filename)) {
+         ilog("Removing old blocks.out file");
+      }
+      fc::cfile new_block_file;
+      new_block_file.set_file_path(new_block_filename);
+      new_block_file.open( LOG_WRITE_C );
+
+      static_assert( block_log::max_supported_version > 0, "a version number of zero is not supported" );
+      uint32_t version = block_log::max_supported_version;
+      new_block_file.seek(0);
+      new_block_file.write((char*)&version, sizeof(version));
+      new_block_file.write((char*)&truncate_at_block, sizeof(truncate_at_block));
+
+      new_block_file << *original_block_log.chain_id;
+
+      // append a totem to indicate the division between blocks and header
+      auto totem = block_log::npos;
+      new_block_file.write((char*)&totem, sizeof(totem));
+
+      const auto new_block_file_first_block_pos = new_block_file.tellp();
+
+      // copy over remainder of block log to new block log
+      auto buffer =  make_unique<char[]>(detail::reverse_iterator::_buf_len);                   //read big chunks of old blocks.log into this buffer
+      char* buf =  buffer.get();
+
+      const uint64_t pos_delta = original_block_log.fpos0 - new_block_file_first_block_pos;     // offset bytes to shift from old blocklog to new blocklog
+      auto status = fseek(original_block_log.blk_in, 0, SEEK_END);
+      EOS_ASSERT( status == 0, block_log_exception, "blocks.log seek failed" );
+      const uint64_t to_write = ftell(original_block_log.blk_in) - original_block_log.fpos0;
+      const auto pos_size = sizeof(uint64_t);
+      uint64_t original_pos = ftell(original_block_log.blk_in) - pos_size;
+
+      fc::path new_index_filename = block_dir / "index.out";
+      const auto num_blocks = original_block_log.last_block - truncate_at_block + 1;
+      detail::index_writer index(new_index_filename, num_blocks);
+      uint64_t read_size = 0;
+      for(uint64_t written = to_write; written > 0; written -= read_size) {
+         read_size = to_write;
+         if (read_size > detail::reverse_iterator::_buf_len) {
+            read_size = detail::reverse_iterator::_buf_len;
+         }
+         const auto start_of_blk_buffer_pos = original_block_log.fpos0 + to_write - read_size;
+         status = fseek(original_block_log.blk_in, start_of_blk_buffer_pos, SEEK_SET);
+         const auto num_read = fread(buf, read_size, 1, original_block_log.blk_in);
+         EOS_ASSERT( num_read == 1, block_log_exception, "blocks.log read failed" );
+         while(original_pos >= start_of_blk_buffer_pos) {
+            const auto buffer_index = original_pos - start_of_blk_buffer_pos;
+            uint64_t& pos_content = *(uint64_t*)(buf + buffer_index);
+            const auto start_of_this_block = pos_content;
+            pos_content = start_of_this_block - pos_delta;
+            index.write(pos_content);
+            original_pos = start_of_this_block - pos_size;
+         }
+         new_block_file.write(buf, read_size);
+      }
+      index.complete();
+      fclose(original_block_log.blk_in);
+      original_block_log.blk_in = nullptr;
+      new_block_file.flush();
+      new_block_file.close();
+
+      fc::path old_log = block_dir / "old.log";
+      rename(original_block_log.block_file_name, old_log);
+      rename(new_block_filename, original_block_log.block_file_name);
+      fc::path old_ind = block_dir / "old.index";
+      rename(original_block_log.index_file_name, old_ind);
+      rename(new_index_filename, original_block_log.index_file_name);
+
+      return true;
+   }
+
    trim_data::trim_data(fc::path block_dir) {
 
       // code should follow logic in block_log::repair_log
diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp
index 739e50e6b2d..db7f3c00b7b 100644
--- a/libraries/chain/include/eosio/chain/block_log.hpp
+++ b/libraries/chain/include/eosio/chain/block_log.hpp
@@ -78,7 +78,10 @@ namespace eosio { namespace chain {
          static bool contains_chain_id(uint32_t version, uint32_t first_block_num);
 
          static bool is_supported_version(uint32_t version);
-      private:
+
+         static bool trim_blocklog_front(const fc::path& block_dir, uint32_t truncate_at_block);
+
+   private:
          void open(const fc::path& data_dir);
          void construct_index();
 
diff --git a/libraries/chain/include/eosio/chain/chain_id_type.hpp b/libraries/chain/include/eosio/chain/chain_id_type.hpp
index 1fb1ac70a94..5a1cc52de86 100644
--- a/libraries/chain/include/eosio/chain/chain_id_type.hpp
+++ b/libraries/chain/include/eosio/chain/chain_id_type.hpp
@@ -42,11 +42,11 @@ namespace chain {
          friend T fc::variant::as()const;
 
          friend class eosio::chain_apis::read_only;
-         friend class chain_plugin;
 
          friend class eosio::net_plugin_impl;
          friend struct eosio::handshake_message;
          friend class block_log;
+         friend struct trim_data;
          friend class controller;
          friend struct controller_impl;
          friend class global_property_object;
diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp
index ab5ac942b1b..55e2b4939a1 100644
--- a/programs/eosio-blocklog/main.cpp
+++ b/programs/eosio-blocklog/main.cpp
@@ -231,188 +231,11 @@ int trim_blocklog_end(bfs::path block_dir, uint32_t n) {       //n is last block
    return 0;
 }
 
-int trim_blocklog_front(bfs::path block_dir, uint32_t n) {        //n is first block to keep (remove prior blocks)
+bool trim_blocklog_front(bfs::path block_dir, uint32_t n) {        //n is first block to keep (remove prior blocks)
    report_time rt("trimming blocklog start");
-   using namespace std;
-   cout << "\nIn directory " << block_dir << " will trim all blocks before block " << n << " from blocks.log and blocks.index.\n";
-   trim_data td(block_dir);
-   if (n <= td.first_block) {
-      cerr << "There are no blocks before block " << n << " so do nothing\n";
-      return 1;
-   }
-   if (n > td.last_block) {
-      cerr << "All blocks are before block " << n << " so do nothing (trim_front would delete entire blocks.log)\n";
-      return 2;
-   }
-   td.find_block_pos(n);
-
-   constexpr uint32_t buf_len{1U<<24};                            //buf_len must be a power of 2
-   auto buffer =  make_unique<char[]>(buf_len);                   //read big chunks of old blocks.log into this buffer
-   char* buf =  buffer.get();
-
-   bfs::path block_out_filename = block_dir / "blocks.out";
-   FILE* blk_out = FOPEN(block_out_filename.c_str(), "wb");
-   EOS_ASSERT( blk_out != nullptr, block_log_not_found, "cannot write ${file}", ("file", block_out_filename.string()) );
-
-   //in version 1 file: version number, no first block number, rest of header length 0x6e, at 0x72 first block
-   //in version 2 file: version number, first block number,    rest of header length 0x6e, at 0x76 totem,  at 0x7e first block
-   *(uint32_t*)buf = 2;
-   auto size = fwrite((void*)buf, sizeof(uint32_t), 1, blk_out);                           //write version number 2
-   EOS_ASSERT( size == 1, block_log_exception, "blocks.out write fails" );
-   size = fwrite((void*)&n, sizeof(n), 1, blk_out);                            //write first block number
-   EOS_ASSERT( size == 1, block_log_exception, "blocks.out write fails" );
-
-   const auto past_version_offset = (td.version == 1) ? 4 : 8;
-   auto status = fseek(td.blk_in, past_version_offset, SEEK_SET) ;                //position past version number and maybe block number
-   EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", td.block_file_name.string())("pos", past_version_offset) );
-   const auto header_len = 0x6e;
-   size = fread((void*)buf, header_len, 1, td.blk_in);
-   EOS_ASSERT( size == 1, block_log_exception, "blocks.log read fails" );
-   const auto totem_len = 8; //copy rest of header length 0x6e
-   memset(buf + header_len, 0xff, totem_len);                                       //totem is 8 bytes of 0xff
-   size = fwrite(buf, header_len + totem_len, 1, blk_out);                          //write header and totem
-   EOS_ASSERT( size == 1, block_log_exception, "blocks.out write fails" );
-
-   //pos_delta is the amount to subtract from every file position record in blocks.log because the blocks < n are removed
-   uint64_t pos_delta = td.fpos0 - 0x7e;                           //bytes removed from the blocklog
-   //bytes removed is file position of block n minus file position 'where file header ends'
-   //even if version 1 blocklog 'where file header ends' is where the new version 2 file header of length 0x7e ends
-
-   //read big chunks of blocks.log into buf, update the file position records, then write to blk_out
-   status = fseek(td.blk_in, 0, SEEK_END);                     //get blocks.log file length
-   EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from end of file", ("file", td.block_file_name.string())("pos", 0) );
-   const uint64_t end = ftell(td.blk_in);
-   uint32_t last_buf_len = (end - td.fpos0 >= buf_len)? buf_len : end - td.fpos0; //bytes to read from blk_in
-   status = fseek(td.blk_in, -(uint64_t)last_buf_len, SEEK_END);        //pos is where read last buf from blk_in
-   EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from end of file", ("file", td.block_file_name.string())("pos", last_buf_len) );
-   uint64_t pos = ftell(td.blk_in);
-   EOS_ASSERT( pos == end - last_buf_len, block_log_exception, "cannot seek to ${file} ${pos} from end of file", ("file", td.block_file_name.string())("pos", last_buf_len) );
-   uint64_t did_read = fread((void*)buf, last_buf_len, 1, td.blk_in);           //read tail of blocks.log file into buf
-   cout << "seek " << td.block_file_name << " to " << pos << " read " << last_buf_len << " bytes\n";//debug
-   EOS_ASSERT( did_read == 1, block_log_exception, "${file} read fails", ("file", td.block_file_name.string()) );
-
-   //prepare to write index_out_filename
-   uint64_t total_fpos_len = (td.last_block + 1 - n) * sizeof(uint64_t);
-   auto fpos_buffer = make_unique<uint64_t[]>(buf_len);
-   uint64_t* fpos_list = fpos_buffer.get();                        //list of file positions, periodically write to blocks.index
-   bfs::path index_out_filename = block_dir / "index.out";
-   FILE* ind_out = FOPEN(index_out_filename.c_str(), "wb");
-   EOS_ASSERT( ind_out != nullptr, block_log_not_found, "cannot write ${file}", ("file",index_out_filename.string()) );
-   uint64_t last_fpos_len = total_fpos_len & ((uint64_t)buf_len - 1);//buf_len is a power of 2 so -1 creates low bits all 1
-   if (!last_fpos_len)                                            //will write integral number of buf_len and one time write last_fpos_len
-      last_fpos_len = buf_len;
-   uint32_t blk_base = td.last_block + 1 - (last_fpos_len >> 3);     //first entry in fpos_list is for block blk_base
-   cout << "******filling index buf blk_base " << blk_base << " last_fpos_len " << last_fpos_len << '\n';//debug
-
-   //as we traverse linked list of blocks in buf (from end toward start), for each block we know this:
-   int32_t index_start;                                           //buf index for block start
-   int32_t index_end;                                             //buf index for block end == buf index for block start file position
-   uint64_t file_pos;                                             //file position of block start
-   uint32_t bnum;                                                 //block number
-
-   //some code here is repeated in the loop below but we do it twice so can print a status message before the loop starts
-   index_end = last_buf_len - 8;                                     //buf index where last block ends and the block file position starts
-   file_pos = *(uint64_t*)(buf + index_end);                         //file pos of block start
-   index_start = file_pos - pos;                                   //buf index for block start
-   bnum = *(uint32_t*)(buf + index_start + blknum_offset);         //block number of previous block (is big endian)
-   bnum = endian_reverse_u32(bnum) + 1;                              //convert from big endian to little endian and add 1
-   EOS_ASSERT( bnum == td.last_block, block_log_exception, "last block from ${index} ${ind} != last block from ${block} ${log}", ("index", td.index_file_name.string())("ind", td.last_block)("block", td.block_file_name.string())("log", bnum) );
-
-   cout << '\n';                                                  //print header for periodic update messages
-   cout << setw(10) << "block" << setw(16) << "old file pos" << setw(16) << "new file pos" << '\n';
-   cout << setw(10) << bnum << setw(16) << file_pos << setw(16) << file_pos-pos_delta << '\n';
-   uint64_t last_file_pos = file_pos + 1;                            //used to check that file_pos is strictly decreasing
-   constexpr uint32_t fk{4096};                                   //buffer shift that keeps buf disk block aligned
-
-   for (;;) {        //invariant: at top of loop index_end and bnum are set and index_end is >= 0
-      file_pos = *(uint64_t *) (buf + index_end);                 //get file pos of block start
-      if (file_pos >= last_file_pos) {                            //check that file_pos decreases
-         cout << '\n';
-         cout << "file pos for block " << bnum + 1 << " is " << last_file_pos << '\n';
-         cout << "file pos for block " << bnum << " is " << file_pos << '\n';
-         EOS_ASSERT(file_pos < last_file_pos, block_log_exception, "${file} linked list of blocks is corrupt", ("file", td.index_file_name.string()) );
-      }
-      last_file_pos = file_pos;
-      *(uint64_t *) (buf + index_end) = file_pos - pos_delta;     //update file position in buf
-      fpos_list[bnum - blk_base] = file_pos - pos_delta;          //save updated file position for new index file
-      index_start = file_pos - pos;                               //will go negative when pass first block in buf
-
-      if ((bnum & 0xfffff) == 0)                                  //periodic progress indicator
-         cout << setw(10) << bnum << setw(16) << file_pos << setw(16) << file_pos-pos_delta << '\n';
-
-      if (bnum == blk_base) {                                     //if fpos_list is full write it to file
-         cout << "****** bnum=" << bnum << " blk_base= " << blk_base << " so write index buf seek to " << (blk_base - n) * sizeof(uint64_t) << " write len " << last_fpos_len << '\n';//debug
-         status = fseek(ind_out, (blk_base - n) * sizeof(uint64_t), SEEK_SET);
-         EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from end of file", ("file", index_out_filename.string())("pos", last_buf_len) );
-         size = fwrite((void *) fpos_list, last_fpos_len, 1, ind_out);       //write fpos_list to index file
-         EOS_ASSERT( size == 1, block_log_exception, "${file} write fails", ("file", index_out_filename.string()) );
-         last_fpos_len = buf_len;
-         if (bnum == n) {                                         //if done with all blocks >= block n
-            cout << setw(10) << bnum << setw(16) << file_pos << setw(16) << file_pos-pos_delta << '\n';
-            EOS_ASSERT( index_start==0, block_log_exception, "block ${n} found with unexpected index_start ${s}", ("n",n) ("s",index_start));
-            EOS_ASSERT( pos==td.fpos0, block_log_exception, "block ${n} found at unexpected file position ${p}", ("n",n) ("p",file_pos));
-            status = fseek(blk_out, pos - pos_delta, SEEK_SET);
-            EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", index_out_filename.string())("pos", pos - pos_delta) );
-            size = fwrite((void*)buf, last_buf_len, 1, blk_out);
-            EOS_ASSERT( size == 1, block_log_exception, "${file} write fails", ("file", block_out_filename.string()) );
-            break;
-         } else {
-            blk_base -= (buf_len >> 3);
-            cout << "******filling index buf blk_base " << blk_base << " fpos_len " << buf_len << '\n';//debug
-         }
-      }
-
-      if (index_start <= 0) {                                     //if buf is ready to write (all file pos are updated)
-         //cout << "index_start = " << index_start << " so write buf len " << last_buf_len << " bnum= " << bnum << '\n';//debug
-         EOS_ASSERT(file_pos > td.fpos0, block_log_exception, "reverse scan of ${file} did not halt at block ${n}", ("file", td.block_file_name.string())("n",n));
-         status = fseek(blk_out, pos - pos_delta, SEEK_SET);
-         EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from end of file", ("file", block_out_filename.string())("pos", pos - pos_delta) );
-         size = fwrite((void*)buf, last_buf_len, 1, blk_out);
-         EOS_ASSERT( size == 1, block_log_exception, "${file} write fails", ("file", block_out_filename.string()) );
-         last_buf_len = (pos - td.fpos0 > buf_len) ? buf_len : pos - td.fpos0;
-         pos -= last_buf_len;
-         index_start += last_buf_len;
-         //cout << "seek blocks to " << pos << " read " << last_buf_len << " bytes\n";//debug
-         status = fseek(td.blk_in, pos, SEEK_SET);
-         EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from end of file", ("file", td.block_file_name.string())("pos", pos) );
-         did_read = fread((void*)buf, last_buf_len, 1, td.blk_in);           //read next buf from blocklog
-         EOS_ASSERT(did_read == 1, block_log_exception, "${file} read fails", ("file", td.index_file_name.string()));
-      }
-
-      index_end = index_start - sizeof(uint64_t);                 //move to prior block in buf
-      --bnum;
-
-      if (index_end < 0) {                                        //the file pos record straddles buf boundary
-         cout << "****  index_end= " << index_end << " so save buf-4K len= " << last_buf_len-fk << " bnum= " << bnum << '\n';//debug
-         status = fseek(blk_out, pos - pos_delta + fk, SEEK_SET);
-         EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from end of file", ("file", index_out_filename.string())("pos", pos - pos_delta + fk) );
-         size = fwrite((void*)(buf + fk), last_buf_len - fk, 1, blk_out);             //skip first 4K of buf write out the rest
-         EOS_ASSERT( size == 1, block_log_exception, "${file} write fails", ("file", index_out_filename.string()) );
-         last_buf_len = (pos - td.fpos0 >= buf_len - fk) ? buf_len - fk : pos - td.fpos0;   //bytes to read from blk_in
-         memcpy(buf + last_buf_len, buf, fk);                     //move first 4K of buf after zone will read
-         pos-= last_buf_len;
-         index_end += last_buf_len;
-         cout << "seek blocks to " << pos << " read " << last_buf_len << " bytes before saved 4k\n";//debug
-         status = fseek(td.blk_in, pos, SEEK_SET);
-         EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from end of file", ("file", td.block_file_name.string())("pos", pos) );
-         did_read = fread(buf, last_buf_len, 1, td.blk_in);
-         EOS_ASSERT(did_read == 1, block_log_exception, "${file} read fails", ("file", td.block_file_name.string()) );
-         last_buf_len += fk;                                       //bytes in buf will eventually write to disk
-      }
-   }
-
-   fclose(blk_out);
-   fclose(ind_out);
-   bfs::path old_log = block_dir / "old.log";
-   bfs::path old_ind = block_dir / "old.index";
-   rename(td.block_file_name, old_log);
-   rename(td.index_file_name, old_ind);
-   rename(block_out_filename, td.block_file_name);
-   rename(index_out_filename, td.index_file_name);
-   cout << "The new " << td.block_file_name << " and " << td.index_file_name << " files contain blocks " << n << " through " << td.last_block << '\n';
-   cout << "The original (before trim front) files have been renamed to " << old_log << " and " << old_ind << ".\n";
+   const bool status = block_log::trim_blocklog_front(block_dir, n);
    rt.report();
-   return 0;
+   return status;
 }
 
 

From f071dc9f97a5272c8135b2a39655a4795f6808fa Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Mon, 30 Sep 2019 16:47:21 -0500
Subject: [PATCH 05/16] Added integration tests for trimming front of block
 log. GH #7939

---
 tests/block_log_util_test.py | 75 +++++++++++++++++++++++++++++++++++-
 1 file changed, 74 insertions(+), 1 deletion(-)

diff --git a/tests/block_log_util_test.py b/tests/block_log_util_test.py
index 2d0a292b7f2..1278808c50b 100755
--- a/tests/block_log_util_test.py
+++ b/tests/block_log_util_test.py
@@ -22,6 +22,16 @@
 
 from core_symbol import CORE_SYMBOL
 
+def verifyBlockLog(expected_block_num, trimmedBlockLog):
+    firstBlockNum = expected_block_num
+    for block in trimmedBlockLog:
+        assert 'block_num' in block, print("ERROR: eosio-blocklog didn't return block output")
+        block_num = block['block_num']
+        assert block_num == expected_block_num
+        expected_block_num += 1
+    Print("Block_log contiguous from block number %d to %d" % (firstBlockNum, expected_block_num - 1))
+
+
 appArgs=AppArgs()
 args = TestHelper.parse_args({"--dump-error-details","--keep-logs","-v","--leave-running","--clean-run"})
 Utils.Debug=args.v
@@ -33,7 +43,7 @@
 prodCount=2
 killAll=args.clean_run
 walletPort=TestHelper.DEFAULT_WALLET_PORT
-totalNodes=pnodes
+totalNodes=pnodes+1
 
 walletMgr=WalletMgr(True, port=walletPort)
 testSuccessful=False
@@ -130,6 +140,69 @@ def checkBlockLog(blockLog, blockNumsToFind, firstBlockNum=1):
     Print("Block num %d will definitely be at least one block behind the most recent entry in block log, so --trim will work" % (beforeEndOfBlockLog))
     output=cluster.getBlockLog(0, blockLogAction=BlockLogAction.trim, last=beforeEndOfBlockLog, throwException=True)
 
+    Print("Kill the non production node, we want to verify its block log")
+    cluster.getNode(2).kill(signal.SIGTERM)
+
+    Print("Trim off block num 1 to remove genesis block from block log.")
+    output=cluster.getBlockLog(2, blockLogAction=BlockLogAction.trim, first=2, throwException=True)
+
+    Print("Smoke test the trimmed block log.")
+    output=cluster.getBlockLog(2, blockLogAction=BlockLogAction.smoke_test)
+
+    Print("Analyze block log.")
+    trimmedBlockLog=cluster.getBlockLog(2, blockLogAction=BlockLogAction.return_blocks)
+
+    verifyBlockLog(2, trimmedBlockLog)
+
+    # relaunch the node with the truncated block log and ensure it catches back up with the producers
+    current_head_block_num = node1.getInfo()["head_block_num"]
+    cluster.getNode(2).relaunch(2, cachePopen=True)
+    assert cluster.getNode(2).waitForBlock(current_head_block_num, timeout=60, reportInterval=15)
+
+    # ensure it continues to advance
+    current_head_block_num = node1.getInfo()["head_block_num"]
+    assert cluster.getNode(2).waitForBlock(current_head_block_num, timeout=60, reportInterval=15)
+    info = cluster.getNode(2).getInfo()
+    block = cluster.getNode(2).getBlock(2)
+    assert block is not None
+    block = cluster.getNode(2).getBlock(1, silentErrors=True)
+    assert block is None
+
+    # verify it shuts down cleanly
+    cluster.getNode(2).interruptAndVerifyExitStatus()
+
+    firstBlock = info["last_irreversible_block_num"]
+    Print("Trim off block num %s." % (firstBlock))
+    output=cluster.getBlockLog(2, blockLogAction=BlockLogAction.trim, first=firstBlock, throwException=True)
+
+    Print("Smoke test the trimmed block log.")
+    output=cluster.getBlockLog(2, blockLogAction=BlockLogAction.smoke_test)
+
+    Print("Analyze block log.")
+    trimmedBlockLog=cluster.getBlockLog(2, blockLogAction=BlockLogAction.return_blocks)
+
+    verifyBlockLog(firstBlock, trimmedBlockLog)
+
+    # relaunch the node with the truncated block log and ensure it catches back up with the producers
+    current_head_block_num = node1.getInfo()["head_block_num"]
+    assert current_head_block_num >= info["head_block_num"]
+    cluster.getNode(2).relaunch(2, cachePopen=True)
+    assert cluster.getNode(2).waitForBlock(current_head_block_num, timeout=60, reportInterval=15)
+
+    # ensure it continues to advance
+    current_head_block_num = node1.getInfo()["head_block_num"]
+    assert cluster.getNode(2).waitForBlock(current_head_block_num, timeout=60, reportInterval=15)
+    info = cluster.getNode(2).getInfo()
+    block = cluster.getNode(2).getBlock(firstBlock)
+    assert block is not None
+    block = cluster.getNode(2).getBlock(firstBlock - 1, silentErrors=True)
+    assert block is None
+    block = cluster.getNode(2).getBlock(1, silentErrors=True)
+    assert block is None
+
+    # verify it shuts down cleanly
+    cluster.getNode(2).interruptAndVerifyExitStatus()
+
     testSuccessful=True
 
 finally:

From 4e9431da2e1400d04896ce90f46f4e7a3fefc3a9 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Tue, 1 Oct 2019 10:11:48 -0500
Subject: [PATCH 06/16] Added comments and cleanup. GH #7939

---
 libraries/chain/block_log.cpp | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index 0f037762f5e..de18f2a9073 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -969,6 +969,7 @@ namespace eosio { namespace chain {
       }
       original_block_log.find_block_pos(truncate_at_block);
 
+      // ****** create the new block log file and write out the header for the file
       fc::path new_block_filename = block_dir / "blocks.out";
       if (fc::remove(new_block_filename)) {
          ilog("Removing old blocks.out file");
@@ -990,31 +991,45 @@ namespace eosio { namespace chain {
       new_block_file.write((char*)&totem, sizeof(totem));
 
       const auto new_block_file_first_block_pos = new_block_file.tellp();
+      // ****** end of new block log header
+
 
       // copy over remainder of block log to new block log
-      auto buffer =  make_unique<char[]>(detail::reverse_iterator::_buf_len);                   //read big chunks of old blocks.log into this buffer
+      auto buffer =  make_unique<char[]>(detail::reverse_iterator::_buf_len);
       char* buf =  buffer.get();
 
-      const uint64_t pos_delta = original_block_log.fpos0 - new_block_file_first_block_pos;     // offset bytes to shift from old blocklog to new blocklog
+      // offset bytes to shift from old blocklog position to new blocklog position
+      const uint64_t pos_delta = original_block_log.fpos0 - new_block_file_first_block_pos;
       auto status = fseek(original_block_log.blk_in, 0, SEEK_END);
       EOS_ASSERT( status == 0, block_log_exception, "blocks.log seek failed" );
+
+      // all blocks to copy to the new blocklog
       const uint64_t to_write = ftell(original_block_log.blk_in) - original_block_log.fpos0;
       const auto pos_size = sizeof(uint64_t);
+
+      // start with the last block's position stored at the end of the block
       uint64_t original_pos = ftell(original_block_log.blk_in) - pos_size;
 
-      fc::path new_index_filename = block_dir / "index.out";
       const auto num_blocks = original_block_log.last_block - truncate_at_block + 1;
+
+      fc::path new_index_filename = block_dir / "index.out";
       detail::index_writer index(new_index_filename, num_blocks);
+
       uint64_t read_size = 0;
       for(uint64_t written = to_write; written > 0; written -= read_size) {
          read_size = to_write;
          if (read_size > detail::reverse_iterator::_buf_len) {
             read_size = detail::reverse_iterator::_buf_len;
          }
+
+         // read in the previous contiguous memory into the read buffer
          const auto start_of_blk_buffer_pos = original_block_log.fpos0 + to_write - read_size;
          status = fseek(original_block_log.blk_in, start_of_blk_buffer_pos, SEEK_SET);
          const auto num_read = fread(buf, read_size, 1, original_block_log.blk_in);
          EOS_ASSERT( num_read == 1, block_log_exception, "blocks.log read failed" );
+
+         // walk this memory section to adjust block position to match the adjusted location
+         // of the block start and store in the new index file
          while(original_pos >= start_of_blk_buffer_pos) {
             const auto buffer_index = original_pos - start_of_blk_buffer_pos;
             uint64_t& pos_content = *(uint64_t*)(buf + buffer_index);

From 79e6396646fa7cc5d55d9a69172dc25b90dc7e8f Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Wed, 2 Oct 2019 10:54:18 -0500
Subject: [PATCH 07/16] Replaced cached trim_data variables with methods that
 determine those values. GH #7939

---
 libraries/chain/block_log.cpp                 | 93 +++++++++----------
 .../chain/include/eosio/chain/block_log.hpp   |  7 +-
 programs/eosio-blocklog/main.cpp              |  8 +-
 3 files changed, 50 insertions(+), 58 deletions(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index de18f2a9073..06f4c2901f6 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -157,15 +157,17 @@ namespace eosio { namespace chain {
          explicit fileptr_datastream( FILE* file, const std::string& filename ) : _file(file), _filename(filename) {}
 
          void skip( size_t s ) {
-            std::vector<char> d( s );
-            read( &d[0], s );
+            auto status = fseek(_file, s, SEEK_SET);
+            EOS_ASSERT( status == 0, block_log_exception,
+                        "Could not seek past ${bytes} bytes in Block log file at '${blocks_log}'. Returned status: ${status}",
+                        ("bytes", s)("blocks_log", _filename)("status", status) );
          }
 
          bool read( char* d, size_t s ) {
             size_t result = fread( d, 1, s, _file );
             EOS_ASSERT( result == s, block_log_exception,
-                        "in file: ${file} only able to read ${act} of the expected ${exp}",
-                        ("file", _filename)("act",result)("exp",s) );
+                        "only able to read ${act} bytes of the expected ${exp} bytes in file: ${file}",
+                        ("act",result)("exp",s)("file", _filename) );
             return true;
          }
 
@@ -967,7 +969,6 @@ namespace eosio { namespace chain {
          ilog("All blocks are before block ${n} so do nothing (trim front would delete entire blocks.log).", ("n", truncate_at_block));
          return false;
       }
-      original_block_log.find_block_pos(truncate_at_block);
 
       // ****** create the new block log file and write out the header for the file
       fc::path new_block_filename = block_dir / "blocks.out";
@@ -978,13 +979,14 @@ namespace eosio { namespace chain {
       new_block_file.set_file_path(new_block_filename);
       new_block_file.open( LOG_WRITE_C );
 
-      static_assert( block_log::max_supported_version > 0, "a version number of zero is not supported" );
+      static_assert( block_log::max_supported_version == 3,
+                     "Code was written to support version 3 format, need to update this code for latest format." );
       uint32_t version = block_log::max_supported_version;
       new_block_file.seek(0);
       new_block_file.write((char*)&version, sizeof(version));
       new_block_file.write((char*)&truncate_at_block, sizeof(truncate_at_block));
 
-      new_block_file << *original_block_log.chain_id;
+      new_block_file << original_block_log.chain_id;
 
       // append a totem to indicate the division between blocks and header
       auto totem = block_log::npos;
@@ -999,12 +1001,13 @@ namespace eosio { namespace chain {
       char* buf =  buffer.get();
 
       // offset bytes to shift from old blocklog position to new blocklog position
-      const uint64_t pos_delta = original_block_log.fpos0 - new_block_file_first_block_pos;
+      const uint64_t original_file_block_pos = original_block_log.block_pos(truncate_at_block);
+      const uint64_t pos_delta = original_file_block_pos - new_block_file_first_block_pos;
       auto status = fseek(original_block_log.blk_in, 0, SEEK_END);
       EOS_ASSERT( status == 0, block_log_exception, "blocks.log seek failed" );
 
       // all blocks to copy to the new blocklog
-      const uint64_t to_write = ftell(original_block_log.blk_in) - original_block_log.fpos0;
+      const uint64_t to_write = ftell(original_block_log.blk_in) - original_file_block_pos;
       const auto pos_size = sizeof(uint64_t);
 
       // start with the last block's position stored at the end of the block
@@ -1023,7 +1026,7 @@ namespace eosio { namespace chain {
          }
 
          // read in the previous contiguous memory into the read buffer
-         const auto start_of_blk_buffer_pos = original_block_log.fpos0 + to_write - read_size;
+         const auto start_of_blk_buffer_pos = original_file_block_pos + to_write - read_size;
          status = fseek(original_block_log.blk_in, start_of_blk_buffer_pos, SEEK_SET);
          const auto num_read = fread(buf, read_size, 1, original_block_log.blk_in);
          EOS_ASSERT( num_read == 1, block_log_exception, "blocks.log read failed" );
@@ -1069,7 +1072,7 @@ namespace eosio { namespace chain {
       EOS_ASSERT( ind_in != nullptr, block_log_not_found, "cannot read file ${file}", ("file",index_file_name.string()) );
       auto size = fread((void*)&version,sizeof(version), 1, blk_in);
       EOS_ASSERT( size == 1, block_log_unsupported_version, "invalid format for file ${file}", ("file",block_file_name.string()));
-      cout << "block log version= " << version << '\n';
+      ilog("block log version= ${version}",("version",version));
       EOS_ASSERT( block_log::is_supported_version(version), block_log_unsupported_version, "block log version ${v} is not supported", ("v",version));
 
       detail::fileptr_datastream ds(blk_in, block_file_name.string());
@@ -1089,14 +1092,13 @@ namespace eosio { namespace chain {
             chain_id = gs.compute_chain_id();
          }
          else if (block_log::contains_chain_id(version, first_block)) {
-            chain_id = chain_id_type{};
-            ds >> *chain_id;
+            ds >> chain_id;
          }
          else {
-            EOS_ASSERT( false, block_log_exception,
-                        "Block log ${file} is not supported. version: ${ver} and first_block: ${first_block} does not contain "
-                        "a genesis_state nor a chain_id.",
-                        ("file", block_file_name.string())("ver", version)("first_block", first_block));
+            EOS_THROW( block_log_exception,
+                       "Block log ${file} is not supported. version: ${ver} and first_block: ${first_block} does not contain "
+                       "a genesis_state nor a chain_id.",
+                       ("file", block_file_name.string())("ver", version)("first_block", first_block));
          }
 
          const auto expected_totem = block_log::npos;
@@ -1117,13 +1119,12 @@ namespace eosio { namespace chain {
                  "indicates the first block is at ${index}",
                  ("file", block_file_name.string())("determined", start_of_blocks)("index",first_block_pos));
 
-      cout << "first block= " << first_block << '\n';
+      ilog("first block= ${first}",("first",first_block));
       const auto status = fseek(ind_in, 0, SEEK_END);                //get length of blocks.index (gives number of blocks)
       EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} end", ("file", index_file_name.string()) );
-      EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} end", ("file", index_file_name.string()) );
       const uint64_t file_end = ftell(ind_in);                //get length of blocks.index (gives number of blocks)
       last_block = first_block + file_end/sizeof(uint64_t) - 1;
-      cout << "last block=  " << last_block << '\n';
+      ilog("last block= ${first}",("first",first_block));
    }
 
    trim_data::~trim_data() {
@@ -1133,9 +1134,21 @@ namespace eosio { namespace chain {
          fclose(ind_in);
    }
 
+   uint64_t trim_data::block_index(uint32_t n) const {
+      using namespace std;
+      EOS_ASSERT( n <= last_block, block_log_exception,
+                  "cannot seek in ${file} to block number ${b}, block number ${last} is the last block",
+                  ("file", index_file_name.string())("b",n)("last",last_block) );
+      return sizeof(uint64_t) * (n - first_block);
+   }
+
    uint64_t trim_data::block_pos(uint32_t n) {
       using namespace std;
-      index_pos = sizeof(uint64_t) * (n - first_block);
+      // can indicate the location of the block after the last block
+      if (n == last_block + 1) {
+         return ftell(blk_in);
+      }
+      const uint64_t index_pos = sizeof(uint64_t) * (n - first_block);
       auto status = fseek(ind_in, index_pos, SEEK_SET);
       EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file for block ${b}", ("file", index_file_name.string())("pos", index_pos)("b",n) );
       const uint64_t pos = ftell(ind_in);
@@ -1143,40 +1156,22 @@ namespace eosio { namespace chain {
       uint64_t block_n_pos;
       auto size = fread((void*)&block_n_pos, sizeof(block_n_pos), 1, ind_in);                   //filepos of block n
       EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}", ("file", index_file_name.string())("b",n) );
-      return block_n_pos;
-   }
 
-   void trim_data::find_block_pos(uint32_t n) {
-      //get file position of block n from blocks.index then confirm block n is found in blocks.log at that position
-      //sets fpos0 and fpos1, throws exception if block at fpos0 is not block n
-      using namespace std;
-      auto temp = block_pos(first_block);
-      fpos0 = block_pos(n);
-      auto size = fread((void*)&fpos1, sizeof(fpos1), 1, ind_in);                   //filepos of block n+1
-      if (n != last_block)
-         EOS_ASSERT( size == 1, block_log_exception, "cannot read ${file} entry for block ${b}, size=${size}",
-                     ("file", index_file_name.string())("b",n + 1)("size",size) );
-
-      cout << "According to blocks.index:\n";
-      cout << "    block " << n << " starts at position " << fpos0 << '\n';
-      cout << "    block " << n + 1;
-
-      if (n != last_block)
-         cout << " starts at position " << fpos1 << '\n';
-      else
-         cout << " is past end\n";
-
-      //read blocks.log and verify block number n is found at file position fpos0
-      auto status = fseek(blk_in, fpos0 + blknum_offset, SEEK_SET);
-      EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", block_file_name.string())("pos", fpos0 + blknum_offset) );
+      //read blocks.log and verify block number n is found at the determined file position
+      const auto calc_blknum_pos = block_n_pos + blknum_offset;
+      status = fseek(blk_in, calc_blknum_pos, SEEK_SET);
+      EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", block_file_name.string())("pos", calc_blknum_pos) );
       const uint64_t block_offset_pos = ftell(blk_in);
-      EOS_ASSERT( block_offset_pos == fpos0 + blknum_offset, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", block_file_name.string())("pos", fpos0 + blknum_offset) );
+      EOS_ASSERT( block_offset_pos == calc_blknum_pos, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file", ("file", block_file_name.string())("pos", calc_blknum_pos) );
       uint32_t prior_blknum;
       size = fread((void*)&prior_blknum, sizeof(prior_blknum), 1, blk_in);     //read bigendian block number of prior block
       EOS_ASSERT( size == 1, block_log_exception, "cannot read prior block");
       const uint32_t bnum = fc::endian_reverse_u32(prior_blknum) + 1;          //convert to little endian, add 1 since prior block
-      cout << "At position " << fpos0 << " in " << index_file_name.generic_string() << " find block " << bnum << (bnum == n ? " as expected\n": " - not good!\n");
-      EOS_ASSERT( bnum == n, block_log_exception, "${index} does not agree with ${blocks}", ("index", index_file_name.string())("blocks", block_file_name.string()) );
+      EOS_ASSERT( bnum == n, block_log_exception,
+                  "At position ${pos} in ${file} expected to find ${exp_bnum} but found ${act_bnum}",
+                  ("pos",block_offset_pos)("file", block_file_name.string())("exp_bnum",n)("act_bnum",bnum) );
+
+      return block_n_pos;
    }
 
    } } /// eosio::chain
diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp
index db7f3c00b7b..31a44d7d5d9 100644
--- a/libraries/chain/include/eosio/chain/block_log.hpp
+++ b/libraries/chain/include/eosio/chain/block_log.hpp
@@ -97,7 +97,7 @@ namespace eosio { namespace chain {
    struct trim_data {            //used by trim_blocklog_front(), trim_blocklog_end(), and smoke_test()
       trim_data(fc::path block_dir);
       ~trim_data();
-      void find_block_pos(uint32_t n);
+      uint64_t block_index(uint32_t n) const;
       uint64_t block_pos(uint32_t n);
       fc::path block_file_name, index_file_name;        //full pathname for blocks.log and blocks.index
       uint32_t version = 0;                              //blocklog version
@@ -106,11 +106,8 @@ namespace eosio { namespace chain {
       FILE* blk_in = nullptr;                            //C style files for reading blocks.log and blocks.index
       FILE* ind_in = nullptr;                            //C style files for reading blocks.log and blocks.index
       //we use low level file IO because it is distinctly faster than C++ filebuf or iostream
-      uint64_t index_pos = 0;                            //filepos in blocks.index for block n, +8 for block n+1
-      uint64_t fpos0 = 0;                                //filepos in blocks.log for block n and block n+1
-      uint64_t fpos1 = 0;                                //filepos in blocks.log for block n and block n+1
       uint64_t first_block_pos = 0;                      //file position in blocks.log for the first block in the log
-      fc::optional<chain_id_type> chain_id;
+      chain_id_type chain_id;
 
       static constexpr int blknum_offset{14};            //offset from start of block to 4 byte block number, valid for the only allowed versions
    };
diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp
index 55e2b4939a1..9da73c7698b 100644
--- a/programs/eosio-blocklog/main.cpp
+++ b/programs/eosio-blocklog/main.cpp
@@ -222,9 +222,9 @@ int trim_blocklog_end(bfs::path block_dir, uint32_t n) {       //n is last block
       cerr << "There are no blocks after block " << n << " so do nothing\n";
       return 2;
    }
-   td.find_block_pos(n);
-   bfs::resize_file(td.block_file_name, td.fpos1);
-   uint64_t index_end= td.index_pos + sizeof(uint64_t);             //advance past record for block n
+   const uint64_t end_of_new_file = td.block_pos(n + 1);
+   bfs::resize_file(td.block_file_name, end_of_new_file);
+   const uint64_t index_end= td.block_index(n) + sizeof(uint64_t);             //advance past record for block n
    bfs::resize_file(td.index_file_name, index_end);
    cout << "blocks.index has been trimmed to " << index_end << " bytes\n";
    rt.report();
@@ -262,7 +262,7 @@ void smoke_test(bfs::path block_dir) {
    for (uint32_t n = td.first_block; ; n += delta) {
       if (n > td.last_block)
          n = td.last_block;
-      td.find_block_pos(n);                                 //check block 'n' is where blocks.index says
+      td.block_pos(n);                                 //check block 'n' is where blocks.index says
       if (n == td.last_block)
          break;
    }

From 303c839387531f66bb4404922381c76658a98261 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Wed, 2 Oct 2019 13:27:35 -0500
Subject: [PATCH 08/16] Added passing in the temp directory for temporary
 storage of new files and eventual storage of old versions. GH #7939

---
 libraries/chain/block_log.cpp                     | 7 ++++---
 libraries/chain/include/eosio/chain/block_log.hpp | 2 +-
 programs/eosio-blocklog/main.cpp                  | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index 06f4c2901f6..f08e90c9088 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -956,8 +956,9 @@ namespace eosio { namespace chain {
       return std::clamp(version, min_supported_version, max_supported_version) == version;
    }
 
-   bool block_log::trim_blocklog_front(const fc::path& block_dir, uint32_t truncate_at_block) {
+   bool block_log::trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block) {
       using namespace std;
+      EOS_ASSERT( block_dir != temp_dir, block_log_exception, "block_dir and temp_dir need to be different directories" );
       ilog("In directory ${dir} will trim all blocks before block ${n} from blocks.log and blocks.index.",
            ("dir", block_dir.generic_string())("n", truncate_at_block));
       trim_data original_block_log(block_dir);
@@ -971,7 +972,7 @@ namespace eosio { namespace chain {
       }
 
       // ****** create the new block log file and write out the header for the file
-      fc::path new_block_filename = block_dir / "blocks.out";
+      fc::path new_block_filename = temp_dir / "blocks.log";
       if (fc::remove(new_block_filename)) {
          ilog("Removing old blocks.out file");
       }
@@ -1015,7 +1016,7 @@ namespace eosio { namespace chain {
 
       const auto num_blocks = original_block_log.last_block - truncate_at_block + 1;
 
-      fc::path new_index_filename = block_dir / "index.out";
+      fc::path new_index_filename = temp_dir / "blocks.index";
       detail::index_writer index(new_index_filename, num_blocks);
 
       uint64_t read_size = 0;
diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp
index 31a44d7d5d9..5fbe3e771a5 100644
--- a/libraries/chain/include/eosio/chain/block_log.hpp
+++ b/libraries/chain/include/eosio/chain/block_log.hpp
@@ -79,7 +79,7 @@ namespace eosio { namespace chain {
 
          static bool is_supported_version(uint32_t version);
 
-         static bool trim_blocklog_front(const fc::path& block_dir, uint32_t truncate_at_block);
+         static bool trim_blocklog_front(const fc::path& block_dir, const fc::path& temp_dir, uint32_t truncate_at_block);
 
    private:
          void open(const fc::path& data_dir);
diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp
index 9da73c7698b..811eb488751 100644
--- a/programs/eosio-blocklog/main.cpp
+++ b/programs/eosio-blocklog/main.cpp
@@ -233,7 +233,7 @@ int trim_blocklog_end(bfs::path block_dir, uint32_t n) {       //n is last block
 
 bool trim_blocklog_front(bfs::path block_dir, uint32_t n) {        //n is first block to keep (remove prior blocks)
    report_time rt("trimming blocklog start");
-   const bool status = block_log::trim_blocklog_front(block_dir, n);
+   const bool status = block_log::trim_blocklog_front(block_dir, blocks_dir / "old", n);
    rt.report();
    return status;
 }

From a607f9b743705716d2387327d2dccf884dad5d4f Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Wed, 2 Oct 2019 13:32:48 -0500
Subject: [PATCH 09/16] Fixed variable name. GH #7939

---
 programs/eosio-blocklog/main.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/eosio-blocklog/main.cpp b/programs/eosio-blocklog/main.cpp
index 811eb488751..bbe368cc96f 100644
--- a/programs/eosio-blocklog/main.cpp
+++ b/programs/eosio-blocklog/main.cpp
@@ -233,7 +233,7 @@ int trim_blocklog_end(bfs::path block_dir, uint32_t n) {       //n is last block
 
 bool trim_blocklog_front(bfs::path block_dir, uint32_t n) {        //n is first block to keep (remove prior blocks)
    report_time rt("trimming blocklog start");
-   const bool status = block_log::trim_blocklog_front(block_dir, blocks_dir / "old", n);
+   const bool status = block_log::trim_blocklog_front(block_dir, block_dir / "old", n);
    rt.report();
    return status;
 }

From 798bece93554d40ea5e2b5cf46f3f6d13a9f61c1 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Wed, 2 Oct 2019 13:41:23 -0500
Subject: [PATCH 10/16] Need to ensure temp directory exists. GH #7939

---
 libraries/chain/block_log.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index f08e90c9088..c7d86205e3c 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -972,6 +972,7 @@ namespace eosio { namespace chain {
       }
 
       // ****** create the new block log file and write out the header for the file
+      fc::create_directories(temp_dir);
       fc::path new_block_filename = temp_dir / "blocks.log";
       if (fc::remove(new_block_filename)) {
          ilog("Removing old blocks.out file");

From 14920bc4899dbe5794a4c1cd76cd2af6e7a626f5 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Thu, 3 Oct 2019 08:31:44 -0500
Subject: [PATCH 11/16] Change assert that always failed to EOS_THROW. GH #7939

---
 libraries/chain/block_log.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index c7d86205e3c..3919c9cc190 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -591,10 +591,10 @@ namespace eosio { namespace chain {
          new_block_stream << chain_id;
       }
       else {
-         EOS_ASSERT( false, block_log_exception,
-                     "Block log ${file} is not supported. version: ${ver} and first_block_num: ${fbn} does not contain "
-                     "a genesis_state nor a chain_id.",
-                     ("file", (backup_dir / "blocks.log").generic_string())("ver", version)("fbn", first_block_num));
+         EOS_THROW( block_log_exception,
+                    "Block log ${file} is not supported. version: ${ver} and first_block_num: ${fbn} does not contain "
+                    "a genesis_state nor a chain_id.",
+                    ("file", (backup_dir / "blocks.log").generic_string())("ver", version)("fbn", first_block_num));
       }
 
       if (version != 1) {

From 936934b46b862e55ecfaae8679ac0bb794954c5f Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Thu, 3 Oct 2019 10:11:16 -0500
Subject: [PATCH 12/16] Fixed skip logic and added assert to verify at or past
 first block when searching for a block number location. GH #7939

---
 libraries/chain/block_log.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index 3919c9cc190..85fe1ab9840 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -157,7 +157,7 @@ namespace eosio { namespace chain {
          explicit fileptr_datastream( FILE* file, const std::string& filename ) : _file(file), _filename(filename) {}
 
          void skip( size_t s ) {
-            auto status = fseek(_file, s, SEEK_SET);
+            auto status = fseek(_file, s, SEEK_CUR);
             EOS_ASSERT( status == 0, block_log_exception,
                         "Could not seek past ${bytes} bytes in Block log file at '${blocks_log}'. Returned status: ${status}",
                         ("bytes", s)("blocks_log", _filename)("status", status) );
@@ -1138,6 +1138,9 @@ namespace eosio { namespace chain {
 
    uint64_t trim_data::block_index(uint32_t n) const {
       using namespace std;
+      EOS_ASSERT( first_block <= n, block_log_exception,
+                  "cannot seek in ${file} to block number ${b}, block number ${first} is the first block",
+                  ("file", index_file_name.string())("b",n)("first",first_block) );
       EOS_ASSERT( n <= last_block, block_log_exception,
                   "cannot seek in ${file} to block number ${b}, block number ${last} is the last block",
                   ("file", index_file_name.string())("b",n)("last",last_block) );
@@ -1150,7 +1153,7 @@ namespace eosio { namespace chain {
       if (n == last_block + 1) {
          return ftell(blk_in);
       }
-      const uint64_t index_pos = sizeof(uint64_t) * (n - first_block);
+      const uint64_t index_pos = block_index(n);
       auto status = fseek(ind_in, index_pos, SEEK_SET);
       EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} ${pos} from beginning of file for block ${b}", ("file", index_file_name.string())("pos", index_pos)("b",n) );
       const uint64_t pos = ftell(ind_in);

From 2e7f7f47c4ca6bb112a8ef335cff7da28445f032 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Thu, 3 Oct 2019 11:20:07 -0500
Subject: [PATCH 13/16] Fixed logic for reverse reading the block log for
 truncation. GH #7939

---
 libraries/chain/block_log.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index 85fe1ab9840..737d052140c 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -1021,14 +1021,14 @@ namespace eosio { namespace chain {
       detail::index_writer index(new_index_filename, num_blocks);
 
       uint64_t read_size = 0;
-      for(uint64_t written = to_write; written > 0; written -= read_size) {
-         read_size = to_write;
+      for(uint64_t to_write_remaining = to_write; to_write_remaining > 0; to_write_remaining -= read_size) {
+         read_size = to_write_remaining;
          if (read_size > detail::reverse_iterator::_buf_len) {
             read_size = detail::reverse_iterator::_buf_len;
          }
 
          // read in the previous contiguous memory into the read buffer
-         const auto start_of_blk_buffer_pos = original_file_block_pos + to_write - read_size;
+         const auto start_of_blk_buffer_pos = original_file_block_pos + to_write_remaining - read_size;
          status = fseek(original_block_log.blk_in, start_of_blk_buffer_pos, SEEK_SET);
          const auto num_read = fread(buf, read_size, 1, original_block_log.blk_in);
          EOS_ASSERT( num_read == 1, block_log_exception, "blocks.log read failed" );

From 018f0c1893e4c909ceb1cc478ea60f724f1d3c82 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Thu, 3 Oct 2019 11:22:40 -0500
Subject: [PATCH 14/16] Fixed code so last_block is set before we call
 block_pos ... GH #7939

which validates that n is less than or equal to last_block + 1.
---
 libraries/chain/block_log.cpp | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index 737d052140c..153a60fac83 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -1115,17 +1115,18 @@ namespace eosio { namespace chain {
       }
 
       const uint64_t start_of_blocks = ftell(blk_in);
+
+      const auto status = fseek(ind_in, 0, SEEK_END);                //get length of blocks.index (gives number of blocks)
+      EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} end", ("file", index_file_name.string()) );
+      const uint64_t file_end = ftell(ind_in);                //get length of blocks.index (gives number of blocks)
+      last_block = first_block + file_end/sizeof(uint64_t) - 1;
+
       first_block_pos = block_pos(first_block);
       EOS_ASSERT(start_of_blocks == first_block_pos, block_log_exception,
                  "Block log ${file} was determined to have its first block at ${determined}, but the block index "
                  "indicates the first block is at ${index}",
                  ("file", block_file_name.string())("determined", start_of_blocks)("index",first_block_pos));
-
       ilog("first block= ${first}",("first",first_block));
-      const auto status = fseek(ind_in, 0, SEEK_END);                //get length of blocks.index (gives number of blocks)
-      EOS_ASSERT( status == 0, block_log_exception, "cannot seek to ${file} end", ("file", index_file_name.string()) );
-      const uint64_t file_end = ftell(ind_in);                //get length of blocks.index (gives number of blocks)
-      last_block = first_block + file_end/sizeof(uint64_t) - 1;
       ilog("last block= ${first}",("first",first_block));
    }
 

From 470873024db432683fa4b4ec5ee9a2c36801c286 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Thu, 3 Oct 2019 17:19:02 -0500
Subject: [PATCH 15/16] Fixed trimmed file to write buffer in reverse order. GH
 #7939

and other fixes
---
 libraries/chain/block_log.cpp | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index 153a60fac83..5285646d7f7 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -979,7 +979,11 @@ namespace eosio { namespace chain {
       }
       fc::cfile new_block_file;
       new_block_file.set_file_path(new_block_filename);
+      // need to open as append in case the file doesn't already exist, then reopen without append to allow writing the
+      // file in any order
       new_block_file.open( LOG_WRITE_C );
+      new_block_file.close();
+      new_block_file.open( LOG_RW_C );
 
       static_assert( block_log::max_supported_version == 3,
                      "Code was written to support version 3 format, need to update this code for latest format." );
@@ -997,7 +1001,6 @@ namespace eosio { namespace chain {
       const auto new_block_file_first_block_pos = new_block_file.tellp();
       // ****** end of new block log header
 
-
       // copy over remainder of block log to new block log
       auto buffer =  make_unique<char[]>(detail::reverse_iterator::_buf_len);
       char* buf =  buffer.get();
@@ -1043,6 +1046,7 @@ namespace eosio { namespace chain {
             index.write(pos_content);
             original_pos = start_of_this_block - pos_size;
          }
+         new_block_file.seek(new_block_file_first_block_pos + to_write_remaining - read_size);
          new_block_file.write(buf, read_size);
       }
       index.complete();
@@ -1051,10 +1055,10 @@ namespace eosio { namespace chain {
       new_block_file.flush();
       new_block_file.close();
 
-      fc::path old_log = block_dir / "old.log";
+      fc::path old_log = temp_dir / "old.log";
       rename(original_block_log.block_file_name, old_log);
       rename(new_block_filename, original_block_log.block_file_name);
-      fc::path old_ind = block_dir / "old.index";
+      fc::path old_ind = temp_dir / "old.index";
       rename(original_block_log.index_file_name, old_ind);
       rename(new_index_filename, original_block_log.index_file_name);
 
@@ -1127,7 +1131,7 @@ namespace eosio { namespace chain {
                  "indicates the first block is at ${index}",
                  ("file", block_file_name.string())("determined", start_of_blocks)("index",first_block_pos));
       ilog("first block= ${first}",("first",first_block));
-      ilog("last block= ${first}",("first",first_block));
+      ilog("last block= ${last}",("last",last_block));
    }
 
    trim_data::~trim_data() {

From 9de6bbb83dca03a83c0d54baa4dc06fc734f1da5 Mon Sep 17 00:00:00 2001
From: "johnsonb@objectcomputing.com" <Brian Johnson>
Date: Thu, 3 Oct 2019 17:24:24 -0500
Subject: [PATCH 16/16] Fixed comment. GH #7939

---
 libraries/chain/block_log.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp
index 5285646d7f7..822de1ea8b8 100644
--- a/libraries/chain/block_log.cpp
+++ b/libraries/chain/block_log.cpp
@@ -979,7 +979,7 @@ namespace eosio { namespace chain {
       }
       fc::cfile new_block_file;
       new_block_file.set_file_path(new_block_filename);
-      // need to open as append in case the file doesn't already exist, then reopen without append to allow writing the
+      // need to open as append since the file doesn't already exist, then reopen without append to allow writing the
       // file in any order
       new_block_file.open( LOG_WRITE_C );
       new_block_file.close();