From 9e32a103968e4d1061a03962958553cb019c2c2a Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Mon, 9 Jan 2023 16:18:28 -0500
Subject: [PATCH 01/48] Added snapshot scheduling calls to schema

---
 .../producer_api_plugin/producer.swagger.yaml | 224 ++++++++++++++++--
 1 file changed, 201 insertions(+), 23 deletions(-)

diff --git a/plugins/producer_api_plugin/producer.swagger.yaml b/plugins/producer_api_plugin/producer.swagger.yaml
index 507c5f555f..54c8dbcc1e 100644
--- a/plugins/producer_api_plugin/producer.swagger.yaml
+++ b/plugins/producer_api_plugin/producer.swagger.yaml
@@ -39,7 +39,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: "#/component/schema/OK"
   /producer/resume:
     post:
       summary: resume
@@ -51,7 +51,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: "#/component/schema/OK"
   /producer/paused:
     post:
       summary: paused
@@ -63,7 +63,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: "#/component/schema/OK"
   /producer/get_runtime_options:
     post:
       summary: get_runtime_options
@@ -75,7 +75,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/Runtime_Options'
+                $ref: "#/component/schema/Runtime_Options"
 
   /producer/update_runtime_options:
     post:
@@ -86,7 +86,7 @@ paths:
         content:
           application/json:
             schema:
-              $ref: '#/component/schema/Runtime_Options'
+              $ref: "#/component/schema/Runtime_Options"
 
       responses:
         "201":
@@ -94,7 +94,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: "#/component/schema/OK"
   /producer/get_greylist:
     post:
       summary: get_greylist
@@ -125,11 +125,11 @@ paths:
             schema:
               type: object
               properties:
-                  accounts:
-                    type: array
-                    description: List of account names to add
-                    items:
-                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+                accounts:
+                  type: array
+                  description: List of account names to add
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
 
       responses:
         "201":
@@ -137,13 +137,13 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: "#/component/schema/OK"
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/Error'
+                $ref: "#/component/schema/Error"
   /producer/remove_greylist_accounts:
     post:
       summary: remove_greylist_accounts
@@ -167,13 +167,13 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: "#/component/schema/OK"
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/Error'
+                $ref: "#/component/schema/Error"
   /producer/get_whitelist_blacklist:
     post:
       summary: get_whitelist_blacklist
@@ -265,14 +265,14 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: "#/component/schema/OK"
 
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/Error'
+                $ref: "#/component/schema/Error"
 
   /producer/create_snapshot:
     post:
@@ -291,7 +291,7 @@ paths:
                     $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
                   head_block_num:
                     type: integer
-                    descripiton: Highest block number on the chain
+                    description: Highest block number on the chain
                     example: 5102
                   head_block_time:
                     type: string
@@ -310,7 +310,186 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/Error'
+                $ref: "#/component/schema/Error"
+
+  /producer/schedule_snapshot:
+    post:
+      summary: schedule_snapshot
+      description: Submits a request to generate a schedule for automated snapshot with given parameters. If request body is empty, triest to execute snapshot immediately. Returns error when unable to create snapshot.
+      operationId: schedule_snapshot
+      requestBody:
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                blocks_count:
+                  type: integer
+                  description: Generate snapshot every blocks_count blocks
+                start_block_num:
+                  type: integer
+                  description: Block number at which schedule starts
+                  example: 5102
+                end_block_num:
+                  type: integer
+                  description: Block number at which schedule ends
+                  example: 15102
+      responses:
+        "201":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  head_block_id:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                  head_block_num:
+                    type: integer
+                    description: Highest block number on the chain
+                    example: 5102
+                  head_block_time:
+                    type: string
+                    description: Highest block unix timestamp
+                    example: 2020-11-16T00:00:00.000
+                  version:
+                    type: integer
+                    description: version number
+                    example: 6
+                  snapshot_id:
+                    type: integer
+                    description: snapshot id
+                  snapshot_name:
+                    type: string
+                    description: The path and file name of the snapshot
+                    example: /home/me/nodes/node-name/snapshots/snapshot-0000999f99999f9f999f99f99ff9999f999f9fff99ff99ffff9f9f9fff9f9999.bin
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: "#/component/schema/Error"
+
+  /producer/get_snapshot_status:
+    post:
+      summary: get_snapshot_status
+      description: Queries a status of the existing scheduled snapshot. Returns error when unable to find snapshot requested or query its status.
+      operationId: get_snapshot_status
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                snapshot_id:
+                  type: integer
+                  description: snapshot id
+      responses:
+        "201":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  head_block_id:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                  head_block_num:
+                    type: integer
+                    description: Highest block number on the chain
+                    example: 5102
+                  head_block_time:
+                    type: string
+                    description: Highest block unix timestamp
+                    example: 2020-11-16T00:00:00.000
+                  version:
+                    type: integer
+                    description: version number
+                    example: 6
+                  snapshot_id:
+                    type: integer
+                    description: unique snapshot id
+                  snapshot_name:
+                    type: string
+                    description: The path and file name of the snapshot
+                    example: /home/me/nodes/node-name/snapshots/snapshot-0000999f99999f9f999f99f99ff9999f999f9fff99ff99ffff9f9f9fff9f9999.bin
+                  blocks_count:
+                    type: integer
+                    description: Generate snapshot every blocks_count blocks
+                  start_block_num:
+                    type: integer
+                    description: Block number at which schedule starts                    
+                  end_block_num:
+                    type: integer
+                    description: Block number at which schedule ends                    
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: "#/component/schema/Error"
+
+  /producer/unschedule_snapshot:
+    post:
+      summary: unschedule_snapshot
+      description: Submits a request to remove identified by id recurring snapshot from schedule. Returns error when unable to create unschedule.
+      operationId: unschedule_snapshot
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                snapshot_id:
+                  type: integer
+                  description: snapshot id
+      responses:
+        "201":
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  head_block_id:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                  head_block_num:
+                    type: integer
+                    description: Highest block number on the chain
+                    example: 5102
+                  head_block_time:
+                    type: string
+                    description: Highest block unix timestamp
+                    example: 2020-11-16T00:00:00.000
+                  version:
+                    type: integer
+                    description: version number
+                    example: 6
+                  snapshot_id:
+                    type: integer
+                    description: unique snapshot id
+                  snapshot_name:
+                    type: string
+                    description: The path and file name of the snapshot
+                    example: /home/me/nodes/node-name/snapshots/snapshot-0000999f99999f9f999f99f99ff9999f999f9fff99ff99ffff9f9f9fff9f9999.bin
+                  blocks_count:
+                    type: integer
+                    description: Generate snapshot every blocks_count blocks
+                  start_block_num:
+                    type: integer
+                    description: Block number at which schedule starts                    
+                  end_block_num:
+                    type: integer
+                    description: Block number at which schedule ends                    
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: "#/component/schema/Error"
+
 
   /producer/get_integrity_hash:
     post:
@@ -354,13 +533,13 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/OK'
+                $ref: "#/component/schema/OK"
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/Error'
+                $ref: "#/component/schema/Error"
 
   /producer/get_supported_protocol_features:
     post:
@@ -424,7 +603,6 @@ paths:
                               value:
                                 type: string
 
-
   /producer/get_account_ram_corrections:
     post:
       summary: get_account_ram_corrections
@@ -536,7 +714,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/component/schema/Error'
+                $ref: "#/component/schema/Error"
 
 component:
   schema:

From 6da9928831a4141b7bef65989b242becd89580a2 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Wed, 11 Jan 2023 12:05:19 -0500
Subject: [PATCH 02/48] Updates to snapshot scheduling api

---
 .../producer_api_plugin/producer.swagger.yaml | 200 ++++++++++++------
 1 file changed, 137 insertions(+), 63 deletions(-)

diff --git a/plugins/producer_api_plugin/producer.swagger.yaml b/plugins/producer_api_plugin/producer.swagger.yaml
index 54c8dbcc1e..1f2173685a 100644
--- a/plugins/producer_api_plugin/producer.swagger.yaml
+++ b/plugins/producer_api_plugin/producer.swagger.yaml
@@ -334,35 +334,104 @@ paths:
                   type: integer
                   description: Block number at which schedule ends
                   example: 15102
+                snapshot_description:
+                    type: string
+                    description: Description of the snapshot
+                    example: Daily snapshot
+
       responses:
         "201":
           description: OK
           content:
             application/json:
               schema:
+                  type: object
+                  properties:
+                    snapshot_request_id:
+                      type: integer
+                      description: Unique id identifying current snapshot request
+                    blocks_count:
+                      type: integer
+                      description: Generate snapshot every blocks_count blocks
+                    start_block_num:
+                      type: integer
+                      description: Block number at which schedule starts
+                      example: 5102
+                    end_block_num:
+                      type: integer
+                      description: Block number at which schedule ends
+                      example: 15102
+                    snapshot_description:
+                        type: string
+                        description: Description of the snapshot
+                        example: Daily snapshot
+        "400":
+          description: client error
+          content:
+            application/json:
+              schema:
+                $ref: "#/component/schema/Error"
+
+  /producer/get_snapshot_requests:
+    post:
+      summary: get_snapshot_request_status
+      description: Returns a list of scheduled snapshots
+      operationId: get_snapshot_status
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              properties:
+                offset:
+                  type: integer
+                  description: Snapshot offset
+                limit:
+                  type: integer
+                  description: Number of snapshots to return
+                
+      responses:
+        "201":
+          description: OK
+          content:
+            application/json:
+             schema:
                 type: object
                 properties:
-                  head_block_id:
-                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
-                  head_block_num:
+                  offset:
                     type: integer
-                    description: Highest block number on the chain
-                    example: 5102
-                  head_block_time:
-                    type: string
-                    description: Highest block unix timestamp
-                    example: 2020-11-16T00:00:00.000
-                  version:
+                    description: Snapshot offset
+                  limit:
                     type: integer
-                    description: version number
-                    example: 6
-                  snapshot_id:
+                    description: Number of snapshots to return
+                  count:
                     type: integer
-                    description: snapshot id
-                  snapshot_name:
-                    type: string
-                    description: The path and file name of the snapshot
-                    example: /home/me/nodes/node-name/snapshots/snapshot-0000999f99999f9f999f99f99ff9999f999f9fff99ff99ffff9f9f9fff9f9999.bin
+                    description: Total amount of snapshots
+                  snapshot_requests:
+                    type: array
+                    description: An array of scheduled snapshots
+                    items:
+                      type: object
+                      properties:
+                          snapshot_request_id:
+                            type: integer
+                            description: Unique id identifying current snapshot request
+                          blocks_count:
+                            type: integer
+                            description: Generate snapshot every blocks_count blocks
+                          start_block_num:
+                            type: integer
+                            description: Block number at which schedule starts
+                            example: 5102
+                          end_block_num:
+                            type: integer
+                            description: Block number at which schedule ends
+                            example: 15102
+                          snapshot_description:
+                              type: string
+                              description: Description of the snapshot
+                              example: Daily snapshot                
         "400":
           description: client error
           content:
@@ -370,9 +439,9 @@ paths:
               schema:
                 $ref: "#/component/schema/Error"
 
-  /producer/get_snapshot_status:
+  /producer/get_snapshot_request_status:
     post:
-      summary: get_snapshot_status
+      summary: get_snapshot_request_status
       description: Queries a status of the existing scheduled snapshot. Returns error when unable to find snapshot requested or query its status.
       operationId: get_snapshot_status
       requestBody:
@@ -393,36 +462,53 @@ paths:
               schema:
                 type: object
                 properties:
-                  head_block_id:
-                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
-                  head_block_num:
+                  snapshot_request_id:
                     type: integer
-                    description: Highest block number on the chain
-                    example: 5102
-                  head_block_time:
+                    description: Unique id identifying current snapshot request
+                  snapshot_request_time:
                     type: string
-                    description: Highest block unix timestamp
+                    description: Snapshot unix timestamp
                     example: 2020-11-16T00:00:00.000
-                  version:
-                    type: integer
-                    description: version number
-                    example: 6
-                  snapshot_id:
-                    type: integer
-                    description: unique snapshot id
-                  snapshot_name:
-                    type: string
-                    description: The path and file name of the snapshot
-                    example: /home/me/nodes/node-name/snapshots/snapshot-0000999f99999f9f999f99f99ff9999f999f9fff99ff99ffff9f9f9fff9f9999.bin
                   blocks_count:
                     type: integer
                     description: Generate snapshot every blocks_count blocks
                   start_block_num:
                     type: integer
-                    description: Block number at which schedule starts                    
+                    description: Block number at which schedule starts
+                    example: 5102
                   end_block_num:
                     type: integer
-                    description: Block number at which schedule ends                    
+                    description: Block number at which schedule ends
+                    example: 15102
+                  snapshot_description:
+                      type: string
+                      description: Description of the snapshot
+                      example: Daily snapshot
+                  pending_snapshots:                  
+                      type: array
+                      description: List of pending snapshots
+                      items:
+                        type: object
+                        properties:  
+                          head_block_id:
+                            $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                          head_block_num:
+                            type: integer
+                            description: Highest block number on the chain
+                            example: 5102
+                          head_block_time:
+                            type: string
+                            description: Highest block unix timestamp
+                            example: 2020-11-16T00:00:00.000
+                          version:
+                            type: integer
+                            description: version number
+                            example: 6
+                          snapshot_name:
+                            type: string
+                            description: The path and file name of the snapshot
+                            example: /home/me/nodes/node-name/snapshots/snapshot-0000999f99999f9f999f99f99ff9999f999f9fff99ff99ffff9f9f9fff9f9999.bin
+                                   
         "400":
           description: client error
           content:
@@ -453,36 +539,24 @@ paths:
               schema:
                 type: object
                 properties:
-                  head_block_id:
-                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
-                  head_block_num:
-                    type: integer
-                    description: Highest block number on the chain
-                    example: 5102
-                  head_block_time:
-                    type: string
-                    description: Highest block unix timestamp
-                    example: 2020-11-16T00:00:00.000
-                  version:
-                    type: integer
-                    description: version number
-                    example: 6
-                  snapshot_id:
-                    type: integer
-                    description: unique snapshot id
-                  snapshot_name:
-                    type: string
-                    description: The path and file name of the snapshot
-                    example: /home/me/nodes/node-name/snapshots/snapshot-0000999f99999f9f999f99f99ff9999f999f9fff99ff99ffff9f9f9fff9f9999.bin
+                  snapshot_request_id:
+                      type: integer
+                      description: Unique id identifying current snapshot request
                   blocks_count:
                     type: integer
                     description: Generate snapshot every blocks_count blocks
                   start_block_num:
                     type: integer
-                    description: Block number at which schedule starts                    
+                    description: Block number at which schedule starts
+                    example: 5102
                   end_block_num:
                     type: integer
-                    description: Block number at which schedule ends                    
+                    description: Block number at which schedule ends
+                    example: 15102
+                  snapshot_description:
+                      type: string
+                      description: Description of the snapshot
+                      example: Daily snapshot                
         "400":
           description: client error
           content:

From 7cb3a08ef36991502140f4fa1a8c225f170c1454 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 17 Jan 2023 14:03:26 -0500
Subject: [PATCH 03/48] Updates to swagger spec

---
 .../producer_api_plugin/producer.swagger.yaml | 34 +++++++++----------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/plugins/producer_api_plugin/producer.swagger.yaml b/plugins/producer_api_plugin/producer.swagger.yaml
index 1f2173685a..997c524167 100644
--- a/plugins/producer_api_plugin/producer.swagger.yaml
+++ b/plugins/producer_api_plugin/producer.swagger.yaml
@@ -374,7 +374,7 @@ paths:
 
   /producer/get_snapshot_requests:
     post:
-      summary: get_snapshot_request_status
+      summary: get_snapshot_requests
       description: Returns a list of scheduled snapshots
       operationId: get_snapshot_status
       requestBody:
@@ -384,13 +384,13 @@ paths:
             schema:
               type: object
               properties:
-                offset:
+                start_snapshot_request_id:
                   type: integer
                   description: Snapshot offset
                 limit:
                   type: integer
                   description: Number of snapshots to return
-                
+
       responses:
         "201":
           description: OK
@@ -399,21 +399,19 @@ paths:
              schema:
                 type: object
                 properties:
-                  offset:
-                    type: integer
-                    description: Snapshot offset
-                  limit:
-                    type: integer
-                    description: Number of snapshots to return
-                  count:
+                  next_request_id:
                     type: integer
-                    description: Total amount of snapshots
+                    description: Next available snapshot ID
                   snapshot_requests:
                     type: array
                     description: An array of scheduled snapshots
                     items:
                       type: object
                       properties:
+                          snapshot_request_time:
+                            type: string
+                            description: Snapshot unix timestamp
+                            example: 2020-11-16T00:00:00.000
                           snapshot_request_id:
                             type: integer
                             description: Unique id identifying current snapshot request
@@ -431,7 +429,7 @@ paths:
                           snapshot_description:
                               type: string
                               description: Description of the snapshot
-                              example: Daily snapshot                
+                              example: Daily snapshot
         "400":
           description: client error
           content:
@@ -451,7 +449,7 @@ paths:
             schema:
               type: object
               properties:
-                snapshot_id:
+                snapshot_request_id:
                   type: integer
                   description: snapshot id
       responses:
@@ -484,12 +482,12 @@ paths:
                       type: string
                       description: Description of the snapshot
                       example: Daily snapshot
-                  pending_snapshots:                  
+                  pending_snapshots:
                       type: array
                       description: List of pending snapshots
                       items:
                         type: object
-                        properties:  
+                        properties:
                           head_block_id:
                             $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
                           head_block_num:
@@ -508,7 +506,7 @@ paths:
                             type: string
                             description: The path and file name of the snapshot
                             example: /home/me/nodes/node-name/snapshots/snapshot-0000999f99999f9f999f99f99ff9999f999f9fff99ff99ffff9f9f9fff9f9999.bin
-                                   
+
         "400":
           description: client error
           content:
@@ -528,7 +526,7 @@ paths:
             schema:
               type: object
               properties:
-                snapshot_id:
+                snapshot_request_id:
                   type: integer
                   description: snapshot id
       responses:
@@ -556,7 +554,7 @@ paths:
                   snapshot_description:
                       type: string
                       description: Description of the snapshot
-                      example: Daily snapshot                
+                      example: Daily snapshot
         "400":
           description: client error
           content:

From e54032ab72b9fd692d66160d79cec81829f315bd Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Wed, 25 Jan 2023 14:30:16 -0500
Subject: [PATCH 04/48] Added snapshot status to get_snapshots_requests

---
 .../producer_api_plugin/producer.swagger.yaml | 101 +++++-------------
 1 file changed, 24 insertions(+), 77 deletions(-)

diff --git a/plugins/producer_api_plugin/producer.swagger.yaml b/plugins/producer_api_plugin/producer.swagger.yaml
index 997c524167..62935e1d9e 100644
--- a/plugins/producer_api_plugin/producer.swagger.yaml
+++ b/plugins/producer_api_plugin/producer.swagger.yaml
@@ -430,83 +430,30 @@ paths:
                               type: string
                               description: Description of the snapshot
                               example: Daily snapshot
-        "400":
-          description: client error
-          content:
-            application/json:
-              schema:
-                $ref: "#/component/schema/Error"
-
-  /producer/get_snapshot_request_status:
-    post:
-      summary: get_snapshot_request_status
-      description: Queries a status of the existing scheduled snapshot. Returns error when unable to find snapshot requested or query its status.
-      operationId: get_snapshot_status
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                snapshot_request_id:
-                  type: integer
-                  description: snapshot id
-      responses:
-        "201":
-          description: OK
-          content:
-            application/json:
-              schema:
-                type: object
-                properties:
-                  snapshot_request_id:
-                    type: integer
-                    description: Unique id identifying current snapshot request
-                  snapshot_request_time:
-                    type: string
-                    description: Snapshot unix timestamp
-                    example: 2020-11-16T00:00:00.000
-                  blocks_count:
-                    type: integer
-                    description: Generate snapshot every blocks_count blocks
-                  start_block_num:
-                    type: integer
-                    description: Block number at which schedule starts
-                    example: 5102
-                  end_block_num:
-                    type: integer
-                    description: Block number at which schedule ends
-                    example: 15102
-                  snapshot_description:
-                      type: string
-                      description: Description of the snapshot
-                      example: Daily snapshot
-                  pending_snapshots:
-                      type: array
-                      description: List of pending snapshots
-                      items:
-                        type: object
-                        properties:
-                          head_block_id:
-                            $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
-                          head_block_num:
-                            type: integer
-                            description: Highest block number on the chain
-                            example: 5102
-                          head_block_time:
-                            type: string
-                            description: Highest block unix timestamp
-                            example: 2020-11-16T00:00:00.000
-                          version:
-                            type: integer
-                            description: version number
-                            example: 6
-                          snapshot_name:
-                            type: string
-                            description: The path and file name of the snapshot
-                            example: /home/me/nodes/node-name/snapshots/snapshot-0000999f99999f9f999f99f99ff9999f999f9fff99ff99ffff9f9f9fff9f9999.bin
-
+                          pending_snapshots:
+                            type: array
+                            description: List of pending snapshots
+                            items:
+                              type: object
+                              properties:
+                                head_block_id:
+                                  $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                                head_block_num:
+                                  type: integer
+                                  description: Highest block number on the chain
+                                  example: 5102
+                                head_block_time:
+                                  type: string
+                                  description: Highest block unix timestamp
+                                  example: 2020-11-16T00:00:00.000
+                                version:
+                                  type: integer
+                                  description: version number
+                                  example: 6
+                                snapshot_name:
+                                  type: string
+                                  description: The path and file name of the snapshot
+                                  example: /home/me/nodes/node-name/snapshots/snapshot-0000999f99999f9f999f99f99ff9999f999f9fff99ff99ffff9f9f9fff9f9999.bin
         "400":
           description: client error
           content:

From e17e772e0c4d64680c46d9c0618fd60453c5bfd4 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Mon, 30 Jan 2023 11:56:56 -0500
Subject: [PATCH 05/48] Updates to schema

---
 .../producer_api_plugin/producer.swagger.yaml | 21 -------------------
 1 file changed, 21 deletions(-)

diff --git a/plugins/producer_api_plugin/producer.swagger.yaml b/plugins/producer_api_plugin/producer.swagger.yaml
index 62935e1d9e..5c418f791a 100644
--- a/plugins/producer_api_plugin/producer.swagger.yaml
+++ b/plugins/producer_api_plugin/producer.swagger.yaml
@@ -377,20 +377,6 @@ paths:
       summary: get_snapshot_requests
       description: Returns a list of scheduled snapshots
       operationId: get_snapshot_status
-      requestBody:
-        required: true
-        content:
-          application/json:
-            schema:
-              type: object
-              properties:
-                start_snapshot_request_id:
-                  type: integer
-                  description: Snapshot offset
-                limit:
-                  type: integer
-                  description: Number of snapshots to return
-
       responses:
         "201":
           description: OK
@@ -399,19 +385,12 @@ paths:
              schema:
                 type: object
                 properties:
-                  next_request_id:
-                    type: integer
-                    description: Next available snapshot ID
                   snapshot_requests:
                     type: array
                     description: An array of scheduled snapshots
                     items:
                       type: object
                       properties:
-                          snapshot_request_time:
-                            type: string
-                            description: Snapshot unix timestamp
-                            example: 2020-11-16T00:00:00.000
                           snapshot_request_id:
                             type: integer
                             description: Unique id identifying current snapshot request

From d865960c075425687be664b1df1a41e52066b4c6 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Wed, 1 Feb 2023 13:39:00 -0500
Subject: [PATCH 06/48] Work in progress - requests scaffold, db

---
 .../snapshot_schedule_db.hpp                  | 20 +++++++++++++++++++
 .../producer_api_plugin/producer.swagger.yaml | 16 +++++++--------
 .../producer_api_plugin.cpp                   |  6 ++++++
 .../eosio/producer_plugin/producer_plugin.hpp | 19 ++++++++++++++++++
 plugins/producer_plugin/producer_plugin.cpp   | 14 +++++++++++++
 5 files changed, 67 insertions(+), 8 deletions(-)
 create mode 100644 plugins/producer_api_plugin/include/eosio/producer_api_plugin/snapshot_schedule_db.hpp

diff --git a/plugins/producer_api_plugin/include/eosio/producer_api_plugin/snapshot_schedule_db.hpp b/plugins/producer_api_plugin/include/eosio/producer_api_plugin/snapshot_schedule_db.hpp
new file mode 100644
index 0000000000..26164fb6cf
--- /dev/null
+++ b/plugins/producer_api_plugin/include/eosio/producer_api_plugin/snapshot_schedule_db.hpp
@@ -0,0 +1,20 @@
+#pragma once
+
+#include <boost/asio.hpp>
+#include <boost/filesystem.hpp>
+
+#include <appbase/application.hpp>
+#include <eosio/chain/exceptions.hpp>
+
+namespace bfs = boost::filesystem;
+
+namespace eosio::producer_api {
+class snapshot_schedule_db {
+private:
+    std::vector<producer_plugin::snapshot_request_information>  snapshot_requests;
+    std::map<uint32_t, snapshot_information>                    pending_snapshots;
+public:
+   snapshot_schedule_db() {}
+   ~snapshot_schedule_db() {}
+};
+}// namespace eosio::producer_api
diff --git a/plugins/producer_api_plugin/producer.swagger.yaml b/plugins/producer_api_plugin/producer.swagger.yaml
index 5c418f791a..12fb7e4152 100644
--- a/plugins/producer_api_plugin/producer.swagger.yaml
+++ b/plugins/producer_api_plugin/producer.swagger.yaml
@@ -323,9 +323,9 @@ paths:
             schema:
               type: object
               properties:
-                blocks_count:
+                block_spacing:
                   type: integer
-                  description: Generate snapshot every blocks_count blocks
+                  description: Generate snapshot every block_spacing blocks
                 start_block_num:
                   type: integer
                   description: Block number at which schedule starts
@@ -350,9 +350,9 @@ paths:
                     snapshot_request_id:
                       type: integer
                       description: Unique id identifying current snapshot request
-                    blocks_count:
+                    block_spacing:
                       type: integer
-                      description: Generate snapshot every blocks_count blocks
+                      description: Generate snapshot every block_spacing blocks
                     start_block_num:
                       type: integer
                       description: Block number at which schedule starts
@@ -394,9 +394,9 @@ paths:
                           snapshot_request_id:
                             type: integer
                             description: Unique id identifying current snapshot request
-                          blocks_count:
+                          block_spacing:
                             type: integer
-                            description: Generate snapshot every blocks_count blocks
+                            description: Generate snapshot every block_spacing blocks
                           start_block_num:
                             type: integer
                             description: Block number at which schedule starts
@@ -466,9 +466,9 @@ paths:
                   snapshot_request_id:
                       type: integer
                       description: Unique id identifying current snapshot request
-                  blocks_count:
+                  block_spacing:
                     type: integer
-                    description: Generate snapshot every blocks_count blocks
+                    description: Generate snapshot every block_spacing blocks
                   start_block_num:
                     type: integer
                     description: Block number at which schedule starts
diff --git a/plugins/producer_api_plugin/producer_api_plugin.cpp b/plugins/producer_api_plugin/producer_api_plugin.cpp
index 634c7da8cd..6658af308d 100644
--- a/plugins/producer_api_plugin/producer_api_plugin.cpp
+++ b/plugins/producer_api_plugin/producer_api_plugin.cpp
@@ -120,6 +120,12 @@ void producer_api_plugin::plugin_startup() {
             INVOKE_R_V(producer, get_integrity_hash), 201),
        CALL_ASYNC(producer, producer, create_snapshot, producer_plugin::snapshot_information,
             INVOKE_R_V_ASYNC(producer, create_snapshot), 201),
+       CALL_WITH_400(producer, producer, schedule_snapshot,
+            INVOKE_V_R(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
+       CALL_WITH_400(producer, producer, get_snapshot_requests,
+            INVOKE_R_V(producer, get_snapshot_requests), 201),
+       CALL_WITH_400(producer, producer, unschedule_snapshot,
+            INVOKE_V_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_information), 201),
        CALL_WITH_400(producer, producer, get_scheduled_protocol_feature_activations,
             INVOKE_R_V(producer, get_scheduled_protocol_feature_activations), 201),
        CALL_WITH_400(producer, producer, schedule_protocol_feature_activations,
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
index 2ee1f132f6..873113a1fe 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
@@ -50,6 +50,19 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
       std::string          snapshot_name;
    };
 
+   struct snapshot_request_information {
+      uint32_t             snapshot_request_id;
+      uint32_t             block_spacing;
+      uint32_t             start_block_num;
+      uint32_t             end_block_num;
+      std::string          snapshot_description;
+   };
+
+   struct snapshot_requests {
+      std::vector<snapshot_request_information> requests;
+      std::map<uint32_t, snapshot_information>  pending_snapshots;
+   };
+
    struct scheduled_protocol_feature_activations {
       std::vector<chain::digest_type> protocol_features_to_activate;
    };
@@ -105,7 +118,11 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
    void set_whitelist_blacklist(const whitelist_blacklist& params);
 
    integrity_hash_information get_integrity_hash() const;
+
    void create_snapshot(next_function<snapshot_information> next);
+   void schedule_snapshot(const snapshot_request_information& schedule);
+   void unschedule_snapshot(const snapshot_request_information& schedule);
+   snapshot_requests get_snapshot_requests() const;
 
    scheduled_protocol_feature_activations get_scheduled_protocol_feature_activations() const;
    void schedule_protocol_feature_activations(const scheduled_protocol_feature_activations& schedule);
@@ -155,6 +172,8 @@ FC_REFLECT(eosio::producer_plugin::greylist_params, (accounts));
 FC_REFLECT(eosio::producer_plugin::whitelist_blacklist, (actor_whitelist)(actor_blacklist)(contract_whitelist)(contract_blacklist)(action_blacklist)(key_blacklist) )
 FC_REFLECT(eosio::producer_plugin::integrity_hash_information, (head_block_id)(integrity_hash))
 FC_REFLECT(eosio::producer_plugin::snapshot_information, (head_block_id)(head_block_num)(head_block_time)(version)(snapshot_name))
+FC_REFLECT(eosio::producer_plugin::snapshot_request_information, (snapshot_request_id)(block_spacing)(start_block_num)(end_block_num)(snapshot_description))
+FC_REFLECT(eosio::producer_plugin::snapshot_requests, (requests)(pending_snapshots))
 FC_REFLECT(eosio::producer_plugin::scheduled_protocol_feature_activations, (protocol_features_to_activate))
 FC_REFLECT(eosio::producer_plugin::get_supported_protocol_features_params, (exclude_disabled)(exclude_unactivatable))
 FC_REFLECT(eosio::producer_plugin::get_account_ram_corrections_params, (lower_bound)(upper_bound)(limit)(reverse))
diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp
index 92acbb2b04..21ed463da0 100644
--- a/plugins/producer_plugin/producer_plugin.cpp
+++ b/plugins/producer_plugin/producer_plugin.cpp
@@ -1321,6 +1321,20 @@ void producer_plugin::create_snapshot(producer_plugin::next_function<producer_pl
    }
 }
 
+void producer_plugin::schedule_snapshot(const snapshot_request_information& schedule)
+{
+}
+
+void producer_plugin::unschedule_snapshot(const snapshot_request_information& schedule)
+{
+}
+
+producer_plugin::snapshot_requests producer_plugin::get_snapshot_requests() const
+{
+   snapshot_requests results;
+   return results;
+}
+
 producer_plugin::scheduled_protocol_feature_activations
 producer_plugin::get_scheduled_protocol_feature_activations()const {
    return {my->_protocol_features_to_activate};

From 5ee9f0f3d0446afd27cc7f4e8f1ce71375716d36 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Sun, 12 Feb 2023 01:03:44 -0500
Subject: [PATCH 07/48] intermediate commit to sync changes

---
 .../chain/include/eosio/chain/exceptions.hpp  |  5 ++
 .../producer_api_plugin.cpp                   |  4 +-
 .../eosio/producer_plugin/producer_plugin.hpp | 18 +++++--
 .../producer_plugin/snapshot_db_json.hpp      | 52 +++++++++++++++++++
 .../producer_plugin/snapshot_scheduler.hpp    | 43 +++++++++++++++
 plugins/producer_plugin/producer_plugin.cpp   |  6 ++-
 6 files changed, 119 insertions(+), 9 deletions(-)
 create mode 100644 plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
 create mode 100644 plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp

diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp
index d0ee22b625..295754d1bd 100644
--- a/libraries/chain/include/eosio/chain/exceptions.hpp
+++ b/libraries/chain/include/eosio/chain/exceptions.hpp
@@ -573,6 +573,11 @@ namespace eosio { namespace chain {
                                     3170011, "The signer returned no valid block signatures" )
       FC_DECLARE_DERIVED_EXCEPTION( unsupported_multiple_block_signatures,  producer_exception,
                                     3170012, "The signer returned multiple signatures but that is not supported" )
+      FC_DECLARE_DERIVED_EXCEPTION( duplicate_snapshot_request,  producer_exception,
+                                    3170013, "Snapshot has been already scheduled with specified parameters" )
+      FC_DECLARE_DERIVED_EXCEPTION( snapshot_request_not_found,  producer_exception,
+                                    3170014, "Snapshot request not found" )
+
 
    FC_DECLARE_DERIVED_EXCEPTION( reversible_blocks_exception,           chain_exception,
                                  3180000, "Reversible Blocks exception" )
diff --git a/plugins/producer_api_plugin/producer_api_plugin.cpp b/plugins/producer_api_plugin/producer_api_plugin.cpp
index 6658af308d..58d1dbc794 100644
--- a/plugins/producer_api_plugin/producer_api_plugin.cpp
+++ b/plugins/producer_api_plugin/producer_api_plugin.cpp
@@ -122,8 +122,8 @@ void producer_api_plugin::plugin_startup() {
             INVOKE_R_V_ASYNC(producer, create_snapshot), 201),
        CALL_WITH_400(producer, producer, schedule_snapshot,
             INVOKE_V_R(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
-       CALL_WITH_400(producer, producer, get_snapshot_requests,
-            INVOKE_R_V(producer, get_snapshot_requests), 201),
+     //  CALL_WITH_400(producer, producer, get_snapshot_requests,
+          //  INVOKE_R_V(producer, get_snapshot_requests), 201),
        CALL_WITH_400(producer, producer, unschedule_snapshot,
             INVOKE_V_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_information), 201),
        CALL_WITH_400(producer, producer, get_scheduled_protocol_feature_activations,
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
index 873113a1fe..46c001b79e 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
@@ -58,9 +58,17 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
       std::string          snapshot_description;
    };
 
-   struct snapshot_requests {
-      std::vector<snapshot_request_information> requests;
-      std::map<uint32_t, snapshot_information>  pending_snapshots;
+   // id and description are not affecting snapshot execution
+   struct snapshot_request_cmp {
+      bool operator()(const snapshot_request_information& sri1, const snapshot_request_information& sri2) const {
+         return sri1.block_spacing <  sri2.block_spacing || sri1.start_block_num <  sri2.start_block_num || sri1.end_block_num <  sri2.end_block_num;
+      }
+   };
+   
+   using snapshot_requests =  std::map<snapshot_request_information, std::variant<std::monostate, snapshot_information>, snapshot_request_cmp>;
+
+   struct get_snapshot_requests_result {
+      snapshot_requests requests;
    };
 
    struct scheduled_protocol_feature_activations {
@@ -122,7 +130,7 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
    void create_snapshot(next_function<snapshot_information> next);
    void schedule_snapshot(const snapshot_request_information& schedule);
    void unschedule_snapshot(const snapshot_request_information& schedule);
-   snapshot_requests get_snapshot_requests() const;
+   get_snapshot_requests_result get_snapshot_requests() const;
 
    scheduled_protocol_feature_activations get_scheduled_protocol_feature_activations() const;
    void schedule_protocol_feature_activations(const scheduled_protocol_feature_activations& schedule);
@@ -173,7 +181,7 @@ FC_REFLECT(eosio::producer_plugin::whitelist_blacklist, (actor_whitelist)(actor_
 FC_REFLECT(eosio::producer_plugin::integrity_hash_information, (head_block_id)(integrity_hash))
 FC_REFLECT(eosio::producer_plugin::snapshot_information, (head_block_id)(head_block_num)(head_block_time)(version)(snapshot_name))
 FC_REFLECT(eosio::producer_plugin::snapshot_request_information, (snapshot_request_id)(block_spacing)(start_block_num)(end_block_num)(snapshot_description))
-FC_REFLECT(eosio::producer_plugin::snapshot_requests, (requests)(pending_snapshots))
+FC_REFLECT(eosio::producer_plugin::get_snapshot_requests_result, (requests))
 FC_REFLECT(eosio::producer_plugin::scheduled_protocol_feature_activations, (protocol_features_to_activate))
 FC_REFLECT(eosio::producer_plugin::get_supported_protocol_features_params, (exclude_disabled)(exclude_unactivatable))
 FC_REFLECT(eosio::producer_plugin::get_account_ram_corrections_params, (lower_bound)(upper_bound)(limit)(reverse))
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
new file mode 100644
index 0000000000..b17f84268c
--- /dev/null
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
@@ -0,0 +1,52 @@
+#pragma once
+
+#include <eosio/chain/types.hpp>
+#include <eosio/chain/exceptions.hpp>
+
+#include <eosio/producer_plugin/producer_plugin.hpp>
+
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/json_parser.hpp>
+#include <boost/date_time/posix_time/posix_time.hpp>
+
+
+namespace eosio {
+
+using namespace eosio::chain;
+
+class snapshot_db_json {
+public:
+   snapshot_db_json(std::string db_path) : db_path(db_path) {}
+ 
+   const snapshot_db_json & operator >> (producer_plugin::snapshot_requests & sri)
+   {
+        boost::property_tree::ptree root;
+        std::ifstream file(db_path);
+        boost::property_tree::read_json(file, root);
+        file.close();
+
+        // parse ptree
+
+        return *this;
+   }
+
+   const snapshot_db_json & operator << (const producer_plugin::snapshot_requests & sri)
+   {
+        boost::property_tree::ptree root;
+
+       // boost::property_tree::ptree node;
+       // node.put("aaa", bbb);
+       // root.push_back(std::make_pair("", node));
+    
+        std::ofstream file(db_path);
+        boost::property_tree::write_json(file, root);
+        file.close();
+
+        return *this;
+   }
+
+private:
+      std::string db_path;
+};
+
+} // namespace eosio
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
new file mode 100644
index 0000000000..16467a0412
--- /dev/null
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -0,0 +1,43 @@
+#pragma once
+
+#include <eosio/chain/types.hpp>
+#include <eosio/chain/exceptions.hpp>
+
+#include <eosio/producer_plugin/producer_plugin.hpp>
+
+
+namespace eosio {
+
+using namespace eosio::chain;
+
+template<typename T>
+class snapshot_scheduler {
+public:
+   snapshot_scheduler(T & snapshot_db) : snapshot_db(snapshot_db)  {
+      snapshot_db >> requests;
+   }
+
+   const producer_plugin::snapshot_request_information & schedule_snapshot(const producer_plugin::snapshot_request_information & sri) {
+      EOS_ASSERT(requests.count(sri), duplicate_snapshot_request, "Duplicate snapshot request");
+      requests[sri]= std::monostate{};
+      snapshot_db << requests;
+      return sri;
+   }
+
+   const producer_plugin::snapshot_request_information & unschedule_snapshot(const producer_plugin::snapshot_request_information & sri) {
+      EOS_ASSERT(!requests.count(sri), snapshot_request_not_found, "Snapshot request not found");
+      requests.erase(sri);
+      snapshot_db << requests;
+      return sri;
+   }
+   
+   const producer_plugin::snapshot_requests & get_snapshots() {
+      return requests;
+   }
+
+private:
+      T                                   snapshot_db;
+      producer_plugin::snapshot_requests  requests;  
+};
+
+} // namespace eosio
diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp
index 21ed463da0..36101de251 100644
--- a/plugins/producer_plugin/producer_plugin.cpp
+++ b/plugins/producer_plugin/producer_plugin.cpp
@@ -1,6 +1,8 @@
 #include <eosio/producer_plugin/producer_plugin.hpp>
 #include <eosio/producer_plugin/pending_snapshot.hpp>
 #include <eosio/producer_plugin/subjective_billing.hpp>
+#include <eosio/producer_plugin/snapshot_scheduler.hpp>
+#include <eosio/producer_plugin/snapshot_db_json.hpp>
 #include <eosio/chain/plugin_interface.hpp>
 #include <eosio/chain/global_property_object.hpp>
 #include <eosio/chain/generated_transaction_object.hpp>
@@ -1329,9 +1331,9 @@ void producer_plugin::unschedule_snapshot(const snapshot_request_information& sc
 {
 }
 
-producer_plugin::snapshot_requests producer_plugin::get_snapshot_requests() const
+producer_plugin::get_snapshot_requests_result producer_plugin::get_snapshot_requests() const
 {
-   snapshot_requests results;
+   get_snapshot_requests_result results;
    return results;
 }
 

From b1654c97631ce22d89118d4dcc4f0a7909e00f8d Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Sun, 12 Feb 2023 20:46:41 -0500
Subject: [PATCH 08/48] wip schedule/unschedule to update json db

---
 .../snapshot_schedule_db.hpp                  | 20 -----
 .../eosio/producer_plugin/producer_plugin.hpp |  5 +-
 .../producer_plugin/snapshot_db_json.hpp      | 81 ++++++++++++-------
 .../producer_plugin/snapshot_scheduler.hpp    | 45 +++++++----
 plugins/producer_plugin/producer_plugin.cpp   | 16 +++-
 5 files changed, 98 insertions(+), 69 deletions(-)
 delete mode 100644 plugins/producer_api_plugin/include/eosio/producer_api_plugin/snapshot_schedule_db.hpp

diff --git a/plugins/producer_api_plugin/include/eosio/producer_api_plugin/snapshot_schedule_db.hpp b/plugins/producer_api_plugin/include/eosio/producer_api_plugin/snapshot_schedule_db.hpp
deleted file mode 100644
index 26164fb6cf..0000000000
--- a/plugins/producer_api_plugin/include/eosio/producer_api_plugin/snapshot_schedule_db.hpp
+++ /dev/null
@@ -1,20 +0,0 @@
-#pragma once
-
-#include <boost/asio.hpp>
-#include <boost/filesystem.hpp>
-
-#include <appbase/application.hpp>
-#include <eosio/chain/exceptions.hpp>
-
-namespace bfs = boost::filesystem;
-
-namespace eosio::producer_api {
-class snapshot_schedule_db {
-private:
-    std::vector<producer_plugin::snapshot_request_information>  snapshot_requests;
-    std::map<uint32_t, snapshot_information>                    pending_snapshots;
-public:
-   snapshot_schedule_db() {}
-   ~snapshot_schedule_db() {}
-};
-}// namespace eosio::producer_api
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
index 46c001b79e..a23bbc0fba 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
@@ -64,7 +64,7 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
          return sri1.block_spacing <  sri2.block_spacing || sri1.start_block_num <  sri2.start_block_num || sri1.end_block_num <  sri2.end_block_num;
       }
    };
-   
+
    using snapshot_requests =  std::map<snapshot_request_information, std::variant<std::monostate, snapshot_information>, snapshot_request_cmp>;
 
    struct get_snapshot_requests_result {
@@ -170,7 +170,8 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
    void log_failed_transaction(const transaction_id_type& trx_id, const chain::packed_transaction_ptr& packed_trx_ptr, const char* reason) const;
 
  private:
-   std::shared_ptr<class producer_plugin_impl> my;
+   std::shared_ptr<class producer_plugin_impl>  my;
+   std::shared_ptr<class snapshot_scheduler>    scheduler;
 };
 
 } //eosio
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
index b17f84268c..78a401d64a 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
@@ -1,13 +1,13 @@
 #pragma once
 
-#include <eosio/chain/types.hpp>
 #include <eosio/chain/exceptions.hpp>
+#include <eosio/chain/types.hpp>
 
 #include <eosio/producer_plugin/producer_plugin.hpp>
 
-#include <boost/property_tree/ptree.hpp>
-#include <boost/property_tree/json_parser.hpp>
 #include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/property_tree/json_parser.hpp>
+#include <boost/property_tree/ptree.hpp>
 
 
 namespace eosio {
@@ -16,37 +16,62 @@ using namespace eosio::chain;
 
 class snapshot_db_json {
 public:
-   snapshot_db_json(std::string db_path) : db_path(db_path) {}
- 
-   const snapshot_db_json & operator >> (producer_plugin::snapshot_requests & sri)
-   {
-        boost::property_tree::ptree root;
-        std::ifstream file(db_path);
-        boost::property_tree::read_json(file, root);
-        file.close();
-
-        // parse ptree
-
-        return *this;
+   snapshot_db_json() {}
+   snapshot_db_json(bfs::path db_path) : db_path(db_path) {}
+   ~snapshot_db_json() {}
+
+   void set_path(bfs::path path) {
+      db_path = path;
    }
 
-   const snapshot_db_json & operator << (const producer_plugin::snapshot_requests & sri)
-   {
-        boost::property_tree::ptree root;
+   bfs::path get_json_path() {
+      return db_path / "snapshot-schedule.json";
+   }
+
+   const snapshot_db_json& operator>>(producer_plugin::snapshot_requests& sr) {
+      boost::property_tree::ptree root;
+      std::ifstream file(get_json_path().string());
+      boost::property_tree::read_json(file, root);
+      file.close();
+
+      // parse ptree
+      for(boost::property_tree::ptree::value_type& req: root.get_child("snapshot_requests")) {
+         producer_plugin::snapshot_request_information sri;
+         sri.snapshot_request_id = req.second.get<uint32_t>("snapshot_request_id");
+         sri.snapshot_description = req.second.get<std::string>("snapshot_description");
+         sri.block_spacing = req.second.get<uint32_t>("block_spacing");
+         sri.start_block_num = req.second.get<uint32_t>("start_block_num");
+         sri.end_block_num = req.second.get<uint32_t>("end_block_num");
+         sr[sri] = std::monostate{};
+      }
+
+      return *this;
+   }
+
+   const snapshot_db_json& operator<<(const producer_plugin::snapshot_requests& sr) {
+      boost::property_tree::ptree root;
+      boost::property_tree::ptree node_srs;
+
+      for(const auto& [key, value]: sr) {
+         boost::property_tree::ptree node;
+         node.put("snapshot_request_id", key.snapshot_request_id);
+         node.put("snapshot_description", key.snapshot_description);
+         node.put("block_spacing", key.block_spacing);
+         node.put("start_block_num", key.start_block_num);
+         node.put("end_block_num", key.end_block_num);
+         node_srs.push_back(std::make_pair("", node));
+      }
+      root.push_back(std::make_pair("snapshot_requests", node_srs));
 
-       // boost::property_tree::ptree node;
-       // node.put("aaa", bbb);
-       // root.push_back(std::make_pair("", node));
-    
-        std::ofstream file(db_path);
-        boost::property_tree::write_json(file, root);
-        file.close();
+      std::ofstream file(get_json_path().string());
+      boost::property_tree::write_json(file, root);
+      file.close();
 
-        return *this;
+      return *this;
    }
 
 private:
-      std::string db_path;
+   bfs::path db_path;
 };
 
-} // namespace eosio
+}// namespace eosio
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 16467a0412..81d1c217e4 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -1,43 +1,58 @@
 #pragma once
 
-#include <eosio/chain/types.hpp>
 #include <eosio/chain/exceptions.hpp>
+#include <eosio/chain/types.hpp>
 
 #include <eosio/producer_plugin/producer_plugin.hpp>
-
+#include <eosio/producer_plugin/snapshot_db_json.hpp>
 
 namespace eosio {
 
 using namespace eosio::chain;
 
-template<typename T>
 class snapshot_scheduler {
 public:
-   snapshot_scheduler(T & snapshot_db) : snapshot_db(snapshot_db)  {
+   snapshot_scheduler() {}
+   ~snapshot_scheduler() {}
+
+   snapshot_db_json& get_db() {
+      return snapshot_db;
+   }
+
+   void load_schedule() {
       snapshot_db >> requests;
    }
 
-   const producer_plugin::snapshot_request_information & schedule_snapshot(const producer_plugin::snapshot_request_information & sri) {
-      EOS_ASSERT(requests.count(sri), duplicate_snapshot_request, "Duplicate snapshot request");
-      requests[sri]= std::monostate{};
+   const producer_plugin::snapshot_request_information& schedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
+      EOS_ASSERT(!requests.count(sri), duplicate_snapshot_request, "Duplicate snapshot request");
+      requests[sri] = std::monostate{};
       snapshot_db << requests;
       return sri;
    }
 
-   const producer_plugin::snapshot_request_information & unschedule_snapshot(const producer_plugin::snapshot_request_information & sri) {
-      EOS_ASSERT(!requests.count(sri), snapshot_request_not_found, "Snapshot request not found");
-      requests.erase(sri);
+   const producer_plugin::snapshot_request_information& unschedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
+      bool bFound = false;
+      for(auto it = requests.begin(); it != requests.end();) {
+         if(it->first.snapshot_request_id == sri.snapshot_request_id) {
+            bFound = true;
+            requests.erase(it++);
+         } else {
+            ++it;
+         }
+      }
+      EOS_ASSERT(bFound, snapshot_request_not_found, "Snapshot request not found");
+
       snapshot_db << requests;
       return sri;
    }
-   
-   const producer_plugin::snapshot_requests & get_snapshots() {
+
+   const producer_plugin::snapshot_requests& get_snapshots() {
       return requests;
    }
 
 private:
-      T                                   snapshot_db;
-      producer_plugin::snapshot_requests  requests;  
+   snapshot_db_json snapshot_db;
+   producer_plugin::snapshot_requests requests;
 };
 
-} // namespace eosio
+}// namespace eosio
diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp
index 36101de251..54c5782d00 100644
--- a/plugins/producer_plugin/producer_plugin.cpp
+++ b/plugins/producer_plugin/producer_plugin.cpp
@@ -2,7 +2,6 @@
 #include <eosio/producer_plugin/pending_snapshot.hpp>
 #include <eosio/producer_plugin/subjective_billing.hpp>
 #include <eosio/producer_plugin/snapshot_scheduler.hpp>
-#include <eosio/producer_plugin/snapshot_db_json.hpp>
 #include <eosio/chain/plugin_interface.hpp>
 #include <eosio/chain/global_property_object.hpp>
 #include <eosio/chain/generated_transaction_object.hpp>
@@ -698,7 +697,8 @@ void new_chain_banner(const eosio::chain::controller& db)
 }
 
 producer_plugin::producer_plugin()
-   : my(new producer_plugin_impl(app().get_io_service())){
+   : my(new producer_plugin_impl(app().get_io_service())),
+     scheduler(new snapshot_scheduler()) {
    }
 
 producer_plugin::~producer_plugin() {}
@@ -997,6 +997,12 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
       }
    }
 
+   scheduler->get_db().set_path(my->_snapshots_dir);
+   // if schedule exists, try to load it
+   if (fc::exists(scheduler->get_db().get_json_path())) {
+      scheduler->load_schedule();
+   }
+
 } FC_LOG_AND_RETHROW() }
 
 void producer_plugin::plugin_startup()
@@ -1323,12 +1329,14 @@ void producer_plugin::create_snapshot(producer_plugin::next_function<producer_pl
    }
 }
 
-void producer_plugin::schedule_snapshot(const snapshot_request_information& schedule)
+void producer_plugin::schedule_snapshot(const snapshot_request_information& sri)
 {
+   scheduler->schedule_snapshot(sri);
 }
 
-void producer_plugin::unschedule_snapshot(const snapshot_request_information& schedule)
+void producer_plugin::unschedule_snapshot(const snapshot_request_information& sri)
 {
+   scheduler->unschedule_snapshot(sri);
 }
 
 producer_plugin::get_snapshot_requests_result producer_plugin::get_snapshot_requests() const

From 576143901abf9e1e319e4439e6f5042f3a4ba4b2 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Mon, 13 Feb 2023 21:47:22 -0500
Subject: [PATCH 09/48] Added ut to a snapshot scheduler

---
 plugins/producer_plugin/test/CMakeLists.txt   |  5 ++
 .../test/test_snapshot_scheduler.cpp          | 55 +++++++++++++++++++
 2 files changed, 60 insertions(+)
 create mode 100644 plugins/producer_plugin/test/test_snapshot_scheduler.cpp

diff --git a/plugins/producer_plugin/test/CMakeLists.txt b/plugins/producer_plugin/test/CMakeLists.txt
index a5dc01df96..aaf4c93404 100644
--- a/plugins/producer_plugin/test/CMakeLists.txt
+++ b/plugins/producer_plugin/test/CMakeLists.txt
@@ -12,3 +12,8 @@ add_executable( test_snapshot_information test_snapshot_information.cpp )
 target_link_libraries( test_snapshot_information producer_plugin eosio_testing )
 
 add_test(NAME test_snapshot_information COMMAND plugins/producer_plugin/test/test_snapshot_information WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+
+add_executable( test_snapshot_scheduler test_snapshot_scheduler.cpp )
+target_link_libraries( test_snapshot_scheduler producer_plugin eosio_testing )
+
+add_test(NAME test_snapshot_scheduler COMMAND plugins/producer_plugin/test/test_snapshot_scheduler WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
new file mode 100644
index 0000000000..bde72fd8a3
--- /dev/null
+++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
@@ -0,0 +1,55 @@
+#define BOOST_TEST_MODULE snapshot_scheduler
+#include <boost/test/included/unit_test.hpp>
+
+#include <eosio/chain/exceptions.hpp>
+#include <eosio/producer_plugin/snapshot_scheduler.hpp>
+
+
+#include <eosio/testing/tester.hpp>
+
+namespace {
+
+using namespace eosio;
+using namespace eosio::chain;
+
+BOOST_AUTO_TEST_SUITE(snapshot_scheduler_test)
+
+BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
+
+   fc::logger log;
+   snapshot_scheduler scheduler;
+
+   {
+      // add/remove test
+      producer_plugin::snapshot_request_information sri1 = {.snapshot_request_id = 0, .block_spacing = 100, .start_block_num = 5000, .end_block_num = 10000, .snapshot_description = "Example of recurring snapshot"};
+      producer_plugin::snapshot_request_information sri2 = {.snapshot_request_id = 1, .block_spacing = 0, .start_block_num = 5200, .end_block_num = 5200, .snapshot_description = "Example of one-time snapshot"};
+
+      scheduler.schedule_snapshot(sri1);
+      scheduler.schedule_snapshot(sri2);
+
+      BOOST_CHECK_EQUAL(2, scheduler.get_snapshots().size());
+
+      BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri1), duplicate_snapshot_request, [](const fc::assert_exception& e) {
+         return e.to_detail_string().find("Duplicate snapshot request") != std::string::npos;
+      });
+
+      producer_plugin::snapshot_request_information sri_delete_1 = {.snapshot_request_id = 0};
+      scheduler.unschedule_snapshot(sri_delete_1);
+
+      BOOST_CHECK_EQUAL(1, scheduler.get_snapshots().size());
+
+      producer_plugin::snapshot_request_information sri_delete_none = {.snapshot_request_id = 2};
+      BOOST_CHECK_EXCEPTION(scheduler.unschedule_snapshot(sri_delete_none), snapshot_request_not_found, [](const fc::assert_exception& e) {
+         return e.to_detail_string().find("Snapshot request not found") != std::string::npos;
+      });
+
+      producer_plugin::snapshot_request_information sri_delete_2 = {.snapshot_request_id = 1};
+      scheduler.unschedule_snapshot(sri_delete_2);
+
+      BOOST_CHECK_EQUAL(0, scheduler.get_snapshots().size());
+   }
+}
+
+BOOST_AUTO_TEST_SUITE_END()
+
+}// namespace

From 2aad1dcaefb400e3349afb138a0758ef78f5f76a Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Wed, 15 Feb 2023 16:47:26 -0500
Subject: [PATCH 10/48] Work in progress, moved things around

---
 .../producer_api_plugin.cpp                   |  4 +--
 .../eosio/producer_plugin/producer_plugin.hpp |  3 +-
 .../producer_plugin/snapshot_scheduler.hpp    |  4 +++
 plugins/producer_plugin/producer_plugin.cpp   | 35 +++++++++++++------
 4 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/plugins/producer_api_plugin/producer_api_plugin.cpp b/plugins/producer_api_plugin/producer_api_plugin.cpp
index 58d1dbc794..6658af308d 100644
--- a/plugins/producer_api_plugin/producer_api_plugin.cpp
+++ b/plugins/producer_api_plugin/producer_api_plugin.cpp
@@ -122,8 +122,8 @@ void producer_api_plugin::plugin_startup() {
             INVOKE_R_V_ASYNC(producer, create_snapshot), 201),
        CALL_WITH_400(producer, producer, schedule_snapshot,
             INVOKE_V_R(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
-     //  CALL_WITH_400(producer, producer, get_snapshot_requests,
-          //  INVOKE_R_V(producer, get_snapshot_requests), 201),
+       CALL_WITH_400(producer, producer, get_snapshot_requests,
+            INVOKE_R_V(producer, get_snapshot_requests), 201),
        CALL_WITH_400(producer, producer, unschedule_snapshot,
             INVOKE_V_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_information), 201),
        CALL_WITH_400(producer, producer, get_scheduled_protocol_feature_activations,
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
index a23bbc0fba..cbc128b746 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
@@ -68,7 +68,7 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
    using snapshot_requests =  std::map<snapshot_request_information, std::variant<std::monostate, snapshot_information>, snapshot_request_cmp>;
 
    struct get_snapshot_requests_result {
-      snapshot_requests requests;
+       std::vector<snapshot_request_information>  requests;
    };
 
    struct scheduled_protocol_feature_activations {
@@ -171,7 +171,6 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
 
  private:
    std::shared_ptr<class producer_plugin_impl>  my;
-   std::shared_ptr<class snapshot_scheduler>    scheduler;
 };
 
 } //eosio
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 81d1c217e4..779ebe0a29 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -50,6 +50,10 @@ class snapshot_scheduler {
       return requests;
    }
 
+   void on_irreversible_block(uint32_t height) {
+
+   }
+
 private:
    snapshot_db_json snapshot_db;
    producer_plugin::snapshot_requests requests;
diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp
index 54c5782d00..1743b0a91d 100644
--- a/plugins/producer_plugin/producer_plugin.cpp
+++ b/plugins/producer_plugin/producer_plugin.cpp
@@ -27,6 +27,8 @@
 #include <boost/multi_index/hashed_index.hpp>
 #include <boost/multi_index/ordered_index.hpp>
 #include <boost/signals2/connection.hpp>
+#include <boost/range/adaptor/map.hpp>
+#include <boost/range/algorithm/copy.hpp>
 
 namespace bmi = boost::multi_index;
 using bmi::indexed_by;
@@ -265,7 +267,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
    public:
       producer_plugin_impl(boost::asio::io_service& io)
       :_timer(io)
-      ,_transaction_ack_channel(app().get_channel<compat::channels::transaction_ack>())
+      ,_transaction_ack_channel(app().get_channel<compat::channels::transaction_ack>()),
+      _snapshot_scheduler(new snapshot_scheduler())
       {
       }
 
@@ -295,6 +298,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       void log_trx_results( const packed_transaction_ptr& trx, const transaction_trace_ptr& trace,
                             const fc::exception_ptr& except_ptr, uint32_t billed_cpu_us, const fc::time_point& start );
 
+      snapshot_scheduler & get_snapshot_scheduler() { return *_snapshot_scheduler; }
+
       boost::program_options::variables_map _options;
       bool     _production_enabled                 = false;
       bool     _pause_production                   = false;
@@ -361,6 +366,9 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       // path to write the snapshots to
       bfs::path _snapshots_dir;
 
+      // snapshot scheduler
+      std::shared_ptr<class snapshot_scheduler>    _snapshot_scheduler;
+
       void consider_new_watermark( account_name producer, uint32_t block_num, block_timestamp_type timestamp) {
          auto itr = _producer_watermarks.find( producer );
          if( itr != _producer_watermarks.end() ) {
@@ -409,6 +417,9 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
 
             snapshots_by_height.erase(snapshots_by_height.begin());
          }
+
+         // notify scheduler
+         _snapshot_scheduler->on_irreversible_block(lib_height);
       }
 
       void abort_block() {
@@ -697,8 +708,8 @@ void new_chain_banner(const eosio::chain::controller& db)
 }
 
 producer_plugin::producer_plugin()
-   : my(new producer_plugin_impl(app().get_io_service())),
-     scheduler(new snapshot_scheduler()) {
+   : my(new producer_plugin_impl(app().get_io_service()))
+   {
    }
 
 producer_plugin::~producer_plugin() {}
@@ -997,12 +1008,15 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
       }
    }
 
-   scheduler->get_db().set_path(my->_snapshots_dir);
+   // now path in scheduler can be updated
+   my->get_snapshot_scheduler().get_db().set_path(my->_snapshots_dir);
+
    // if schedule exists, try to load it
-   if (fc::exists(scheduler->get_db().get_json_path())) {
-      scheduler->load_schedule();
+   if (fc::exists(my->get_snapshot_scheduler().get_db().get_json_path())) {
+      my->get_snapshot_scheduler().load_schedule();
    }
 
+
 } FC_LOG_AND_RETHROW() }
 
 void producer_plugin::plugin_startup()
@@ -1331,18 +1345,19 @@ void producer_plugin::create_snapshot(producer_plugin::next_function<producer_pl
 
 void producer_plugin::schedule_snapshot(const snapshot_request_information& sri)
 {
-   scheduler->schedule_snapshot(sri);
+   my->get_snapshot_scheduler().schedule_snapshot(sri);
 }
 
 void producer_plugin::unschedule_snapshot(const snapshot_request_information& sri)
 {
-   scheduler->unschedule_snapshot(sri);
+   my->get_snapshot_scheduler().unschedule_snapshot(sri);
 }
 
 producer_plugin::get_snapshot_requests_result producer_plugin::get_snapshot_requests() const
 {
-   get_snapshot_requests_result results;
-   return results;
+   producer_plugin::get_snapshot_requests_result result;
+   boost::copy(my->get_snapshot_scheduler().get_snapshots() | boost::adaptors::map_keys, std::back_inserter(result.requests));
+   return result;
 }
 
 producer_plugin::scheduled_protocol_feature_activations

From 46502e5814ae74eaa741c687aeff5cf01bfb3ebe Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Wed, 15 Feb 2023 21:38:52 -0500
Subject: [PATCH 11/48] Added execution handlers, several updates

---
 .../chain/include/eosio/chain/exceptions.hpp  |  3 +-
 .../eosio/producer_plugin/producer_plugin.hpp | 10 +++----
 .../producer_plugin/snapshot_scheduler.hpp    | 28 +++++++++++++++++++
 plugins/producer_plugin/producer_plugin.cpp   |  3 +-
 .../test/test_snapshot_scheduler.cpp          | 10 +++++++
 5 files changed, 46 insertions(+), 8 deletions(-)

diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp
index 295754d1bd..2ea1da6acb 100644
--- a/libraries/chain/include/eosio/chain/exceptions.hpp
+++ b/libraries/chain/include/eosio/chain/exceptions.hpp
@@ -577,7 +577,8 @@ namespace eosio { namespace chain {
                                     3170013, "Snapshot has been already scheduled with specified parameters" )
       FC_DECLARE_DERIVED_EXCEPTION( snapshot_request_not_found,  producer_exception,
                                     3170014, "Snapshot request not found" )
-
+      FC_DECLARE_DERIVED_EXCEPTION( invalid_snapshot_request,  producer_exception,
+                                    3170015, "Invalid snapshot request" )
 
    FC_DECLARE_DERIVED_EXCEPTION( reversible_blocks_exception,           chain_exception,
                                  3180000, "Reversible Blocks exception" )
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
index cbc128b746..ba0c11d7e0 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
@@ -51,11 +51,11 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
    };
 
    struct snapshot_request_information {
-      uint32_t             snapshot_request_id;
-      uint32_t             block_spacing;
-      uint32_t             start_block_num;
-      uint32_t             end_block_num;
-      std::string          snapshot_description;
+      uint32_t             snapshot_request_id = 0;
+      uint32_t             block_spacing = 0;
+      uint32_t             start_block_num = 0;
+      uint32_t             end_block_num = 0;
+      std::string          snapshot_description = "";
    };
 
    // id and description are not affecting snapshot execution
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 779ebe0a29..c596bc7960 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -25,6 +25,9 @@ class snapshot_scheduler {
 
    const producer_plugin::snapshot_request_information& schedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
       EOS_ASSERT(!requests.count(sri), duplicate_snapshot_request, "Duplicate snapshot request");
+      EOS_ASSERT(sri.end_block_num==0 || ((sri.end_block_num>0) && (sri.start_block_num <= sri.end_block_num)), invalid_snapshot_request, "End block number should be greater or equal to start block number");
+      EOS_ASSERT(sri.block_spacing==0 || ((sri.block_spacing>0) && (sri.start_block_num + sri.block_spacing <= sri.end_block_num)), invalid_snapshot_request, "Block spacing exceeds defined by start and end range");
+            
       requests[sri] = std::monostate{};
       snapshot_db << requests;
       return sri;
@@ -54,7 +57,32 @@ class snapshot_scheduler {
 
    }
 
+   void on_block(uint32_t height) {
+      for (const auto& [req, snap]: requests) {
+         // execute "asap" or if matches spacing
+         if ((req.start_block_num == 0) || 
+            (!((height - req.start_block_num) % req.block_spacing))) {
+            x_execute_snapshot();
+         }
+         // assume "asap" for snapshot with missed/zero start, it can have spacing
+         if (req.start_block_num == 0) {
+            auto node = requests.extract(req);
+            node.key().start_block_num = height;
+            requests.insert(std::move(node));
+         }
+         // remove expired request
+         if (req.end_block_num > 0 && height >= req.end_block_num) {
+            requests.extract(req);         
+         }      
+      }
+   }
+
 private:
+   void x_execute_snapshot() {
+     // producer_plugin::next_function<producer_plugin::snapshot_information> next;
+     ilog("Executing snapshot according to schedule...");
+   }
+
    snapshot_db_json snapshot_db;
    producer_plugin::snapshot_requests requests;
 };
diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp
index 1743b0a91d..9cdda443d9 100644
--- a/plugins/producer_plugin/producer_plugin.cpp
+++ b/plugins/producer_plugin/producer_plugin.cpp
@@ -391,6 +391,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          auto before = _unapplied_transactions.size();
          _unapplied_transactions.clear_applied( bsp );
          _subjective_billing.on_block( _log, bsp, fc::time_point::now() );
+         _snapshot_scheduler->on_block(bsp->block_num);
          fc_dlog( _log, "Removed applied transactions before: ${before}, after: ${after}",
                   ("before", before)("after", _unapplied_transactions.size()) );
       }
@@ -417,8 +418,6 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
 
             snapshots_by_height.erase(snapshots_by_height.begin());
          }
-
-         // notify scheduler
          _snapshot_scheduler->on_irreversible_block(lib_height);
       }
 
diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
index bde72fd8a3..24c01d8f4f 100644
--- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
+++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
@@ -47,6 +47,16 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
       scheduler.unschedule_snapshot(sri_delete_2);
 
       BOOST_CHECK_EQUAL(0, scheduler.get_snapshots().size());
+
+      producer_plugin::snapshot_request_information sri_large_spacing = {.snapshot_request_id = 0, .block_spacing = 1000, .start_block_num = 5000, .end_block_num = 5010 };
+      BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri_large_spacing), invalid_snapshot_request, [](const fc::assert_exception& e) {
+         return e.to_detail_string().find("Block spacing exceeds defined by start and end range") != std::string::npos;
+      });
+
+      producer_plugin::snapshot_request_information sri_start_end = {.snapshot_request_id = 0, .block_spacing = 1000, .start_block_num = 50000, .end_block_num = 5000 };
+      BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri_start_end), invalid_snapshot_request, [](const fc::assert_exception& e) {
+         return e.to_detail_string().find("End block number should be greater or equal to start block number") != std::string::npos;
+      });
    }
 }
 

From 8301c6bd7bb815eb7388e3f4c9daf86c1483f90c Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Sun, 19 Feb 2023 22:16:49 -0500
Subject: [PATCH 12/48] Improved data structures

---
 .../eosio/producer_plugin/producer_plugin.hpp |  10 +-
 .../producer_plugin/snapshot_db_json.hpp      |   8 +-
 .../producer_plugin/snapshot_scheduler.hpp    |  90 ----------------
 plugins/producer_plugin/producer_plugin.cpp   | 100 ++++++++++++++----
 .../test/test_snapshot_scheduler.cpp          |  10 +-
 5 files changed, 89 insertions(+), 129 deletions(-)
 delete mode 100644 plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
index ba0c11d7e0..e5fd19893e 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
@@ -56,17 +56,9 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
       uint32_t             start_block_num = 0;
       uint32_t             end_block_num = 0;
       std::string          snapshot_description = "";
+      std::optional<snapshot_information> pending_snapshot;
    };
 
-   // id and description are not affecting snapshot execution
-   struct snapshot_request_cmp {
-      bool operator()(const snapshot_request_information& sri1, const snapshot_request_information& sri2) const {
-         return sri1.block_spacing <  sri2.block_spacing || sri1.start_block_num <  sri2.start_block_num || sri1.end_block_num <  sri2.end_block_num;
-      }
-   };
-
-   using snapshot_requests =  std::map<snapshot_request_information, std::variant<std::monostate, snapshot_information>, snapshot_request_cmp>;
-
    struct get_snapshot_requests_result {
        std::vector<snapshot_request_information>  requests;
    };
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
index 78a401d64a..d6748b47b0 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
@@ -28,7 +28,7 @@ class snapshot_db_json {
       return db_path / "snapshot-schedule.json";
    }
 
-   const snapshot_db_json& operator>>(producer_plugin::snapshot_requests& sr) {
+   const snapshot_db_json& operator>>(std::vector<producer_plugin::snapshot_request_information> & sr) {
       boost::property_tree::ptree root;
       std::ifstream file(get_json_path().string());
       boost::property_tree::read_json(file, root);
@@ -42,17 +42,17 @@ class snapshot_db_json {
          sri.block_spacing = req.second.get<uint32_t>("block_spacing");
          sri.start_block_num = req.second.get<uint32_t>("start_block_num");
          sri.end_block_num = req.second.get<uint32_t>("end_block_num");
-         sr[sri] = std::monostate{};
+         sr.push_back(sri);
       }
 
       return *this;
    }
 
-   const snapshot_db_json& operator<<(const producer_plugin::snapshot_requests& sr) {
+   const snapshot_db_json& operator<<(std::vector<producer_plugin::snapshot_request_information> & sr) {
       boost::property_tree::ptree root;
       boost::property_tree::ptree node_srs;
 
-      for(const auto& [key, value]: sr) {
+      for(const auto& key: sr) {
          boost::property_tree::ptree node;
          node.put("snapshot_request_id", key.snapshot_request_id);
          node.put("snapshot_description", key.snapshot_description);
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
deleted file mode 100644
index c596bc7960..0000000000
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ /dev/null
@@ -1,90 +0,0 @@
-#pragma once
-
-#include <eosio/chain/exceptions.hpp>
-#include <eosio/chain/types.hpp>
-
-#include <eosio/producer_plugin/producer_plugin.hpp>
-#include <eosio/producer_plugin/snapshot_db_json.hpp>
-
-namespace eosio {
-
-using namespace eosio::chain;
-
-class snapshot_scheduler {
-public:
-   snapshot_scheduler() {}
-   ~snapshot_scheduler() {}
-
-   snapshot_db_json& get_db() {
-      return snapshot_db;
-   }
-
-   void load_schedule() {
-      snapshot_db >> requests;
-   }
-
-   const producer_plugin::snapshot_request_information& schedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
-      EOS_ASSERT(!requests.count(sri), duplicate_snapshot_request, "Duplicate snapshot request");
-      EOS_ASSERT(sri.end_block_num==0 || ((sri.end_block_num>0) && (sri.start_block_num <= sri.end_block_num)), invalid_snapshot_request, "End block number should be greater or equal to start block number");
-      EOS_ASSERT(sri.block_spacing==0 || ((sri.block_spacing>0) && (sri.start_block_num + sri.block_spacing <= sri.end_block_num)), invalid_snapshot_request, "Block spacing exceeds defined by start and end range");
-            
-      requests[sri] = std::monostate{};
-      snapshot_db << requests;
-      return sri;
-   }
-
-   const producer_plugin::snapshot_request_information& unschedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
-      bool bFound = false;
-      for(auto it = requests.begin(); it != requests.end();) {
-         if(it->first.snapshot_request_id == sri.snapshot_request_id) {
-            bFound = true;
-            requests.erase(it++);
-         } else {
-            ++it;
-         }
-      }
-      EOS_ASSERT(bFound, snapshot_request_not_found, "Snapshot request not found");
-
-      snapshot_db << requests;
-      return sri;
-   }
-
-   const producer_plugin::snapshot_requests& get_snapshots() {
-      return requests;
-   }
-
-   void on_irreversible_block(uint32_t height) {
-
-   }
-
-   void on_block(uint32_t height) {
-      for (const auto& [req, snap]: requests) {
-         // execute "asap" or if matches spacing
-         if ((req.start_block_num == 0) || 
-            (!((height - req.start_block_num) % req.block_spacing))) {
-            x_execute_snapshot();
-         }
-         // assume "asap" for snapshot with missed/zero start, it can have spacing
-         if (req.start_block_num == 0) {
-            auto node = requests.extract(req);
-            node.key().start_block_num = height;
-            requests.insert(std::move(node));
-         }
-         // remove expired request
-         if (req.end_block_num > 0 && height >= req.end_block_num) {
-            requests.extract(req);         
-         }      
-      }
-   }
-
-private:
-   void x_execute_snapshot() {
-     // producer_plugin::next_function<producer_plugin::snapshot_information> next;
-     ilog("Executing snapshot according to schedule...");
-   }
-
-   snapshot_db_json snapshot_db;
-   producer_plugin::snapshot_requests requests;
-};
-
-}// namespace eosio
diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp
index 9cdda443d9..a52cb048f3 100644
--- a/plugins/producer_plugin/producer_plugin.cpp
+++ b/plugins/producer_plugin/producer_plugin.cpp
@@ -1,7 +1,7 @@
 #include <eosio/producer_plugin/producer_plugin.hpp>
 #include <eosio/producer_plugin/pending_snapshot.hpp>
 #include <eosio/producer_plugin/subjective_billing.hpp>
-#include <eosio/producer_plugin/snapshot_scheduler.hpp>
+#include <eosio/producer_plugin/snapshot_db_json.hpp>
 #include <eosio/chain/plugin_interface.hpp>
 #include <eosio/chain/global_property_object.hpp>
 #include <eosio/chain/generated_transaction_object.hpp>
@@ -26,6 +26,7 @@
 #include <boost/multi_index/member.hpp>
 #include <boost/multi_index/hashed_index.hpp>
 #include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index/random_access_index.hpp>
 #include <boost/signals2/connection.hpp>
 #include <boost/range/adaptor/map.hpp>
 #include <boost/range/algorithm/copy.hpp>
@@ -132,6 +133,25 @@ enum class pending_block_mode {
    speculating
 };
 
+struct by_snapshot_id;
+struct by_snapshot_value;
+struct as_vector;
+
+using snapshot_requests = multi_index_container<
+   producer_plugin::snapshot_request_information,
+   indexed_by<
+      hashed_unique<tag<by_snapshot_id>, BOOST_MULTI_INDEX_MEMBER(producer_plugin::snapshot_request_information, uint32_t, snapshot_request_id)>,
+      random_access<tag<as_vector>>,
+      ordered_unique<tag<by_snapshot_value>,
+         composite_key< producer_plugin::snapshot_request_information,
+            BOOST_MULTI_INDEX_MEMBER(producer_plugin::snapshot_request_information, uint32_t, block_spacing),
+            BOOST_MULTI_INDEX_MEMBER(producer_plugin::snapshot_request_information, uint32_t, start_block_num),
+            BOOST_MULTI_INDEX_MEMBER(producer_plugin::snapshot_request_information, uint32_t, end_block_num)
+         >
+      >
+   >
+>;
+
 namespace {
 
 // track multiple failures on unapplied transactions
@@ -267,8 +287,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
    public:
       producer_plugin_impl(boost::asio::io_service& io)
       :_timer(io)
-      ,_transaction_ack_channel(app().get_channel<compat::channels::transaction_ack>()),
-      _snapshot_scheduler(new snapshot_scheduler())
+      ,_transaction_ack_channel(app().get_channel<compat::channels::transaction_ack>())
       {
       }
 
@@ -298,8 +317,6 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       void log_trx_results( const packed_transaction_ptr& trx, const transaction_trace_ptr& trace,
                             const fc::exception_ptr& except_ptr, uint32_t billed_cpu_us, const fc::time_point& start );
 
-      snapshot_scheduler & get_snapshot_scheduler() { return *_snapshot_scheduler; }
-
       boost::program_options::variables_map _options;
       bool     _production_enabled                 = false;
       bool     _pause_production                   = false;
@@ -366,8 +383,10 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       // path to write the snapshots to
       bfs::path _snapshots_dir;
 
-      // snapshot scheduler
-      std::shared_ptr<class snapshot_scheduler>    _snapshot_scheduler;
+      // snapshot schedule
+      snapshot_requests _snapshot_requests;
+      snapshot_db_json  _snapshot_db;
+      uint32_t          _snapshot_id = 0;
 
       void consider_new_watermark( account_name producer, uint32_t block_num, block_timestamp_type timestamp) {
          auto itr = _producer_watermarks.find( producer );
@@ -391,11 +410,36 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          auto before = _unapplied_transactions.size();
          _unapplied_transactions.clear_applied( bsp );
          _subjective_billing.on_block( _log, bsp, fc::time_point::now() );
-         _snapshot_scheduler->on_block(bsp->block_num);
+         scheduler_on_block(bsp);
          fc_dlog( _log, "Removed applied transactions before: ${before}, after: ${after}",
                   ("before", before)("after", _unapplied_transactions.size()) );
       }
 
+      void scheduler_on_block( const block_state_ptr& bsp ) {
+
+         auto height =bsp->block_num;
+         for (const auto & req : _snapshot_requests.get<0>()) {
+            // execute "asap" or if matches spacing
+            if ((req.start_block_num == 0) ||
+               (!((height - req.start_block_num) % req.block_spacing))) {
+              // x_execute_snapshot();
+            }
+            // assume "asap" for snapshot with missed/zero start, it can have spacing
+           /*
+            if (req.start_block_num == 0) {
+               auto node = requests.extract(req);
+               node.key().start_block_num = height;
+               requests.insert(std::move(node));
+            }
+            */
+            // remove expired request
+            if (req.end_block_num > 0 && height >= req.end_block_num) {
+              auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
+              _snapshot_requests.erase(snapshot_by_id.find(req.snapshot_request_id));
+            }
+         }
+      }
+
       void on_block_header( const block_state_ptr& bsp ) {
          consider_new_watermark( bsp->header.producer, bsp->block_num, bsp->block->timestamp );
       }
@@ -418,7 +462,6 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
 
             snapshots_by_height.erase(snapshots_by_height.begin());
          }
-         _snapshot_scheduler->on_irreversible_block(lib_height);
       }
 
       void abort_block() {
@@ -1006,16 +1049,12 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
          my->_subjective_billing.disable_account( account_name(a) );
       }
    }
-
-   // now path in scheduler can be updated
-   my->get_snapshot_scheduler().get_db().set_path(my->_snapshots_dir);
-
-   // if schedule exists, try to load it
-   if (fc::exists(my->get_snapshot_scheduler().get_db().get_json_path())) {
-      my->get_snapshot_scheduler().load_schedule();
+   my->_snapshot_db.set_path(my->_snapshots_dir);
+   if (fc::exists(my->_snapshot_db.get_json_path())) {
+      std::vector<producer_plugin::snapshot_request_information>  sr;
+      my->_snapshot_db >> sr;
+      my->_snapshot_requests.insert(sr.begin(), sr.end());
    }
-
-
 } FC_LOG_AND_RETHROW() }
 
 void producer_plugin::plugin_startup()
@@ -1344,18 +1383,37 @@ void producer_plugin::create_snapshot(producer_plugin::next_function<producer_pl
 
 void producer_plugin::schedule_snapshot(const snapshot_request_information& sri)
 {
-   my->get_snapshot_scheduler().schedule_snapshot(sri);
+   auto& snapshot_by_value = my->_snapshot_requests.get<by_snapshot_value>();
+   auto existing = snapshot_by_value.find(std::make_tuple(sri.block_spacing, sri.start_block_num, sri.end_block_num));
+   EOS_ASSERT(existing == snapshot_by_value.end() , duplicate_snapshot_request, "Duplicate snapshot request");
+   if (sri.end_block_num>0) EOS_ASSERT(sri.start_block_num <= sri.end_block_num, invalid_snapshot_request, "End block number should be greater or equal to start block number");
+   if (sri.block_spacing>0) EOS_ASSERT(sri.start_block_num + sri.block_spacing <= sri.end_block_num, invalid_snapshot_request, "Block spacing exceeds defined by start and end range");
+
+   auto request_id = sri.snapshot_request_id ? sri.snapshot_request_id : my->_snapshot_id++ ;
+   my->_snapshot_requests.emplace(producer_plugin::snapshot_request_information{request_id, sri.block_spacing, sri.start_block_num, sri.end_block_num,{}});
+
+   auto & vec = my->_snapshot_requests.get<as_vector>();
+   std::vector<producer_plugin::snapshot_request_information>  sr(vec.begin(), vec.end());
+   my->_snapshot_db << sr;
 }
 
 void producer_plugin::unschedule_snapshot(const snapshot_request_information& sri)
 {
-   my->get_snapshot_scheduler().unschedule_snapshot(sri);
+   auto& snapshot_by_id = my->_snapshot_requests.get<by_snapshot_id>();
+   auto existing = snapshot_by_id.find(sri.snapshot_request_id);
+   EOS_ASSERT(existing != snapshot_by_id.end(), snapshot_request_not_found, "Snapshot request not found");
+   my->_snapshot_requests.erase(existing);
+
+   auto & vec = my->_snapshot_requests.get<as_vector>();
+   std::vector<producer_plugin::snapshot_request_information>  sr(vec.begin(), vec.end());
+   my->_snapshot_db << sr;
 }
 
 producer_plugin::get_snapshot_requests_result producer_plugin::get_snapshot_requests() const
 {
    producer_plugin::get_snapshot_requests_result result;
-   boost::copy(my->get_snapshot_scheduler().get_snapshots() | boost::adaptors::map_keys, std::back_inserter(result.requests));
+   auto & asvector = my->_snapshot_requests.get<as_vector>();
+   result.requests.insert(result.requests.begin(),  asvector.begin(), asvector.end());
    return result;
 }
 
diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
index 24c01d8f4f..8e94d9c796 100644
--- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
+++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
@@ -2,7 +2,7 @@
 #include <boost/test/included/unit_test.hpp>
 
 #include <eosio/chain/exceptions.hpp>
-#include <eosio/producer_plugin/snapshot_scheduler.hpp>
+#include <eosio/producer_plugin/producer_plugin.hpp>
 
 
 #include <eosio/testing/tester.hpp>
@@ -17,7 +17,7 @@ BOOST_AUTO_TEST_SUITE(snapshot_scheduler_test)
 BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
 
    fc::logger log;
-   snapshot_scheduler scheduler;
+   producer_plugin scheduler;
 
    {
       // add/remove test
@@ -27,7 +27,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
       scheduler.schedule_snapshot(sri1);
       scheduler.schedule_snapshot(sri2);
 
-      BOOST_CHECK_EQUAL(2, scheduler.get_snapshots().size());
+      BOOST_CHECK_EQUAL(2, scheduler.get_snapshot_requests().requests.size());
 
       BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri1), duplicate_snapshot_request, [](const fc::assert_exception& e) {
          return e.to_detail_string().find("Duplicate snapshot request") != std::string::npos;
@@ -36,7 +36,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
       producer_plugin::snapshot_request_information sri_delete_1 = {.snapshot_request_id = 0};
       scheduler.unschedule_snapshot(sri_delete_1);
 
-      BOOST_CHECK_EQUAL(1, scheduler.get_snapshots().size());
+      BOOST_CHECK_EQUAL(1, scheduler.get_snapshot_requests().requests.size());
 
       producer_plugin::snapshot_request_information sri_delete_none = {.snapshot_request_id = 2};
       BOOST_CHECK_EXCEPTION(scheduler.unschedule_snapshot(sri_delete_none), snapshot_request_not_found, [](const fc::assert_exception& e) {
@@ -46,7 +46,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
       producer_plugin::snapshot_request_information sri_delete_2 = {.snapshot_request_id = 1};
       scheduler.unschedule_snapshot(sri_delete_2);
 
-      BOOST_CHECK_EQUAL(0, scheduler.get_snapshots().size());
+      BOOST_CHECK_EQUAL(0, scheduler.get_snapshot_requests().requests.size());
 
       producer_plugin::snapshot_request_information sri_large_spacing = {.snapshot_request_id = 0, .block_spacing = 1000, .start_block_num = 5000, .end_block_num = 5010 };
       BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri_large_spacing), invalid_snapshot_request, [](const fc::assert_exception& e) {

From 34612f2dab3b2b5e40af77bf79d215c72b096d41 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 21 Feb 2023 11:05:27 -0500
Subject: [PATCH 13/48] Little refactoring

---
 .../producer_plugin/snapshot_db_json.hpp      |   4 +-
 .../producer_plugin/snapshot_scheduler.hpp    | 140 ++++++++++++++++++
 plugins/producer_plugin/producer_plugin.cpp   |  90 ++---------
 3 files changed, 152 insertions(+), 82 deletions(-)
 create mode 100644 plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
index d6748b47b0..d87aecbc64 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
@@ -28,7 +28,7 @@ class snapshot_db_json {
       return db_path / "snapshot-schedule.json";
    }
 
-   const snapshot_db_json& operator>>(std::vector<producer_plugin::snapshot_request_information> & sr) {
+   const snapshot_db_json& operator>>(std::vector<producer_plugin::snapshot_request_information>& sr) {
       boost::property_tree::ptree root;
       std::ifstream file(get_json_path().string());
       boost::property_tree::read_json(file, root);
@@ -48,7 +48,7 @@ class snapshot_db_json {
       return *this;
    }
 
-   const snapshot_db_json& operator<<(std::vector<producer_plugin::snapshot_request_information> & sr) {
+   const snapshot_db_json& operator<<(std::vector<producer_plugin::snapshot_request_information>& sr) {
       boost::property_tree::ptree root;
       boost::property_tree::ptree node_srs;
 
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
new file mode 100644
index 0000000000..a84b320e6e
--- /dev/null
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -0,0 +1,140 @@
+#pragma once
+
+#include <eosio/chain/block_state.hpp>
+#include <eosio/chain/config.hpp>
+#include <eosio/chain/exceptions.hpp>
+#include <eosio/chain/resource_limits.hpp>
+#include <eosio/chain/resource_limits_private.hpp>
+#include <eosio/chain/transaction.hpp>
+#include <eosio/chain/types.hpp>
+
+#include <boost/multi_index/hashed_index.hpp>
+#include <boost/multi_index/member.hpp>
+#include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index/random_access_index.hpp>
+#include <boost/multi_index_container.hpp>
+
+#include <eosio/producer_plugin/snapshot_db_json.hpp>
+
+namespace eosio {
+
+namespace bmi = boost::multi_index;
+using chain::account_name;
+using chain::block_state_ptr;
+using chain::packed_transaction;
+using chain::transaction_id_type;
+
+using snapshot_request_information = producer_plugin::snapshot_request_information;
+using get_snapshot_requests_result = producer_plugin::get_snapshot_requests_result;
+
+class snapshot_scheduler_listener {
+   virtual void on_block(const block_state_ptr& bsp) = 0;
+   virtual void on_irreversible_block(const signed_block_ptr& lib) = 0;
+   virtual void on_abort_block() = 0;
+};
+
+class snapshot_scheduler_handler {
+   virtual void schedule_snapshot(const snapshot_request_information& sri) = 0;
+   virtual void unschedule_snapshot(uint32_t sri) = 0;
+   virtual get_snapshot_requests_result get_snapshot_requests() = 0;
+};
+
+class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_scheduler_listener {
+private:
+   struct by_snapshot_id;
+   struct by_snapshot_value;
+   struct as_vector;
+
+   using snapshot_requests = bmi::multi_index_container<
+         snapshot_request_information,
+         indexed_by<
+               bmi::hashed_unique<tag<by_snapshot_id>, BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, snapshot_request_id)>,
+               bmi::random_access<tag<as_vector>>,
+               bmi::ordered_unique<tag<by_snapshot_value>,
+                                   composite_key<snapshot_request_information,
+                                                 BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, block_spacing),
+                                                 BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, start_block_num),
+                                                 BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, end_block_num)>>>>;
+   snapshot_requests _snapshot_requests;
+   snapshot_db_json _snapshot_db;
+   uint32_t _snapshot_id = 0;
+
+public:
+   snapshot_scheduler() {}
+
+   // snapshot_scheduler_listener
+   virtual void on_block(const block_state_ptr& bsp) {
+      auto height = bsp->block_num;
+      for(const auto& req: _snapshot_requests.get<0>()) {
+         // execute "asap" or if matches spacing
+         if((req.start_block_num == 0) ||
+            (!((height - req.start_block_num) % req.block_spacing))) {
+            ilog("BOOM!!!!");
+            // x_execute_snapshot();
+         }
+         // assume "asap" for snapshot with missed/zero start, it can have spacing
+         if(req.start_block_num == 0) {
+            auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
+            auto it = snapshot_by_id.find(req.snapshot_request_id);
+            _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height; });
+         }
+         // remove expired request
+         if(req.end_block_num > 0 && height >= req.end_block_num) {
+            auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
+            _snapshot_requests.erase(snapshot_by_id.find(req.snapshot_request_id));
+         }
+      }
+   }
+
+   virtual void on_irreversible_block(const signed_block_ptr& lib) {
+   }
+
+   virtual void on_abort_block() {
+   }
+
+   // snapshot_scheduler_handler
+   void schedule_snapshot(const snapshot_request_information& sri) {
+      auto& snapshot_by_value = _snapshot_requests.get<by_snapshot_value>();
+      auto existing = snapshot_by_value.find(std::make_tuple(sri.block_spacing, sri.start_block_num, sri.end_block_num));
+      EOS_ASSERT(existing == snapshot_by_value.end(), duplicate_snapshot_request, "Duplicate snapshot request");
+      if(sri.end_block_num > 0) EOS_ASSERT(sri.start_block_num <= sri.end_block_num, invalid_snapshot_request, "End block number should be greater or equal to start block number");
+      if(sri.block_spacing > 0) EOS_ASSERT(sri.start_block_num + sri.block_spacing <= sri.end_block_num, invalid_snapshot_request, "Block spacing exceeds defined by start and end range");
+
+      auto request_id = sri.snapshot_request_id ? sri.snapshot_request_id : _snapshot_id++;
+      _snapshot_requests.emplace(producer_plugin::snapshot_request_information{request_id, sri.block_spacing, sri.start_block_num, sri.end_block_num, {}});
+
+      auto& vec = _snapshot_requests.get<as_vector>();
+      std::vector<producer_plugin::snapshot_request_information> sr(vec.begin(), vec.end());
+      _snapshot_db << sr;
+   }
+
+   virtual void unschedule_snapshot(uint32_t sri) {
+      auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
+      auto existing = snapshot_by_id.find(sri);
+      EOS_ASSERT(existing != snapshot_by_id.end(), snapshot_request_not_found, "Snapshot request not found");
+      _snapshot_requests.erase(existing);
+
+      auto& vec = _snapshot_requests.get<as_vector>();
+      std::vector<producer_plugin::snapshot_request_information> sr(vec.begin(), vec.end());
+      _snapshot_db << sr;
+   }
+
+   virtual get_snapshot_requests_result get_snapshot_requests() {
+      get_snapshot_requests_result result;
+      auto& asvector = _snapshot_requests.get<as_vector>();
+      result.requests.insert(result.requests.begin(), asvector.begin(), asvector.end());
+      return result;
+   }
+
+   // initialize with storage
+   void set_db_path(bfs::path db_path) {
+      _snapshot_db.set_path(db_path);
+      // init from db
+      if(fc::exists(_snapshot_db.get_json_path())) {
+         std::vector<producer_plugin::snapshot_request_information> sr;
+         _snapshot_db >> sr;
+         _snapshot_requests.insert(sr.begin(), sr.end());
+      }
+   }
+};
+}// namespace eosio
diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp
index a52cb048f3..bf3f9fc613 100644
--- a/plugins/producer_plugin/producer_plugin.cpp
+++ b/plugins/producer_plugin/producer_plugin.cpp
@@ -1,7 +1,7 @@
 #include <eosio/producer_plugin/producer_plugin.hpp>
 #include <eosio/producer_plugin/pending_snapshot.hpp>
 #include <eosio/producer_plugin/subjective_billing.hpp>
-#include <eosio/producer_plugin/snapshot_db_json.hpp>
+#include <eosio/producer_plugin/snapshot_scheduler.hpp>
 #include <eosio/chain/plugin_interface.hpp>
 #include <eosio/chain/global_property_object.hpp>
 #include <eosio/chain/generated_transaction_object.hpp>
@@ -133,25 +133,6 @@ enum class pending_block_mode {
    speculating
 };
 
-struct by_snapshot_id;
-struct by_snapshot_value;
-struct as_vector;
-
-using snapshot_requests = multi_index_container<
-   producer_plugin::snapshot_request_information,
-   indexed_by<
-      hashed_unique<tag<by_snapshot_id>, BOOST_MULTI_INDEX_MEMBER(producer_plugin::snapshot_request_information, uint32_t, snapshot_request_id)>,
-      random_access<tag<as_vector>>,
-      ordered_unique<tag<by_snapshot_value>,
-         composite_key< producer_plugin::snapshot_request_information,
-            BOOST_MULTI_INDEX_MEMBER(producer_plugin::snapshot_request_information, uint32_t, block_spacing),
-            BOOST_MULTI_INDEX_MEMBER(producer_plugin::snapshot_request_information, uint32_t, start_block_num),
-            BOOST_MULTI_INDEX_MEMBER(producer_plugin::snapshot_request_information, uint32_t, end_block_num)
-         >
-      >
-   >
->;
-
 namespace {
 
 // track multiple failures on unapplied transactions
@@ -383,10 +364,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       // path to write the snapshots to
       bfs::path _snapshots_dir;
 
-      // snapshot schedule
-      snapshot_requests _snapshot_requests;
-      snapshot_db_json  _snapshot_db;
-      uint32_t          _snapshot_id = 0;
+      // async snapshot scheduler
+      snapshot_scheduler _snapshot_scheduler;
 
       void consider_new_watermark( account_name producer, uint32_t block_num, block_timestamp_type timestamp) {
          auto itr = _producer_watermarks.find( producer );
@@ -410,36 +389,11 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          auto before = _unapplied_transactions.size();
          _unapplied_transactions.clear_applied( bsp );
          _subjective_billing.on_block( _log, bsp, fc::time_point::now() );
-         scheduler_on_block(bsp);
+         _snapshot_scheduler.on_block(bsp);
          fc_dlog( _log, "Removed applied transactions before: ${before}, after: ${after}",
                   ("before", before)("after", _unapplied_transactions.size()) );
       }
 
-      void scheduler_on_block( const block_state_ptr& bsp ) {
-
-         auto height =bsp->block_num;
-         for (const auto & req : _snapshot_requests.get<0>()) {
-            // execute "asap" or if matches spacing
-            if ((req.start_block_num == 0) ||
-               (!((height - req.start_block_num) % req.block_spacing))) {
-              // x_execute_snapshot();
-            }
-            // assume "asap" for snapshot with missed/zero start, it can have spacing
-           /*
-            if (req.start_block_num == 0) {
-               auto node = requests.extract(req);
-               node.key().start_block_num = height;
-               requests.insert(std::move(node));
-            }
-            */
-            // remove expired request
-            if (req.end_block_num > 0 && height >= req.end_block_num) {
-              auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
-              _snapshot_requests.erase(snapshot_by_id.find(req.snapshot_request_id));
-            }
-         }
-      }
-
       void on_block_header( const block_state_ptr& bsp ) {
          consider_new_watermark( bsp->header.producer, bsp->block_num, bsp->block->timestamp );
       }
@@ -1049,12 +1003,9 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
          my->_subjective_billing.disable_account( account_name(a) );
       }
    }
-   my->_snapshot_db.set_path(my->_snapshots_dir);
-   if (fc::exists(my->_snapshot_db.get_json_path())) {
-      std::vector<producer_plugin::snapshot_request_information>  sr;
-      my->_snapshot_db >> sr;
-      my->_snapshot_requests.insert(sr.begin(), sr.end());
-   }
+
+   my->_snapshot_scheduler.set_db_path(my->_snapshots_dir);
+
 } FC_LOG_AND_RETHROW() }
 
 void producer_plugin::plugin_startup()
@@ -1383,38 +1334,17 @@ void producer_plugin::create_snapshot(producer_plugin::next_function<producer_pl
 
 void producer_plugin::schedule_snapshot(const snapshot_request_information& sri)
 {
-   auto& snapshot_by_value = my->_snapshot_requests.get<by_snapshot_value>();
-   auto existing = snapshot_by_value.find(std::make_tuple(sri.block_spacing, sri.start_block_num, sri.end_block_num));
-   EOS_ASSERT(existing == snapshot_by_value.end() , duplicate_snapshot_request, "Duplicate snapshot request");
-   if (sri.end_block_num>0) EOS_ASSERT(sri.start_block_num <= sri.end_block_num, invalid_snapshot_request, "End block number should be greater or equal to start block number");
-   if (sri.block_spacing>0) EOS_ASSERT(sri.start_block_num + sri.block_spacing <= sri.end_block_num, invalid_snapshot_request, "Block spacing exceeds defined by start and end range");
-
-   auto request_id = sri.snapshot_request_id ? sri.snapshot_request_id : my->_snapshot_id++ ;
-   my->_snapshot_requests.emplace(producer_plugin::snapshot_request_information{request_id, sri.block_spacing, sri.start_block_num, sri.end_block_num,{}});
-
-   auto & vec = my->_snapshot_requests.get<as_vector>();
-   std::vector<producer_plugin::snapshot_request_information>  sr(vec.begin(), vec.end());
-   my->_snapshot_db << sr;
+   my->_snapshot_scheduler.schedule_snapshot(sri);
 }
 
 void producer_plugin::unschedule_snapshot(const snapshot_request_information& sri)
 {
-   auto& snapshot_by_id = my->_snapshot_requests.get<by_snapshot_id>();
-   auto existing = snapshot_by_id.find(sri.snapshot_request_id);
-   EOS_ASSERT(existing != snapshot_by_id.end(), snapshot_request_not_found, "Snapshot request not found");
-   my->_snapshot_requests.erase(existing);
-
-   auto & vec = my->_snapshot_requests.get<as_vector>();
-   std::vector<producer_plugin::snapshot_request_information>  sr(vec.begin(), vec.end());
-   my->_snapshot_db << sr;
+   my->_snapshot_scheduler.unschedule_snapshot(sri.snapshot_request_id);
 }
 
 producer_plugin::get_snapshot_requests_result producer_plugin::get_snapshot_requests() const
 {
-   producer_plugin::get_snapshot_requests_result result;
-   auto & asvector = my->_snapshot_requests.get<as_vector>();
-   result.requests.insert(result.requests.begin(),  asvector.begin(), asvector.end());
-   return result;
+   return my->_snapshot_scheduler.get_snapshot_requests();
 }
 
 producer_plugin::scheduled_protocol_feature_activations

From f1affa6b8ec7413502a1740bb25952be3204eed5 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Wed, 22 Feb 2023 12:48:16 -0500
Subject: [PATCH 14/48] Hook snapshot generation to block_start

---
 .../producer_plugin/snapshot_scheduler.hpp    | 39 +++++++++++++++----
 plugins/producer_plugin/producer_plugin.cpp   |  7 +++-
 2 files changed, 37 insertions(+), 9 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index a84b320e6e..f8863c1185 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -28,8 +28,8 @@ using snapshot_request_information = producer_plugin::snapshot_request_informati
 using get_snapshot_requests_result = producer_plugin::get_snapshot_requests_result;
 
 class snapshot_scheduler_listener {
-   virtual void on_block(const block_state_ptr& bsp) = 0;
-   virtual void on_irreversible_block(const signed_block_ptr& lib) = 0;
+   virtual void on_block(uint32_t height) = 0;
+   virtual void on_irreversible_block(uint32_t height) = 0;
    virtual void on_abort_block() = 0;
 };
 
@@ -58,19 +58,18 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
    snapshot_requests _snapshot_requests;
    snapshot_db_json _snapshot_db;
    uint32_t _snapshot_id = 0;
+   std::function<void(producer_plugin::next_function<producer_plugin::snapshot_information>)> _create_snapshot;
 
 public:
    snapshot_scheduler() {}
 
    // snapshot_scheduler_listener
-   virtual void on_block(const block_state_ptr& bsp) {
-      auto height = bsp->block_num;
+   virtual void on_block(uint32_t height) {
       for(const auto& req: _snapshot_requests.get<0>()) {
          // execute "asap" or if matches spacing
          if((req.start_block_num == 0) ||
             (!((height - req.start_block_num) % req.block_spacing))) {
-            ilog("BOOM!!!!");
-            // x_execute_snapshot();
+            execute_snapshot();
          }
          // assume "asap" for snapshot with missed/zero start, it can have spacing
          if(req.start_block_num == 0) {
@@ -86,7 +85,7 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
       }
    }
 
-   virtual void on_irreversible_block(const signed_block_ptr& lib) {
+   virtual void on_irreversible_block(uint32_t height) {
    }
 
    virtual void on_abort_block() {
@@ -136,5 +135,31 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
          _snapshot_requests.insert(sr.begin(), sr.end());
       }
    }
+
+   // snapshot executor
+   void set_create_snapshot_fn(std::function<void(producer_plugin::next_function<producer_plugin::snapshot_information>)> fn) {
+      _create_snapshot = fn;
+   }
+
+
+   void execute_snapshot() {
+      auto next = [](const std::variant<fc::exception_ptr, producer_plugin::snapshot_information>& result) {
+         if(std::holds_alternative<fc::exception_ptr>(result)) {
+            try {
+               std::get<fc::exception_ptr>(result)->dynamic_rethrow_exception();
+            } catch(const fc::exception& e) {
+               edump((e.to_detail_string()));
+               throw;
+            } catch(const std::exception& e) {
+               edump((e.what()));
+               throw;
+            }
+         } else {
+            // success
+         }
+      };
+      _create_snapshot(next);
+      //auto theasync=std::async([&,next]{ return _create_snapshot(next);});
+   }
 };
 }// namespace eosio
diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp
index bf3f9fc613..455c9a90e1 100644
--- a/plugins/producer_plugin/producer_plugin.cpp
+++ b/plugins/producer_plugin/producer_plugin.cpp
@@ -345,6 +345,7 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
       std::optional<scoped_connection>                          _accepted_block_connection;
       std::optional<scoped_connection>                          _accepted_block_header_connection;
       std::optional<scoped_connection>                          _irreversible_block_connection;
+      std::optional<scoped_connection>                          _block_start_connection;
 
       /*
        * HACK ALERT
@@ -389,7 +390,8 @@ class producer_plugin_impl : public std::enable_shared_from_this<producer_plugin
          auto before = _unapplied_transactions.size();
          _unapplied_transactions.clear_applied( bsp );
          _subjective_billing.on_block( _log, bsp, fc::time_point::now() );
-         _snapshot_scheduler.on_block(bsp);
+         //_snapshot_scheduler.on_block(bsp->block_num);
+
          fc_dlog( _log, "Removed applied transactions before: ${before}, after: ${after}",
                   ("before", before)("after", _unapplied_transactions.size()) );
       }
@@ -1005,6 +1007,7 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
    }
 
    my->_snapshot_scheduler.set_db_path(my->_snapshots_dir);
+   my->_snapshot_scheduler.set_create_snapshot_fn(std::bind(&producer_plugin::create_snapshot, this, std::placeholders::_1));
 
 } FC_LOG_AND_RETHROW() }
 
@@ -1028,6 +1031,7 @@ void producer_plugin::plugin_startup()
    my->_accepted_block_connection.emplace(chain.accepted_block.connect( [this]( const auto& bsp ){ my->on_block( bsp ); } ));
    my->_accepted_block_header_connection.emplace(chain.accepted_block_header.connect( [this]( const auto& bsp ){ my->on_block_header( bsp ); } ));
    my->_irreversible_block_connection.emplace(chain.irreversible_block.connect( [this]( const auto& bsp ){ my->on_irreversible_block( bsp->block ); } ));
+   my->_block_start_connection.emplace(chain.block_start.connect( [this]( uint32_t bs ){ my->_snapshot_scheduler.on_block(bs); } ));
 
    const auto lib_num = chain.last_irreversible_block_num();
    const auto lib = chain.fetch_block_by_number(lib_num);
@@ -1326,7 +1330,6 @@ void producer_plugin::create_snapshot(producer_plugin::next_function<producer_pl
                ("bn", head_block_num)
                ("ec", ec.value())
                ("message", ec.message()));
-
          my->_pending_snapshot_index.emplace(head_id, next, pending_path.generic_string(), snapshot_path.generic_string());
       } CATCH_AND_CALL (next);
    }

From 82ae03985dfb8d7592795e06f8e8679ff7b32631 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Thu, 23 Feb 2023 09:39:49 -0500
Subject: [PATCH 15/48] Included light boost integration test

---
 .../producer_plugin/snapshot_scheduler.hpp    | 11 +++-
 .../test/test_snapshot_scheduler.cpp          | 61 +++++++++++++++++--
 2 files changed, 66 insertions(+), 6 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index f8863c1185..67e03c2697 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -96,8 +96,15 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
       auto& snapshot_by_value = _snapshot_requests.get<by_snapshot_value>();
       auto existing = snapshot_by_value.find(std::make_tuple(sri.block_spacing, sri.start_block_num, sri.end_block_num));
       EOS_ASSERT(existing == snapshot_by_value.end(), duplicate_snapshot_request, "Duplicate snapshot request");
-      if(sri.end_block_num > 0) EOS_ASSERT(sri.start_block_num <= sri.end_block_num, invalid_snapshot_request, "End block number should be greater or equal to start block number");
-      if(sri.block_spacing > 0) EOS_ASSERT(sri.start_block_num + sri.block_spacing <= sri.end_block_num, invalid_snapshot_request, "Block spacing exceeds defined by start and end range");
+
+      if(sri.end_block_num > 0) {
+         // if "end" is specified, it should be greater then start
+         EOS_ASSERT(sri.start_block_num <= sri.end_block_num, invalid_snapshot_request, "End block number should be greater or equal to start block number");
+         // if also block_spacing specified, check it
+         if(sri.block_spacing > 0) {
+            EOS_ASSERT(sri.start_block_num + sri.block_spacing <= sri.end_block_num, invalid_snapshot_request, "Block spacing exceeds defined by start and end range");
+         }
+      }
 
       auto request_id = sri.snapshot_request_id ? sri.snapshot_request_id : _snapshot_id++;
       _snapshot_requests.emplace(producer_plugin::snapshot_request_information{request_id, sri.block_spacing, sri.start_block_num, sri.end_block_num, {}});
diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
index 8e94d9c796..555b1e36f9 100644
--- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
+++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
@@ -48,18 +48,71 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
 
       BOOST_CHECK_EQUAL(0, scheduler.get_snapshot_requests().requests.size());
 
-      producer_plugin::snapshot_request_information sri_large_spacing = {.snapshot_request_id = 0, .block_spacing = 1000, .start_block_num = 5000, .end_block_num = 5010 };
+      producer_plugin::snapshot_request_information sri_large_spacing = {.snapshot_request_id = 0, .block_spacing = 1000, .start_block_num = 5000, .end_block_num = 5010};
       BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri_large_spacing), invalid_snapshot_request, [](const fc::assert_exception& e) {
          return e.to_detail_string().find("Block spacing exceeds defined by start and end range") != std::string::npos;
       });
 
-      producer_plugin::snapshot_request_information sri_start_end = {.snapshot_request_id = 0, .block_spacing = 1000, .start_block_num = 50000, .end_block_num = 5000 };
+      producer_plugin::snapshot_request_information sri_start_end = {.snapshot_request_id = 0, .block_spacing = 1000, .start_block_num = 50000, .end_block_num = 5000};
       BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri_start_end), invalid_snapshot_request, [](const fc::assert_exception& e) {
          return e.to_detail_string().find("End block number should be greater or equal to start block number") != std::string::npos;
       });
    }
-}
+   {
+      boost::filesystem::path temp = boost::filesystem::temp_directory_path() / boost::filesystem::unique_path();
+
+      try {
+         std::promise<std::tuple<producer_plugin*, chain_plugin*>> plugin_promise;
+         std::future<std::tuple<producer_plugin*, chain_plugin*>> plugin_fut = plugin_promise.get_future();
+         std::thread app_thread([&]() {
+            fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
+            std::vector<const char*> argv =
+                  {"test", "--data-dir", temp.c_str(), "--config-dir", temp.c_str(),
+                   "-p", "eosio", "-e", "--max-transaction-time", "475", "--disable-subjective-billing=true"};
+            appbase::app().initialize<chain_plugin, producer_plugin>(argv.size(), (char**) &argv[0]);
+            appbase::app().startup();
+            plugin_promise.set_value(
+                  {appbase::app().find_plugin<producer_plugin>(), appbase::app().find_plugin<chain_plugin>()});
+            appbase::app().exec();
+         });
+
+         auto [prod_plug, chain_plug] = plugin_fut.get();
+         // auto chain_id = chain_plug->get_chain_id();
+         std::deque<block_state_ptr> all_blocks;
+         std::promise<void> empty_blocks_promise;
+         std::future<void> empty_blocks_fut = empty_blocks_promise.get_future();
+         auto ab = chain_plug->chain().accepted_block.connect([&](const block_state_ptr& bsp) {
+            static int num_empty = std::numeric_limits<int>::max();
+            all_blocks.push_back(bsp);
+            if(bsp->block->transactions.empty()) {
+               --num_empty;
+               if(num_empty == 0) empty_blocks_promise.set_value();
+            } else {// we want a few empty blocks after we have some non-empty blocks
+               num_empty = 10;
+            }
+         });
+         auto bs = chain_plug->chain().block_start.connect([&](uint32_t bn) {
+         });
+
+         producer_plugin::snapshot_request_information sri = {.block_spacing = 1, .start_block_num = 0, .end_block_num = 1, .snapshot_description = "Example of recurring snapshot"};
+         auto pp = appbase::app().find_plugin<producer_plugin>();
+         pp->schedule_snapshot(sri);
+
+         empty_blocks_fut.wait_for(std::chrono::seconds(5));
+
+         // snapshot is done here and request should be deleted
+         BOOST_CHECK_EQUAL(0, pp->get_snapshot_requests().requests.size());
+
+         appbase::app().quit();
+         app_thread.join();
+      } catch(...) {
+         bfs::remove_all(temp);
+         throw;
+      }
+      bfs::remove_all(temp);
+   }
 
-BOOST_AUTO_TEST_SUITE_END()
+   BOOST_AUTO_TEST_SUITE_END()
+}
 
 }// namespace

From 21630d3a5c17b6f684c235416b5aadfb17f50694 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 28 Feb 2023 13:05:23 -0500
Subject: [PATCH 16/48] Added second snapshot request to test

---
 .../test/test_snapshot_scheduler.cpp              | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
index 555b1e36f9..9870b73b33 100644
--- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
+++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
@@ -94,15 +94,21 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          auto bs = chain_plug->chain().block_start.connect([&](uint32_t bn) {
          });
 
-         producer_plugin::snapshot_request_information sri = {.block_spacing = 1, .start_block_num = 0, .end_block_num = 1, .snapshot_description = "Example of recurring snapshot"};
+         producer_plugin::snapshot_request_information sri1 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 1"};
+         producer_plugin::snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2"};
          auto pp = appbase::app().find_plugin<producer_plugin>();
-         pp->schedule_snapshot(sri);
+         pp->schedule_snapshot(sri1);
+         pp->schedule_snapshot(sri2);
+
+         // both snapshot requests should be present now
+         BOOST_CHECK_EQUAL(2, pp->get_snapshot_requests().requests.size());
 
          empty_blocks_fut.wait_for(std::chrono::seconds(5));
 
-         // snapshot is done here and request should be deleted
-         BOOST_CHECK_EQUAL(0, pp->get_snapshot_requests().requests.size());
+         // one of the snapshots is done here and request, corresponding to it should be deleted
+         BOOST_CHECK_EQUAL(1, pp->get_snapshot_requests().requests.size());
 
+         // quit app
          appbase::app().quit();
          app_thread.join();
       } catch(...) {
@@ -111,7 +117,6 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
       }
       bfs::remove_all(temp);
    }
-
    BOOST_AUTO_TEST_SUITE_END()
 }
 

From 2eae5267eb2fa35dbbba10331e503619c0a18a6a Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 28 Feb 2023 16:43:31 -0500
Subject: [PATCH 17/48] Updated yaml formatting

---
 .../producer_api_plugin/producer.swagger.yaml | 217 +++++++++---------
 1 file changed, 103 insertions(+), 114 deletions(-)

diff --git a/plugins/producer_api_plugin/producer.swagger.yaml b/plugins/producer_api_plugin/producer.swagger.yaml
index f8d51f2cf8..de6d334b38 100644
--- a/plugins/producer_api_plugin/producer.swagger.yaml
+++ b/plugins/producer_api_plugin/producer.swagger.yaml
@@ -37,13 +37,13 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/OK'
+                $ref: "#/components/schemas/OK"
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
+                $ref: "#/components/schemas/Error"
   /producer/resume:
     post:
       summary: resume
@@ -55,13 +55,13 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/OK'
+                $ref: "#/components/schemas/OK"
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
+                $ref: "#/components/schemas/Error"
   /producer/paused:
     post:
       summary: paused
@@ -81,7 +81,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
+                $ref: "#/components/schemas/Error"
   /producer/get_runtime_options:
     post:
       summary: get_runtime_options
@@ -93,13 +93,13 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Runtime_Options'
+                $ref: "#/components/schemas/Runtime_Options"
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
+                $ref: "#/components/schemas/Error"
   /producer/update_runtime_options:
     post:
       summary: update_runtime_options
@@ -109,20 +109,20 @@ paths:
         content:
           application/json:
             schema:
-              $ref: '#/components/schemas/Runtime_Options'
+              $ref: "#/components/schemas/Runtime_Options"
       responses:
         "201":
           description: OK
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/OK'
+                $ref: "#/components/schemas/OK"
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
+                $ref: "#/components/schemas/Error"
   /producer/get_greylist:
     post:
       summary: get_greylist
@@ -146,8 +146,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
-
+                $ref: "#/components/schemas/Error"
   /producer/add_greylist_accounts:
     post:
       summary: add_greylist_accounts
@@ -159,24 +158,24 @@ paths:
             schema:
               type: object
               properties:
-                  accounts:
-                    type: array
-                    description: List of account names to add
-                    items:
-                      $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
+                accounts:
+                  type: array
+                  description: List of account names to add
+                  items:
+                    $ref: "https://docs.eosnetwork.com/openapi/v2.0/Name.yaml"
       responses:
         "201":
           description: OK
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/OK'
+                $ref: "#/components/schemas/OK"
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
+                $ref: "#/components/schemas/Error"
   /producer/remove_greylist_accounts:
     post:
       summary: remove_greylist_accounts
@@ -199,15 +198,14 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/OK'
+                $ref: "#/components/schemas/OK"
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
-
-/producer/get_whitelist_blacklist:
+                $ref: "#/components/schemas/Error"
+  /producer/get_whitelist_blacklist:
     post:
       summary: get_whitelist_blacklist
       description: Retrieves the whitelist and blacklist for producer node. A JSON object containing whitelist and blacklist information.
@@ -254,8 +252,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
-
+                $ref: "#/components/schemas/Error"
   /producer/set_whitelist_blacklist:
     post:
       summary: set_whitelist_blacklist
@@ -302,13 +299,13 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/OK'
+                $ref: "#/components/schemas/OK"
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
+                $ref: "#/components/schemas/Error"
   /producer/create_snapshot:
     post:
       summary: create_snapshot
@@ -346,7 +343,6 @@ paths:
             application/json:
               schema:
                 $ref: "#/component/schema/Error"
-
   /producer/schedule_snapshot:
     post:
       summary: schedule_snapshot
@@ -370,9 +366,9 @@ paths:
                   description: Block number at which schedule ends
                   example: 15102
                 snapshot_description:
-                    type: string
-                    description: Description of the snapshot
-                    example: Daily snapshot
+                  type: string
+                  description: Description of the snapshot
+                  example: Daily snapshot
 
       responses:
         "201":
@@ -380,33 +376,32 @@ paths:
           content:
             application/json:
               schema:
-                  type: object
-                  properties:
-                    snapshot_request_id:
-                      type: integer
-                      description: Unique id identifying current snapshot request
-                    block_spacing:
-                      type: integer
-                      description: Generate snapshot every block_spacing blocks
-                    start_block_num:
-                      type: integer
-                      description: Block number at which schedule starts
-                      example: 5102
-                    end_block_num:
-                      type: integer
-                      description: Block number at which schedule ends
-                      example: 15102
-                    snapshot_description:
-                        type: string
-                        description: Description of the snapshot
-                        example: Daily snapshot
+                type: object
+                properties:
+                  snapshot_request_id:
+                    type: integer
+                    description: Unique id identifying current snapshot request
+                  block_spacing:
+                    type: integer
+                    description: Generate snapshot every block_spacing blocks
+                  start_block_num:
+                    type: integer
+                    description: Block number at which schedule starts
+                    example: 5102
+                  end_block_num:
+                    type: integer
+                    description: Block number at which schedule ends
+                    example: 15102
+                  snapshot_description:
+                    type: string
+                    description: Description of the snapshot
+                    example: Daily snapshot
         "400":
           description: client error
           content:
             application/json:
               schema:
                 $ref: "#/component/schema/Error"
-
   /producer/get_snapshot_requests:
     post:
       summary: get_snapshot_requests
@@ -417,7 +412,7 @@ paths:
           description: OK
           content:
             application/json:
-             schema:
+              schema:
                 type: object
                 properties:
                   snapshot_requests:
@@ -426,55 +421,54 @@ paths:
                     items:
                       type: object
                       properties:
-                          snapshot_request_id:
-                            type: integer
-                            description: Unique id identifying current snapshot request
-                          block_spacing:
-                            type: integer
-                            description: Generate snapshot every block_spacing blocks
-                          start_block_num:
-                            type: integer
-                            description: Block number at which schedule starts
-                            example: 5102
-                          end_block_num:
-                            type: integer
-                            description: Block number at which schedule ends
-                            example: 15102
-                          snapshot_description:
-                              type: string
-                              description: Description of the snapshot
-                              example: Daily snapshot
-                          pending_snapshots:
-                            type: array
-                            description: List of pending snapshots
-                            items:
-                              type: object
-                              properties:
-                                head_block_id:
-                                  $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
-                                head_block_num:
-                                  type: integer
-                                  description: Highest block number on the chain
-                                  example: 5102
-                                head_block_time:
-                                  type: string
-                                  description: Highest block unix timestamp
-                                  example: 2020-11-16T00:00:00.000
-                                version:
-                                  type: integer
-                                  description: version number
-                                  example: 6
-                                snapshot_name:
-                                  type: string
-                                  description: The path and file name of the snapshot
-                                  example: /home/me/nodes/node-name/snapshots/snapshot-0000999f99999f9f999f99f99ff9999f999f9fff99ff99ffff9f9f9fff9f9999.bin
+                        snapshot_request_id:
+                          type: integer
+                          description: Unique id identifying current snapshot request
+                        block_spacing:
+                          type: integer
+                          description: Generate snapshot every block_spacing blocks
+                        start_block_num:
+                          type: integer
+                          description: Block number at which schedule starts
+                          example: 5102
+                        end_block_num:
+                          type: integer
+                          description: Block number at which schedule ends
+                          example: 15102
+                        snapshot_description:
+                          type: string
+                          description: Description of the snapshot
+                          example: Daily snapshot
+                        pending_snapshots:
+                          type: array
+                          description: List of pending snapshots
+                          items:
+                            type: object
+                            properties:
+                              head_block_id:
+                                $ref: "https://docs.eosnetwork.com/openapi/v2.0/Sha256.yaml"
+                              head_block_num:
+                                type: integer
+                                description: Highest block number on the chain
+                                example: 5102
+                              head_block_time:
+                                type: string
+                                description: Highest block unix timestamp
+                                example: 2020-11-16T00:00:00.000
+                              version:
+                                type: integer
+                                description: version number
+                                example: 6
+                              snapshot_name:
+                                type: string
+                                description: The path and file name of the snapshot
+                                example: /home/me/nodes/node-name/snapshots/snapshot-0000999f99999f9f999f99f99ff9999f999f9fff99ff99ffff9f9f9fff9f9999.bin
         "400":
           description: client error
           content:
             application/json:
               schema:
                 $ref: "#/component/schema/Error"
-
   /producer/unschedule_snapshot:
     post:
       summary: unschedule_snapshot
@@ -499,8 +493,8 @@ paths:
                 type: object
                 properties:
                   snapshot_request_id:
-                      type: integer
-                      description: Unique id identifying current snapshot request
+                    type: integer
+                    description: Unique id identifying current snapshot request
                   block_spacing:
                     type: integer
                     description: Generate snapshot every block_spacing blocks
@@ -513,16 +507,15 @@ paths:
                     description: Block number at which schedule ends
                     example: 15102
                   snapshot_description:
-                      type: string
-                      description: Description of the snapshot
-                      example: Daily snapshot
+                    type: string
+                    description: Description of the snapshot
+                    example: Daily snapshot
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
-
+                $ref: "#/components/schemas/Error"
   /producer/get_integrity_hash:
     post:
       summary: get_integrity_hash
@@ -546,8 +539,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
-
+                $ref: "#/components/schemas/Error"
   /producer/schedule_protocol_feature_activations:
     post:
       summary: schedule_protocol_feature_activations
@@ -570,15 +562,14 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/OK'
+                $ref: "#/components/schemas/OK"
         "400":
           description: client error
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
-
-/producer/get_supported_protocol_features:
+                $ref: "#/components/schemas/Error"
+  /producer/get_supported_protocol_features:
     post:
       summary: get_supported_protocol_features
       description: Retrieves supported protocol features for producer node. Pass filters in as part of the request body.
@@ -647,8 +638,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
-
+                $ref: "#/components/schemas/Error"
   /producer/get_account_ram_corrections:
     post:
       summary: get_account_ram_corrections
@@ -699,8 +689,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
-
+                $ref: "#/components/schemas/Error"
   /producer/get_unapplied_transactions:
     post:
       summary: get_unapplied_transactions
@@ -780,7 +769,7 @@ paths:
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/Error'
+                $ref: "#/components/schemas/Error"
 components:
   securitySchemes: {}
   schemas:

From 962f0a975ddfd0a6f43318b334189657700c80c5 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Wed, 1 Mar 2023 13:49:55 -0500
Subject: [PATCH 18/48] Addressing feedback comments 1/2

---
 .../producer_api_plugin.cpp                   |  2 +-
 .../eosio/producer_plugin/producer_plugin.hpp | 16 +++--
 .../producer_plugin/snapshot_db_json.hpp      | 61 +++++++++++--------
 .../producer_plugin/snapshot_scheduler.hpp    | 34 ++++++-----
 plugins/producer_plugin/producer_plugin.cpp   |  2 +-
 .../test/test_snapshot_scheduler.cpp          | 14 ++---
 6 files changed, 75 insertions(+), 54 deletions(-)

diff --git a/plugins/producer_api_plugin/producer_api_plugin.cpp b/plugins/producer_api_plugin/producer_api_plugin.cpp
index d0fd200ad8..5ffcdea67d 100644
--- a/plugins/producer_api_plugin/producer_api_plugin.cpp
+++ b/plugins/producer_api_plugin/producer_api_plugin.cpp
@@ -125,7 +125,7 @@ void producer_api_plugin::plugin_startup() {
        CALL_WITH_400(producer, producer, get_snapshot_requests,
             INVOKE_R_V(producer, get_snapshot_requests), 201),
        CALL_WITH_400(producer, producer, unschedule_snapshot,
-            INVOKE_V_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_information), 201),
+            INVOKE_V_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201),
        CALL_WITH_400(producer, producer, get_scheduled_protocol_feature_activations,
             INVOKE_R_V(producer, get_scheduled_protocol_feature_activations), 201),
        CALL_WITH_400(producer, producer, schedule_protocol_feature_activations,
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
index 95f032034e..3884e034c4 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
@@ -51,11 +51,18 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
    };
 
    struct snapshot_request_information {
-      uint32_t             snapshot_request_id = 0;
       uint32_t             block_spacing = 0;
       uint32_t             start_block_num = 0;
       uint32_t             end_block_num = 0;
       std::string          snapshot_description = "";
+
+   };
+
+   struct snapshot_request_id_information {
+      uint32_t                            snapshot_request_id = 0;
+   };
+
+   struct snapshot_schedule_information : public snapshot_request_id_information, public snapshot_request_information {
       std::optional<snapshot_information> pending_snapshot;
    };
 
@@ -121,7 +128,7 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
 
    void create_snapshot(next_function<snapshot_information> next);
    void schedule_snapshot(const snapshot_request_information& schedule);
-   void unschedule_snapshot(const snapshot_request_information& schedule);
+   void unschedule_snapshot(const snapshot_request_id_information& schedule);
    get_snapshot_requests_result get_snapshot_requests() const;
 
    scheduled_protocol_feature_activations get_scheduled_protocol_feature_activations() const;
@@ -165,7 +172,7 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
    void received_block();
 
  private:
-   std::shared_ptr<class producer_plugin_impl>  my;
+   std::shared_ptr<class producer_plugin_impl> my;
 };
 
 } //eosio
@@ -175,7 +182,8 @@ FC_REFLECT(eosio::producer_plugin::greylist_params, (accounts));
 FC_REFLECT(eosio::producer_plugin::whitelist_blacklist, (actor_whitelist)(actor_blacklist)(contract_whitelist)(contract_blacklist)(action_blacklist)(key_blacklist) )
 FC_REFLECT(eosio::producer_plugin::integrity_hash_information, (head_block_id)(integrity_hash))
 FC_REFLECT(eosio::producer_plugin::snapshot_information, (head_block_id)(head_block_num)(head_block_time)(version)(snapshot_name))
-FC_REFLECT(eosio::producer_plugin::snapshot_request_information, (snapshot_request_id)(block_spacing)(start_block_num)(end_block_num)(snapshot_description))
+FC_REFLECT(eosio::producer_plugin::snapshot_request_information, (block_spacing)(start_block_num)(end_block_num)(snapshot_description))
+FC_REFLECT(eosio::producer_plugin::snapshot_request_id_information, (snapshot_request_id))
 FC_REFLECT(eosio::producer_plugin::get_snapshot_requests_result, (requests))
 FC_REFLECT(eosio::producer_plugin::scheduled_protocol_feature_activations, (protocol_features_to_activate))
 FC_REFLECT(eosio::producer_plugin::get_supported_protocol_features_params, (exclude_disabled)(exclude_unactivatable))
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
index d87aecbc64..1d8979c6f1 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
@@ -9,46 +9,50 @@
 #include <boost/property_tree/json_parser.hpp>
 #include <boost/property_tree/ptree.hpp>
 
-
 namespace eosio {
 
-using namespace eosio::chain;
-
+/// this class designed to serialize/deserialize snapshot schedule to a filesystem so it can be restored after restart
 class snapshot_db_json {
 public:
-   snapshot_db_json() {}
-   snapshot_db_json(bfs::path db_path) : db_path(db_path) {}
-   ~snapshot_db_json() {}
+   snapshot_db_json()   = default;
+   ~snapshot_db_json()  = default;
 
    void set_path(bfs::path path) {
-      db_path = path;
+      db_path = std::move(path);
    }
 
    bfs::path get_json_path() {
       return db_path / "snapshot-schedule.json";
    }
 
-   const snapshot_db_json& operator>>(std::vector<producer_plugin::snapshot_request_information>& sr) {
+   const snapshot_db_json& operator>>(std::vector<producer_plugin::snapshot_schedule_information>& sr) {
       boost::property_tree::ptree root;
-      std::ifstream file(get_json_path().string());
-      boost::property_tree::read_json(file, root);
-      file.close();
-
-      // parse ptree
-      for(boost::property_tree::ptree::value_type& req: root.get_child("snapshot_requests")) {
-         producer_plugin::snapshot_request_information sri;
-         sri.snapshot_request_id = req.second.get<uint32_t>("snapshot_request_id");
-         sri.snapshot_description = req.second.get<std::string>("snapshot_description");
-         sri.block_spacing = req.second.get<uint32_t>("block_spacing");
-         sri.start_block_num = req.second.get<uint32_t>("start_block_num");
-         sri.end_block_num = req.second.get<uint32_t>("end_block_num");
-         sr.push_back(sri);
+
+      try {
+         std::ifstream file(get_json_path().string());
+         boost::property_tree::read_json(file, root);
+         file.close();
+
+         // parse ptree
+         for(boost::property_tree::ptree::value_type& req: root.get_child("snapshot_requests")) {
+            producer_plugin::snapshot_schedule_information ssi;
+            ssi.snapshot_request_id = req.second.get<uint32_t>("snapshot_request_id");
+            ssi.snapshot_description = req.second.get<std::string>("snapshot_description");
+            ssi.block_spacing = req.second.get<uint32_t>("block_spacing");
+            ssi.start_block_num = req.second.get<uint32_t>("start_block_num");
+            ssi.end_block_num = req.second.get<uint32_t>("end_block_num");
+            sr.push_back(ssi);
+         }
+      }
+      catch (std::ifstream::failure e) {
+          wlog( "unable to restore snapshots schedule from filesystem: ${details}",
+                  ("details",e.what()) );
       }
 
       return *this;
    }
 
-   const snapshot_db_json& operator<<(std::vector<producer_plugin::snapshot_request_information>& sr) {
+   const snapshot_db_json& operator<<(std::vector<producer_plugin::snapshot_schedule_information>& sr) {
       boost::property_tree::ptree root;
       boost::property_tree::ptree node_srs;
 
@@ -61,11 +65,18 @@ class snapshot_db_json {
          node.put("end_block_num", key.end_block_num);
          node_srs.push_back(std::make_pair("", node));
       }
+
       root.push_back(std::make_pair("snapshot_requests", node_srs));
 
-      std::ofstream file(get_json_path().string());
-      boost::property_tree::write_json(file, root);
-      file.close();
+      try {
+         std::ofstream file(get_json_path().string());
+         boost::property_tree::write_json(file, root);
+         file.close();
+      }
+      catch (std::ofstream::failure e) {
+          wlog( "unable to store snapshots schedule to filesystem: ${details}",
+                  ("details",e.what()) );
+      }
 
       return *this;
    }
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 67e03c2697..23d3afc5b7 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <eosio/producer_plugin/snapshot_db_json.hpp>
+
 #include <eosio/chain/block_state.hpp>
 #include <eosio/chain/config.hpp>
 #include <eosio/chain/exceptions.hpp>
@@ -14,8 +16,6 @@
 #include <boost/multi_index/random_access_index.hpp>
 #include <boost/multi_index_container.hpp>
 
-#include <eosio/producer_plugin/snapshot_db_json.hpp>
-
 namespace eosio {
 
 namespace bmi = boost::multi_index;
@@ -23,9 +23,14 @@ using chain::account_name;
 using chain::block_state_ptr;
 using chain::packed_transaction;
 using chain::transaction_id_type;
+using chain::duplicate_snapshot_request;
+using chain::invalid_snapshot_request;
+using chain::snapshot_request_not_found;
 
-using snapshot_request_information = producer_plugin::snapshot_request_information;
-using get_snapshot_requests_result = producer_plugin::get_snapshot_requests_result;
+using snapshot_schedule_information    = producer_plugin::snapshot_schedule_information;
+using snapshot_request_information     = producer_plugin::snapshot_request_information;
+using get_snapshot_requests_result     = producer_plugin::get_snapshot_requests_result;
+using snapshot_request_id_information  = producer_plugin::snapshot_request_id_information;
 
 class snapshot_scheduler_listener {
    virtual void on_block(uint32_t height) = 0;
@@ -46,12 +51,12 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
    struct as_vector;
 
    using snapshot_requests = bmi::multi_index_container<
-         snapshot_request_information,
+         snapshot_schedule_information,
          indexed_by<
-               bmi::hashed_unique<tag<by_snapshot_id>, BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, snapshot_request_id)>,
+               bmi::hashed_unique<tag<by_snapshot_id>, BOOST_MULTI_INDEX_MEMBER(snapshot_request_id_information, uint32_t, snapshot_request_id)>,
                bmi::random_access<tag<as_vector>>,
                bmi::ordered_unique<tag<by_snapshot_value>,
-                                   composite_key<snapshot_request_information,
+                                   composite_key<snapshot_schedule_information,
                                                  BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, block_spacing),
                                                  BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, start_block_num),
                                                  BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, end_block_num)>>>>;
@@ -61,7 +66,7 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
    std::function<void(producer_plugin::next_function<producer_plugin::snapshot_information>)> _create_snapshot;
 
 public:
-   snapshot_scheduler() {}
+   snapshot_scheduler() = default;
 
    // snapshot_scheduler_listener
    virtual void on_block(uint32_t height) {
@@ -79,8 +84,7 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
          }
          // remove expired request
          if(req.end_block_num > 0 && height >= req.end_block_num) {
-            auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
-            _snapshot_requests.erase(snapshot_by_id.find(req.snapshot_request_id));
+            _snapshot_requests.erase(req.snapshot_request_id);
          }
       }
    }
@@ -106,11 +110,10 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
          }
       }
 
-      auto request_id = sri.snapshot_request_id ? sri.snapshot_request_id : _snapshot_id++;
-      _snapshot_requests.emplace(producer_plugin::snapshot_request_information{request_id, sri.block_spacing, sri.start_block_num, sri.end_block_num, {}});
+      _snapshot_requests.emplace(producer_plugin::snapshot_schedule_information {{_snapshot_id++},{sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description},{}});
 
       auto& vec = _snapshot_requests.get<as_vector>();
-      std::vector<producer_plugin::snapshot_request_information> sr(vec.begin(), vec.end());
+      std::vector<producer_plugin::snapshot_schedule_information> sr(vec.begin(), vec.end());
       _snapshot_db << sr;
    }
 
@@ -121,7 +124,7 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
       _snapshot_requests.erase(existing);
 
       auto& vec = _snapshot_requests.get<as_vector>();
-      std::vector<producer_plugin::snapshot_request_information> sr(vec.begin(), vec.end());
+      std::vector<producer_plugin::snapshot_schedule_information> sr(vec.begin(), vec.end());
       _snapshot_db << sr;
    }
 
@@ -137,7 +140,7 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
       _snapshot_db.set_path(db_path);
       // init from db
       if(fc::exists(_snapshot_db.get_json_path())) {
-         std::vector<producer_plugin::snapshot_request_information> sr;
+         std::vector<producer_plugin::snapshot_schedule_information> sr;
          _snapshot_db >> sr;
          _snapshot_requests.insert(sr.begin(), sr.end());
       }
@@ -166,7 +169,6 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
          }
       };
       _create_snapshot(next);
-      //auto theasync=std::async([&,next]{ return _create_snapshot(next);});
    }
 };
 }// namespace eosio
diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp
index 27463d467a..c118e52f68 100644
--- a/plugins/producer_plugin/producer_plugin.cpp
+++ b/plugins/producer_plugin/producer_plugin.cpp
@@ -1334,7 +1334,7 @@ void producer_plugin::schedule_snapshot(const snapshot_request_information& sri)
    my->_snapshot_scheduler.schedule_snapshot(sri);
 }
 
-void producer_plugin::unschedule_snapshot(const snapshot_request_information& sri)
+void producer_plugin::unschedule_snapshot(const snapshot_request_id_information& sri)
 {
    my->_snapshot_scheduler.unschedule_snapshot(sri.snapshot_request_id);
 }
diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
index 9870b73b33..331620eec3 100644
--- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
+++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
@@ -21,8 +21,8 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
 
    {
       // add/remove test
-      producer_plugin::snapshot_request_information sri1 = {.snapshot_request_id = 0, .block_spacing = 100, .start_block_num = 5000, .end_block_num = 10000, .snapshot_description = "Example of recurring snapshot"};
-      producer_plugin::snapshot_request_information sri2 = {.snapshot_request_id = 1, .block_spacing = 0, .start_block_num = 5200, .end_block_num = 5200, .snapshot_description = "Example of one-time snapshot"};
+      producer_plugin::snapshot_request_information sri1 = {.block_spacing = 100, .start_block_num = 5000, .end_block_num = 10000, .snapshot_description = "Example of recurring snapshot"};
+      producer_plugin::snapshot_request_information sri2 = {.block_spacing = 0, .start_block_num = 5200, .end_block_num = 5200, .snapshot_description = "Example of one-time snapshot"};
 
       scheduler.schedule_snapshot(sri1);
       scheduler.schedule_snapshot(sri2);
@@ -33,27 +33,27 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          return e.to_detail_string().find("Duplicate snapshot request") != std::string::npos;
       });
 
-      producer_plugin::snapshot_request_information sri_delete_1 = {.snapshot_request_id = 0};
+      producer_plugin::snapshot_request_id_information sri_delete_1 = {.snapshot_request_id = 0};
       scheduler.unschedule_snapshot(sri_delete_1);
 
       BOOST_CHECK_EQUAL(1, scheduler.get_snapshot_requests().requests.size());
 
-      producer_plugin::snapshot_request_information sri_delete_none = {.snapshot_request_id = 2};
+      producer_plugin::snapshot_request_id_information sri_delete_none = {.snapshot_request_id = 2};
       BOOST_CHECK_EXCEPTION(scheduler.unschedule_snapshot(sri_delete_none), snapshot_request_not_found, [](const fc::assert_exception& e) {
          return e.to_detail_string().find("Snapshot request not found") != std::string::npos;
       });
 
-      producer_plugin::snapshot_request_information sri_delete_2 = {.snapshot_request_id = 1};
+      producer_plugin::snapshot_request_id_information sri_delete_2 = {.snapshot_request_id = 1};
       scheduler.unschedule_snapshot(sri_delete_2);
 
       BOOST_CHECK_EQUAL(0, scheduler.get_snapshot_requests().requests.size());
 
-      producer_plugin::snapshot_request_information sri_large_spacing = {.snapshot_request_id = 0, .block_spacing = 1000, .start_block_num = 5000, .end_block_num = 5010};
+      producer_plugin::snapshot_request_information sri_large_spacing = {.block_spacing = 1000, .start_block_num = 5000, .end_block_num = 5010};
       BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri_large_spacing), invalid_snapshot_request, [](const fc::assert_exception& e) {
          return e.to_detail_string().find("Block spacing exceeds defined by start and end range") != std::string::npos;
       });
 
-      producer_plugin::snapshot_request_information sri_start_end = {.snapshot_request_id = 0, .block_spacing = 1000, .start_block_num = 50000, .end_block_num = 5000};
+      producer_plugin::snapshot_request_information sri_start_end = {.block_spacing = 1000, .start_block_num = 50000, .end_block_num = 5000};
       BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri_start_end), invalid_snapshot_request, [](const fc::assert_exception& e) {
          return e.to_detail_string().find("End block number should be greater or equal to start block number") != std::string::npos;
       });

From b14fc5a4b454d9046541cc11dccd8cd6342b0068 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Wed, 1 Mar 2023 21:15:11 -0500
Subject: [PATCH 19/48] Addressing feedback (2/2 minus refactoring)

---
 .../producer_plugin/snapshot_db_json.hpp      |  2 +
 .../producer_plugin/snapshot_scheduler.hpp    | 63 +++++++------------
 plugins/producer_plugin/producer_plugin.cpp   |  7 +--
 3 files changed, 28 insertions(+), 44 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
index 1d8979c6f1..80a29d090b 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
@@ -30,6 +30,7 @@ class snapshot_db_json {
 
       try {
          std::ifstream file(get_json_path().string());
+         file.exceptions(std::istream::failbit|std::istream::eofbit);
          boost::property_tree::read_json(file, root);
          file.close();
 
@@ -70,6 +71,7 @@ class snapshot_db_json {
 
       try {
          std::ofstream file(get_json_path().string());
+         file.exceptions(std::istream::failbit|std::istream::eofbit);
          boost::property_tree::write_json(file, root);
          file.close();
       }
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 23d3afc5b7..1d6027eb35 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -19,29 +19,18 @@
 namespace eosio {
 
 namespace bmi = boost::multi_index;
-using chain::account_name;
-using chain::block_state_ptr;
-using chain::packed_transaction;
-using chain::transaction_id_type;
-using chain::duplicate_snapshot_request;
-using chain::invalid_snapshot_request;
-using chain::snapshot_request_not_found;
-
-using snapshot_schedule_information    = producer_plugin::snapshot_schedule_information;
-using snapshot_request_information     = producer_plugin::snapshot_request_information;
-using get_snapshot_requests_result     = producer_plugin::get_snapshot_requests_result;
-using snapshot_request_id_information  = producer_plugin::snapshot_request_id_information;
 
 class snapshot_scheduler_listener {
+   virtual void on_start_block(uint32_t height) = 0;
    virtual void on_block(uint32_t height) = 0;
    virtual void on_irreversible_block(uint32_t height) = 0;
    virtual void on_abort_block() = 0;
 };
 
 class snapshot_scheduler_handler {
-   virtual void schedule_snapshot(const snapshot_request_information& sri) = 0;
+   virtual void schedule_snapshot(const producer_plugin::snapshot_request_information& sri) = 0;
    virtual void unschedule_snapshot(uint32_t sri) = 0;
-   virtual get_snapshot_requests_result get_snapshot_requests() = 0;
+   virtual producer_plugin::get_snapshot_requests_result get_snapshot_requests() = 0;
 };
 
 class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_scheduler_listener {
@@ -51,15 +40,15 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
    struct as_vector;
 
    using snapshot_requests = bmi::multi_index_container<
-         snapshot_schedule_information,
+         producer_plugin::snapshot_schedule_information,
          indexed_by<
-               bmi::hashed_unique<tag<by_snapshot_id>, BOOST_MULTI_INDEX_MEMBER(snapshot_request_id_information, uint32_t, snapshot_request_id)>,
+               bmi::hashed_unique<tag<by_snapshot_id>, BOOST_MULTI_INDEX_MEMBER(producer_plugin::snapshot_request_id_information, uint32_t, snapshot_request_id)>,
                bmi::random_access<tag<as_vector>>,
                bmi::ordered_unique<tag<by_snapshot_value>,
-                                   composite_key<snapshot_schedule_information,
-                                                 BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, block_spacing),
-                                                 BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, start_block_num),
-                                                 BOOST_MULTI_INDEX_MEMBER(snapshot_request_information, uint32_t, end_block_num)>>>>;
+                                   composite_key<producer_plugin::snapshot_schedule_information,
+                                                 BOOST_MULTI_INDEX_MEMBER(producer_plugin::snapshot_request_information, uint32_t, block_spacing),
+                                                 BOOST_MULTI_INDEX_MEMBER(producer_plugin::snapshot_request_information, uint32_t, start_block_num),
+                                                 BOOST_MULTI_INDEX_MEMBER(producer_plugin::snapshot_request_information, uint32_t, end_block_num)>>>>;
    snapshot_requests _snapshot_requests;
    snapshot_db_json _snapshot_db;
    uint32_t _snapshot_id = 0;
@@ -69,11 +58,10 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
    snapshot_scheduler() = default;
 
    // snapshot_scheduler_listener
-   virtual void on_block(uint32_t height) {
+   void on_start_block(uint32_t height) {
       for(const auto& req: _snapshot_requests.get<0>()) {
-         // execute "asap" or if matches spacing
-         if((req.start_block_num == 0) ||
-            (!((height - req.start_block_num) % req.block_spacing))) {
+         // execute snapshot, -1 since its called from start block
+         if(!((height - req.start_block_num - 1) % req.block_spacing)) {
             execute_snapshot();
          }
          // assume "asap" for snapshot with missed/zero start, it can have spacing
@@ -88,25 +76,22 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
          }
       }
    }
-
-   virtual void on_irreversible_block(uint32_t height) {
-   }
-
-   virtual void on_abort_block() {
-   }
+   void on_block(uint32_t height) {}
+   void on_irreversible_block(uint32_t height) {}
+   void on_abort_block() {}
 
    // snapshot_scheduler_handler
-   void schedule_snapshot(const snapshot_request_information& sri) {
+   void schedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
       auto& snapshot_by_value = _snapshot_requests.get<by_snapshot_value>();
       auto existing = snapshot_by_value.find(std::make_tuple(sri.block_spacing, sri.start_block_num, sri.end_block_num));
-      EOS_ASSERT(existing == snapshot_by_value.end(), duplicate_snapshot_request, "Duplicate snapshot request");
+      EOS_ASSERT(existing == snapshot_by_value.end(), chain::duplicate_snapshot_request, "Duplicate snapshot request");
 
       if(sri.end_block_num > 0) {
          // if "end" is specified, it should be greater then start
-         EOS_ASSERT(sri.start_block_num <= sri.end_block_num, invalid_snapshot_request, "End block number should be greater or equal to start block number");
+         EOS_ASSERT(sri.start_block_num <= sri.end_block_num, chain::invalid_snapshot_request, "End block number should be greater or equal to start block number");
          // if also block_spacing specified, check it
          if(sri.block_spacing > 0) {
-            EOS_ASSERT(sri.start_block_num + sri.block_spacing <= sri.end_block_num, invalid_snapshot_request, "Block spacing exceeds defined by start and end range");
+            EOS_ASSERT(sri.start_block_num + sri.block_spacing <= sri.end_block_num, chain::invalid_snapshot_request, "Block spacing exceeds defined by start and end range");
          }
       }
 
@@ -120,7 +105,7 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
    virtual void unschedule_snapshot(uint32_t sri) {
       auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
       auto existing = snapshot_by_id.find(sri);
-      EOS_ASSERT(existing != snapshot_by_id.end(), snapshot_request_not_found, "Snapshot request not found");
+      EOS_ASSERT(existing != snapshot_by_id.end(), chain::snapshot_request_not_found, "Snapshot request not found");
       _snapshot_requests.erase(existing);
 
       auto& vec = _snapshot_requests.get<as_vector>();
@@ -128,8 +113,8 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
       _snapshot_db << sr;
    }
 
-   virtual get_snapshot_requests_result get_snapshot_requests() {
-      get_snapshot_requests_result result;
+   virtual producer_plugin::get_snapshot_requests_result get_snapshot_requests() {
+      producer_plugin::get_snapshot_requests_result result;
       auto& asvector = _snapshot_requests.get<as_vector>();
       result.requests.insert(result.requests.begin(), asvector.begin(), asvector.end());
       return result;
@@ -158,10 +143,10 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
             try {
                std::get<fc::exception_ptr>(result)->dynamic_rethrow_exception();
             } catch(const fc::exception& e) {
-               edump((e.to_detail_string()));
+               wlog( "snapshot creation error: ${details}", ("details",e.to_detail_string()) );
                throw;
             } catch(const std::exception& e) {
-               edump((e.what()));
+               wlog( "snapshot creation error: ${details}", ("details",e.what()) );
                throw;
             }
          } else {
diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp
index c118e52f68..becee6168d 100644
--- a/plugins/producer_plugin/producer_plugin.cpp
+++ b/plugins/producer_plugin/producer_plugin.cpp
@@ -28,8 +28,6 @@
 #include <boost/multi_index/ordered_index.hpp>
 #include <boost/multi_index/random_access_index.hpp>
 #include <boost/signals2/connection.hpp>
-#include <boost/range/adaptor/map.hpp>
-#include <boost/range/algorithm/copy.hpp>
 
 namespace bmi = boost::multi_index;
 using bmi::indexed_by;
@@ -1000,8 +998,7 @@ void producer_plugin::plugin_initialize(const boost::program_options::variables_
    }
 
    my->_snapshot_scheduler.set_db_path(my->_snapshots_dir);
-   my->_snapshot_scheduler.set_create_snapshot_fn(std::bind(&producer_plugin::create_snapshot, this, std::placeholders::_1));
-
+   my->_snapshot_scheduler.set_create_snapshot_fn([this](producer_plugin::next_function<producer_plugin::snapshot_information> next){create_snapshot(next);});
 } FC_LOG_AND_RETHROW() }
 
 void producer_plugin::plugin_startup()
@@ -1027,7 +1024,7 @@ void producer_plugin::plugin_startup()
    my->_accepted_block_connection.emplace(chain.accepted_block.connect( [this]( const auto& bsp ){ my->on_block( bsp ); } ));
    my->_accepted_block_header_connection.emplace(chain.accepted_block_header.connect( [this]( const auto& bsp ){ my->on_block_header( bsp ); } ));
    my->_irreversible_block_connection.emplace(chain.irreversible_block.connect( [this]( const auto& bsp ){ my->on_irreversible_block( bsp->block ); } ));
-   my->_block_start_connection.emplace(chain.block_start.connect( [this]( uint32_t bs ){ my->_snapshot_scheduler.on_block(bs); } ));
+   my->_block_start_connection.emplace(chain.block_start.connect( [this]( uint32_t bs ){ my->_snapshot_scheduler.on_start_block(bs); } ));
 
    const auto lib_num = chain.last_irreversible_block_num();
    const auto lib = chain.fetch_block_by_number(lib_num);

From cd0d56ab7fb9ae15c14a2025d54f89a1293fbc31 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Wed, 1 Mar 2023 22:01:21 -0500
Subject: [PATCH 20/48] Make sure snapshot state preserved

---
 .../eosio/producer_plugin/snapshot_scheduler.hpp  |  2 +-
 .../test/test_snapshot_scheduler.cpp              | 15 ++++++++++++++-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 1d6027eb35..26a9ad82e8 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -72,7 +72,7 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
          }
          // remove expired request
          if(req.end_block_num > 0 && height >= req.end_block_num) {
-            _snapshot_requests.erase(req.snapshot_request_id);
+            unschedule_snapshot(req.snapshot_request_id);
          }
       }
    }
diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
index 331620eec3..4f7c20455c 100644
--- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
+++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
@@ -3,7 +3,7 @@
 
 #include <eosio/chain/exceptions.hpp>
 #include <eosio/producer_plugin/producer_plugin.hpp>
-
+#include <eosio/producer_plugin/snapshot_db_json.hpp>
 
 #include <eosio/testing/tester.hpp>
 
@@ -64,6 +64,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
       try {
          std::promise<std::tuple<producer_plugin*, chain_plugin*>> plugin_promise;
          std::future<std::tuple<producer_plugin*, chain_plugin*>> plugin_fut = plugin_promise.get_future();
+
          std::thread app_thread([&]() {
             fc::logger::get(DEFAULT_LOGGER).set_log_level(fc::log_level::debug);
             std::vector<const char*> argv =
@@ -111,12 +112,24 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          // quit app
          appbase::app().quit();
          app_thread.join();
+
+         // lets check whether schedule can be read back after restart
+         snapshot_db_json db;
+         std::vector<producer_plugin::snapshot_schedule_information> ssi;
+         db.set_path(temp / "snapshots");
+         db >> ssi;
+         BOOST_CHECK_EQUAL(1, ssi.size());
+         BOOST_CHECK_EQUAL(ssi.begin()->block_spacing, sri2.block_spacing);
       } catch(...) {
          bfs::remove_all(temp);
          throw;
       }
       bfs::remove_all(temp);
    }
+
+
+
+
    BOOST_AUTO_TEST_SUITE_END()
 }
 

From 5819266db85634af4a4a13d157af4a1ab8b79072 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Thu, 2 Mar 2023 10:24:21 -0500
Subject: [PATCH 21/48] Addressing feedback

---
 .../include/eosio/producer_plugin/producer_plugin.hpp    | 1 +
 .../include/eosio/producer_plugin/snapshot_db_json.hpp   | 8 ++++----
 .../include/eosio/producer_plugin/snapshot_scheduler.hpp | 9 +++++----
 plugins/producer_plugin/test/test_snapshot_scheduler.cpp | 3 ---
 4 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
index 3884e034c4..4255b0c942 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
@@ -185,6 +185,7 @@ FC_REFLECT(eosio::producer_plugin::snapshot_information, (head_block_id)(head_bl
 FC_REFLECT(eosio::producer_plugin::snapshot_request_information, (block_spacing)(start_block_num)(end_block_num)(snapshot_description))
 FC_REFLECT(eosio::producer_plugin::snapshot_request_id_information, (snapshot_request_id))
 FC_REFLECT(eosio::producer_plugin::get_snapshot_requests_result, (requests))
+FC_REFLECT_DERIVED(eosio::producer_plugin::snapshot_schedule_information, (eosio::producer_plugin::snapshot_request_id_information)(eosio::producer_plugin::snapshot_request_information), (pending_snapshot))
 FC_REFLECT(eosio::producer_plugin::scheduled_protocol_feature_activations, (protocol_features_to_activate))
 FC_REFLECT(eosio::producer_plugin::get_supported_protocol_features_params, (exclude_disabled)(exclude_unactivatable))
 FC_REFLECT(eosio::producer_plugin::get_account_ram_corrections_params, (lower_bound)(upper_bound)(limit)(reverse))
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
index 80a29d090b..420bc5b850 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
@@ -46,8 +46,8 @@ class snapshot_db_json {
          }
       }
       catch (std::ifstream::failure e) {
-          wlog( "unable to restore snapshots schedule from filesystem: ${details}",
-                  ("details",e.what()) );
+         elog( "unable to restore snapshots schedule from filesystem ${jsonpath}, details: ${details}",
+            ("jsonpath", get_json_path().string()) ("details",e.what()) );
       }
 
       return *this;
@@ -76,8 +76,8 @@ class snapshot_db_json {
          file.close();
       }
       catch (std::ofstream::failure e) {
-          wlog( "unable to store snapshots schedule to filesystem: ${details}",
-                  ("details",e.what()) );
+         elog( "unable to store snapshots schedule to filesystem to ${jsonpath}, details: ${details}",
+            ("jsonpath", get_json_path().string()) ("details", e.what()) );
       }
 
       return *this;
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 26a9ad82e8..41dfec108b 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -116,13 +116,14 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
    virtual producer_plugin::get_snapshot_requests_result get_snapshot_requests() {
       producer_plugin::get_snapshot_requests_result result;
       auto& asvector = _snapshot_requests.get<as_vector>();
+      result.requests.reserve(asvector.size());
       result.requests.insert(result.requests.begin(), asvector.begin(), asvector.end());
       return result;
    }
 
    // initialize with storage
    void set_db_path(bfs::path db_path) {
-      _snapshot_db.set_path(db_path);
+      _snapshot_db.set_path(std::move(db_path));
       // init from db
       if(fc::exists(_snapshot_db.get_json_path())) {
          std::vector<producer_plugin::snapshot_schedule_information> sr;
@@ -133,7 +134,7 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
 
    // snapshot executor
    void set_create_snapshot_fn(std::function<void(producer_plugin::next_function<producer_plugin::snapshot_information>)> fn) {
-      _create_snapshot = fn;
+      _create_snapshot = std::move(fn);
    }
 
 
@@ -143,10 +144,10 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
             try {
                std::get<fc::exception_ptr>(result)->dynamic_rethrow_exception();
             } catch(const fc::exception& e) {
-               wlog( "snapshot creation error: ${details}", ("details",e.to_detail_string()) );
+               elog( "snapshot creation error: ${details}", ("details",e.to_detail_string()) );
                throw;
             } catch(const std::exception& e) {
-               wlog( "snapshot creation error: ${details}", ("details",e.what()) );
+               elog( "snapshot creation error: ${details}", ("details",e.what()) );
                throw;
             }
          } else {
diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
index 4f7c20455c..3fc282a47d 100644
--- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
+++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
@@ -127,9 +127,6 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
       bfs::remove_all(temp);
    }
 
-
-
-
    BOOST_AUTO_TEST_SUITE_END()
 }
 

From 9178e5ef3b8f01dfc9a91b2b500a1ecd606fbe4e Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Thu, 2 Mar 2023 11:39:20 -0500
Subject: [PATCH 22/48] Handle situation where mlock_spacing is not specified

---
 .../include/eosio/producer_plugin/snapshot_scheduler.hpp      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 41dfec108b..32ef6e58db 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -61,7 +61,7 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
    void on_start_block(uint32_t height) {
       for(const auto& req: _snapshot_requests.get<0>()) {
          // execute snapshot, -1 since its called from start block
-         if(!((height - req.start_block_num - 1) % req.block_spacing)) {
+         if(!req.block_spacing || (!((height - req.start_block_num - 1) % req.block_spacing))) {
             execute_snapshot();
          }
          // assume "asap" for snapshot with missed/zero start, it can have spacing
@@ -71,7 +71,7 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
             _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height; });
          }
          // remove expired request
-         if(req.end_block_num > 0 && height >= req.end_block_num) {
+         if(!req.block_spacing || (req.end_block_num > 0 && height >= req.end_block_num)) {
             unschedule_snapshot(req.snapshot_request_id);
          }
       }

From 407e0ba5fef28faf1d7573f0017cbec92c6970f6 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Thu, 2 Mar 2023 23:22:52 -0500
Subject: [PATCH 23/48] Multiple fixes: pendings, tests, namings

---
 .../eosio/producer_plugin/producer_plugin.hpp |  8 +--
 .../producer_plugin/snapshot_db_json.hpp      |  4 +-
 .../producer_plugin/snapshot_scheduler.hpp    | 67 +++++++++++++------
 plugins/producer_plugin/producer_plugin.cpp   |  1 +
 .../test/test_snapshot_scheduler.cpp          | 35 +++++++---
 5 files changed, 78 insertions(+), 37 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
index 4255b0c942..4a35b49fbe 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
@@ -63,11 +63,11 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
    };
 
    struct snapshot_schedule_information : public snapshot_request_id_information, public snapshot_request_information {
-      std::optional<snapshot_information> pending_snapshot;
+      std::optional<std::vector<snapshot_information>> pending_snapshots;
    };
 
    struct get_snapshot_requests_result {
-       std::vector<snapshot_request_information>  requests;
+       std::vector<snapshot_schedule_information>  snapshot_requests;
    };
 
    struct scheduled_protocol_feature_activations {
@@ -184,8 +184,8 @@ FC_REFLECT(eosio::producer_plugin::integrity_hash_information, (head_block_id)(i
 FC_REFLECT(eosio::producer_plugin::snapshot_information, (head_block_id)(head_block_num)(head_block_time)(version)(snapshot_name))
 FC_REFLECT(eosio::producer_plugin::snapshot_request_information, (block_spacing)(start_block_num)(end_block_num)(snapshot_description))
 FC_REFLECT(eosio::producer_plugin::snapshot_request_id_information, (snapshot_request_id))
-FC_REFLECT(eosio::producer_plugin::get_snapshot_requests_result, (requests))
-FC_REFLECT_DERIVED(eosio::producer_plugin::snapshot_schedule_information, (eosio::producer_plugin::snapshot_request_id_information)(eosio::producer_plugin::snapshot_request_information), (pending_snapshot))
+FC_REFLECT(eosio::producer_plugin::get_snapshot_requests_result, (snapshot_requests))
+FC_REFLECT_DERIVED(eosio::producer_plugin::snapshot_schedule_information, (eosio::producer_plugin::snapshot_request_id_information)(eosio::producer_plugin::snapshot_request_information), (pending_snapshots))
 FC_REFLECT(eosio::producer_plugin::scheduled_protocol_feature_activations, (protocol_features_to_activate))
 FC_REFLECT(eosio::producer_plugin::get_supported_protocol_features_params, (exclude_disabled)(exclude_unactivatable))
 FC_REFLECT(eosio::producer_plugin::get_account_ram_corrections_params, (lower_bound)(upper_bound)(limit)(reverse))
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
index 420bc5b850..283858168c 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
@@ -45,7 +45,7 @@ class snapshot_db_json {
             sr.push_back(ssi);
          }
       }
-      catch (std::ifstream::failure e) {
+      catch (std::ifstream::failure & e) {
          elog( "unable to restore snapshots schedule from filesystem ${jsonpath}, details: ${details}",
             ("jsonpath", get_json_path().string()) ("details",e.what()) );
       }
@@ -75,7 +75,7 @@ class snapshot_db_json {
          boost::property_tree::write_json(file, root);
          file.close();
       }
-      catch (std::ofstream::failure e) {
+      catch (std::ofstream::failure & e) {
          elog( "unable to store snapshots schedule to filesystem to ${jsonpath}, details: ${details}",
             ("jsonpath", get_json_path().string()) ("details", e.what()) );
       }
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 32ef6e58db..e81bc35157 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -22,9 +22,6 @@ namespace bmi = boost::multi_index;
 
 class snapshot_scheduler_listener {
    virtual void on_start_block(uint32_t height) = 0;
-   virtual void on_block(uint32_t height) = 0;
-   virtual void on_irreversible_block(uint32_t height) = 0;
-   virtual void on_abort_block() = 0;
 };
 
 class snapshot_scheduler_handler {
@@ -51,9 +48,16 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
                                                  BOOST_MULTI_INDEX_MEMBER(producer_plugin::snapshot_request_information, uint32_t, end_block_num)>>>>;
    snapshot_requests _snapshot_requests;
    snapshot_db_json _snapshot_db;
-   uint32_t _snapshot_id = 0;
+   uint32_t _snapshot_id   = 0;
+   uint32_t _inflight_sid  = 0;
    std::function<void(producer_plugin::next_function<producer_plugin::snapshot_information>)> _create_snapshot;
 
+   void x_serialize() {
+      auto& vec = _snapshot_requests.get<as_vector>();
+      std::vector<producer_plugin::snapshot_schedule_information> sr(vec.begin(), vec.end());
+      _snapshot_db << sr;   
+   }
+
 public:
    snapshot_scheduler() = default;
 
@@ -62,7 +66,7 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
       for(const auto& req: _snapshot_requests.get<0>()) {
          // execute snapshot, -1 since its called from start block
          if(!req.block_spacing || (!((height - req.start_block_num - 1) % req.block_spacing))) {
-            execute_snapshot();
+            execute_snapshot(req.snapshot_request_id);
          }
          // assume "asap" for snapshot with missed/zero start, it can have spacing
          if(req.start_block_num == 0) {
@@ -76,9 +80,6 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
          }
       }
    }
-   void on_block(uint32_t height) {}
-   void on_irreversible_block(uint32_t height) {}
-   void on_abort_block() {}
 
    // snapshot_scheduler_handler
    void schedule_snapshot(const producer_plugin::snapshot_request_information& sri) {
@@ -96,10 +97,7 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
       }
 
       _snapshot_requests.emplace(producer_plugin::snapshot_schedule_information {{_snapshot_id++},{sri.block_spacing, sri.start_block_num, sri.end_block_num, sri.snapshot_description},{}});
-
-      auto& vec = _snapshot_requests.get<as_vector>();
-      std::vector<producer_plugin::snapshot_schedule_information> sr(vec.begin(), vec.end());
-      _snapshot_db << sr;
+      x_serialize();
    }
 
    virtual void unschedule_snapshot(uint32_t sri) {
@@ -107,17 +105,14 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
       auto existing = snapshot_by_id.find(sri);
       EOS_ASSERT(existing != snapshot_by_id.end(), chain::snapshot_request_not_found, "Snapshot request not found");
       _snapshot_requests.erase(existing);
-
-      auto& vec = _snapshot_requests.get<as_vector>();
-      std::vector<producer_plugin::snapshot_schedule_information> sr(vec.begin(), vec.end());
-      _snapshot_db << sr;
+      x_serialize();
    }
 
    virtual producer_plugin::get_snapshot_requests_result get_snapshot_requests() {
       producer_plugin::get_snapshot_requests_result result;
       auto& asvector = _snapshot_requests.get<as_vector>();
-      result.requests.reserve(asvector.size());
-      result.requests.insert(result.requests.begin(), asvector.begin(), asvector.end());
+      result.snapshot_requests.reserve(asvector.size());
+      result.snapshot_requests.insert(result.snapshot_requests.begin(), asvector.begin(), asvector.end());
       return result;
    }
 
@@ -132,14 +127,29 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
       }
    }
 
+   // add pending snapshot info to inflight snapshot request
+   void add_pending_snapshot_info(const producer_plugin::snapshot_information & si) {
+      auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
+      auto  snapshot_req   = snapshot_by_id.find(_inflight_sid);
+      if (snapshot_req != snapshot_by_id.end()) {
+         _snapshot_requests.modify(snapshot_req, [&si](auto& p) { 
+            if (!p.pending_snapshots) {
+               p.pending_snapshots = std::vector<producer_plugin::snapshot_information>();
+            }
+            p.pending_snapshots->emplace_back(si);       
+         });
+      }
+   }
+
    // snapshot executor
    void set_create_snapshot_fn(std::function<void(producer_plugin::next_function<producer_plugin::snapshot_information>)> fn) {
       _create_snapshot = std::move(fn);
    }
 
-
-   void execute_snapshot() {
-      auto next = [](const std::variant<fc::exception_ptr, producer_plugin::snapshot_information>& result) {
+   void execute_snapshot(uint32_t srid) {
+       elog( "execute: ${details}", ("details", srid) );
+      _inflight_sid = srid;
+      auto next = [srid, this](const std::variant<fc::exception_ptr, producer_plugin::snapshot_information>& result) {
          if(std::holds_alternative<fc::exception_ptr>(result)) {
             try {
                std::get<fc::exception_ptr>(result)->dynamic_rethrow_exception();
@@ -151,7 +161,20 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
                throw;
             }
          } else {
-            // success
+            // success, snapshot finalized
+            auto snapshot_info   = std::get<producer_plugin::snapshot_information>(result);
+            auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
+            auto  snapshot_req   = snapshot_by_id.find(srid);
+
+            if (snapshot_req != snapshot_by_id.end()) {               
+               if (auto pending = snapshot_req->pending_snapshots; pending) {                 
+                  auto it = std::remove_if(pending->begin(), pending->end(), [&snapshot_info](const producer_plugin::snapshot_information & s){ return s.head_block_num <=  snapshot_info.head_block_num; });
+                  pending->erase(it, pending->end());
+                  _snapshot_requests.modify(snapshot_req, [&pending](auto& p) { 
+                     p.pending_snapshots = std::move(pending);
+                  });
+               }
+            }
          }
       };
       _create_snapshot(next);
diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp
index becee6168d..1076854980 100644
--- a/plugins/producer_plugin/producer_plugin.cpp
+++ b/plugins/producer_plugin/producer_plugin.cpp
@@ -1322,6 +1322,7 @@ void producer_plugin::create_snapshot(producer_plugin::next_function<producer_pl
                ("ec", ec.value())
                ("message", ec.message()));
          my->_pending_snapshot_index.emplace(head_id, next, pending_path.generic_string(), snapshot_path.generic_string());
+         my->_snapshot_scheduler.add_pending_snapshot_info( producer_plugin::snapshot_information{head_id, head_block_num, head_block_time, chain_snapshot_header::current_version, pending_path.generic_string()} );
       } CATCH_AND_CALL (next);
    }
 }
diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
index 3fc282a47d..0f61d1a987 100644
--- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
+++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
@@ -27,7 +27,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
       scheduler.schedule_snapshot(sri1);
       scheduler.schedule_snapshot(sri2);
 
-      BOOST_CHECK_EQUAL(2, scheduler.get_snapshot_requests().requests.size());
+      BOOST_CHECK_EQUAL(2, scheduler.get_snapshot_requests().snapshot_requests.size());
 
       BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri1), duplicate_snapshot_request, [](const fc::assert_exception& e) {
          return e.to_detail_string().find("Duplicate snapshot request") != std::string::npos;
@@ -36,7 +36,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
       producer_plugin::snapshot_request_id_information sri_delete_1 = {.snapshot_request_id = 0};
       scheduler.unschedule_snapshot(sri_delete_1);
 
-      BOOST_CHECK_EQUAL(1, scheduler.get_snapshot_requests().requests.size());
+      BOOST_CHECK_EQUAL(1, scheduler.get_snapshot_requests().snapshot_requests.size());
 
       producer_plugin::snapshot_request_id_information sri_delete_none = {.snapshot_request_id = 2};
       BOOST_CHECK_EXCEPTION(scheduler.unschedule_snapshot(sri_delete_none), snapshot_request_not_found, [](const fc::assert_exception& e) {
@@ -46,7 +46,7 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
       producer_plugin::snapshot_request_id_information sri_delete_2 = {.snapshot_request_id = 1};
       scheduler.unschedule_snapshot(sri_delete_2);
 
-      BOOST_CHECK_EQUAL(0, scheduler.get_snapshot_requests().requests.size());
+      BOOST_CHECK_EQUAL(0, scheduler.get_snapshot_requests().snapshot_requests.size());
 
       producer_plugin::snapshot_request_information sri_large_spacing = {.block_spacing = 1000, .start_block_num = 5000, .end_block_num = 5010};
       BOOST_CHECK_EXCEPTION(scheduler.schedule_snapshot(sri_large_spacing), invalid_snapshot_request, [](const fc::assert_exception& e) {
@@ -95,19 +95,36 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          auto bs = chain_plug->chain().block_start.connect([&](uint32_t bn) {
          });
 
-         producer_plugin::snapshot_request_information sri1 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 1"};
+        
+       
+         producer_plugin::snapshot_request_information sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2"};
          producer_plugin::snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2"};
+         producer_plugin::snapshot_request_information sri3 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 1"};
+
          auto pp = appbase::app().find_plugin<producer_plugin>();
+         
          pp->schedule_snapshot(sri1);
          pp->schedule_snapshot(sri2);
+         pp->schedule_snapshot(sri3);
+
+         // all three snapshot requests should be present now
+         BOOST_CHECK_EQUAL(3, pp->get_snapshot_requests().snapshot_requests.size());
+
+         empty_blocks_fut.wait_for(std::chrono::seconds(4));
 
-         // both snapshot requests should be present now
-         BOOST_CHECK_EQUAL(2, pp->get_snapshot_requests().requests.size());
+         // there should be a single pending snapshot here
+         BOOST_CHECK_EQUAL(1, pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots->size());
+
+         // lets check the head block num of it
+         BOOST_CHECK_EQUAL(9, pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots->begin()->head_block_num);         
 
          empty_blocks_fut.wait_for(std::chrono::seconds(5));
 
          // one of the snapshots is done here and request, corresponding to it should be deleted
-         BOOST_CHECK_EQUAL(1, pp->get_snapshot_requests().requests.size());
+         BOOST_CHECK_EQUAL(2, pp->get_snapshot_requests().snapshot_requests.size());
+
+         // check whether no pending snapshots present
+         BOOST_CHECK_EQUAL(0, pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots->size());
 
          // quit app
          appbase::app().quit();
@@ -118,8 +135,8 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          std::vector<producer_plugin::snapshot_schedule_information> ssi;
          db.set_path(temp / "snapshots");
          db >> ssi;
-         BOOST_CHECK_EQUAL(1, ssi.size());
-         BOOST_CHECK_EQUAL(ssi.begin()->block_spacing, sri2.block_spacing);
+         BOOST_CHECK_EQUAL(2, ssi.size());
+         BOOST_CHECK_EQUAL(ssi.begin()->block_spacing, sri1.block_spacing);
       } catch(...) {
          bfs::remove_all(temp);
          throw;

From 4be2e1cd1350635c5378008b5066e929549c955d Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Thu, 2 Mar 2023 23:41:34 -0500
Subject: [PATCH 24/48] Removed debug message

---
 .../include/eosio/producer_plugin/snapshot_scheduler.hpp         | 1 -
 1 file changed, 1 deletion(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index e81bc35157..0c9b0b605a 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -147,7 +147,6 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
    }
 
    void execute_snapshot(uint32_t srid) {
-       elog( "execute: ${details}", ("details", srid) );
       _inflight_sid = srid;
       auto next = [srid, this](const std::variant<fc::exception_ptr, producer_plugin::snapshot_information>& result) {
          if(std::holds_alternative<fc::exception_ptr>(result)) {

From c7a2f6130e6daa8579fa4ba6124520d81d802390 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Thu, 2 Mar 2023 23:49:54 -0500
Subject: [PATCH 25/48] Made schedule_snapshot params optional

---
 plugins/producer_api_plugin/producer_api_plugin.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/plugins/producer_api_plugin/producer_api_plugin.cpp b/plugins/producer_api_plugin/producer_api_plugin.cpp
index 5ffcdea67d..0d84ed6b03 100644
--- a/plugins/producer_api_plugin/producer_api_plugin.cpp
+++ b/plugins/producer_api_plugin/producer_api_plugin.cpp
@@ -82,6 +82,11 @@ struct async_result_visitor : public fc::visitor<fc::variant> {
      api_handle.call_name(std::move(params)); \
      eosio::detail::producer_api_plugin_response result{"ok"};
 
+#define INVOKE_V_R_II(api_handle, call_name, in_param) \
+     auto params = parse_params<in_param, http_params_types::possible_no_params>(body);\
+     api_handle.call_name(std::move(params)); \
+     eosio::detail::producer_api_plugin_response result{"ok"};
+
 #define INVOKE_V_V(api_handle, call_name) \
      body = parse_params<std::string, http_params_types::no_params>(body); \
      api_handle.call_name(); \
@@ -121,7 +126,7 @@ void producer_api_plugin::plugin_startup() {
        CALL_ASYNC(producer, producer, create_snapshot, producer_plugin::snapshot_information,
             INVOKE_R_V_ASYNC(producer, create_snapshot), 201),
        CALL_WITH_400(producer, producer, schedule_snapshot,
-            INVOKE_V_R(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
+            INVOKE_V_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
        CALL_WITH_400(producer, producer, get_snapshot_requests,
             INVOKE_R_V(producer, get_snapshot_requests), 201),
        CALL_WITH_400(producer, producer, unschedule_snapshot,

From d2aeeb7e3ee6c6ea97749b09955d9b97728e311c Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Sun, 5 Mar 2023 21:41:33 -0500
Subject: [PATCH 26/48] Fixed pending snapshot test

---
 .../test/test_snapshot_scheduler.cpp          | 28 ++++++-------------
 1 file changed, 9 insertions(+), 19 deletions(-)

diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
index 0f61d1a987..6356393d47 100644
--- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
+++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
@@ -4,9 +4,7 @@
 #include <eosio/chain/exceptions.hpp>
 #include <eosio/producer_plugin/producer_plugin.hpp>
 #include <eosio/producer_plugin/snapshot_db_json.hpp>
-
 #include <eosio/testing/tester.hpp>
-
 namespace {
 
 using namespace eosio;
@@ -78,7 +76,6 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          });
 
          auto [prod_plug, chain_plug] = plugin_fut.get();
-         // auto chain_id = chain_plug->get_chain_id();
          std::deque<block_state_ptr> all_blocks;
          std::promise<void> empty_blocks_promise;
          std::future<void> empty_blocks_fut = empty_blocks_promise.get_future();
@@ -92,17 +89,20 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
                num_empty = 10;
             }
          });
-         auto bs = chain_plug->chain().block_start.connect([&](uint32_t bn) {
+         auto pp = appbase::app().find_plugin<producer_plugin>();
+         auto bs = chain_plug->chain().block_start.connect([&pp](uint32_t bn) {
+            // catching pending snapshot
+            if (pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots && pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots->size()==1) {
+               // lets check the head block num of it, it should be 8 + 1 = 9
+               // this means we are getting a snapshot for correct block # as well
+               BOOST_CHECK_EQUAL(9, pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots->begin()->head_block_num);   
+            }
          });
-
-        
        
          producer_plugin::snapshot_request_information sri1 = {.block_spacing = 8, .start_block_num = 1, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2"};
          producer_plugin::snapshot_request_information sri2 = {.block_spacing = 5000, .start_block_num = 100000, .end_block_num = 300000, .snapshot_description = "Example of recurring snapshot 2"};
          producer_plugin::snapshot_request_information sri3 = {.block_spacing = 2, .start_block_num = 0, .end_block_num = 3, .snapshot_description = "Example of recurring snapshot 1"};
 
-         auto pp = appbase::app().find_plugin<producer_plugin>();
-         
          pp->schedule_snapshot(sri1);
          pp->schedule_snapshot(sri2);
          pp->schedule_snapshot(sri3);
@@ -110,14 +110,6 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          // all three snapshot requests should be present now
          BOOST_CHECK_EQUAL(3, pp->get_snapshot_requests().snapshot_requests.size());
 
-         empty_blocks_fut.wait_for(std::chrono::seconds(4));
-
-         // there should be a single pending snapshot here
-         BOOST_CHECK_EQUAL(1, pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots->size());
-
-         // lets check the head block num of it
-         BOOST_CHECK_EQUAL(9, pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots->begin()->head_block_num);         
-
          empty_blocks_fut.wait_for(std::chrono::seconds(5));
 
          // one of the snapshots is done here and request, corresponding to it should be deleted
@@ -143,8 +135,6 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
       }
       bfs::remove_all(temp);
    }
-
-   BOOST_AUTO_TEST_SUITE_END()
 }
-
+   BOOST_AUTO_TEST_SUITE_END()
 }// namespace

From d0a21f78285cb6d46d48a7fc394ff3810f74d08c Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Mon, 6 Mar 2023 20:42:54 -0500
Subject: [PATCH 27/48] Added multinode py test, fixed bug in scheduler

---
 .../producer_plugin/snapshot_scheduler.hpp    |   8 +-
 tests/CMakeLists.txt                          |   3 +
 tests/TestHarness/Node.py                     |   3 +
 tests/nodeos_snapshot_programmable_test.py    | 210 ++++++++++++++++++
 4 files changed, 220 insertions(+), 4 deletions(-)
 create mode 100755 tests/nodeos_snapshot_programmable_test.py

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 0c9b0b605a..b4fbd61f71 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -64,16 +64,16 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
    // snapshot_scheduler_listener
    void on_start_block(uint32_t height) {
       for(const auto& req: _snapshot_requests.get<0>()) {
-         // execute snapshot, -1 since its called from start block
-         if(!req.block_spacing || (!((height - req.start_block_num - 1) % req.block_spacing))) {
-            execute_snapshot(req.snapshot_request_id);
-         }
          // assume "asap" for snapshot with missed/zero start, it can have spacing
          if(req.start_block_num == 0) {
             auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
             auto it = snapshot_by_id.find(req.snapshot_request_id);
             _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height; });
          }
+         // execute snapshot, -1 since its called from start block
+         if(!req.block_spacing || (!((height - req.start_block_num - 1) % req.block_spacing))) {
+            execute_snapshot(req.snapshot_request_id);
+         }        
          // remove expired request
          if(!req.block_spacing || (req.end_block_num > 0 && height >= req.end_block_num)) {
             unschedule_snapshot(req.snapshot_request_id);
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index d93a17731f..b3b2eeb801 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -25,6 +25,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/restart-scenarios-test.py ${CMAKE_CUR
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/terminate-scenarios-test.py ${CMAKE_CURRENT_BINARY_DIR}/terminate-scenarios-test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_startup_catchup.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_startup_catchup.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_snapshot_diff_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_snapshot_diff_test.py COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_snapshot_programmable_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_snapshot_programmable_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_forked_chain_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_forked_chain_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_short_fork_take_over_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_short_fork_take_over_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_run_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_run_test.py COPYONLY)
@@ -148,6 +149,8 @@ add_test(NAME keosd_auto_launch_test COMMAND tests/keosd_auto_launch_test.py WOR
 set_property(TEST keosd_auto_launch_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME nodeos_snapshot_diff_test COMMAND tests/nodeos_snapshot_diff_test.py -v --clean-run WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST nodeos_snapshot_diff_test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME nodeos_snapshot_programmable_test COMMAND tests/nodeos_snapshot_programmable_test.py -v --clean-run WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST nodeos_snapshot_programmable_test PROPERTY LABELS nonparallelizable_tests)
 
 add_test(NAME trx_finality_status_test COMMAND tests/trx_finality_status_test.py -v --clean-run WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST trx_finality_status_test PROPERTY LABELS nonparallelizable_tests)
diff --git a/tests/TestHarness/Node.py b/tests/TestHarness/Node.py
index a2d8c1bd91..14289b1a71 100644
--- a/tests/TestHarness/Node.py
+++ b/tests/TestHarness/Node.py
@@ -1581,6 +1581,9 @@ def modifyBuiltinPFSubjRestrictions(self, featureCodename, subjectiveRestriction
     def createSnapshot(self):
         return self.processUrllibRequest("producer", "create_snapshot")
 
+    def scheduleSnapshot(self):
+        return self.processUrllibRequest("producer", "schedule_snapshot")
+
     # kill all existing nodeos in case lingering from previous test
     @staticmethod
     def killAllNodeos():
diff --git a/tests/nodeos_snapshot_programmable_test.py b/tests/nodeos_snapshot_programmable_test.py
new file mode 100755
index 0000000000..1b5d226d55
--- /dev/null
+++ b/tests/nodeos_snapshot_programmable_test.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+
+import os
+import time
+import signal
+import decimal
+import math
+import re
+import shutil
+
+from TestHarness import Cluster, Node, TestHelper, Utils, WalletMgr
+from TestHarness.Node import BlockType
+from TestHarness.TestHelper import AppArgs
+from TestHarness.testUtils import BlockLogAction
+
+###############################################################
+# nodeos_snapshot_programmable_test
+#
+#  Test configures a producing node and 2 non-producing nodes.
+#  Configures trx_generator(s) and starts generating transactions and sending them
+#  to the producing node.
+#  - Schedule a snapshot from producing node
+#  - Create a snapshot from producing node
+#  - Convert snapshot to JSON
+#  - Trim blocklog to head block of snapshot
+#  - Start nodeos in irreversible mode on blocklog
+#  - Generate snapshot and convert to JSON
+#  - Compare JSON snapshot to original snapshot JSON
+#
+###############################################################
+
+Print=Utils.Print
+errorExit=Utils.errorExit
+
+appArgs=AppArgs()
+args = TestHelper.parse_args({"--dump-error-details","--keep-logs","-v","--leave-running","--clean-run","--wallet-port"},
+                             applicationSpecificArgs=appArgs)
+
+relaunchTimeout = 30
+Utils.Debug=args.v
+pnodes=1
+testAccounts = 2
+trxGeneratorCnt=2
+startedNonProdNodes = 2
+cluster=Cluster(walletd=True)
+dumpErrorDetails=args.dump_error_details
+keepLogs=args.keep_logs
+dontKill=args.leave_running
+prodCount=2
+killAll=args.clean_run
+walletPort=args.wallet_port
+totalNodes=startedNonProdNodes+pnodes
+
+walletMgr=WalletMgr(True, port=walletPort)
+testSuccessful=False
+killEosInstances=not dontKill
+killWallet=not dontKill
+
+WalletdName=Utils.EosWalletName
+ClientName="cleos"
+
+trxGenLauncher=None
+
+snapshotScheduleDB = "snapshot-schedule.json"
+
+def getLatestSnapshot(nodeId):
+    snapshotDir = os.path.join(Utils.getNodeDataDir(nodeId), "snapshots")
+    snapshotDirContents = os.listdir(snapshotDir)
+    assert len(snapshotDirContents) > 0
+    # disregard snapshot schedule config in same folder
+    if snapshotScheduleDB in snapshotDirContents: snapshotDirContents.remove(snapshotScheduleDB)
+    snapshotDirContents.sort()
+    return os.path.join(snapshotDir, snapshotDirContents[-1])
+
+def removeState(nodeId):
+    dataDir = Utils.getNodeDataDir(nodeId)
+    state = os.path.join(dataDir, "state")
+    shutil.rmtree(state, ignore_errors=True)
+
+try:
+    TestHelper.printSystemInfo("BEGIN")
+    cluster.setWalletMgr(walletMgr)
+
+    cluster.killall(allInstances=killAll)
+    cluster.cleanup()
+    specificExtraNodeosArgs={}
+    Print("Stand up cluster")
+    if cluster.launch(prodCount=prodCount, onlyBios=False, pnodes=pnodes, totalNodes=totalNodes, totalProducers=pnodes*prodCount,
+                      useBiosBootFile=False, specificExtraNodeosArgs=specificExtraNodeosArgs, loadSystemContract=True, maximumP2pPerHost=totalNodes+trxGeneratorCnt) is False:
+        Utils.errorExit("Failed to stand up eos cluster.")
+
+    Print("Create test wallet")
+    wallet = walletMgr.create('txntestwallet')
+    cluster.populateWallet(2, wallet)
+
+    Print("Create test accounts for transactions.")
+    cluster.createAccounts(cluster.eosioAccount, stakedDeposit=0, validationNodeIndex=0)
+
+    account1Name = cluster.accounts[0].name
+    account2Name = cluster.accounts[1].name
+
+    account1PrivKey = cluster.accounts[0].activePrivateKey
+    account2PrivKey = cluster.accounts[1].activePrivateKey
+
+    Print("Validating system accounts after bootstrap")
+    cluster.validateAccounts([cluster.accounts[0], cluster.accounts[1]])
+
+    def waitForBlock(node, blockNum, blockType=BlockType.head, timeout=None, reportInterval=20):
+        if not node.waitForBlock(blockNum, timeout=timeout, blockType=blockType, reportInterval=reportInterval):
+            info=node.getInfo()
+            headBlockNum=info["head_block_num"]
+            libBlockNum=info["last_irreversible_block_num"]
+            Utils.errorExit("Failed to get to %s block number %d. Last had head block number %d and lib %d" % (blockType, blockNum, headBlockNum, libBlockNum))
+
+
+    snapshotNodeId = 0
+    node0=cluster.getNode(snapshotNodeId)
+    irrNodeId = snapshotNodeId+1
+
+    nodeSnap=cluster.getNode(snapshotNodeId)
+    nodeIrr=cluster.getNode(irrNodeId)
+
+    Print("Wait for account creation to be irreversible")
+    blockNum=node0.getBlockNum(BlockType.head)
+    waitForBlock(node0, blockNum, blockType=BlockType.lib)
+
+    Print("Configure and launch txn generators")
+    targetTpsPerGenerator = 10
+    testTrxGenDurationSec=60*30
+    cluster.launchTrxGenerators(contractOwnerAcctName=cluster.eosioAccount.name, acctNamesList=[account1Name, account2Name],
+                                acctPrivKeysList=[account1PrivKey,account2PrivKey], nodeId=snapshotNodeId, tpsPerGenerator=targetTpsPerGenerator,
+                                numGenerators=trxGeneratorCnt, durationSec=testTrxGenDurationSec, waitToComplete=False)
+
+    cluster.waitForTrxGeneratorsSpinup(nodeId=snapshotNodeId, numGenerators=trxGeneratorCnt)
+
+    blockNum=node0.getBlockNum(BlockType.head)
+    timePerBlock=500
+    transactionsPerBlock=targetTpsPerGenerator*trxGeneratorCnt*timePerBlock/1000
+    steadyStateWait=30
+    startBlockNum=blockNum+steadyStateWait
+    numBlocks=30
+    endBlockNum=startBlockNum+numBlocks
+    waitForBlock(node0, endBlockNum)
+    steadyStateWindowTrxs=0
+    steadyStateAvg=0
+    steadyStateWindowBlks=0
+    for bNum in range(startBlockNum, endBlockNum):
+        steadyStateWindowBlks=steadyStateWindowBlks+1
+        block=node0.getBlock(bNum)
+        steadyStateWindowTrxs+=len(block["transactions"])
+
+    steadyStateAvg=steadyStateWindowTrxs / steadyStateWindowBlks
+
+    Print("Validate transactions are generating")
+    minReqPctLeeway=0.60
+    minRequiredTransactions=minReqPctLeeway*transactionsPerBlock
+    assert steadyStateAvg>=minRequiredTransactions, "Expected to at least receive %s transactions per block, but only getting %s" % (minRequiredTransactions, steadyStateAvg)
+
+    Print("Create snapshot")
+    ret = nodeSnap.scheduleSnapshot()
+    assert ret is not None, "Snapshot creation failed"
+  
+    # scheduleSnapshot is async, not returning block info
+    Print("Wait for snapshot node lib to advance")
+    ret_head_block_num = nodeSnap.getBlockNum(BlockType.head)
+    waitForBlock(nodeSnap, ret_head_block_num+1, blockType=BlockType.lib)
+
+    Print("Kill snapshot node")
+    nodeSnap.kill(signal.SIGTERM)
+
+    Print("Convert snapshot to JSON")
+    snapshotFile = getLatestSnapshot(snapshotNodeId)
+    Utils.processLeapUtilCmd("snapshot to-json --input-file {}".format(snapshotFile), "snapshot to-json", silentErrors=False)
+    snapshotFile = snapshotFile + ".json"
+
+    Print("Trim irreversible blocklog to snapshot head block num")
+    nodeIrr.kill(signal.SIGTERM)
+    output=cluster.getBlockLog(irrNodeId, blockLogAction=BlockLogAction.trim, last=ret_head_block_num, throwException=True)
+
+    Print("Relaunch irreversible node in irreversible mode")
+    removeState(irrNodeId)
+    Utils.rmFromFile(Utils.getNodeConfigDir(irrNodeId, "config.ini"), "p2p-peer-address")
+    swapFlags = {"--read-mode":"irreversible", "--p2p-max-nodes-per-host":"0", "--max-clients":"0", "--allowed-connection":"none"}
+    isRelaunchSuccess = nodeIrr.relaunch(chainArg="--replay", addSwapFlags=swapFlags, timeout=relaunchTimeout, cachePopen=True)
+    assert isRelaunchSuccess, "Failed to relaunch snapshot node"
+
+    Print("Create snapshot from irreversible")
+    ret = nodeIrr.createSnapshot()
+    assert ret is not None, "Snapshot creation failed"
+    ret_irr_head_block_num = ret["payload"]["head_block_num"]
+    Print(f"Snapshot head block number {ret_irr_head_block_num}")
+    assert ret_irr_head_block_num == ret_head_block_num, f"Snapshot head block numbers do not match: {ret_irr_head_block_num} != {ret_head_block_num}"
+
+    Print("Kill snapshot node")
+    nodeIrr.kill(signal.SIGTERM)
+
+    Print("Convert snapshot to JSON")
+    irrSnapshotFile = getLatestSnapshot(irrNodeId)
+    Utils.processLeapUtilCmd("snapshot to-json --input-file {}".format(irrSnapshotFile), "snapshot to-json", silentErrors=False)
+    irrSnapshotFile = irrSnapshotFile + ".json"
+
+    assert Utils.compareFiles(snapshotFile, irrSnapshotFile), f"Snapshot files differ {snapshotFile} != {irrSnapshotFile}"
+
+    testSuccessful=True
+
+finally:
+    TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=killEosInstances, killWallet=killWallet, keepLogs=keepLogs, cleanRun=killAll, dumpErrorDetails=dumpErrorDetails)
+
+exitCode = 0 if testSuccessful else 1
+exit(exitCode)
\ No newline at end of file

From 7cf95895eb815b037c2507b13854009fcb8901a6 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 7 Mar 2023 08:42:21 -0500
Subject: [PATCH 28/48] Better handling of "now" snapshots

---
 .../producer_plugin/snapshot_scheduler.hpp    | 21 ++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index b4fbd61f71..1abd15e4fe 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -65,16 +65,23 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
    void on_start_block(uint32_t height) {
       for(const auto& req: _snapshot_requests.get<0>()) {
          // assume "asap" for snapshot with missed/zero start, it can have spacing
-         if(req.start_block_num == 0) {
-            auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
-            auto it = snapshot_by_id.find(req.snapshot_request_id);
-            _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height; });
+         if(!req.start_block_num) {
+            // update start_block_num with current height only if this is recurring
+            // if non recurring, will be executed and unscheduled
+            if (req.block_spacing && height) {
+               auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
+               auto it = snapshot_by_id.find(req.snapshot_request_id);
+               _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height; });
+               x_serialize();
+            }
+            execute_snapshot(req.snapshot_request_id);
          }
          // execute snapshot, -1 since its called from start block
-         if(!req.block_spacing || (!((height - req.start_block_num - 1) % req.block_spacing))) {
+         else if(!req.block_spacing || (!((height - req.start_block_num - 1) % req.block_spacing))) {
             execute_snapshot(req.snapshot_request_id);
-         }        
-         // remove expired request
+         }       
+
+         // cleanup - remove expired request
          if(!req.block_spacing || (req.end_block_num > 0 && height >= req.end_block_num)) {
             unschedule_snapshot(req.snapshot_request_id);
          }

From dc5c15426fb32203bf9d60d8714b595ff8e08c6f Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 7 Mar 2023 10:01:24 -0500
Subject: [PATCH 29/48] Updated snapshot_diff_test to use scheduler

---
 tests/nodeos_snapshot_diff_test.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tests/nodeos_snapshot_diff_test.py b/tests/nodeos_snapshot_diff_test.py
index 13f2570d77..c0e7e81acf 100755
--- a/tests/nodeos_snapshot_diff_test.py
+++ b/tests/nodeos_snapshot_diff_test.py
@@ -19,7 +19,7 @@
 #  Test configures a producing node and 2 non-producing nodes.
 #  Configures trx_generator(s) and starts generating transactions and sending them
 #  to the producing node.
-#  - Create a snapshot from producing node
+#  - Create a snapshot from producing node using snapshot scheduler
 #  - Convert snapshot to JSON
 #  - Trim blocklog to head block of snapshot
 #  - Start nodeos in irreversible mode on blocklog
@@ -60,10 +60,14 @@
 
 trxGenLauncher=None
 
+snapshotScheduleDB = "snapshot-schedule.json"
+
 def getLatestSnapshot(nodeId):
     snapshotDir = os.path.join(Utils.getNodeDataDir(nodeId), "snapshots")
     snapshotDirContents = os.listdir(snapshotDir)
     assert len(snapshotDirContents) > 0
+    # disregard snapshot schedule config in same folder
+    if snapshotScheduleDB in snapshotDirContents: snapshotDirContents.remove(snapshotScheduleDB)
     snapshotDirContents.sort()
     return os.path.join(snapshotDir, snapshotDirContents[-1])
 
@@ -152,10 +156,9 @@ def waitForBlock(node, blockNum, blockType=BlockType.head, timeout=None, reportI
     assert steadyStateAvg>=minRequiredTransactions, "Expected to at least receive %s transactions per block, but only getting %s" % (minRequiredTransactions, steadyStateAvg)
 
     Print("Create snapshot")
-    ret = nodeSnap.createSnapshot()
+    ret = nodeSnap.scheduleSnapshot()
     assert ret is not None, "Snapshot creation failed"
-    ret_head_block_num = ret["payload"]["head_block_num"]
-    Print(f"Snapshot head block number {ret_head_block_num}")
+    ret_head_block_num = nodeSnap.getBlockNum(BlockType.head) + 1
 
     Print("Wait for snapshot node lib to advance")
     waitForBlock(nodeSnap, ret_head_block_num+1, blockType=BlockType.lib)

From 2c5bb903e716e02b3f1adef523e11416f61ce518 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 7 Mar 2023 10:02:21 -0500
Subject: [PATCH 30/48] Cleanup requests with invalid start

---
 .../include/eosio/producer_plugin/snapshot_scheduler.hpp      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 1abd15e4fe..1154178e57 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -81,8 +81,8 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
             execute_snapshot(req.snapshot_request_id);
          }       
 
-         // cleanup - remove expired request
-         if(!req.block_spacing || (req.end_block_num > 0 && height >= req.end_block_num)) {
+         // cleanup - remove expired (or invalid) request
+         if(!req.start_block_num || !req.block_spacing || (req.end_block_num > 0 && height >= req.end_block_num)) {
             unschedule_snapshot(req.snapshot_request_id);
          }
       }

From dec7c1103aeb06ebc0fd5e149b1759e9d4f54598 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 7 Mar 2023 14:01:18 -0500
Subject: [PATCH 31/48] Fixed whitespace plus additional test

---
 .../eosio/producer_plugin/producer_plugin.hpp |   2 +-
 tests/nodeos_snapshot_programmable_test.py    | 304 +++++++++---------
 2 files changed, 150 insertions(+), 156 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
index ca8d73e132..27d8726c6f 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/producer_plugin.hpp
@@ -60,7 +60,7 @@ class producer_plugin : public appbase::plugin<producer_plugin> {
    };
 
    struct snapshot_request_id_information {
-      uint32_t                            snapshot_request_id = 0;
+      uint32_t snapshot_request_id = 0;
    };
 
    struct snapshot_schedule_information : public snapshot_request_id_information, public snapshot_request_information {
diff --git a/tests/nodeos_snapshot_programmable_test.py b/tests/nodeos_snapshot_programmable_test.py
index 1b5d226d55..44433aa11a 100755
--- a/tests/nodeos_snapshot_programmable_test.py
+++ b/tests/nodeos_snapshot_programmable_test.py
@@ -2,54 +2,42 @@
 
 import os
 import time
-import signal
 import decimal
+import json
 import math
 import re
-import shutil
+import signal
 
-from TestHarness import Cluster, Node, TestHelper, Utils, WalletMgr
+from TestHarness import Account, Cluster, Node, TestHelper, Utils, WalletMgr
 from TestHarness.Node import BlockType
-from TestHarness.TestHelper import AppArgs
-from TestHarness.testUtils import BlockLogAction
 
 ###############################################################
-# nodeos_snapshot_programmable_test
+# nodeos_snapshot_programmable
 #
-#  Test configures a producing node and 2 non-producing nodes.
-#  Configures trx_generator(s) and starts generating transactions and sending them
-#  to the producing node.
-#  - Schedule a snapshot from producing node
-#  - Create a snapshot from producing node
-#  - Convert snapshot to JSON
-#  - Trim blocklog to head block of snapshot
-#  - Start nodeos in irreversible mode on blocklog
-#  - Generate snapshot and convert to JSON
-#  - Compare JSON snapshot to original snapshot JSON
+#  Test to verify that programmable snapshot functionality is
+#  working appropriately when forks occur
 #
 ###############################################################
-
 Print=Utils.Print
 errorExit=Utils.errorExit
 
-appArgs=AppArgs()
-args = TestHelper.parse_args({"--dump-error-details","--keep-logs","-v","--leave-running","--clean-run","--wallet-port"},
-                             applicationSpecificArgs=appArgs)
+from core_symbol import CORE_SYMBOL
+
 
-relaunchTimeout = 30
+args = TestHelper.parse_args({"--prod-count","--dump-error-details","--keep-logs","-v","--leave-running","--clean-run",
+                              "--wallet-port"})
 Utils.Debug=args.v
-pnodes=1
-testAccounts = 2
-trxGeneratorCnt=2
-startedNonProdNodes = 2
+totalProducerNodes=2
+totalNonProducerNodes=1
+totalNodes=totalProducerNodes+totalNonProducerNodes
+maxActiveProducers=3
+totalProducers=maxActiveProducers
 cluster=Cluster(walletd=True)
 dumpErrorDetails=args.dump_error_details
 keepLogs=args.keep_logs
 dontKill=args.leave_running
-prodCount=2
 killAll=args.clean_run
 walletPort=args.wallet_port
-totalNodes=startedNonProdNodes+pnodes
 
 walletMgr=WalletMgr(True, port=walletPort)
 testSuccessful=False
@@ -59,152 +47,158 @@
 WalletdName=Utils.EosWalletName
 ClientName="cleos"
 
-trxGenLauncher=None
-
 snapshotScheduleDB = "snapshot-schedule.json"
 
-def getLatestSnapshot(nodeId):
+EOSIO_ACCT_PRIVATE_DEFAULT_KEY = "5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3"
+EOSIO_ACCT_PUBLIC_DEFAULT_KEY = "EOS6MRyAjQq8ud7hVNYcfnVPJqcVpscN5So8BhtHuGYqET5GDW5CV"
+
+def waitForBlock(node, blockNum, blockType=BlockType.head, timeout=None, reportInterval=20):
+        if not node.waitForBlock(blockNum, timeout=timeout, blockType=blockType, reportInterval=reportInterval):
+            info=node.getInfo()
+            headBlockNum=info["head_block_num"]
+            libBlockNum=info["last_irreversible_block_num"]
+            Utils.errorExit("Failed to get to %s block number %d. Last had head block number %d and lib %d" % (blockType, blockNum, headBlockNum, libBlockNum))
+
+def getSnapshotsCount(nodeId):
     snapshotDir = os.path.join(Utils.getNodeDataDir(nodeId), "snapshots")
     snapshotDirContents = os.listdir(snapshotDir)
     assert len(snapshotDirContents) > 0
     # disregard snapshot schedule config in same folder
     if snapshotScheduleDB in snapshotDirContents: snapshotDirContents.remove(snapshotScheduleDB)
-    snapshotDirContents.sort()
-    return os.path.join(snapshotDir, snapshotDirContents[-1])
-
-def removeState(nodeId):
-    dataDir = Utils.getNodeDataDir(nodeId)
-    state = os.path.join(dataDir, "state")
-    shutil.rmtree(state, ignore_errors=True)
+    return len(snapshotDirContents)
 
 try:
     TestHelper.printSystemInfo("BEGIN")
-    cluster.setWalletMgr(walletMgr)
 
+    cluster.setWalletMgr(walletMgr)
     cluster.killall(allInstances=killAll)
     cluster.cleanup()
-    specificExtraNodeosArgs={}
     Print("Stand up cluster")
-    if cluster.launch(prodCount=prodCount, onlyBios=False, pnodes=pnodes, totalNodes=totalNodes, totalProducers=pnodes*prodCount,
-                      useBiosBootFile=False, specificExtraNodeosArgs=specificExtraNodeosArgs, loadSystemContract=True, maximumP2pPerHost=totalNodes+trxGeneratorCnt) is False:
+    specificExtraNodeosArgs={}
+    # producer nodes will be mapped to 0 through totalProducerNodes-1, so the number totalProducerNodes will be the non-producing node
+    specificExtraNodeosArgs[totalProducerNodes]="--plugin eosio::test_control_api_plugin"
+
+    # ensure that transactions don't get cleaned up too early
+    successDuration = 360
+    failure_duration = 360
+    extraNodeosArgs=" --transaction-finality-status-max-storage-size-gb 1 " + \
+                   f"--transaction-finality-status-success-duration-sec {successDuration} --transaction-finality-status-failure-duration-sec {failure_duration}"
+    extraNodeosArgs+=" --http-max-response-time-ms 990000"
+
+
+    # ***   setup topogrophy   ***
+
+    # "bridge" shape connects defprocera through defproducerb (in node0) to each other and defproducerc is alone (in node01)
+    # and the only connection between those 2 groups is through the bridge node
+    if cluster.launch(prodCount=2, topo="bridge", pnodes=totalProducerNodes,
+                      totalNodes=totalNodes, totalProducers=totalProducers,
+                      useBiosBootFile=False, specificExtraNodeosArgs=specificExtraNodeosArgs,
+                      extraNodeosArgs=extraNodeosArgs) is False:
+        Utils.cmdError("launcher")
         Utils.errorExit("Failed to stand up eos cluster.")
-
-    Print("Create test wallet")
-    wallet = walletMgr.create('txntestwallet')
-    cluster.populateWallet(2, wallet)
-
-    Print("Create test accounts for transactions.")
-    cluster.createAccounts(cluster.eosioAccount, stakedDeposit=0, validationNodeIndex=0)
-
-    account1Name = cluster.accounts[0].name
-    account2Name = cluster.accounts[1].name
-
-    account1PrivKey = cluster.accounts[0].activePrivateKey
-    account2PrivKey = cluster.accounts[1].activePrivateKey
-
     Print("Validating system accounts after bootstrap")
-    cluster.validateAccounts([cluster.accounts[0], cluster.accounts[1]])
-
-    def waitForBlock(node, blockNum, blockType=BlockType.head, timeout=None, reportInterval=20):
-        if not node.waitForBlock(blockNum, timeout=timeout, blockType=blockType, reportInterval=reportInterval):
-            info=node.getInfo()
-            headBlockNum=info["head_block_num"]
-            libBlockNum=info["last_irreversible_block_num"]
-            Utils.errorExit("Failed to get to %s block number %d. Last had head block number %d and lib %d" % (blockType, blockNum, headBlockNum, libBlockNum))
-
-
-    snapshotNodeId = 0
-    node0=cluster.getNode(snapshotNodeId)
-    irrNodeId = snapshotNodeId+1
-
-    nodeSnap=cluster.getNode(snapshotNodeId)
-    nodeIrr=cluster.getNode(irrNodeId)
-
-    Print("Wait for account creation to be irreversible")
-    blockNum=node0.getBlockNum(BlockType.head)
-    waitForBlock(node0, blockNum, blockType=BlockType.lib)
-
-    Print("Configure and launch txn generators")
-    targetTpsPerGenerator = 10
-    testTrxGenDurationSec=60*30
-    cluster.launchTrxGenerators(contractOwnerAcctName=cluster.eosioAccount.name, acctNamesList=[account1Name, account2Name],
-                                acctPrivKeysList=[account1PrivKey,account2PrivKey], nodeId=snapshotNodeId, tpsPerGenerator=targetTpsPerGenerator,
-                                numGenerators=trxGeneratorCnt, durationSec=testTrxGenDurationSec, waitToComplete=False)
-
-    cluster.waitForTrxGeneratorsSpinup(nodeId=snapshotNodeId, numGenerators=trxGeneratorCnt)
-
-    blockNum=node0.getBlockNum(BlockType.head)
-    timePerBlock=500
-    transactionsPerBlock=targetTpsPerGenerator*trxGeneratorCnt*timePerBlock/1000
-    steadyStateWait=30
-    startBlockNum=blockNum+steadyStateWait
-    numBlocks=30
-    endBlockNum=startBlockNum+numBlocks
-    waitForBlock(node0, endBlockNum)
-    steadyStateWindowTrxs=0
-    steadyStateAvg=0
-    steadyStateWindowBlks=0
-    for bNum in range(startBlockNum, endBlockNum):
-        steadyStateWindowBlks=steadyStateWindowBlks+1
-        block=node0.getBlock(bNum)
-        steadyStateWindowTrxs+=len(block["transactions"])
-
-    steadyStateAvg=steadyStateWindowTrxs / steadyStateWindowBlks
-
-    Print("Validate transactions are generating")
-    minReqPctLeeway=0.60
-    minRequiredTransactions=minReqPctLeeway*transactionsPerBlock
-    assert steadyStateAvg>=minRequiredTransactions, "Expected to at least receive %s transactions per block, but only getting %s" % (minRequiredTransactions, steadyStateAvg)
-
-    Print("Create snapshot")
-    ret = nodeSnap.scheduleSnapshot()
-    assert ret is not None, "Snapshot creation failed"
-  
-    # scheduleSnapshot is async, not returning block info
-    Print("Wait for snapshot node lib to advance")
-    ret_head_block_num = nodeSnap.getBlockNum(BlockType.head)
-    waitForBlock(nodeSnap, ret_head_block_num+1, blockType=BlockType.lib)
-
-    Print("Kill snapshot node")
-    nodeSnap.kill(signal.SIGTERM)
-
-    Print("Convert snapshot to JSON")
-    snapshotFile = getLatestSnapshot(snapshotNodeId)
-    Utils.processLeapUtilCmd("snapshot to-json --input-file {}".format(snapshotFile), "snapshot to-json", silentErrors=False)
-    snapshotFile = snapshotFile + ".json"
-
-    Print("Trim irreversible blocklog to snapshot head block num")
-    nodeIrr.kill(signal.SIGTERM)
-    output=cluster.getBlockLog(irrNodeId, blockLogAction=BlockLogAction.trim, last=ret_head_block_num, throwException=True)
-
-    Print("Relaunch irreversible node in irreversible mode")
-    removeState(irrNodeId)
-    Utils.rmFromFile(Utils.getNodeConfigDir(irrNodeId, "config.ini"), "p2p-peer-address")
-    swapFlags = {"--read-mode":"irreversible", "--p2p-max-nodes-per-host":"0", "--max-clients":"0", "--allowed-connection":"none"}
-    isRelaunchSuccess = nodeIrr.relaunch(chainArg="--replay", addSwapFlags=swapFlags, timeout=relaunchTimeout, cachePopen=True)
-    assert isRelaunchSuccess, "Failed to relaunch snapshot node"
-
-    Print("Create snapshot from irreversible")
-    ret = nodeIrr.createSnapshot()
-    assert ret is not None, "Snapshot creation failed"
-    ret_irr_head_block_num = ret["payload"]["head_block_num"]
-    Print(f"Snapshot head block number {ret_irr_head_block_num}")
-    assert ret_irr_head_block_num == ret_head_block_num, f"Snapshot head block numbers do not match: {ret_irr_head_block_num} != {ret_head_block_num}"
-
-    Print("Kill snapshot node")
-    nodeIrr.kill(signal.SIGTERM)
-
-    Print("Convert snapshot to JSON")
-    irrSnapshotFile = getLatestSnapshot(irrNodeId)
-    Utils.processLeapUtilCmd("snapshot to-json --input-file {}".format(irrSnapshotFile), "snapshot to-json", silentErrors=False)
-    irrSnapshotFile = irrSnapshotFile + ".json"
-
-    assert Utils.compareFiles(snapshotFile, irrSnapshotFile), f"Snapshot files differ {snapshotFile} != {irrSnapshotFile}"
+    cluster.validateAccounts(None)
+
+    # ***   identify each node (producers and non-producing node)   ***
+
+    nonProdNode=None
+    prodNodes=[]
+    producers=[]
+    for i, node in enumerate(cluster.getNodes()):
+        node.producers=Cluster.parseProducers(node.nodeId)
+        numProducers=len(node.producers)
+        Print(f"node {i} has producers={node.producers}")
+        if numProducers==0:
+            if nonProdNode is None:
+                nonProdNode=node
+            else:
+                Utils.errorExit("More than one non-producing nodes")
+        else:
+            prodNodes.append(node)
+            producers.extend(node.producers)
+
+    prodAB=prodNodes[0]  # defproducera, defproducerb
+    prodC=prodNodes[1]   # defproducerc
+
+    # ***   Identify a block where production is stable   ***
+
+    #verify nodes are in sync and advancing
+    cluster.biosNode.kill(signal.SIGTERM)
+    cluster.waitOnClusterSync(blockAdvancing=5)
+
+    Print("Creating account1")
+    account1 = Account('account1')
+    account1.ownerPublicKey = EOSIO_ACCT_PUBLIC_DEFAULT_KEY
+    account1.activePublicKey = EOSIO_ACCT_PUBLIC_DEFAULT_KEY
+    cluster.createAccountAndVerify(account1, cluster.eosioAccount, stakedDeposit=1000)
+
+    Print("Validating accounts after bootstrap")
+    cluster.validateAccounts([account1])
+
+    # ***   Schedule snapshot, it should become pending, then wait for finality
+    prodAB.scheduleSnapshot()
+    blockNum=prodAB.getBlockNum(BlockType.head) + 1
+    waitForBlock(prodAB, blockNum + 1, blockType=BlockType.lib)
+
+   
+    # ***   Killing the "bridge" node   ***
+    Print("Sending command to kill \"bridge\" node to separate the 2 producer groups.")
+    # kill at the beginning of the production window for defproducera, so there is time for the fork for
+    # defproducerc to grow before it would overtake the fork for defproducera and defproducerb
+    killAtProducer="defproducera"
+    nonProdNode.killNodeOnProducer(producer=killAtProducer, whereInSequence=1)
+
+    #verify that the non producing node is not alive (and populate the producer nodes with current getInfo data to report if
+    #an error occurs)
+    numPasses = 2
+    blocksPerProducer = 12
+    blocksPerRound = totalProducers * blocksPerProducer
+    count = blocksPerRound * numPasses
+    while nonProdNode.verifyAlive() and count > 0:
+        # wait on prodNode 0 since it will continue to advance, since defproducera and defproducerb are its producers
+        Print("Wait for next block")
+        assert prodAB.waitForNextBlock(timeout=6), "Production node AB should continue to advance, even after bridge node is killed"
+        count -= 1
+   
+    # schedule a snapshot that should get finalized
+    prodC.scheduleSnapshot()
+
+    assert not nonProdNode.verifyAlive(), "Bridge node should have been killed if test was functioning correctly."
+
+    def getState(status):
+        assert status is not None, "ERROR: getTransactionStatus failed to return any status"
+        assert "state" in status, \
+            f"ERROR: getTransactionStatus returned a status object that didn't have a \"state\" field. state: {json.dumps(status, indent=1)}"
+        return status["state"]
+   
+    assert prodC.waitForNextBlock(), "Production node C should continue to advance, even after bridge node is killed"
+
+    Print("Relaunching the non-producing bridge node to connect the nodes")
+    if not nonProdNode.relaunch():
+        errorExit(f"Failure - (non-production) node {nonProdNode.nodeNum} should have restarted")
+
+    Print("Wait for LIB to move, which indicates prodC has forked out the branch")
+    assert prodC.waitForLibToAdvance(), \
+        "ERROR: Network did not reach consensus after bridge node was restarted."
+ 
+    for prodNode in prodNodes:
+        info=prodNode.getInfo()
+        Print(f"node info: {json.dumps(info, indent=1)}")
+
+    assert prodC.waitForProducer("defproducerc"), \
+        f"Waiting for prodC to produce, but it never happened" + \
+        f"\n\nprod AB info: {json.dumps(prodAB.getInfo(), indent=1)}\n\nprod C info: {json.dumps(prodC.getInfo(), indent=1)}"
+    
+    blockNum=prodC.getBlockNum(BlockType.head) + 1
+    waitForBlock(prodC, blockNum + 1, blockType=BlockType.lib)
+
+    # AB & C compare counts, should be same
+    assert getSnapshotsCount(0) == getSnapshotsCount(1), \
+        "ERROR: Snapshot generation failed."
 
     testSuccessful=True
-
 finally:
     TestHelper.shutdown(cluster, walletMgr, testSuccessful=testSuccessful, killEosInstances=killEosInstances, killWallet=killWallet, keepLogs=keepLogs, cleanRun=killAll, dumpErrorDetails=dumpErrorDetails)
 
-exitCode = 0 if testSuccessful else 1
-exit(exitCode)
\ No newline at end of file
+errorCode = 0 if testSuccessful else 1
+exit(errorCode)
\ No newline at end of file

From 041d897df06978391b1ee7025bb6c5b75758594d Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Thu, 9 Mar 2023 13:55:08 -0500
Subject: [PATCH 32/48] Addressing feedback

---
 .../include/eosio/producer_plugin/snapshot_scheduler.hpp | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 1154178e57..45f805a928 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -63,6 +63,8 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
 
    // snapshot_scheduler_listener
    void on_start_block(uint32_t height) {
+      bool serialize_needed = false;
+
       for(const auto& req: _snapshot_requests.get<0>()) {
          // assume "asap" for snapshot with missed/zero start, it can have spacing
          if(!req.start_block_num) {
@@ -72,7 +74,7 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
                auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
                auto it = snapshot_by_id.find(req.snapshot_request_id);
                _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height; });
-               x_serialize();
+               serialize_needed = true;              
             }
             execute_snapshot(req.snapshot_request_id);
          }
@@ -86,6 +88,9 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
             unschedule_snapshot(req.snapshot_request_id);
          }
       }
+
+      // store db to filesystem
+      if (serialize_needed) x_serialize();
    }
 
    // snapshot_scheduler_handler
@@ -130,6 +135,8 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
       if(fc::exists(_snapshot_db.get_json_path())) {
          std::vector<producer_plugin::snapshot_schedule_information> sr;
          _snapshot_db >> sr;
+         // if db read succeeded, clear/load
+         _snapshot_requests.get<by_snapshot_id>().clear();
          _snapshot_requests.insert(sr.begin(), sr.end());
       }
    }

From 24cd36d37c574a58b5dc97eec8b6c15d48f36081 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Mon, 13 Mar 2023 00:16:59 -0400
Subject: [PATCH 33/48] Fix for the flakiness of snapshot test

---
 tests/nodeos_snapshot_diff_test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/nodeos_snapshot_diff_test.py b/tests/nodeos_snapshot_diff_test.py
index c0e7e81acf..6ca85ecaac 100755
--- a/tests/nodeos_snapshot_diff_test.py
+++ b/tests/nodeos_snapshot_diff_test.py
@@ -156,6 +156,7 @@ def waitForBlock(node, blockNum, blockType=BlockType.head, timeout=None, reportI
     assert steadyStateAvg>=minRequiredTransactions, "Expected to at least receive %s transactions per block, but only getting %s" % (minRequiredTransactions, steadyStateAvg)
 
     Print("Create snapshot")
+    waitForBlock(nodeSnap, nodeSnap.getBlockNum(BlockType.head) + 30)
     ret = nodeSnap.scheduleSnapshot()
     assert ret is not None, "Snapshot creation failed"
     ret_head_block_num = nodeSnap.getBlockNum(BlockType.head) + 1

From ec624b5dcc597358e85a63ca5d9ecaec629fc737 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Mon, 13 Mar 2023 21:24:40 -0400
Subject: [PATCH 34/48] Improved snapshot_diff test

---
 tests/nodeos_snapshot_diff_test.py | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/tests/nodeos_snapshot_diff_test.py b/tests/nodeos_snapshot_diff_test.py
index 6ca85ecaac..417346f780 100755
--- a/tests/nodeos_snapshot_diff_test.py
+++ b/tests/nodeos_snapshot_diff_test.py
@@ -19,7 +19,7 @@
 #  Test configures a producing node and 2 non-producing nodes.
 #  Configures trx_generator(s) and starts generating transactions and sending them
 #  to the producing node.
-#  - Create a snapshot from producing node using snapshot scheduler
+#  - Create a snapshot from producing node
 #  - Convert snapshot to JSON
 #  - Trim blocklog to head block of snapshot
 #  - Start nodeos in irreversible mode on blocklog
@@ -40,7 +40,7 @@
 pnodes=1
 testAccounts = 2
 trxGeneratorCnt=2
-startedNonProdNodes = 2
+startedNonProdNodes = 3
 cluster=Cluster(walletd=True)
 dumpErrorDetails=args.dump_error_details
 keepLogs=args.keep_logs
@@ -115,9 +115,11 @@ def waitForBlock(node, blockNum, blockType=BlockType.head, timeout=None, reportI
     snapshotNodeId = 0
     node0=cluster.getNode(snapshotNodeId)
     irrNodeId = snapshotNodeId+1
+    progNodeId = irrNodeId+1
 
     nodeSnap=cluster.getNode(snapshotNodeId)
     nodeIrr=cluster.getNode(irrNodeId)
+    nodeProg=cluster.getNode(progNodeId)
 
     Print("Wait for account creation to be irreversible")
     blockNum=node0.getBlockNum(BlockType.head)
@@ -156,10 +158,13 @@ def waitForBlock(node, blockNum, blockType=BlockType.head, timeout=None, reportI
     assert steadyStateAvg>=minRequiredTransactions, "Expected to at least receive %s transactions per block, but only getting %s" % (minRequiredTransactions, steadyStateAvg)
 
     Print("Create snapshot")
-    waitForBlock(nodeSnap, nodeSnap.getBlockNum(BlockType.head) + 30)
-    ret = nodeSnap.scheduleSnapshot()
+    ret = nodeProg.scheduleSnapshot()
     assert ret is not None, "Snapshot creation failed"
-    ret_head_block_num = nodeSnap.getBlockNum(BlockType.head) + 1
+    
+    ret = nodeSnap.createSnapshot()
+    assert ret is not None, "Snapshot creation failed"
+    ret_head_block_num = ret["payload"]["head_block_num"]
+    Print(f"Snapshot head block number {ret_head_block_num}")
 
     Print("Wait for snapshot node lib to advance")
     waitForBlock(nodeSnap, ret_head_block_num+1, blockType=BlockType.lib)
@@ -199,6 +204,16 @@ def waitForBlock(node, blockNum, blockType=BlockType.head, timeout=None, reportI
     irrSnapshotFile = irrSnapshotFile + ".json"
 
     assert Utils.compareFiles(snapshotFile, irrSnapshotFile), f"Snapshot files differ {snapshotFile} != {irrSnapshotFile}"
+  
+    Print("Kill programmable node")
+    nodeProg.kill(signal.SIGTERM)
+
+    Print("Convert snapshot to JSON")
+    progSnapshotFile = getLatestSnapshot(progNodeId)
+    Utils.processLeapUtilCmd("snapshot to-json --input-file {}".format(progSnapshotFile), "snapshot to-json", silentErrors=False)
+    progSnapshotFile = progSnapshotFile + ".json"
+
+    assert Utils.compareFiles(progSnapshotFile, irrSnapshotFile), f"Snapshot files differ {progSnapshotFile} != {irrSnapshotFile}"
 
     testSuccessful=True
 

From dbf901a55f651b9ddc2bdaff24cce68c6f09452a Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Mon, 13 Mar 2023 22:15:40 -0400
Subject: [PATCH 35/48] Quit node gracefully on scheduler errors

---
 .../include/eosio/producer_plugin/snapshot_db_json.hpp        | 2 ++
 .../include/eosio/producer_plugin/snapshot_scheduler.hpp      | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
index 283858168c..10634a85d7 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
@@ -48,6 +48,7 @@ class snapshot_db_json {
       catch (std::ifstream::failure & e) {
          elog( "unable to restore snapshots schedule from filesystem ${jsonpath}, details: ${details}",
             ("jsonpath", get_json_path().string()) ("details",e.what()) );
+         appbase::app().quit();
       }
 
       return *this;
@@ -78,6 +79,7 @@ class snapshot_db_json {
       catch (std::ofstream::failure & e) {
          elog( "unable to store snapshots schedule to filesystem to ${jsonpath}, details: ${details}",
             ("jsonpath", get_json_path().string()) ("details", e.what()) );
+         appbase::app().quit();
       }
 
       return *this;
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 45f805a928..57c18d1418 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -168,10 +168,10 @@ class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_sc
                std::get<fc::exception_ptr>(result)->dynamic_rethrow_exception();
             } catch(const fc::exception& e) {
                elog( "snapshot creation error: ${details}", ("details",e.to_detail_string()) );
-               throw;
+               appbase::app().quit();
             } catch(const std::exception& e) {
                elog( "snapshot creation error: ${details}", ("details",e.what()) );
-               throw;
+               appbase::app().quit();
             }
          } else {
             // success, snapshot finalized

From c8487beb17e1f87c1e73bb07d74e3ed1724e9aa8 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 14 Mar 2023 12:38:28 -0400
Subject: [PATCH 36/48] Renamed and cleaned up last test

---
 tests/CMakeLists.txt                               |  6 +++---
 ...able_test.py => nodeos_snapshot_forked_test.py} | 14 +++++++-------
 2 files changed, 10 insertions(+), 10 deletions(-)
 rename tests/{nodeos_snapshot_programmable_test.py => nodeos_snapshot_forked_test.py} (96%)

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 5a8967ce2b..a2dfe96a60 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -25,7 +25,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/restart-scenarios-test.py ${CMAKE_CUR
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/terminate-scenarios-test.py ${CMAKE_CURRENT_BINARY_DIR}/terminate-scenarios-test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_startup_catchup.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_startup_catchup.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_snapshot_diff_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_snapshot_diff_test.py COPYONLY)
-configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_snapshot_programmable_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_snapshot_programmable_test.py COPYONLY)
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_snapshot_forked_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_snapshot_forked_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_forked_chain_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_forked_chain_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_short_fork_take_over_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_short_fork_take_over_test.py COPYONLY)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_run_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_run_test.py COPYONLY)
@@ -151,8 +151,8 @@ add_test(NAME keosd_auto_launch_test COMMAND tests/keosd_auto_launch_test.py WOR
 set_property(TEST keosd_auto_launch_test PROPERTY LABELS nonparallelizable_tests)
 add_test(NAME nodeos_snapshot_diff_test COMMAND tests/nodeos_snapshot_diff_test.py -v --clean-run WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST nodeos_snapshot_diff_test PROPERTY LABELS nonparallelizable_tests)
-add_test(NAME nodeos_snapshot_programmable_test COMMAND tests/nodeos_snapshot_programmable_test.py -v --clean-run WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
-set_property(TEST nodeos_snapshot_programmable_test PROPERTY LABELS nonparallelizable_tests)
+add_test(NAME nodeos_snapshot_forked_test COMMAND tests/nodeos_snapshot_forked_test.py -v --clean-run WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+set_property(TEST nodeos_snapshot_forked_test PROPERTY LABELS nonparallelizable_tests)
 
 add_test(NAME trx_finality_status_test COMMAND tests/trx_finality_status_test.py -v --clean-run WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
 set_property(TEST trx_finality_status_test PROPERTY LABELS nonparallelizable_tests)
diff --git a/tests/nodeos_snapshot_programmable_test.py b/tests/nodeos_snapshot_forked_test.py
similarity index 96%
rename from tests/nodeos_snapshot_programmable_test.py
rename to tests/nodeos_snapshot_forked_test.py
index 44433aa11a..c35af7cd81 100755
--- a/tests/nodeos_snapshot_programmable_test.py
+++ b/tests/nodeos_snapshot_forked_test.py
@@ -12,11 +12,14 @@
 from TestHarness.Node import BlockType
 
 ###############################################################
-# nodeos_snapshot_programmable
+# nodeos_snapshot_forked
 #
 #  Test to verify that programmable snapshot functionality is
 #  working appropriately when forks occur
 #
+#  Setup involves constructing bridge-mode node topology and 
+#  killing a "bridge" node
+#
 ###############################################################
 Print=Utils.Print
 errorExit=Utils.errorExit
@@ -135,12 +138,9 @@ def getSnapshotsCount(nodeId):
     Print("Validating accounts after bootstrap")
     cluster.validateAccounts([account1])
 
-    # ***   Schedule snapshot, it should become pending, then wait for finality
+    # ***   Schedule snapshot, it should become pending
     prodAB.scheduleSnapshot()
-    blockNum=prodAB.getBlockNum(BlockType.head) + 1
-    waitForBlock(prodAB, blockNum + 1, blockType=BlockType.lib)
-
-   
+      
     # ***   Killing the "bridge" node   ***
     Print("Sending command to kill \"bridge\" node to separate the 2 producer groups.")
     # kill at the beginning of the production window for defproducera, so there is time for the fork for
@@ -194,7 +194,7 @@ def getState(status):
 
     # AB & C compare counts, should be same
     assert getSnapshotsCount(0) == getSnapshotsCount(1), \
-        "ERROR: Snapshot generation failed."
+        "ERROR: Pre-fork and post-fork snapshots failed to finalize."
 
     testSuccessful=True
 finally:

From dfc8d629f3ddc35a259d1095ca53f95b75d6f457 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 14 Mar 2023 14:03:06 -0400
Subject: [PATCH 37/48] cleaned up listeners/handlers

---
 .../eosio/producer_plugin/snapshot_scheduler.hpp     | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 57c18d1418..21feccc790 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -20,17 +20,7 @@ namespace eosio {
 
 namespace bmi = boost::multi_index;
 
-class snapshot_scheduler_listener {
-   virtual void on_start_block(uint32_t height) = 0;
-};
-
-class snapshot_scheduler_handler {
-   virtual void schedule_snapshot(const producer_plugin::snapshot_request_information& sri) = 0;
-   virtual void unschedule_snapshot(uint32_t sri) = 0;
-   virtual producer_plugin::get_snapshot_requests_result get_snapshot_requests() = 0;
-};
-
-class snapshot_scheduler : public snapshot_scheduler_handler, public snapshot_scheduler_listener {
+class snapshot_scheduler {
 private:
    struct by_snapshot_id;
    struct by_snapshot_value;

From dfe1c55ed65e5ac1afe565f0fc3d1f45caf6e161 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 14 Mar 2023 19:20:46 -0400
Subject: [PATCH 38/48] Addressing feedback

---
 .../producer_plugin/snapshot_scheduler.hpp    | 31 +++++++++++++++----
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 21feccc790..341362a469 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -53,9 +53,24 @@ class snapshot_scheduler {
 
    // snapshot_scheduler_listener
    void on_start_block(uint32_t height) {
-      bool serialize_needed = false;
+      bool serialize_needed  = false;
+      bool snapshot_executed = false; 
+
+      auto execute_snapshot_with_log = [this, &height](const auto & req) {
+         dlog("snapshot scheduler creating a snapshot from the request [start_block_num:${start_block_num}, end_block_num=${end_block_num}, block_spacing=${block_spacing}], height=${height}",
+             ("start_block_num", req.start_block_num)
+             ("end_block_num",   req.end_block_num)
+             ("block_spacing",   req.block_spacing)
+             ("height",          height));
+             
+         execute_snapshot(req.snapshot_request_id);
+      };    
 
       for(const auto& req: _snapshot_requests.get<0>()) {
+         // -1 since its called from start block
+         bool recurring_snapshot =  req.block_spacing &&  (!((height - req.start_block_num - 1) % req.block_spacing));
+         bool onetime_snapshot   = (!req.block_spacing) && (height == req.start_block_num + 1);
+        
          // assume "asap" for snapshot with missed/zero start, it can have spacing
          if(!req.start_block_num) {
             // update start_block_num with current height only if this is recurring
@@ -66,17 +81,21 @@ class snapshot_scheduler {
                _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height; });
                serialize_needed = true;              
             }
-            execute_snapshot(req.snapshot_request_id);
-         }
-         // execute snapshot, -1 since its called from start block
-         else if(!req.block_spacing || (!((height - req.start_block_num - 1) % req.block_spacing))) {
-            execute_snapshot(req.snapshot_request_id);
+            execute_snapshot_with_log(req);
+            snapshot_executed = true;
+         }        
+         else if(recurring_snapshot || onetime_snapshot) {
+            execute_snapshot_with_log(req);
          }       
 
          // cleanup - remove expired (or invalid) request
          if(!req.start_block_num || !req.block_spacing || (req.end_block_num > 0 && height >= req.end_block_num)) {
             unschedule_snapshot(req.snapshot_request_id);
+            snapshot_executed = true;
          }
+
+         // stop iterating snapshot requests after snapshot execution
+         if (snapshot_executed) break;
       }
 
       // store db to filesystem

From 41601acac4bf2d1d1a1cb5488a02ea8a558ddfa0 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 14 Mar 2023 20:08:43 -0400
Subject: [PATCH 39/48] make sure future scheduled snapshots treated correctly

---
 .../include/eosio/producer_plugin/snapshot_scheduler.hpp        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 341362a469..b8a03985b2 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -68,7 +68,7 @@ class snapshot_scheduler {
 
       for(const auto& req: _snapshot_requests.get<0>()) {
          // -1 since its called from start block
-         bool recurring_snapshot =  req.block_spacing &&  (!((height - req.start_block_num - 1) % req.block_spacing));
+         bool recurring_snapshot =  req.block_spacing &&  (height >= req.start_block_num + 1) && (!((height - req.start_block_num - 1) % req.block_spacing));
          bool onetime_snapshot   = (!req.block_spacing) && (height == req.start_block_num + 1);
         
          // assume "asap" for snapshot with missed/zero start, it can have spacing

From 3ec37d2eeec4ef1864da229ce36f9df857cfe60c Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 14 Mar 2023 21:38:54 -0400
Subject: [PATCH 40/48] fixed ups

---
 .../include/eosio/producer_plugin/snapshot_scheduler.hpp     | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index b8a03985b2..92af9ef260 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -56,7 +56,7 @@ class snapshot_scheduler {
       bool serialize_needed  = false;
       bool snapshot_executed = false; 
 
-      auto execute_snapshot_with_log = [this, &height](const auto & req) {
+      auto execute_snapshot_with_log = [this, &height, &snapshot_executed](const auto & req) {
          dlog("snapshot scheduler creating a snapshot from the request [start_block_num:${start_block_num}, end_block_num=${end_block_num}, block_spacing=${block_spacing}], height=${height}",
              ("start_block_num", req.start_block_num)
              ("end_block_num",   req.end_block_num)
@@ -64,6 +64,7 @@ class snapshot_scheduler {
              ("height",          height));
              
          execute_snapshot(req.snapshot_request_id);
+         snapshot_executed = true;
       };    
 
       for(const auto& req: _snapshot_requests.get<0>()) {
@@ -82,7 +83,6 @@ class snapshot_scheduler {
                serialize_needed = true;              
             }
             execute_snapshot_with_log(req);
-            snapshot_executed = true;
          }        
          else if(recurring_snapshot || onetime_snapshot) {
             execute_snapshot_with_log(req);
@@ -91,7 +91,6 @@ class snapshot_scheduler {
          // cleanup - remove expired (or invalid) request
          if(!req.start_block_num || !req.block_spacing || (req.end_block_num > 0 && height >= req.end_block_num)) {
             unschedule_snapshot(req.snapshot_request_id);
-            snapshot_executed = true;
          }
 
          // stop iterating snapshot requests after snapshot execution

From 104160055850207b93370b68cc654f1ddd0e4191 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 14 Mar 2023 22:05:39 -0400
Subject: [PATCH 41/48] logic fix

---
 .../include/eosio/producer_plugin/snapshot_scheduler.hpp      | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 92af9ef260..a7a5ecb2f4 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -89,7 +89,9 @@ class snapshot_scheduler {
          }       
 
          // cleanup - remove expired (or invalid) request
-         if(!req.start_block_num || !req.block_spacing || (req.end_block_num > 0 && height >= req.end_block_num)) {
+         if(!req.start_block_num ||         
+            (!req.block_spacing && height >= (req.start_block_num + 1)) || 
+            (req.end_block_num > 0 && height >= (req.end_block_num + 1))) {
             unschedule_snapshot(req.snapshot_request_id);
          }
 

From d0d5d02f0d068a6cc2a3a5e86f5941b07c4eb6ca Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Tue, 14 Mar 2023 22:30:59 -0400
Subject: [PATCH 42/48] one more fix

---
 .../producer_plugin/snapshot_scheduler.hpp    | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index a7a5ecb2f4..344737e3fd 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -57,14 +57,17 @@ class snapshot_scheduler {
       bool snapshot_executed = false; 
 
       auto execute_snapshot_with_log = [this, &height, &snapshot_executed](const auto & req) {
-         dlog("snapshot scheduler creating a snapshot from the request [start_block_num:${start_block_num}, end_block_num=${end_block_num}, block_spacing=${block_spacing}], height=${height}",
-             ("start_block_num", req.start_block_num)
-             ("end_block_num",   req.end_block_num)
-             ("block_spacing",   req.block_spacing)
-             ("height",          height));
-             
-         execute_snapshot(req.snapshot_request_id);
-         snapshot_executed = true;
+         // one snapshot per height
+         if (!snapshot_executed) {
+            dlog("snapshot scheduler creating a snapshot from the request [start_block_num:${start_block_num}, end_block_num=${end_block_num}, block_spacing=${block_spacing}], height=${height}",
+               ("start_block_num", req.start_block_num)
+               ("end_block_num",   req.end_block_num)
+               ("block_spacing",   req.block_spacing)
+               ("height",          height));
+               
+            execute_snapshot(req.snapshot_request_id);
+            snapshot_executed = true;
+         }
       };    
 
       for(const auto& req: _snapshot_requests.get<0>()) {
@@ -79,7 +82,7 @@ class snapshot_scheduler {
             if (req.block_spacing && height) {
                auto& snapshot_by_id = _snapshot_requests.get<by_snapshot_id>();
                auto it = snapshot_by_id.find(req.snapshot_request_id);
-               _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height; });
+               _snapshot_requests.modify(it, [&height](auto& p) { p.start_block_num = height - 1; });
                serialize_needed = true;              
             }
             execute_snapshot_with_log(req);
@@ -94,9 +97,6 @@ class snapshot_scheduler {
             (req.end_block_num > 0 && height >= (req.end_block_num + 1))) {
             unschedule_snapshot(req.snapshot_request_id);
          }
-
-         // stop iterating snapshot requests after snapshot execution
-         if (snapshot_executed) break;
       }
 
       // store db to filesystem

From b25ee3c306ef7b06ffe803f2ff7c89776c1e9706 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Wed, 15 Mar 2023 11:27:55 -0400
Subject: [PATCH 43/48] break after unschedule

---
 .../include/eosio/producer_plugin/snapshot_scheduler.hpp         | 1 +
 1 file changed, 1 insertion(+)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 344737e3fd..410350a12a 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -96,6 +96,7 @@ class snapshot_scheduler {
             (!req.block_spacing && height >= (req.start_block_num + 1)) || 
             (req.end_block_num > 0 && height >= (req.end_block_num + 1))) {
             unschedule_snapshot(req.snapshot_request_id);
+            if (snapshot_executed) break;
          }
       }
 

From 696f30f8ed9cdbdd6af55c196efc49985a6670fc Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Wed, 15 Mar 2023 17:22:52 -0400
Subject: [PATCH 44/48] more selectability for invalid request

---
 .../include/eosio/producer_plugin/snapshot_scheduler.hpp        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 410350a12a..b88bcfc0f5 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -92,7 +92,7 @@ class snapshot_scheduler {
          }       
 
          // cleanup - remove expired (or invalid) request
-         if(!req.start_block_num ||         
+         if((!req.start_block_num && !req.block_spacing) ||         
             (!req.block_spacing && height >= (req.start_block_num + 1)) || 
             (req.end_block_num > 0 && height >= (req.end_block_num + 1))) {
             unschedule_snapshot(req.snapshot_request_id);

From b72027cf22ccab7954e20b23c24975c92e16f8a8 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Wed, 15 Mar 2023 18:03:48 -0400
Subject: [PATCH 45/48] should allow loop to run from start to finish

---
 .../include/eosio/producer_plugin/snapshot_scheduler.hpp         | 1 -
 1 file changed, 1 deletion(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index b88bcfc0f5..8302e09bb7 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -96,7 +96,6 @@ class snapshot_scheduler {
             (!req.block_spacing && height >= (req.start_block_num + 1)) || 
             (req.end_block_num > 0 && height >= (req.end_block_num + 1))) {
             unschedule_snapshot(req.snapshot_request_id);
-            if (snapshot_executed) break;
          }
       }
 

From 6e0672ece401e4578b5b9951d6f970f9258f155e Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Thu, 16 Mar 2023 11:52:01 -0400
Subject: [PATCH 46/48] Addressing feedback

---
 .../producer_api_plugin.cpp                    | 18 ++++++------------
 .../eosio/producer_plugin/snapshot_db_json.hpp |  8 +++-----
 .../producer_plugin/snapshot_scheduler.hpp     |  8 +++++---
 3 files changed, 14 insertions(+), 20 deletions(-)

diff --git a/plugins/producer_api_plugin/producer_api_plugin.cpp b/plugins/producer_api_plugin/producer_api_plugin.cpp
index 53ab61a134..14b6b19006 100644
--- a/plugins/producer_api_plugin/producer_api_plugin.cpp
+++ b/plugins/producer_api_plugin/producer_api_plugin.cpp
@@ -109,18 +109,6 @@ void producer_api_plugin::plugin_startup() {
             INVOKE_R_V(producer, get_greylist), 201),
        CALL_WITH_400(producer, producer, get_whitelist_blacklist,
             INVOKE_R_V(producer, get_whitelist_blacklist), 201),
-       CALL_WITH_400(producer, producer, set_whitelist_blacklist,
-            INVOKE_V_R(producer, set_whitelist_blacklist, producer_plugin::whitelist_blacklist), 201),
-       CALL_WITH_400(producer, producer, get_integrity_hash,
-            INVOKE_R_V(producer, get_integrity_hash), 201),
-       CALL_ASYNC(producer, producer, create_snapshot, producer_plugin::snapshot_information,
-            INVOKE_R_V_ASYNC(producer, create_snapshot), 201),
-       CALL_WITH_400(producer, producer, schedule_snapshot,
-            INVOKE_V_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
-       CALL_WITH_400(producer, producer, get_snapshot_requests,
-            INVOKE_R_V(producer, get_snapshot_requests), 201),
-       CALL_WITH_400(producer, producer, unschedule_snapshot,
-            INVOKE_V_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201),
        CALL_WITH_400(producer, producer, get_scheduled_protocol_feature_activations,
             INVOKE_R_V(producer, get_scheduled_protocol_feature_activations), 201),
        CALL_WITH_400(producer, producer, get_supported_protocol_features,
@@ -148,6 +136,12 @@ void producer_api_plugin::plugin_startup() {
             INVOKE_V_R(producer, set_whitelist_blacklist, producer_plugin::whitelist_blacklist), 201),
        CALL_ASYNC(producer, producer, create_snapshot, producer_plugin::snapshot_information,
             INVOKE_R_V_ASYNC(producer, create_snapshot), 201),
+       CALL_WITH_400(producer, producer, schedule_snapshot,
+            INVOKE_V_R_II(producer, schedule_snapshot, producer_plugin::snapshot_request_information), 201),
+       CALL_WITH_400(producer, producer, get_snapshot_requests,
+            INVOKE_R_V(producer, get_snapshot_requests), 201),
+       CALL_WITH_400(producer, producer, unschedule_snapshot,
+            INVOKE_V_R(producer, unschedule_snapshot, producer_plugin::snapshot_request_id_information), 201),
        CALL_WITH_400(producer, producer, get_integrity_hash,
             INVOKE_R_V(producer, get_integrity_hash), 201),
        CALL_WITH_400(producer, producer, schedule_protocol_feature_activations,
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
index 10634a85d7..db9e84837a 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_db_json.hpp
@@ -21,7 +21,7 @@ class snapshot_db_json {
       db_path = std::move(path);
    }
 
-   bfs::path get_json_path() {
+   bfs::path get_json_path() const {
       return db_path / "snapshot-schedule.json";
    }
 
@@ -32,8 +32,7 @@ class snapshot_db_json {
          std::ifstream file(get_json_path().string());
          file.exceptions(std::istream::failbit|std::istream::eofbit);
          boost::property_tree::read_json(file, root);
-         file.close();
-
+       
          // parse ptree
          for(boost::property_tree::ptree::value_type& req: root.get_child("snapshot_requests")) {
             producer_plugin::snapshot_schedule_information ssi;
@@ -54,7 +53,7 @@ class snapshot_db_json {
       return *this;
    }
 
-   const snapshot_db_json& operator<<(std::vector<producer_plugin::snapshot_schedule_information>& sr) {
+   const snapshot_db_json& operator<<(std::vector<producer_plugin::snapshot_schedule_information>& sr) const {
       boost::property_tree::ptree root;
       boost::property_tree::ptree node_srs;
 
@@ -74,7 +73,6 @@ class snapshot_db_json {
          std::ofstream file(get_json_path().string());
          file.exceptions(std::istream::failbit|std::istream::eofbit);
          boost::property_tree::write_json(file, root);
-         file.close();
       }
       catch (std::ofstream::failure & e) {
          elog( "unable to store snapshots schedule to filesystem to ${jsonpath}, details: ${details}",
diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 8302e09bb7..97277b0069 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -56,7 +56,7 @@ class snapshot_scheduler {
       bool serialize_needed  = false;
       bool snapshot_executed = false; 
 
-      auto execute_snapshot_with_log = [this, &height, &snapshot_executed](const auto & req) {
+      auto execute_snapshot_with_log = [this, height, &snapshot_executed](const auto & req) {
          // one snapshot per height
          if (!snapshot_executed) {
             dlog("snapshot scheduler creating a snapshot from the request [start_block_num:${start_block_num}, end_block_num=${end_block_num}, block_spacing=${block_spacing}], height=${height}",
@@ -86,8 +86,7 @@ class snapshot_scheduler {
                serialize_needed = true;              
             }
             execute_snapshot_with_log(req);
-         }        
-         else if(recurring_snapshot || onetime_snapshot) {
+         } else if(recurring_snapshot || onetime_snapshot) {
             execute_snapshot_with_log(req);
          }       
 
@@ -197,6 +196,9 @@ class snapshot_scheduler {
                      p.pending_snapshots = std::move(pending);
                   });
                }
+            } else {     
+               // should not happen, but if happens - not a terminal error          
+               elog( "unable to finalize pending snapshot request: ${details}", ("details", srid) );
             }
          }
       };

From af808b72e6fea87cfdf1cd1b0d5c7ac07ffe6e1d Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Thu, 16 Mar 2023 16:20:18 -0400
Subject: [PATCH 47/48] feedback addressed

---
 .../include/eosio/producer_plugin/snapshot_scheduler.hpp     | 4 ++--
 plugins/producer_plugin/test/test_snapshot_scheduler.cpp     | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index 97277b0069..b302b4b8f5 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -197,8 +197,8 @@ class snapshot_scheduler {
                   });
                }
             } else {     
-               // should not happen, but if happens - not a terminal error          
-               elog( "unable to finalize pending snapshot request: ${details}", ("details", srid) );
+               // this could happen when request is already "expired" so there is no value in reporting it          
+               // dlog( "unable to finalize pending snapshot request: ${details}", ("details", srid) );
             }
          }
       };
diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
index 6356393d47..e03149bec1 100644
--- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
+++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
@@ -92,10 +92,11 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          auto pp = appbase::app().find_plugin<producer_plugin>();
          auto bs = chain_plug->chain().block_start.connect([&pp](uint32_t bn) {
             // catching pending snapshot
-            if (pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots && pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots->size()==1) {
+            auto& pending = pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots;
+            if (pending && pending->size()==1) {
                // lets check the head block num of it, it should be 8 + 1 = 9
                // this means we are getting a snapshot for correct block # as well
-               BOOST_CHECK_EQUAL(9, pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots->begin()->head_block_num);   
+               BOOST_CHECK_EQUAL(9, pending->begin()->head_block_num);   
             }
          });
        

From 17c4ce3a5c1ed9dc2c5eb4502073d4b985b4d021 Mon Sep 17 00:00:00 2001
From: 766C6164 <valterdaw@gmail.com>
Date: Thu, 16 Mar 2023 16:46:43 -0400
Subject: [PATCH 48/48] feedback addressed

---
 .../eosio/producer_plugin/snapshot_scheduler.hpp     |  3 ---
 .../producer_plugin/test/test_snapshot_scheduler.cpp | 12 +++++++-----
 2 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
index b302b4b8f5..53f17d5e0f 100644
--- a/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
+++ b/plugins/producer_plugin/include/eosio/producer_plugin/snapshot_scheduler.hpp
@@ -196,9 +196,6 @@ class snapshot_scheduler {
                      p.pending_snapshots = std::move(pending);
                   });
                }
-            } else {     
-               // this could happen when request is already "expired" so there is no value in reporting it          
-               // dlog( "unable to finalize pending snapshot request: ${details}", ("details", srid) );
             }
          }
       };
diff --git a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
index e03149bec1..eab3995a32 100644
--- a/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
+++ b/plugins/producer_plugin/test/test_snapshot_scheduler.cpp
@@ -92,11 +92,13 @@ BOOST_AUTO_TEST_CASE(snapshot_scheduler_test) {
          auto pp = appbase::app().find_plugin<producer_plugin>();
          auto bs = chain_plug->chain().block_start.connect([&pp](uint32_t bn) {
             // catching pending snapshot
-            auto& pending = pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots;
-            if (pending && pending->size()==1) {
-               // lets check the head block num of it, it should be 8 + 1 = 9
-               // this means we are getting a snapshot for correct block # as well
-               BOOST_CHECK_EQUAL(9, pending->begin()->head_block_num);   
+            if (!pp->get_snapshot_requests().snapshot_requests.empty()) {
+               auto& pending = pp->get_snapshot_requests().snapshot_requests.begin()->pending_snapshots;
+               if (pending && pending->size()==1) {
+                  // lets check the head block num of it, it should be 8 + 1 = 9
+                  // this means we are getting a snapshot for correct block # as well
+                  BOOST_CHECK_EQUAL(9, pending->begin()->head_block_num);   
+               }
             }
          });