From db55b091048607ba0f6aafe4d666d090f061ce76 Mon Sep 17 00:00:00 2001
From: Kevin Heifner <heifnerk@objectcomputing.com>
Date: Tue, 20 Oct 2020 08:28:16 -0500
Subject: [PATCH 1/9] Use new zipkin trace macros

---
 libraries/fc                                  |  2 +-
 libraries/rodeos/rodeos.cpp                   | 13 +++++
 plugins/chain_plugin/chain_plugin.cpp         | 58 +++++++++++++++++--
 plugins/net_plugin/net_plugin.cpp             | 15 +++++
 .../state_history_plugin.cpp                  | 18 ++++++
 programs/nodeos/logging.json                  | 18 ++++++
 6 files changed, 117 insertions(+), 7 deletions(-)

diff --git a/libraries/fc b/libraries/fc
index 8637f75b300..e7ad1fb0abd 160000
--- a/libraries/fc
+++ b/libraries/fc
@@ -1 +1 @@
-Subproject commit 8637f75b30028688d6ce6926879f0cea8a36e2c7
+Subproject commit e7ad1fb0abd4f2e322397fe6db9eb2c61912c3f7
diff --git a/libraries/rodeos/rodeos.cpp b/libraries/rodeos/rodeos.cpp
index c9b443e67db..abddfb6f8fd 100644
--- a/libraries/rodeos/rodeos.cpp
+++ b/libraries/rodeos/rodeos.cpp
@@ -172,6 +172,12 @@ void rodeos_db_snapshot::write_block_info(const ship_protocol::get_blocks_result
    signed_block_header block;
    from_bin(block, bin);
 
+   auto blk_trace = fc_create_trace_with_id( "Block", result.this_block->block_id );
+   auto blk_span = fc_create_span( blk_trace, "rodeos-received" );
+   fc_add_tag( blk_span, "block_id", to_string( result.this_block->block_id ) );
+   fc_add_tag( blk_span, "block_num", block_num );
+   fc_add_tag( blk_span, "block_time", block.timestamp.to_time_point().elapsed.count() );
+
    write_block_info(block_num, result.this_block->block_id, block);
 }
 
@@ -184,6 +190,13 @@ void rodeos_db_snapshot::write_block_info(const ship_protocol::get_blocks_result
 
    const signed_block_header& header =
          std::visit([](const auto& blk) { return static_cast<const signed_block_header&>(blk); }, *result.block);
+
+   auto blk_trace = fc_create_trace_with_id( "Block", result.this_block->block_id );
+   auto blk_span = fc_create_span( blk_trace, "rodeos-received" );
+   fc_add_tag( blk_span, "block_id", to_string( result.this_block->block_id ) );
+   fc_add_tag( blk_span, "block_num", block_num );
+   fc_add_tag( blk_span, "block_time", eosio::microseconds_to_str( header.timestamp.to_time_point().elapsed.count() ) );
+
    write_block_info(block_num, result.this_block->block_id, header);
 }
 
diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp
index ecd4959e456..83e2eede207 100644
--- a/plugins/chain_plugin/chain_plugin.cpp
+++ b/plugins/chain_plugin/chain_plugin.cpp
@@ -41,6 +41,7 @@ FC_REFLECT(chainbase::environment, (debug)(os)(arch)(boost_version)(compiler) )
 
 const fc::string deep_mind_logger_name("deep-mind");
 fc::logger _deep_mind_log;
+fc::logger _zipkin_logger;
 
 namespace eosio {
 
@@ -1351,10 +1352,12 @@ void chain_plugin::plugin_shutdown() {
    if(app().is_quiting())
       my->chain->get_wasm_interface().indicate_shutting_down();
    my->chain.reset();
+   fc::log_config::shutdown_appenders();
 }
 
 void chain_plugin::handle_sighup() {
    fc::logger::update( deep_mind_logger_name, _deep_mind_log );
+   fc::logger::update( fc::zipkin_logger_name, _zipkin_logger );
 }
 
 chain_apis::read_write::read_write(controller& db, const fc::microseconds& abi_serializer_max_time, bool api_accept_transactions)
@@ -2768,12 +2771,34 @@ void read_write::push_transaction(const read_write::push_transaction_params& par
          input_trx = std::make_shared<packed_transaction>( std::move( input_trx_v0 ), true );
       } EOS_RETHROW_EXCEPTIONS(chain::packed_transaction_type_exception, "Invalid packed transaction")
 
+      auto trx_trace = fc_create_trace_with_id("Transaction", input_trx->id());
+      auto trx_span = fc_create_span(trx_trace, "HTTP Received");
+      fc_add_tag(trx_span, "trx_id", input_trx->id());
+      fc_add_tag(trx_span, "method", "push_transaction");
+
       app().get_method<incoming::methods::transaction_async>()(input_trx, true,
-            [this, next](const std::variant<fc::exception_ptr, transaction_trace_ptr>& result) -> void {
+            [this, token=trx_trace.get_token(), input_trx, next]
+            (const std::variant<fc::exception_ptr, transaction_trace_ptr>& result) -> void {
+
+         auto trx_span = fc_create_span_from_token(token, "Processed");
+         fc_add_tag(trx_span, "trx_id", input_trx->id());
+
          if (std::holds_alternative<fc::exception_ptr>(result)) {
-            next(std::get<fc::exception_ptr>(result));
+            auto& eptr = std::get<fc::exception_ptr>(result);
+            fc_add_tag(trx_span, "error", eptr->to_string());
+            next(eptr);
          } else {
-            auto trx_trace_ptr = std::get<transaction_trace_ptr>(result);
+            auto& trx_trace_ptr = std::get<transaction_trace_ptr>(result);
+
+            fc_add_tag(trx_span, "block_num", trx_trace_ptr->block_num);
+            fc_add_tag(trx_span, "block_time", trx_trace_ptr->block_time.to_time_point());
+            fc_add_tag(trx_span, "elapsed", trx_trace_ptr->elapsed.count());
+            if( trx_trace_ptr->receipt ) {
+               fc_add_tag(trx_span, "status", std::string(trx_trace_ptr->receipt->status));
+            }
+            if( trx_trace_ptr->except ) {
+               fc_add_tag(trx_span, "error", trx_trace_ptr->except->to_string());
+            }
 
             try {
                fc::variant output;
@@ -2889,12 +2914,33 @@ void read_write::send_transaction(const read_write::send_transaction_params& par
          input_trx = std::make_shared<packed_transaction>( std::move( input_trx_v0 ), true );
       } EOS_RETHROW_EXCEPTIONS(chain::packed_transaction_type_exception, "Invalid packed transaction")
 
+      auto trx_trace = fc_create_trace_with_id("Transaction", input_trx->id());
+      auto trx_span = fc_create_span(trx_trace, "HTTP Received");
+      fc_add_tag(trx_span, "trx_id", input_trx->id());
+      fc_add_tag(trx_span, "method", "send_transaction");
+
       app().get_method<incoming::methods::transaction_async>()(input_trx, true,
-            [this, next](const std::variant<fc::exception_ptr, transaction_trace_ptr>& result) -> void {
+            [this, token=trx_trace.get_token(), input_trx, next]
+            (const std::variant<fc::exception_ptr, transaction_trace_ptr>& result) -> void {
+         auto trx_span = fc_create_span_from_token(token, "Processed");
+         fc_add_tag(trx_span, "trx_id", input_trx->id());
+
          if (std::holds_alternative<fc::exception_ptr>(result)) {
-            next(std::get<fc::exception_ptr>(result));
+            auto& eptr = std::get<fc::exception_ptr>(result);
+            fc_add_tag(trx_span, "error", eptr->to_string());
+            next(eptr);
          } else {
-            auto trx_trace_ptr = std::get<transaction_trace_ptr>(result);
+            auto& trx_trace_ptr = std::get<transaction_trace_ptr>(result);
+
+            fc_add_tag(trx_span, "block_num", trx_trace_ptr->block_num);
+            fc_add_tag(trx_span, "block_time", trx_trace_ptr->block_time.to_time_point());
+            fc_add_tag(trx_span, "elapsed", trx_trace_ptr->elapsed.count());
+            if( trx_trace_ptr->receipt ) {
+               fc_add_tag(trx_span, "status", std::string(trx_trace_ptr->receipt->status));
+            }
+            if( trx_trace_ptr->except ) {
+               fc_add_tag(trx_span, "error", trx_trace_ptr->except->to_string());
+            }
 
             try {
                fc::variant output;
diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp
index 350650f88b0..0436e1caaf7 100644
--- a/plugins/net_plugin/net_plugin.cpp
+++ b/plugins/net_plugin/net_plugin.cpp
@@ -3313,6 +3313,13 @@ namespace eosio {
       controller& cc = chain_plug->chain();
       dispatcher->strand.post( [this, bs]() {
          fc_dlog( logger, "signaled accepted_block, blk num = ${num}, id = ${id}", ("num", bs->block_num)("id", bs->id) );
+
+         auto blk_trace = fc_create_trace_with_id( "Block", bs->id );
+         auto blk_span = fc_create_span( blk_trace, "Accepted" );
+         fc_add_tag( blk_span, "block_id", bs->id );
+         fc_add_tag( blk_span, "block_num", bs->block_num );
+         fc_add_tag( blk_span, "block_time", bs->block->timestamp.to_time_point() );
+
          dispatcher->bcast_block( bs->block, bs->id );
       });
    }
@@ -3325,6 +3332,14 @@ namespace eosio {
          dispatcher->strand.post( [this, block]() {
             auto id = block->calculate_id();
             fc_dlog( logger, "signaled pre_accepted_block, blk num = ${num}, id = ${id}", ("num", block->block_num())("id", id) );
+
+            auto blk_trace = fc_create_trace_with_id("Block", id);
+            auto blk_span = fc_create_span(blk_trace, "PreAccepted");
+            fc_add_tag(blk_span, "block_id", id);
+            fc_add_tag(blk_span, "block_num", block->block_num());
+            fc_add_tag(blk_span, "block_time", block->timestamp.to_time_point());
+            fc_add_tag(blk_span, "producer", block->producer.to_string());
+
             dispatcher->bcast_block( block, id );
          });
       }
diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp
index 3278512b4d5..97b8e456ab2 100644
--- a/plugins/state_history_plugin/state_history_plugin.cpp
+++ b/plugins/state_history_plugin/state_history_plugin.cpp
@@ -237,6 +237,19 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
          --current_request->max_messages_in_flight;
          need_to_send_update = current_request->start_block_num <= current &&
                                current_request->start_block_num < current_request->end_block_num;
+
+         std::visit( []( auto&& ptr ) {
+            if( ptr ) {
+               if (fc::logger::get(fc::zipkin_logger_name).is_enabled( ::fc::log_level::debug )) {
+                  auto id = ptr->calculate_id();
+                  auto blk_trace = fc_create_trace_with_id( "Block", id );
+                  auto blk_span = fc_create_span( blk_trace, "SHiP-Send" );
+                  fc_add_tag( blk_span, "block_id", id );
+                  fc_add_tag( blk_span, "block_num", ptr->block_num() );
+                  fc_add_tag( blk_span, "block_time", ptr->timestamp.to_time_point() );
+               }
+            }
+         }, result.block );
       }
 
       void send_update(const block_state_ptr& block_state) {
@@ -352,6 +365,11 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
    }
 
    void on_accepted_block(const block_state_ptr& block_state) {
+      auto blk_trace = fc_create_trace_with_id("Block", block_state->id);
+      auto blk_span = fc_create_span(blk_trace, "SHiP-Accepted");
+      fc_add_tag(blk_span, "block_id", block_state->id);
+      fc_add_tag(blk_span, "block_num", block_state->block_num);
+      fc_add_tag(blk_span, "block_time", block_state->block->timestamp.to_time_point());
       if (trace_log)
          trace_log->store(chain_plug->chain().db(), block_state);
       if (chain_state_log)
diff --git a/programs/nodeos/logging.json b/programs/nodeos/logging.json
index d2da3457ca1..d106677acce 100644
--- a/programs/nodeos/logging.json
+++ b/programs/nodeos/logging.json
@@ -46,6 +46,16 @@
         "host": "host_name"
       },
       "enabled": true
+    },{
+      "name": "zip",
+      "type": "zipkin",
+      "args": {
+        "endpoint": "http://127.0.0.1:9411",
+        "path": "/api/v2/spans",
+        "service_name": "nodeos",
+        "timeout_us": 200000
+      },
+      "enabled": true
     }
   ],
   "loggers": [{
@@ -57,6 +67,14 @@
         "stderr",
         "net"
       ]
+    },{
+      "name": "zipkin",
+      "level": "debug",
+      "enabled": true,
+      "additivity": false,
+      "appenders": [
+        "zip"
+      ]
     },{
       "name": "net_plugin_impl",
       "level": "info",

From 0802379d080aaba47579846e3e971bab0557d758 Mon Sep 17 00:00:00 2001
From: Kevin Heifner <heifnerk@objectcomputing.com>
Date: Tue, 20 Oct 2020 10:03:17 -0500
Subject: [PATCH 2/9] Add zipkin logger to rodeos

---
 programs/rodeos/cloner_plugin.cpp | 19 +++++++++++++++++--
 programs/rodeos/cloner_plugin.hpp |  1 +
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/programs/rodeos/cloner_plugin.cpp b/programs/rodeos/cloner_plugin.cpp
index 09f700dcd49..2d4afa22a54 100644
--- a/programs/rodeos/cloner_plugin.cpp
+++ b/programs/rodeos/cloner_plugin.cpp
@@ -2,6 +2,11 @@
 #include "ship_client.hpp"
 
 #include <b1/rodeos/rodeos.hpp>
+
+#include <fc/log/logger.hpp>
+#include <fc/log/logger_config.hpp>
+#include <fc/log/trace.hpp>
+
 #include <boost/asio/connect.hpp>
 #include <boost/asio/ip/tcp.hpp>
 #include <boost/beast/core.hpp>
@@ -28,6 +33,8 @@ using rodeos::rodeos_filter;
 
 struct cloner_session;
 
+fc::logger _zipkin_logger;
+
 struct cloner_config : ship_client::connection_config {
    uint32_t    skip_to     = 0;
    uint32_t    stop_before = 0;
@@ -247,17 +254,25 @@ void cloner_plugin::plugin_initialize(const variables_map& options) {
    FC_LOG_AND_RETHROW()
 }
 
-void cloner_plugin::plugin_startup() { my->start(); }
+void cloner_plugin::plugin_startup() {
+   handle_sighup();
+   my->start();
+}
 
 void cloner_plugin::plugin_shutdown() {
    if (my->session)
       my->session->connection->close(false);
    my->timer.cancel();
+   fc::log_config::shutdown_appenders();
    ilog("cloner_plugin stopped");
 }
 
 void cloner_plugin::set_streamer(std::function<void(const char* data, uint64_t data_size)> streamer_func) {
-   my->streamer = streamer_func;
+   my->streamer = std::move(streamer_func);
+}
+
+void cloner_plugin::handle_sighup() {
+   fc::logger::update( fc::zipkin_logger_name, _zipkin_logger );
 }
 
 } // namespace b1
diff --git a/programs/rodeos/cloner_plugin.hpp b/programs/rodeos/cloner_plugin.hpp
index fb24b8a1aca..3b2b63b5dce 100644
--- a/programs/rodeos/cloner_plugin.hpp
+++ b/programs/rodeos/cloner_plugin.hpp
@@ -14,6 +14,7 @@ class cloner_plugin : public appbase::plugin<cloner_plugin> {
    void         plugin_initialize(const appbase::variables_map& options);
    void         plugin_startup();
    void         plugin_shutdown();
+   void         handle_sighup() override;
 
    void set_streamer(std::function<void(const char* data, uint64_t data_size)> streamer_function);
 

From 25b409dcfd889171610b3a7d22b9165cefa1de48 Mon Sep 17 00:00:00 2001
From: Kevin Heifner <heifnerk@objectcomputing.com>
Date: Fri, 30 Oct 2020 07:59:51 -0500
Subject: [PATCH 3/9] Moved shutdown to logger

---
 plugins/chain_plugin/chain_plugin.cpp | 2 +-
 programs/rodeos/cloner_plugin.cpp     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp
index 83e2eede207..064afdd8101 100644
--- a/plugins/chain_plugin/chain_plugin.cpp
+++ b/plugins/chain_plugin/chain_plugin.cpp
@@ -1352,7 +1352,7 @@ void chain_plugin::plugin_shutdown() {
    if(app().is_quiting())
       my->chain->get_wasm_interface().indicate_shutting_down();
    my->chain.reset();
-   fc::log_config::shutdown_appenders();
+   _zipkin_logger.shutdown();
 }
 
 void chain_plugin::handle_sighup() {
diff --git a/programs/rodeos/cloner_plugin.cpp b/programs/rodeos/cloner_plugin.cpp
index 2d4afa22a54..c1b6798d543 100644
--- a/programs/rodeos/cloner_plugin.cpp
+++ b/programs/rodeos/cloner_plugin.cpp
@@ -263,7 +263,7 @@ void cloner_plugin::plugin_shutdown() {
    if (my->session)
       my->session->connection->close(false);
    my->timer.cancel();
-   fc::log_config::shutdown_appenders();
+   _zipkin_logger.shutdown();
    ilog("cloner_plugin stopped");
 }
 

From 3272f84c636bb25562563a5a4ff051c4d3b5d11d Mon Sep 17 00:00:00 2001
From: Kevin Heifner <heifnerk@objectcomputing.com>
Date: Tue, 3 Nov 2020 07:39:38 -0600
Subject: [PATCH 4/9] Fix merge issues

---
 libraries/rodeos/rodeos.cpp                           | 8 ++++++++
 plugins/chain_plugin/chain_plugin.cpp                 | 1 +
 plugins/net_plugin/net_plugin.cpp                     | 1 +
 plugins/state_history_plugin/state_history_plugin.cpp | 2 ++
 4 files changed, 12 insertions(+)

diff --git a/libraries/rodeos/rodeos.cpp b/libraries/rodeos/rodeos.cpp
index abddfb6f8fd..d8067bc7517 100644
--- a/libraries/rodeos/rodeos.cpp
+++ b/libraries/rodeos/rodeos.cpp
@@ -2,6 +2,7 @@
 
 #include <b1/rodeos/callbacks/kv.hpp>
 #include <b1/rodeos/rodeos_tables.hpp>
+#include <fc/log/trace.hpp>
 
 namespace b1::rodeos {
 
@@ -162,6 +163,13 @@ void rodeos_db_snapshot::write_block_info(uint32_t block_num, const eosio::check
    table.put(info);
 }
 
+namespace {
+   std::string to_string( const eosio::checksum256& cs ) {
+      auto bytes = cs.extract_as_byte_array();
+      return fc::to_hex((const char*)bytes.data(), bytes.size());
+   }
+}
+
 void rodeos_db_snapshot::write_block_info(const ship_protocol::get_blocks_result_v0& result) {
    check_write(result);
    if (!result.block)
diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp
index 064afdd8101..2c1720e2e36 100644
--- a/plugins/chain_plugin/chain_plugin.cpp
+++ b/plugins/chain_plugin/chain_plugin.cpp
@@ -29,6 +29,7 @@
 
 #include <fc/io/json.hpp>
 #include <fc/variant.hpp>
+#include <fc/log/trace.hpp>
 #include <signal.h>
 #include <cstdlib>
 
diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp
index 0436e1caaf7..c0db39df5a7 100644
--- a/plugins/net_plugin/net_plugin.cpp
+++ b/plugins/net_plugin/net_plugin.cpp
@@ -16,6 +16,7 @@
 #include <fc/io/raw.hpp>
 #include <fc/log/appender.hpp>
 #include <fc/log/logger_config.hpp>
+#include <fc/log/trace.hpp>
 #include <fc/reflect/variant.hpp>
 #include <fc/crypto/rand.hpp>
 #include <fc/exception/exception.hpp>
diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp
index 97b8e456ab2..65f1d28c53c 100644
--- a/plugins/state_history_plugin/state_history_plugin.cpp
+++ b/plugins/state_history_plugin/state_history_plugin.cpp
@@ -4,6 +4,8 @@
 #include <eosio/state_history/serialization.hpp>
 #include <eosio/state_history_plugin/state_history_plugin.hpp>
 
+#include <fc/log/trace.hpp>
+
 #include <boost/asio/bind_executor.hpp>
 #include <boost/asio/ip/host_name.hpp>
 #include <boost/asio/ip/tcp.hpp>

From 4709cba3391f552bc22d0c07e22d1c39bc05aec0 Mon Sep 17 00:00:00 2001
From: Kevin Heifner <heifnerk@objectcomputing.com>
Date: Fri, 6 Nov 2020 08:31:25 -0600
Subject: [PATCH 5/9] zipkin is no longer a logger

---
 plugins/chain_plugin/chain_plugin.cpp           | 17 ++++++++++++++---
 .../state_history_plugin.cpp                    |  2 +-
 programs/rodeos/cloner_plugin.cpp               | 17 +++++++++++++----
 3 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp
index 2c1720e2e36..03643f77582 100644
--- a/plugins/chain_plugin/chain_plugin.cpp
+++ b/plugins/chain_plugin/chain_plugin.cpp
@@ -42,7 +42,6 @@ FC_REFLECT(chainbase::environment, (debug)(os)(arch)(boost_version)(compiler) )
 
 const fc::string deep_mind_logger_name("deep-mind");
 fc::logger _deep_mind_log;
-fc::logger _zipkin_logger;
 
 namespace eosio {
 
@@ -324,6 +323,12 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip
           "print contract's output to console")
          ("deep-mind", bpo::bool_switch()->default_value(false),
           "print deeper information about chain operations")
+         ("telemetry-url", bpo::value<std::string>(),
+          "Send Zipkin spans to url. e.g. http://127.0.0.1:9411/api/v2/spans" )
+         ("telemetry-service-name", bpo::value<std::string>()->default_value("nodeos"),
+          "Zipkin localEndpoint.serviceName sent with each span" )
+         ("telemetry-timeout-us", bpo::value<uint32_t>()->default_value(200000),
+          "Timeout for sending Zipkin span." )
          ("actor-whitelist", boost::program_options::value<vector<string>>()->composing()->multitoken(),
           "Account added to actor whitelist (may specify multiple times)")
          ("actor-blacklist", boost::program_options::value<vector<string>>()->composing()->multitoken(),
@@ -1207,6 +1212,13 @@ void chain_plugin::plugin_initialize(const variables_map& options) {
          my->chain->enable_deep_mind( &_deep_mind_log );
       }
 
+      if (options.count("telemetry-url")) {
+         fc::zipkin_config::init( options["telemetry-url"].as<std::string>(),
+                                  options["telemetry-service-name"].as<std::string>(),
+                                  options["telemetry-timeout-us"].as<uint32_t>() );
+      }
+
+
       // set up method providers
       my->get_block_by_number_provider = app().get_method<methods::get_block_by_number>().register_provider(
             [this]( uint32_t block_num ) -> signed_block_ptr {
@@ -1353,12 +1365,11 @@ void chain_plugin::plugin_shutdown() {
    if(app().is_quiting())
       my->chain->get_wasm_interface().indicate_shutting_down();
    my->chain.reset();
-   _zipkin_logger.shutdown();
+   zipkin_config::shutdown();
 }
 
 void chain_plugin::handle_sighup() {
    fc::logger::update( deep_mind_logger_name, _deep_mind_log );
-   fc::logger::update( fc::zipkin_logger_name, _zipkin_logger );
 }
 
 chain_apis::read_write::read_write(controller& db, const fc::microseconds& abi_serializer_max_time, bool api_accept_transactions)
diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp
index 65f1d28c53c..99bfb7cc398 100644
--- a/plugins/state_history_plugin/state_history_plugin.cpp
+++ b/plugins/state_history_plugin/state_history_plugin.cpp
@@ -242,7 +242,7 @@ struct state_history_plugin_impl : std::enable_shared_from_this<state_history_pl
 
          std::visit( []( auto&& ptr ) {
             if( ptr ) {
-               if (fc::logger::get(fc::zipkin_logger_name).is_enabled( ::fc::log_level::debug )) {
+               if (fc::zipkin_config::is_enabled()) {
                   auto id = ptr->calculate_id();
                   auto blk_trace = fc_create_trace_with_id( "Block", id );
                   auto blk_span = fc_create_span( blk_trace, "SHiP-Send" );
diff --git a/programs/rodeos/cloner_plugin.cpp b/programs/rodeos/cloner_plugin.cpp
index c1b6798d543..af53fe4160d 100644
--- a/programs/rodeos/cloner_plugin.cpp
+++ b/programs/rodeos/cloner_plugin.cpp
@@ -1,5 +1,6 @@
 #include "cloner_plugin.hpp"
 #include "ship_client.hpp"
+#include "config.hpp"
 
 #include <b1/rodeos/rodeos.hpp>
 
@@ -33,8 +34,6 @@ using rodeos::rodeos_filter;
 
 struct cloner_session;
 
-fc::logger _zipkin_logger;
-
 struct cloner_config : ship_client::connection_config {
    uint32_t    skip_to     = 0;
    uint32_t    stop_before = 0;
@@ -226,6 +225,12 @@ void cloner_plugin::set_program_options(options_description& cli, options_descri
    clop("clone-stop,x", bpo::value<uint32_t>(), "Stop before block [arg]");
    op("clone-exit-on-filter-wasm-error", bpo::bool_switch()->default_value(false),
       "Shutdown application if filter wasm throws an exception");
+   op("telemetry-url", bpo::value<std::string>(),
+      "Send Zipkin spans to url. e.g. http://127.0.0.1:9411/api/v2/spans" );
+   op("telemetry-service-name", bpo::value<std::string>()->default_value(b1::rodeos::config::rodeos_executable_name),
+      "Zipkin localEndpoint.serviceName sent with each span" );
+   op("telemetry-timeout-us", bpo::value<uint32_t>()->default_value(200000),
+      "Timeout for sending Zipkin span." );
    // todo: remove
    op("filter-name", bpo::value<std::string>(), "Filter name");
    op("filter-wasm", bpo::value<std::string>(), "Filter wasm");
@@ -250,6 +255,11 @@ void cloner_plugin::plugin_initialize(const variables_map& options) {
       } else if (options.count("filter-name") || options.count("filter-wasm")) {
          throw std::runtime_error("filter-name and filter-wasm must be used together");
       }
+      if (options.count("telemetry-url")) {
+         fc::zipkin_config::init( options["telemetry-url"].as<std::string>(),
+                                  options["telemetry-service-name"].as<std::string>(),
+                                  options["telemetry-timeout-us"].as<uint32_t>() );
+      }
    }
    FC_LOG_AND_RETHROW()
 }
@@ -263,7 +273,7 @@ void cloner_plugin::plugin_shutdown() {
    if (my->session)
       my->session->connection->close(false);
    my->timer.cancel();
-   _zipkin_logger.shutdown();
+   fc::zipkin_config::shutdown();
    ilog("cloner_plugin stopped");
 }
 
@@ -272,7 +282,6 @@ void cloner_plugin::set_streamer(std::function<void(const char* data, uint64_t d
 }
 
 void cloner_plugin::handle_sighup() {
-   fc::logger::update( fc::zipkin_logger_name, _zipkin_logger );
 }
 
 } // namespace b1

From bcbf691d4e51373c5719ff15272c131436d396bc Mon Sep 17 00:00:00 2001
From: Kevin Heifner <heifnerk@objectcomputing.com>
Date: Fri, 6 Nov 2020 08:31:49 -0600
Subject: [PATCH 6/9] Update to fc with latest zipkin

---
 libraries/fc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/fc b/libraries/fc
index e7ad1fb0abd..588f7bdfed5 160000
--- a/libraries/fc
+++ b/libraries/fc
@@ -1 +1 @@
-Subproject commit e7ad1fb0abd4f2e322397fe6db9eb2c61912c3f7
+Subproject commit 588f7bdfed50e5ac9f8f552bdb1a811ad13178c7

From 31b23f5a269e6cccb7c466f13b78cb1ef6774b54 Mon Sep 17 00:00:00 2001
From: Kevin Heifner <heifnerk@objectcomputing.com>
Date: Fri, 6 Nov 2020 10:35:49 -0600
Subject: [PATCH 7/9] Update to fc with fix

---
 libraries/fc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/fc b/libraries/fc
index 588f7bdfed5..40313ca3e3c 160000
--- a/libraries/fc
+++ b/libraries/fc
@@ -1 +1 @@
-Subproject commit 588f7bdfed50e5ac9f8f552bdb1a811ad13178c7
+Subproject commit 40313ca3e3cdabb1d9cac389fbb2288d3fea834c

From cc0ba46f0a2f2a7ae391da937cc141589948bbf1 Mon Sep 17 00:00:00 2001
From: Kevin Heifner <heifnerk@objectcomputing.com>
Date: Mon, 9 Nov 2020 07:26:13 -0600
Subject: [PATCH 8/9] Update to fc with latest

---
 libraries/fc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/fc b/libraries/fc
index 40313ca3e3c..bfed8803e89 160000
--- a/libraries/fc
+++ b/libraries/fc
@@ -1 +1 @@
-Subproject commit 40313ca3e3cdabb1d9cac389fbb2288d3fea834c
+Subproject commit bfed8803e8964c967c7fbf317dc5a8c65bd32ad4

From 74108786c4f55179d6efaa759a36bcb3a858d68c Mon Sep 17 00:00:00 2001
From: Kevin Heifner <heifnerk@objectcomputing.com>
Date: Mon, 9 Nov 2020 14:46:41 -0600
Subject: [PATCH 9/9] Update to eosio fc

---
 libraries/fc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libraries/fc b/libraries/fc
index bfed8803e89..fae2261e794 160000
--- a/libraries/fc
+++ b/libraries/fc
@@ -1 +1 @@
-Subproject commit bfed8803e8964c967c7fbf317dc5a8c65bd32ad4
+Subproject commit fae2261e7944a883607daeb280d92d559b01b984