CAD-2907: add locli

cabal.project

@@ -11,6 +11,7 @@ packages:
     cardano-testnet
     nix/workbench/cardano-topology
     bench/tx-generator
+    nix/workbench/locli
 
 package cardano-api
   ghc-options: -Werror

nix/haskell.nix

@@ -84,7 +84,7 @@ let
       }
       {
         # Stamp executables with the git revision and add shell completion
-        packages = lib.genAttrs ["cardano-node" "cardano-cli" "cardano-topology"] (name: {
+        packages = lib.genAttrs ["cardano-node" "cardano-cli" "cardano-topology" "locli" ] (name: {
           components.exes.${name}.postInstall = ''
             ${lib.optionalString stdenv.hostPlatform.isWindows setLibSodium}
             ${setGitRev}

nix/pkgs.nix

@@ -66,6 +66,7 @@ final: prev: with final;
   inherit (cardanoNodeHaskellPackages.cardano-cli.components.exes) cardano-cli;
   inherit (cardanoNodeHaskellPackages.cardano-topology.components.exes) cardano-topology;
   inherit (cardanoNodeHaskellPackages.tx-generator.components.exes) tx-generator;
+  inherit (cardanoNodeHaskellPackages.locli.components.exes) locli;
   inherit (cardanoNodeHaskellPackages.bech32.components.exes) bech32;
   inherit (cardanoNodeHaskellPackages.cardano-submit-api.components.exes) cardano-submit-api;
   cardano-node-profiled = cardanoNodeProfiledHaskellPackages.cardano-node.components.exes.cardano-node;

nix/supervisord-cluster/default.nix

@@ -148,7 +148,7 @@ let
     wb_run_start_args=(
         --supervisor-conf      "${backend.supervisord.mkSupervisorConf profile}"
       )
-    wb run start $(wb run current-name) "''${wb_run_start_args[@]}"
+    wb run start $(wb run current-tag) "''${wb_run_start_args[@]}"
 
     echo 'workbench:  cluster started. Run `stop-cluster` to stop'
   '';

nix/workbench/analyse.sh

@@ -0,0 +1,105 @@
+usage_analyse() {
+     usage "analyse" "Analyse cluster runs" <<EOF
+    block-propagation RUN-NAME
+                          Block propagation analysis for the entire cluster
+
+    machine-timeline RUN-NAME MACH-NAME
+                          Produce a general performance timeline for MACH-NAME
+
+EOF
+}
+
+analyse() {
+local op=${1:-$(usage_analyse)}; shift
+
+case "$op" in
+    block-propagation | bp )
+        local usage="USAGE: wb analyse $op [RUN-NAME=current]"
+        local name=${1:-current}
+        local dir=$(run get "$name")
+        local adir=$dir/analysis
+
+        mkdir -p "$adir"
+
+        ## 0. subset what we care about into the keyfile
+        local keyfile=$adir/substring-keys
+        locli analyse substring-keys | grep -v 'Temporary modify' > "$keyfile"
+        cat >>"$keyfile" <<EOF
+TraceForgedBlock
+AddedToCurrentChain
+TraceChainSyncServerReadBlocked.AddBlock
+TraceChainSyncServerRead.AddBlock
+TraceBlockFetchServerSendBlock
+TraceDownloadedHeader
+CompletedBlockFetch
+EOF
+        ## 1. enumerate logs, filter by keyfile & consolidate
+        local logdirs=("$dir"/node-*/)
+
+        msg "filtering logs in: $dir/node-* "
+        local jq_args=(
+            --sort-keys
+            --compact-output
+            $(wb backend lostream-fixup-jqargs "$dir")
+            ' delpaths([["app"],["env"],["loc"],["msg"],["ns"],["sev"]])
+            '"$(wb backend lostream-fixup-jqexpr)"
+        )
+        for d in "${logdirs[@]}"
+        do ## TODO: supervisor-specific logfile layout
+           grep -hFf "$keyfile" $(ls "$d"/stdout* | tac) | jq "${jq_args[@]}" > \
+                "$adir"/logs-$(basename "$d").flt.json &
+        done
+        wait
+
+        msg "log sizes:  (files: $(ls "$adir"/*.flt.json | wc -l), lines: $(cat "$adir"/*.flt.json | wc -l))"
+
+        msg "analysing.."
+        local locli_args=(
+            --genesis         "$dir"/genesis/genesis.json
+            --run-metafile    "$dir"/meta.json
+            ## ->
+            # --logobjects-json "$adir"/logs-cluster.logobjects.json
+            --analysis-json   "$adir"/block-event-stream.json
+        )
+
+        locli 'analyse' 'block-propagation' \
+            "${locli_args[@]}" "$adir"/*.flt.json;;
+
+    machine-timeline | machine | mt )
+        local usage="USAGE: wb analyse $op [RUN-NAME=current] [MACH-NAME=node-1]"
+        local name=${1:-current}
+        local mach=${2:-node-1}
+        local dir=$(run get "$name")
+        local adir=$dir/analysis
+
+        mkdir -p "$adir"
+
+        ## 0. subset what we care about into the keyfile
+        local keyfile=$adir/substring-keys
+        locli analyse substring-keys | grep -v 'Temporary modify' > "$keyfile"
+
+        ## 1. enumerate logs, filter by keyfile & consolidate
+        local logs=("$dir"/$mach/stdout) consolidated="$adir"/logs-$mach.json
+        grep -hFf "$keyfile" "${logs[@]}"  > "$consolidated"
+        msg "analysing logs of:  $mach  (lines: $(wc -l "$consolidated"))"
+
+        local locli_args=(
+            --genesis         "$dir"/genesis/genesis.json
+            --run-metafile    "$dir"/meta.json
+            ## ->
+            --logobjects-json "$adir"/logs-$mach.logobjects.json
+            --slotstats-json  "$adir"/logs-$mach.slotstats.json
+            --timeline-pretty "$adir"/logs-$mach.timeline.txt
+            --stats-csv       "$adir"/logs-$mach.stats.csv
+            --analysis-json   "$adir"/logs-$mach.analysis.json
+            # --timeline-csv            "$adir"/logs-$mach.timeline.csv
+            # --cpu-spans-histogram-png "$adir"/logs-"$mach".cpu85-span-lens.png
+            # --derived-vectors-0-csv   "$adir"/logs-$mach".derived.1.csv
+            # --derived-vectors-1-csv   "$adir"/logs-$mach.derived.1.csv
+        )
+
+        locli 'analyse' 'machine-timeline' \
+            "${locli_args[@]}" "$consolidated";;
+
+    * ) usage_analyse;; esac
+}

nix/workbench/backend.sh

@@ -6,12 +6,17 @@ usage_backend() {
                      Given a run directory, print the node socket path
                        for 'cardano-cli'
 
-    start RUNDIR     Start an allocated run
-
     record-extended-env-config ENV-JSON [ENV-CONFIG-OPTS..]
                      Extend the environment JSON file with backend-specific
                        environment config
 
+    describe-run RUNDIR
+    pre-run-hook RUNDIR
+    lostream-fixup-jqargs RUNDIR
+    lostream-fixup-jqexpr
+
+    start-run RUNDIR Start an allocated run
+
     assert-is BACKEND-NAME
                      Check that the current backend is as expected
 
@@ -25,11 +30,15 @@ local op=${1:-$(usage_backend)} # No need to shift -- backends will use the op.
 case "${op}" in
     is-running )                 $WORKBENCH_BACKEND "$@";;
     get-node-socket-path )       $WORKBENCH_BACKEND "$@";;
-    wait-for-local-node-socket ) $WORKBENCH_BACKEND "$@";;
     record-extended-env-config ) $WORKBENCH_BACKEND "$@";;
     describe-run )               $WORKBENCH_BACKEND "$@";;
     pre-run-hook )               $WORKBENCH_BACKEND "$@";;
     start-run )                  $WORKBENCH_BACKEND "$@";;
+    lostream-fixup-jqargs )      $WORKBENCH_BACKEND "$@";;
+    lostream-fixup-jqexpr )      $WORKBENCH_BACKEND "$@";;
+
+    ## Handle non-generic calls:
+    passthrough | pass )         $WORKBENCH_BACKEND "$@";;
 
     assert-is )
         local usage="USAGE: wb run $op BACKEND-NAME"

nix/workbench/default.nix

@@ -10,6 +10,7 @@
 , customConfig
 , cardano-cli
 , cardano-topology
+, locli
 
 , useCabalRun
 }:
@@ -56,6 +57,7 @@ let
 
       cardano-cli
       cardano-topology
+      locli
     ];
 
   runWorkbench =
@@ -120,7 +122,11 @@ let
       ${exeCabalOp "run" "cardano-topology"} "$@"
     }
 
-    export -f cardano-cli cardano-node cardano-topology
+    function locli() {
+      ${exeCabalOp "run" "locli"} "$@"
+    }
+
+    export -f cardano-cli cardano-node cardano-topology locli
 
     ''}
 
@@ -129,7 +135,7 @@ let
         ''
       git log -n1 --alternate-refs --pretty=format:"%Cblue%h %Cred%cr %Cgreen%D %Cblue%s%Creset"
       echo -n "workbench:  prebuilding executables (because of useCabalRun):"
-      for exe in cardano-cli cardano-node cardano-topology
+      for exe in cardano-cli cardano-node cardano-topology locli
       do echo -n " $exe"
          ${exeCabalOp "run" "$exe"} --help >/dev/null || return 1
       done

nix/workbench/lib.sh

@@ -52,3 +52,16 @@ fatal() {
 jqtest() {
     jq --exit-status "$@" > /dev/null
 }
+
+git_repo_commit_description() {
+    local repo=$1
+    local commit=$(git -C "$repo" rev-parse 'HEAD' 2>/dev/null || true)
+
+    test -d "$repo/.git" && {
+        git -C "$repo" fetch
+        git -C "$repo" describe --match '[0-9].*' --tags $commit 2>/dev/null |
+            cut -d'-' -f1,2 | tr -d '\n'
+        git -C "$repo" diff --exit-code --quiet || echo '-modified'
+    } || true
+}
+

nix/workbench/locli/NOTICE

@@ -0,0 +1,13 @@
+Copyright 2020 Input Output (Hong Kong) Ltd.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

nix/workbench/locli/app/locli.hs

@@ -0,0 +1,18 @@
+{-# LANGUAGE OverloadedStrings #-}
+
+import           Cardano.Prelude hiding (option)
+
+import           Control.Monad.Trans.Except.Exit (orDie)
+import qualified Options.Applicative as Opt
+
+import           Cardano.Analysis.TopHandler
+import           Cardano.Unlog.Parsers (opts, pref)
+import           Cardano.Unlog.Run (renderCommandError, runCommand)
+
+
+main :: IO ()
+main = toplevelExceptionHandler $ do
+
+  co <- Opt.customExecParser pref opts
+
+  orDie renderCommandError $ runCommand co

nix/workbench/locli/locli.cabal

@@ -0,0 +1,96 @@
+cabal-version: 2.4
+
+name:                  locli
+version:               1.27
+description:           Cardano log analysis CLI.
+author:                IOHK
+maintainer:            operations@iohk.io
+license:               Apache-2.0
+license-files:
+  NOTICE
+build-type:            Simple
+
+library
+
+  hs-source-dirs:      src
+
+  exposed-modules:     Data.Accum
+                       Data.Distribution
+
+                       Cardano.Analysis.Profile
+                       Cardano.Analysis.TopHandler
+
+                       Cardano.Unlog.BlockProp
+                       Cardano.Unlog.Commands
+                       Cardano.Unlog.LogObject
+                       Cardano.Unlog.Parsers
+                       Cardano.Unlog.Resources
+                       Cardano.Unlog.Run
+                       Cardano.Unlog.SlotStats
+                       Cardano.Unlog.Summary
+                       Cardano.Unlog.Timeline
+
+  other-modules:       Paths_locli
+
+  build-depends:       base
+                     , aeson
+                     , aeson-pretty
+                     , async
+                     , attoparsec
+                     , attoparsec-iso8601
+                     , bytestring
+                     , cardano-config
+                     , cardano-prelude
+                     , containers
+                     , directory
+                     , filepath
+                     , file-embed
+                     , gnuplot
+                     , Histogram
+                     , iohk-monitoring
+                     , optparse-applicative
+                     , optparse-generic
+                     , ouroboros-network
+                     , process
+                     , scientific
+                     , split
+                     , template-haskell
+                     , text
+                     , text-short
+                     , time
+                     , transformers
+                     , transformers-except
+                     , unordered-containers
+                     , utf8-string
+                     , vector
+
+  default-language:    Haskell2010
+  default-extensions:  NoImplicitPrelude
+                       OverloadedStrings
+                       TupleSections
+
+  ghc-options:         -Wall
+                       -Wincomplete-record-updates
+                       -Wincomplete-uni-patterns
+                       -Wredundant-constraints
+                       -Wpartial-fields
+                       -Wcompat
+                       -Wno-all-missed-specialisations
+
+executable locli
+  hs-source-dirs:      app
+  main-is:             locli.hs
+  default-language:    Haskell2010
+  ghc-options:         -threaded
+                       -Wall
+                       -rtsopts
+                       "-with-rtsopts=-T"
+  build-depends:       base
+                     , cardano-prelude
+                     , locli
+                     , optparse-applicative
+                     , text
+                     , text-short
+                     , transformers
+                     , transformers-except
+  default-extensions:  NoImplicitPrelude