Bugfixes: Handle empty timelines (#3456)

Also:

* fixed Demo.Plugins.GUI artifact as monitoring() plugin now requires
client id parameter.
* removed dead code in expand.go
* keep building the initial notebook even if some of the cells failed.
Otherwise the notebook will be half built and subsequent cells are lost.
* Updated docs and config file references.

artifacts/definitions/Demo/Plugins/GUI.yaml

@@ -265,6 +265,7 @@ sources:
           {{ define "Q" }}
             SELECT _ts, CPUPercent
             FROM monitoring(
+                  client_id="server",
                   artifact="Server.Monitor.Health/Prometheus",
                   start_time=now() - 10 * 60)
             LIMIT 100
@@ -283,6 +284,7 @@ sources:
           */
           SELECT timestamp(epoch=_ts) AS Timestamp, CPUPercent
           FROM monitoring(
+            client_id="server",
             source="Prometheus",
             artifact="Server.Monitor.Health",
             start_time=now() - 10 * 60)
@@ -291,6 +293,7 @@ sources:
                timestamp(epoch=_ts) AS Timestamp,
                dict(X=CPUPercent, Y=1) AS Dict
           FROM monitoring(
+            client_id="server",
             source="Prometheus",
             artifact="Server.Monitor.Health",
             start_time=now() - 10 * 60)

artifacts/definitions/Windows/Packs/Persistence.yaml

@@ -7,26 +7,13 @@ precondition:
 
 sources:
   - name: WMI Event Filters
-    description: |
-      {{ DocFrom "Windows.Persistence.PermanentWMIEvents" }}
-
     query: |
         SELECT * FROM Artifact.Windows.Persistence.PermanentWMIEvents()
 
   - name: Startup Items
-    description: |
-      {{ DocFrom "Windows.Sys.StartupItems" }}
-
     query: |
         SELECT * FROM Artifact.Windows.Sys.StartupItems()
 
   - name: Debug Bootstraping
-    description: |
-      {{ DocFrom "Windows.Persistence.Debug" }}
-
-      If there are any rows in the table below then executing the
-      program will also launch the program listed under the Debugger
-      column.
-
     query: |
       SELECT * FROM Artifact.Windows.Persistence.Debug()

bin/fuse_unix.go

@@ -1,5 +1,5 @@
-//go:build !windows
-// +build !windows
+//go:build !windows && !freebsd
+// +build !windows,!freebsd
 
 package main
 

docs/references/server.config.yaml

@@ -304,6 +304,19 @@ Client:
     - Label1
     - Label1
 
+  ## The client normally does not write any logs on the
+  ## endpoint. However this makes it hard to debug any issues so you
+  ## can choose to have the client write its logs in a file on the
+  ## endpoint. The file will be written in an encrypted form which can
+  ## only be decrypted by the Generic.Client.LocalLogsRetrieve
+  ## artifact.
+
+  ## Path relative to the relevant tmpdir above where client side logs
+  ## are kept. These logs are encrypted client side and need to be
+  ## decrypted on the server to read.
+  logfile_name: "logfile.log"
+  logfile_size: 10000000
+
   ## Velociraptor keeps a local buffer file to store query results
   ## while they are being shipped across the network. There are two
   ## types of buffers - an in memory buffer and a local file based
@@ -1158,6 +1171,10 @@ defaults:
   # Minion.notebook_number_of_local_workers to 5.
   notebook_number_of_local_workers: 5
 
+  # Wait this long for a worker to become available before giving
+  # up. The default is 10 seconds.
+  notebook_wait_time_for_worker_ms: 10000
+
   # The default priority of notebook processors (Higher priority will
   # receive jobs over lower priority).
   notebook_worker_priority: 10
@@ -1254,6 +1271,12 @@ defaults:
   # Maximum length of the line that will be parsed (16kb)
   watch_plugin_buffer_size: 16384
 
+  # Period in seconds when to produce a backup. Velociraptor will
+  # generate a backup of important metadata about the server. By
+  # default this happens daily but you can change it here (set to -1)
+  # to disable backups.
+  backup_period_seconds: 86400
+
 # The Velociraptor server may be placed into "lockdown" mode. While in
 # lockdown mode certain permissions are denied - even for
 # administrators. This additional protection mode helps to mitigate

docs/references/vql.yaml

@@ -3696,15 +3696,10 @@
   - name: client_id
     type: string
     description: The client id to extract
-  - name: flow_id
-    type: string
-    description: A flow ID (client or server artifacts)
-  - name: hunt_id
-    type: string
-    description: Retrieve sources from this hunt (combines all results from all clients)
+    required: true
   - name: artifact
     type: string
-    description: The name of the artifact collection to fetch
+    description: The name of the event artifact to read
   - name: source
     type: string
     description: An optional named source within the artifact
@@ -3714,29 +3709,12 @@
   - name: end_time
     type: Any
     description: Stop end events reach this time (event sources).
-  - name: notebook_id
-    type: string
-    description: The notebook to read from (should also include cell id)
-  - name: notebook_cell_id
-    type: string
-    description: The notebook cell read from (should also include notebook id)
-  - name: notebook_cell_version
-    type: string
-    description: The notebook cell version to read from (should also include notebook
-      id and notebook cell)
-  - name: notebook_cell_table
-    type: int64
-    description: A notebook cell can have multiple tables.)
   - name: start_row
     type: int64
     description: Start reading the result set from this row
   - name: count
     type: int64
     description: Maximum number of clients to fetch (default unlimited)'
-  - name: orgs
-    type: string
-    description: Run the query over these orgs. If empty use the current org.'
-    repeated: true
   category: server
   metadata:
     permissions: READ_RESULTS
@@ -5131,7 +5109,6 @@
 
     This plugin is useful for evaluating a query in a different
     environment or context, or turning a string into a query.
-
   type: Plugin
   args:
   - name: query
@@ -7068,6 +7045,7 @@
   - name: artifact
     type: string
     description: The artifact to watch
+    required: true
   category: event
   metadata:
     permissions: READ_RESULTS
@@ -7378,3 +7356,4 @@
   category: plugin
   metadata:
     permissions: FILESYSTEM_READ
+

docs/wix/velociraptor_amd64.xml

@@ -3,7 +3,7 @@
 <?define PackageDescription="Velociraptor Service Installer" ?>
 <?define Manufacturer="Velocidex" ?>
 <?define Name="Velociraptor" ?>
-<?define Version="0.68.1" ?>
+<?define Version="0.72.3" ?>
 <?define BinaryName="Velociraptor.exe" ?>
 
 <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi"

docs/wix/velociraptor_x86.xml

@@ -3,7 +3,7 @@
 <?define PackageDescription="Velociraptor Service Installer" ?>
 <?define Manufacturer="Velocidex" ?>
 <?define Name="Velociraptor" ?>
-<?define Version="0.68.1" ?>
+<?define Version="0.72.3" ?>
 <?define BinaryName="Velociraptor.exe" ?>
 
 <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi"

reporting/expand.go

@@ -3,20 +3,10 @@
 package reporting
 
 import (
-	"bytes"
 	"context"
-	"fmt"
 	"io"
-	"log"
-	"os"
 
-	"text/template"
-
-	"github.com/Velocidex/ordereddict"
 	"github.com/olekukonko/tablewriter"
-	"www.velocidex.com/golang/velociraptor/actions"
-	config_proto "www.velocidex.com/golang/velociraptor/config/proto"
-	"www.velocidex.com/golang/velociraptor/services"
 	"www.velocidex.com/golang/velociraptor/utils"
 	"www.velocidex.com/golang/vfilter"
 )
@@ -55,87 +45,3 @@ func EvalQueryToTable(ctx context.Context,
 
 	return table
 }
-
-type Expansions struct {
-	config_obj *config_proto.Config
-	rows       []vfilter.Row
-	scope      vfilter.Scope
-}
-
-// Support a number of expansions in description strings.
-func FormatDescription(
-	ctx context.Context,
-	config_obj *config_proto.Config,
-	description string,
-	rows []vfilter.Row) string {
-
-	expansion := &Expansions{
-		config_obj: config_obj,
-		rows:       rows,
-	}
-
-	tmpl, err := template.New("description").Funcs(
-		template.FuncMap{
-			"DocFrom": expansion.DocFrom,
-			"Query":   expansion.Query,
-		}).Parse(description)
-	if err != nil {
-		return description
-	}
-
-	buffer := &bytes.Buffer{}
-
-	err = tmpl.Execute(buffer, expansion)
-	if err != nil {
-		return description
-	}
-
-	return buffer.String()
-}
-
-func (self *Expansions) DocFrom(
-	ctx context.Context, artifact string) string {
-	manager, err := services.GetRepositoryManager(self.config_obj)
-	if err != nil {
-		return ""
-	}
-
-	repository, err := manager.GetGlobalRepository(self.config_obj)
-	if err != nil {
-		return ""
-	}
-
-	artifact_definition, pres := repository.Get(ctx, self.config_obj, artifact)
-	if !pres {
-		return ""
-	}
-
-	return artifact_definition.Description
-}
-
-func (self *Expansions) Query(queries ...string) string {
-	result := &bytes.Buffer{}
-
-	env := ordereddict.NewDict().Set("Rows", self.rows)
-	scope := self.scope.Copy().AppendVars(env)
-
-	defer scope.Close()
-
-	scope.SetLogger(log.New(os.Stderr, "", 0))
-
-	for _, query := range queries {
-		query_log := actions.QueryLog.AddQuery(query)
-		vql, err := vfilter.Parse(query)
-		if err != nil {
-			return fmt.Sprintf("Error: %v", err)
-		}
-		ctx, cancel := context.WithCancel(context.Background())
-		defer cancel()
-
-		table := EvalQueryToTable(ctx, scope, vql, result)
-		table.Render()
-		query_log.Close()
-	}
-
-	return result.String()
-}

services/notebook/initial.go

@@ -254,6 +254,8 @@ func (self *NotebookManager) CreateInitialNotebook(ctx context.Context,
 			notebook_metadata.CellMetadata, cell_metadata)
 	}
 
+	var final_err error
+
 	// When we create the notebook we need to wait for all the cells
 	// to be calculated otherwise we will overwhelm the workers.
 	for _, cell_req := range new_cell_requests {
@@ -265,11 +267,16 @@ func (self *NotebookManager) CreateInitialNotebook(ctx context.Context,
 		_, err = self.UpdateNotebookCell(
 			ctx, notebook_metadata, principal, cell_req)
 		if err != nil {
-			return err
+			// We failed to create this cell but we should not stop
+			// because this will ignore the next cells. Keep going
+			// anyway otherwise the next cells will be lost.
+			if final_err == nil {
+				final_err = err
+			}
 		}
 	}
 
-	return err
+	return final_err
 }
 
 func getCellsForEvents(ctx context.Context,

timelines/reader.go

@@ -55,7 +55,11 @@ func (self *TimelineReader) getIndex(i int) (*IndexRecord, error) {
 
 func (self *TimelineReader) Stat() *timelines_proto.Timeline {
 	first_record, _ := self.getIndex(0)
-	last_record, _ := self.getIndex(int(self.index_stat.Size()/IndexRecordSize - 1))
+	last_record := first_record
+	last_idx := int(self.index_stat.Size()/IndexRecordSize - 1)
+	if last_idx >= 0 {
+		last_record, _ = self.getIndex(last_idx)
+	}
 
 	if first_record == nil || last_record == nil {
 		return &timelines_proto.Timeline{Id: self.id}

utils/errors.go

@@ -18,6 +18,7 @@ var (
 	InvalidConfigError  = errors.New("InvalidConfigError")
 	NotFoundError       = Wrap(os.ErrNotExist, "NotFoundError")
 	InvalidArgError     = errors.New("InvalidArgError")
+	IOError             = errors.New("IOError")
 )
 
 // This is a custom error type that wraps an inner error but does not

utils/read_seek_reader_adapter.go

@@ -33,6 +33,10 @@ func (self ReadSeekReaderAdapter) Close() error {
 }
 
 func (self *ReadSeekReaderAdapter) Read(buf []byte) (int, error) {
+	if self.offset < 0 {
+		return 0, IOError
+	}
+
 	n, err := self.reader.ReadAt(buf, self.offset)
 	self.offset += int64(n)
 	return n, err

vql/server/flows/monitoring.go

@@ -38,7 +38,7 @@ import (
 type MonitoringPluginArgs struct {
 	ClientId string `vfilter:"required,field=client_id,doc=The client id to extract"`
 
-	Artifact string `vfilter:"optional,field=artifact,doc=The name of the event artifact to read"`
+	Artifact string `vfilter:"required,field=artifact,doc=The name of the event artifact to read"`
 	Source   string `vfilter:"optional,field=source,doc=An optional named source within the artifact"`
 
 	StartTime vfilter.Any `vfilter:"optional,field=start_time,doc=Start return events from this date (for event sources)"`