Add the 'prometheus_textfile` option.

- When a file is passed, netbackup will generate a node-exporter
  compatible textfile. This allows easy reporting and alerting
  of backup conditions.

README.md

@@ -302,7 +302,7 @@ exclude = [
 ]
 ```
 
-This will use rsync to *only* copy the contents of the directories above. Notice the use of "***" for rsync.
+This will use rsync to *only* copy the contents of the directories above. Notice the use of `***` for rsync.
 Without this, we'd need to explicitly include the full path to the last directory element. See the rsync(1) manpage for further details.
 
 Another example using rclone:
@@ -329,6 +329,27 @@ The directory where netbackup will save the command output. The files are named
 
 Override the automatic filename generation and logging directory. Netbackup will send output directly into this file.
 
+### prometheus_textfile (string)
+
+If set, `netbackup` will generate node-exporter textfile compatible metrics in this file.
+The time series looks like:
+
+```
+backup{name="backupname", job="netbackup", status="success"} <unix_timestamp>
+```
+
+There are some important points to note:
+
+1. You must enable the `textfile` exporter in your `node-exporter` ([documentation](https://github.com/prometheus/node_exporter)).
+   This is usually the default.
+2. The file *must* be in the directory used by `node-exporter` to import textfiles. Under Debian, this is
+   `/var/lib/prometheus/node-exporter`. Other distributions may use a different directory. Check the
+   node-exporter documentation to figure out the correct directory.
+3. The file *must* end in `.prom` and contain the full path. E.g: `/var/lib/prometheus/node-exporter/netbackup.prom`.
+4. To alert on missed backups, create a prometheus alert that fires when the current timestamp minus the
+   timestamp in the timeseries is over the desired threshold (in seconds). I may add an example of this here
+   in the future.
+
 ## Suggestions and bug reports
 
 Feel free to open bug reports or suggest features in the [Issues](https://github.com/marcopaganini/netbackup/issues) page. PRs are always welcome, but please discuss your feature/bugfix first by creating an issue.

config/config.go

@@ -39,6 +39,7 @@ type Config struct {
 	LogDir             string   `toml:"log_dir"`
 	Logfile            string   `toml:"log_file"`
 	CustomBin          string   `toml:"custom_bin"`
+	PromTextFile       string   `toml:"prometheus_textfile"`
 	// LUKS specific options
 	LuksDestDev string `toml:"luks_dest_dev"`
 	LuksKeyFile string `toml:"luks_keyfile"`
@@ -112,7 +113,7 @@ func ParseConfig(r io.Reader) (*Config, error) {
 		return nil, fmt.Errorf("dest_dev must be an absolute path")
 	case config.LuksDestDev != "" && !strings.HasPrefix(config.LuksDestDev, "/"):
 		return nil, fmt.Errorf("dest_luks_dev must be an absolute path")
-	// Specific checks
+	// Specific checks.
 	case config.LuksDestDev != "" && config.LuksKeyFile == "":
 		return nil, fmt.Errorf("dest_luks_dev requires luks_key_file")
 	}

main.go

@@ -189,5 +189,13 @@ func main() {
 	if err = b.Run(ctx); err != nil {
 		log.Fatalln(err)
 	}
+	// Save node (prometheus) compatible textfile, if requested.
+	if config.PromTextFile != "" {
+		log.Verbosef(1, "Writing node-exporter (prometheus) textfile to: %s\n", config.PromTextFile)
+		if err := writeNodeTextFile(config.PromTextFile, config.Name); err != nil {
+			log.Verbosef(1, "Warning: Unable to write node (prometheus) textfile: %v\n", err)
+		}
+	}
+
 	log.Verboseln(1, "*** Backup Result: Success")
 }

prom.go

@@ -0,0 +1,107 @@
+// netbackup - Consistent multi-method backup tool
+//
+// See instructions in the README.md file that accompanies this program.
+//
+// (C) 2015-2024 by Marco Paganini <paganini AT paganini DOT net>
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"regexp"
+	"syscall"
+	"time"
+)
+
+// writeNodeTextFile writes a record in a prometheus node-exporter
+// compatible "textfile" format. The record is formatted as:
+//
+// backup{name="foobar", job="netbackup", status="success"} <timestamp>
+//
+// Existing lines with the same format and name will be overwritten.
+// All other lines will remain intact.
+//
+// The function employs FLock() on a separate lockfile to prevent race
+// conditions when modifying to the original file. All writes go into a
+// temporary file that is atomically renamed to the final name once work is
+// done.
+func writeNodeTextFile(filename string, name string) error {
+	lockfile := filename + ".lock"
+	lock, err := os.Create(lockfile)
+	if err != nil {
+		return err
+	}
+	defer lock.Close()
+
+	if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
+		return err
+	}
+	defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN)
+
+	// Read contents from original filename.
+	file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0755)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	data, err := io.ReadAll(file)
+	if err != nil {
+		return err
+	}
+
+	// Rebuild output without any previous lines with the same name
+	// and the new line added with the current unix timestamp.
+	matchname, err := regexp.Compile(`backup[\s]*{.*name="` + name + `".*`)
+	if err != nil {
+		return err
+	}
+
+	output := []byte{}
+	for _, line := range bytes.Split(data, []byte("\n")) {
+		// See https://github.com/golang/go/issues/35130
+		// To understand why this BS is needed here.
+		if len(line) == 0 {
+			continue
+		}
+		// Don't copy our own lines.
+		if matchname.Match(line) {
+			continue
+		}
+		output = append(output, line...)
+		output = append(output, byte('\n'))
+	}
+	// Add our line.
+	now := time.Now().Unix()
+	s := fmt.Sprintf("backup{name=%q, job=\"netbackup\", status=\"success\"} %d\n", name, now)
+	output = append(output, []byte(s)...)
+
+	// Write to temporary file and rename it to the original file name.
+	dirname, fname := filepath.Split(filename)
+	if dirname == "" {
+		dirname = "./"
+	}
+
+	temp, err := os.CreateTemp(dirname, fname)
+	if err != nil {
+		return err
+	}
+	defer os.Remove(temp.Name())
+	defer temp.Close()
+
+	_, err = temp.Write(output)
+	if err != nil {
+		return err
+	}
+	temp.Close()
+
+	if err := os.Rename(temp.Name(), filename); err != nil {
+		return err
+	}
+
+	return nil
+}

prom_test.go

@@ -0,0 +1,107 @@
+// netbackup - Consistent multi-method backup tool
+//
+// See instructions in the README.md file that accompanies this program.
+//
+// (C) 2015-2024 by Marco Paganini <paganini AT paganini DOT net>
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+// Number of records to create/test.
+const numRecords = 20
+
+// generate creates multiple node compatible backup records in parallel.
+func generate(tmpfile string, ch chan error) {
+	// Generate multiple backup records.
+	for i := 0; i < numRecords; i++ {
+		go func(ch chan error, name string) {
+			err := writeNodeTextFile(tmpfile, name)
+			ch <- err
+		}(ch, fmt.Sprintf("backup%03.3d", i))
+	}
+}
+
+// errcheck returns the first error found in a slice of error channels.
+func errcheck(ch chan error) error {
+	var saved error
+	for i := 0; i < numRecords; i++ {
+		err := <-ch
+		if err != nil && saved != nil {
+			saved = err
+		}
+	}
+	return saved
+}
+
+// filecheck parses the generated file and makes sure we have exactly
+// numRecords properly formatted records.
+func filecheck(t *testing.T, tmpfile string) error {
+	data, err := os.ReadFile(tmpfile)
+	if err != nil {
+		return err
+	}
+	lines := bytes.Split(data, []byte("\n"))
+
+	t.Log("Generated file contents")
+	for i, v := range lines {
+		t.Logf("%d: %s\n", i, v)
+	}
+
+	// Make sure we have exactly numRecords lines.
+	numlines := len(lines) - 1 // Skip the last blank line caused by a newline.
+	if numlines != numRecords {
+		return fmt.Errorf("number of lines mismatch: expected %d, found %d", numRecords, numlines)
+	}
+
+	// Fill in the "names" map with all names found in the file.
+	re := regexp.MustCompile(`backup[\s]*{name="([^"]*)", job="netbackup", status="success"} [0-9]+`)
+	names := map[string]bool{}
+	for _, line := range lines {
+		// Skip blank line at the end.
+		if len(line) == 0 {
+			continue
+		}
+		matches := re.FindSubmatch(line)
+		if matches != nil {
+			names[string(matches[1])] = true
+		}
+	}
+
+	// Make sure all names are present.
+	missing := []string{}
+	for i := 0; i < numRecords; i++ {
+		name := fmt.Sprintf("backup%03.3d", i)
+		_, ok := names[name]
+		if !ok {
+			missing = append(missing, name)
+		}
+	}
+	if len(missing) > 0 {
+		return fmt.Errorf("missing backup names in output: %s", strings.Join(missing, ", "))
+	}
+
+	return nil
+}
+
+func TestMulti(t *testing.T) {
+	ch := make(chan error, numRecords)
+
+	tmpfile := filepath.Join(t.TempDir(), "testfile")
+	generate(tmpfile, ch)
+
+	if err := errcheck(ch); err != nil {
+		t.Errorf("TestMulti: error writing textfile: %v", err)
+	}
+	if err := filecheck(t, tmpfile); err != nil {
+		t.Errorf("TestMulti: file contents error: %v", err)
+	}
+}