From 0a8832e5eae5d2323ffb7c1afd9d849c9e606331 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Thu, 25 Feb 2016 13:55:02 +0100
Subject: [PATCH 01/24] Add a collector for ZFS, currently focussed on ARC
 stats.

It is tested on FreeBSD 10.2-RELEASE and Linux (ZFS on Linux 0.6.5.4).

On FreeBSD, Solaris, etc. ZFS metrics are exposed through sysctls.
ZFS on Linux exposes the same metrics through procfs `/proc/spl/...`.

In addition to sysctl metrics, 'computed metrics' are exposed by
the collector, which are based on several sysctl values.
There is some conditional logic involved in computing these metrics
which cannot be easily mapped to PromQL.

Not all 92 ARC sysctls are exposed right now but this can be changed
with one additional LOC each.
---
 collector/zfs.go         | 179 +++++++++++++++++++++++++++++++++++++++
 collector/zfs_freebsd.go |  56 ++++++++++++
 collector/zfs_linux.go   |  83 ++++++++++++++++++
 3 files changed, 318 insertions(+)
 create mode 100644 collector/zfs.go
 create mode 100644 collector/zfs_freebsd.go
 create mode 100644 collector/zfs_linux.go

diff --git a/collector/zfs.go b/collector/zfs.go
new file mode 100644
index 0000000000..c4314485f8
--- /dev/null
+++ b/collector/zfs.go
@@ -0,0 +1,179 @@
+package collector
+
+// +build !nofilesystem
+
+import (
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/log"
+)
+
+type zfsSysctl string
+type zfsComputedMetric string
+type zfsMetricValue int
+
+const zfsErrorValue = zfsMetricValue(-1)
+
+type zfsMetricProvider struct {
+	fetchedResults  map[zfsSysctl]zfsMetricValue
+	computedResults map[zfsComputedMetric]zfsMetricValue
+}
+
+type zfsCollector struct {
+	metricProvider  *zfsMetricProvider
+	fetchedMetrics  map[zfsSysctl]prometheus.Gauge
+	computedMetrics map[zfsComputedMetric]prometheus.Gauge
+}
+
+//------------------------------------------------------------------------------
+//                                 Collector
+//------------------------------------------------------------------------------
+
+func init() {
+	Factories["zfs"] = NewZFSCollector
+}
+
+const zfsArcSubsystemName = "zfs_arc"
+
+func NewZFSCollector() (Collector, error) {
+	if err := zfsInitialize(); err != nil {
+		return &zfsCollector{}, err
+	}
+
+	fetchedMetrics := make(map[zfsSysctl]prometheus.Gauge)
+
+	insertGauge := func(sysctl, name string) {
+		fetchedMetrics[zfsSysctl(sysctl)] = prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Namespace: Namespace,
+				Subsystem: zfsArcSubsystemName,
+				Name:      name,
+				Help:      sysctl,
+			},
+		)
+	}
+
+	insertGauge("kstat.zfs.misc.arcstats.p", "mru_size")
+	insertGauge("kstat.zfs.misc.arcstats.size", "size")
+	insertGauge("kstat.zfs.misc.arcstats.c_min", "target_min_size")
+	insertGauge("kstat.zfs.misc.arcstats.c", "target_size")
+	insertGauge("kstat.zfs.misc.arcstats.c_max", "target_max_size")
+	insertGauge("kstat.zfs.misc.arcstats.hits", "hits")
+	insertGauge("kstat.zfs.misc.arcstats.misses", "misses")
+
+	return &zfsCollector{
+		metricProvider: NewZfsMetricProvider(),
+		fetchedMetrics: fetchedMetrics,
+		computedMetrics: map[zfsComputedMetric]prometheus.Gauge{
+			zfsComputedMetric("computed.mfu_size"): prometheus.NewGauge(
+				prometheus.GaugeOpts{
+					Namespace: Namespace,
+					Subsystem: zfsArcSubsystemName,
+					Name:      "mfu_size",
+					Help:      "mfu_size",
+				},
+			),
+		},
+	}, nil
+}
+
+func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
+
+	log.Debug("Preparing metrics update")
+	if err := c.metricProvider.PrepareUpdate(); err != nil {
+		return err
+	}
+	defer c.metricProvider.InvalidateCache()
+
+	log.Debugf("Fetching %d metrics", len(c.fetchedMetrics))
+	for metric := range c.fetchedMetrics {
+
+		value, err := c.metricProvider.GetFetchedMetric(metric)
+		if err != nil {
+			return err
+		}
+		c.fetchedMetrics[metric].Set(float64(value))
+		c.fetchedMetrics[metric].Collect(ch)
+
+	}
+
+	log.Debugf("Computing %d metrics", len(c.computedMetrics))
+	for metric := range c.computedMetrics {
+
+		value, err := c.metricProvider.GetComputedMetric(metric)
+		if err != nil {
+			return err
+		}
+		c.computedMetrics[metric].Set(float64(value))
+		c.computedMetrics[metric].Collect(ch)
+
+	}
+
+	return err
+}
+
+//------------------------------------------------------------------------------
+//                                 Metrics Provider
+// Platform-dependend parts implemented in zfs_${platform} files.
+//------------------------------------------------------------------------------
+
+func NewZfsMetricProvider() *zfsMetricProvider {
+	return &zfsMetricProvider{
+		fetchedResults:  make(map[zfsSysctl]zfsMetricValue),
+		computedResults: make(map[zfsComputedMetric]zfsMetricValue),
+	}
+
+}
+
+func (p *zfsMetricProvider) InvalidateCache() {
+	p.fetchedResults = make(map[zfsSysctl]zfsMetricValue)
+	p.computedResults = make(map[zfsComputedMetric]zfsMetricValue)
+}
+
+func (p *zfsMetricProvider) GetFetchedMetric(s zfsSysctl) (value zfsMetricValue, err error) {
+
+	var ok bool
+	value = zfsErrorValue
+
+	value, ok = p.fetchedResults[s]
+	if !ok {
+		value, err = p.handleFetchedMetricCacheMiss(s)
+		if err != nil {
+			return value, err
+		}
+	}
+
+	return value, err
+}
+
+func (p *zfsMetricProvider) GetComputedMetric(c zfsComputedMetric) (value zfsMetricValue, err error) {
+
+	if c == "computed.mfu_size" {
+
+		value = zfsErrorValue
+
+		arc_size, err := p.GetFetchedMetric(zfsSysctl("kstat.zfs.misc.arcstats.size"))
+		if err != nil {
+			return value, err
+		}
+		target_size, err := p.GetFetchedMetric(zfsSysctl("kstat.zfs.misc.arcstats.c"))
+		if err != nil {
+			return value, err
+		}
+		mru_size, err := p.GetFetchedMetric(zfsSysctl("kstat.zfs.misc.arcstats.p"))
+		if err != nil {
+			return value, err
+		}
+
+		// See zfs-stats implementation
+		if arc_size > target_size {
+			return arc_size - mru_size, nil
+		} else {
+			return target_size - mru_size, nil
+		}
+
+	}
+
+	return zfsErrorValue, fmt.Errorf("no implementation for computed metric '%s'", c)
+}
diff --git a/collector/zfs_freebsd.go b/collector/zfs_freebsd.go
new file mode 100644
index 0000000000..d65dfc3b9b
--- /dev/null
+++ b/collector/zfs_freebsd.go
@@ -0,0 +1,56 @@
+package collector
+
+// +build freebsd
+
+import (
+	"fmt"
+	"unsafe"
+)
+
+/*
+#cgo LDFLAGS:
+#include <fcntl.h>
+#include <stdlib.h>
+#include <sys/param.h>
+#include <sys/pcpu.h>
+#include <sys/resource.h>
+#include <sys/sysctl.h>
+#include <sys/time.h>
+
+int zfsIntegerSysctl(const char *name) {
+
+	int value;
+	size_t value_size = sizeof(value);
+	if (sysctlbyname(name,  &value, &value_size, NULL, 0) != -1 ||
+	    value_size != sizeof(value)) {
+		return -1;
+	}
+	return value;
+
+}
+
+*/
+import "C"
+
+func zfsInitialize() error {
+	return nil
+}
+
+func (c *zfsMetricProvider) PrepareUpdate() error {
+	return nil
+}
+
+func (p *zfsMetricProvider) handleFetchedMetricCacheMiss(c zfsSysctl) (zfsMetricValue, error) {
+
+	sysctlCString := C.CString(string(c))
+	defer C.free(unsafe.Pointer(sysctlCString))
+
+	value := int(C.zfsIntegerSysctl(sysctlCString))
+
+	if value == -1 {
+		return zfsErrorValue, fmt.Errorf("Could not retrieve sysctl '%s'", c)
+	}
+
+	return zfsMetricValue(value), nil
+
+}
diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
new file mode 100644
index 0000000000..089124d7c8
--- /dev/null
+++ b/collector/zfs_linux.go
@@ -0,0 +1,83 @@
+package collector
+
+// +build linux
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/prometheus/common/log"
+)
+
+const zfsArcstatsProcpath = "spl/kstat/zfs/arcstats"
+
+func zfsInitialize() error {
+	path := procFilePath(zfsArcstatsProcpath)
+	fd, err := os.Open(path)
+	if err != nil {
+		log.Errorf("Cannot open ZFS arcstats logpath: %s", path)
+		return err
+	}
+	return fd.Close()
+}
+
+func (p *zfsMetricProvider) PrepareUpdate() (err error) {
+
+	err = p.prepareUpdateArcstats(zfsArcstatsProcpath)
+	if err != nil {
+		return
+	}
+	return nil
+}
+
+func (p *zfsMetricProvider) handleFetchedMetricCacheMiss(s zfsSysctl) (value zfsMetricValue, err error) {
+
+	// all values are fetched in PrepareUpdate()
+	return zfsErrorValue, fmt.Errorf("sysctl '%s' found")
+}
+
+func (p *zfsMetricProvider) prepareUpdateArcstats(zfsArcstatsProcpath string) (err error) {
+
+	file, err := os.Open(procFilePath(zfsArcstatsProcpath))
+	if err != nil {
+		log.Errorf("Cannot open ZFS arcstats procfs file for reading. " +
+			" Is the kernel module mounted?")
+		return err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+
+	parseLine := false
+	for scanner.Scan() {
+		parts := strings.Fields(scanner.Text())
+
+		if !parseLine && parts[0] == "name" && parts[1] == "type" && parts[2] == "data" {
+			// Start parsing from here
+			parseLine = true
+			continue
+		}
+
+		if !parseLine || len(parts) < 3 {
+			continue
+		}
+
+		key := fmt.Sprintf("kstat.zfs.misc.arcstats.%s", parts[0])
+
+		value, err := strconv.Atoi(parts[2])
+		if err != nil {
+			return fmt.Errorf("could not parse expected integer value for '%s'", key)
+		}
+		log.Debugf("%s = %d", key, value)
+		p.fetchedResults[zfsSysctl(key)] = zfsMetricValue(value)
+	}
+	if !parseLine {
+		return errors.New("did not parse a single arcstat metrics")
+	}
+
+	return scanner.Err()
+}

From 538183f99abad0aedd228dd5a12e4b284394613d Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 00:06:59 +0100
Subject: [PATCH 02/24] Incorporate Feedback from brian-brazil.

Use ConstMetric with UntypedValue.
Handle nozfs build flag.
Remove computed values.
---
 collector/zfs.go         | 165 ++++++++++++++++-----------------------
 collector/zfs_freebsd.go |   6 +-
 collector/zfs_linux.go   |   5 +-
 3 files changed, 71 insertions(+), 105 deletions(-)

diff --git a/collector/zfs.go b/collector/zfs.go
index c4314485f8..c822eaae02 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -1,29 +1,46 @@
 package collector
 
 // +build !nofilesystem
+// +build !nozfs
 
 import (
-	"fmt"
-
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/log"
 )
 
-type zfsSysctl string
-type zfsComputedMetric string
 type zfsMetricValue int
-
 const zfsErrorValue = zfsMetricValue(-1)
 
-type zfsMetricProvider struct {
-	fetchedResults  map[zfsSysctl]zfsMetricValue
-	computedResults map[zfsComputedMetric]zfsMetricValue
+type zfsSysctl string
+type zfsSubsystemName string
+const (
+	arc = zfsSubsystemName("zfs_arc")
+)
+
+
+//------------------------------------------------------------------------------
+//                                    Metrics
+//------------------------------------------------------------------------------
+
+type zfsMetric struct {
+	subsystem zfsSubsystemName // The Prometheus subsystem name
+	name string // The Prometheus name of the metric
+	sysctl zfsSysctl // The sysctl of the ZFS metric
 }
 
-type zfsCollector struct {
-	metricProvider  *zfsMetricProvider
-	fetchedMetrics  map[zfsSysctl]prometheus.Gauge
-	computedMetrics map[zfsComputedMetric]prometheus.Gauge
+func NewZFSMetric(subsystem zfsSubsystemName, sysctl, name string) zfsMetric {
+	return zfsMetric{
+		sysctl: zfsSysctl(sysctl),
+		subsystem: subsystem,
+		name: name,
+	}
+}
+func (m *zfsMetric) BuildFQName() string {
+	return prometheus.BuildFQName(Namespace, string(m.subsystem), m.name)
+}
+
+func (m *zfsMetric) HelpString() string {
+	return m.name
 }
 
 //------------------------------------------------------------------------------
@@ -34,47 +51,27 @@ func init() {
 	Factories["zfs"] = NewZFSCollector
 }
 
-const zfsArcSubsystemName = "zfs_arc"
+type zfsCollector struct {
+	zfsMetrics []zfsMetric
+	metricProvider zfsMetricProvider
+}
 
 func NewZFSCollector() (Collector, error) {
 	if err := zfsInitialize(); err != nil {
 		return &zfsCollector{}, err
 	}
 
-	fetchedMetrics := make(map[zfsSysctl]prometheus.Gauge)
-
-	insertGauge := func(sysctl, name string) {
-		fetchedMetrics[zfsSysctl(sysctl)] = prometheus.NewGauge(
-			prometheus.GaugeOpts{
-				Namespace: Namespace,
-				Subsystem: zfsArcSubsystemName,
-				Name:      name,
-				Help:      sysctl,
-			},
-		)
-	}
-
-	insertGauge("kstat.zfs.misc.arcstats.p", "mru_size")
-	insertGauge("kstat.zfs.misc.arcstats.size", "size")
-	insertGauge("kstat.zfs.misc.arcstats.c_min", "target_min_size")
-	insertGauge("kstat.zfs.misc.arcstats.c", "target_size")
-	insertGauge("kstat.zfs.misc.arcstats.c_max", "target_max_size")
-	insertGauge("kstat.zfs.misc.arcstats.hits", "hits")
-	insertGauge("kstat.zfs.misc.arcstats.misses", "misses")
-
 	return &zfsCollector{
-		metricProvider: NewZfsMetricProvider(),
-		fetchedMetrics: fetchedMetrics,
-		computedMetrics: map[zfsComputedMetric]prometheus.Gauge{
-			zfsComputedMetric("computed.mfu_size"): prometheus.NewGauge(
-				prometheus.GaugeOpts{
-					Namespace: Namespace,
-					Subsystem: zfsArcSubsystemName,
-					Name:      "mfu_size",
-					Help:      "mfu_size",
-				},
-			),
+		zfsMetrics: []zfsMetric{
+			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.p", "mru_size"),
+			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.size", "size"),
+			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.c_min", "target_min_size"),
+			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.c", "target_size"),
+			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.c_max", "target_max_size"),
+			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.hits", "hits"),
+			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.misses", "misses"),
 		},
+		metricProvider: NewZFSMetricProvider(),
 	}, nil
 }
 
@@ -86,27 +83,25 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 	}
 	defer c.metricProvider.InvalidateCache()
 
-	log.Debugf("Fetching %d metrics", len(c.fetchedMetrics))
-	for metric := range c.fetchedMetrics {
+	log.Debugf("Fetching %d metrics", len(c.zfsMetrics))
+	for _, metric := range c.zfsMetrics {
 
-		value, err := c.metricProvider.GetFetchedMetric(metric)
+		value, err := c.metricProvider.Value(metric.sysctl)
 		if err != nil {
 			return err
 		}
-		c.fetchedMetrics[metric].Set(float64(value))
-		c.fetchedMetrics[metric].Collect(ch)
-
-	}
 
-	log.Debugf("Computing %d metrics", len(c.computedMetrics))
-	for metric := range c.computedMetrics {
+		ch <- prometheus.MustNewConstMetric(
+			prometheus.NewDesc(
+				metric.BuildFQName(),
+				metric.HelpString(),
+				nil,
+				nil,
+			),
+			prometheus.UntypedValue,
+			float64(value),
+		)
 
-		value, err := c.metricProvider.GetComputedMetric(metric)
-		if err != nil {
-			return err
-		}
-		c.computedMetrics[metric].Set(float64(value))
-		c.computedMetrics[metric].Collect(ch)
 
 	}
 
@@ -118,27 +113,29 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 // Platform-dependend parts implemented in zfs_${platform} files.
 //------------------------------------------------------------------------------
 
-func NewZfsMetricProvider() *zfsMetricProvider {
-	return &zfsMetricProvider{
-		fetchedResults:  make(map[zfsSysctl]zfsMetricValue),
-		computedResults: make(map[zfsComputedMetric]zfsMetricValue),
+type zfsMetricProvider struct {
+	values map[zfsSysctl]zfsMetricValue
+}
+
+func NewZFSMetricProvider() zfsMetricProvider {
+	return zfsMetricProvider{
+		values:  make(map[zfsSysctl]zfsMetricValue),
 	}
 
 }
 
 func (p *zfsMetricProvider) InvalidateCache() {
-	p.fetchedResults = make(map[zfsSysctl]zfsMetricValue)
-	p.computedResults = make(map[zfsComputedMetric]zfsMetricValue)
+	p.values = make(map[zfsSysctl]zfsMetricValue)
 }
 
-func (p *zfsMetricProvider) GetFetchedMetric(s zfsSysctl) (value zfsMetricValue, err error) {
+func (p *zfsMetricProvider) Value(s zfsSysctl) (value zfsMetricValue, err error) {
 
 	var ok bool
 	value = zfsErrorValue
 
-	value, ok = p.fetchedResults[s]
+	value, ok = p.values[s]
 	if !ok {
-		value, err = p.handleFetchedMetricCacheMiss(s)
+		value, err = p.handleMiss(s)
 		if err != nil {
 			return value, err
 		}
@@ -147,33 +144,3 @@ func (p *zfsMetricProvider) GetFetchedMetric(s zfsSysctl) (value zfsMetricValue,
 	return value, err
 }
 
-func (p *zfsMetricProvider) GetComputedMetric(c zfsComputedMetric) (value zfsMetricValue, err error) {
-
-	if c == "computed.mfu_size" {
-
-		value = zfsErrorValue
-
-		arc_size, err := p.GetFetchedMetric(zfsSysctl("kstat.zfs.misc.arcstats.size"))
-		if err != nil {
-			return value, err
-		}
-		target_size, err := p.GetFetchedMetric(zfsSysctl("kstat.zfs.misc.arcstats.c"))
-		if err != nil {
-			return value, err
-		}
-		mru_size, err := p.GetFetchedMetric(zfsSysctl("kstat.zfs.misc.arcstats.p"))
-		if err != nil {
-			return value, err
-		}
-
-		// See zfs-stats implementation
-		if arc_size > target_size {
-			return arc_size - mru_size, nil
-		} else {
-			return target_size - mru_size, nil
-		}
-
-	}
-
-	return zfsErrorValue, fmt.Errorf("no implementation for computed metric '%s'", c)
-}
diff --git a/collector/zfs_freebsd.go b/collector/zfs_freebsd.go
index d65dfc3b9b..6c044fddd2 100644
--- a/collector/zfs_freebsd.go
+++ b/collector/zfs_freebsd.go
@@ -40,15 +40,15 @@ func (c *zfsMetricProvider) PrepareUpdate() error {
 	return nil
 }
 
-func (p *zfsMetricProvider) handleFetchedMetricCacheMiss(c zfsSysctl) (zfsMetricValue, error) {
+func (p *zfsMetricProvider) handleMiss(s zfsSysctl) (zfsMetricValue, error) {
 
-	sysctlCString := C.CString(string(c))
+	sysctlCString := C.CString(string(s))
 	defer C.free(unsafe.Pointer(sysctlCString))
 
 	value := int(C.zfsIntegerSysctl(sysctlCString))
 
 	if value == -1 {
-		return zfsErrorValue, fmt.Errorf("Could not retrieve sysctl '%s'", c)
+		return zfsErrorValue, fmt.Errorf("Could not retrieve sysctl '%s'", s)
 	}
 
 	return zfsMetricValue(value), nil
diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
index 089124d7c8..807444a943 100644
--- a/collector/zfs_linux.go
+++ b/collector/zfs_linux.go
@@ -34,8 +34,7 @@ func (p *zfsMetricProvider) PrepareUpdate() (err error) {
 	return nil
 }
 
-func (p *zfsMetricProvider) handleFetchedMetricCacheMiss(s zfsSysctl) (value zfsMetricValue, err error) {
-
+func (p *zfsMetricProvider) handleMiss(s zfsSysctl) (value zfsMetricValue, err error) {
 	// all values are fetched in PrepareUpdate()
 	return zfsErrorValue, fmt.Errorf("sysctl '%s' found")
 }
@@ -73,7 +72,7 @@ func (p *zfsMetricProvider) prepareUpdateArcstats(zfsArcstatsProcpath string) (e
 			return fmt.Errorf("could not parse expected integer value for '%s'", key)
 		}
 		log.Debugf("%s = %d", key, value)
-		p.fetchedResults[zfsSysctl(key)] = zfsMetricValue(value)
+		p.values[zfsSysctl(key)] = zfsMetricValue(value)
 	}
 	if !parseLine {
 		return errors.New("did not parse a single arcstat metrics")

From 62e4a7d21647b17d4ee0c54caa8013fe49c8dcc9 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 01:02:04 +0100
Subject: [PATCH 03/24] Change behavior on errors.

collector now does not expose metrics if stats cannot be fetched.
---
 collector/zfs.go         | 16 ++++++++++++++--
 collector/zfs_freebsd.go |  9 +++++++++
 collector/zfs_linux.go   | 14 ++++----------
 3 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/collector/zfs.go b/collector/zfs.go
index c822eaae02..e71292128a 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -4,6 +4,8 @@ package collector
 // +build !nozfs
 
 import (
+	"errors"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/log"
 )
@@ -11,6 +13,8 @@ import (
 type zfsMetricValue int
 const zfsErrorValue = zfsMetricValue(-1)
 
+var zfsNotAvailableError = errors.New("ZFS / ZFS statistics are not available")
+
 type zfsSysctl string
 type zfsSubsystemName string
 const (
@@ -57,7 +61,11 @@ type zfsCollector struct {
 }
 
 func NewZFSCollector() (Collector, error) {
-	if err := zfsInitialize(); err != nil {
+	err := zfsInitialize()
+	switch {
+		case err == zfsNotAvailableError:
+			log.Debug(err)
+			break
 		return &zfsCollector{}, err
 	}
 
@@ -78,7 +86,11 @@ func NewZFSCollector() (Collector, error) {
 func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 
 	log.Debug("Preparing metrics update")
-	if err := c.metricProvider.PrepareUpdate(); err != nil {
+	err = c.metricProvider.PrepareUpdate()
+	switch {
+		case err == zfsNotAvailableError:
+			log.Debug(err)
+			return nil
 		return err
 	}
 	defer c.metricProvider.InvalidateCache()
diff --git a/collector/zfs_freebsd.go b/collector/zfs_freebsd.go
index 6c044fddd2..73a045b854 100644
--- a/collector/zfs_freebsd.go
+++ b/collector/zfs_freebsd.go
@@ -12,6 +12,7 @@ import (
 #include <fcntl.h>
 #include <stdlib.h>
 #include <sys/param.h>
+#include <sys/module.h>
 #include <sys/pcpu.h>
 #include <sys/resource.h>
 #include <sys/sysctl.h>
@@ -29,6 +30,11 @@ int zfsIntegerSysctl(const char *name) {
 
 }
 
+int zfsModuleLoaded() {
+	int modid = modfind("zfs");
+	return modid < 0 ? 0 : -1;
+}
+
 */
 import "C"
 
@@ -37,6 +43,9 @@ func zfsInitialize() error {
 }
 
 func (c *zfsMetricProvider) PrepareUpdate() error {
+	if (C.zfsModuleLoaded() == 0) {
+		return zfsNotAvailableError
+	}
 	return nil
 }
 
diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
index 807444a943..8d4f2c0bc1 100644
--- a/collector/zfs_linux.go
+++ b/collector/zfs_linux.go
@@ -16,13 +16,7 @@ import (
 const zfsArcstatsProcpath = "spl/kstat/zfs/arcstats"
 
 func zfsInitialize() error {
-	path := procFilePath(zfsArcstatsProcpath)
-	fd, err := os.Open(path)
-	if err != nil {
-		log.Errorf("Cannot open ZFS arcstats logpath: %s", path)
-		return err
-	}
-	return fd.Close()
+	return nil
 }
 
 func (p *zfsMetricProvider) PrepareUpdate() (err error) {
@@ -43,9 +37,9 @@ func (p *zfsMetricProvider) prepareUpdateArcstats(zfsArcstatsProcpath string) (e
 
 	file, err := os.Open(procFilePath(zfsArcstatsProcpath))
 	if err != nil {
-		log.Errorf("Cannot open ZFS arcstats procfs file for reading. " +
-			" Is the kernel module mounted?")
-		return err
+		log.Debugf("Cannot open ZFS arcstats procfs file for reading. " +
+			" Is the kernel module loaded?")
+		return zfsNotAvailableError
 	}
 	defer file.Close()
 

From e92e9a265411b1aa52a5b21ef92c478329a30751 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 01:08:30 +0100
Subject: [PATCH 04/24] Enable ZFS exporter by default and update README.

---
 README.md        | 2 +-
 node_exporter.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index c53d33d08e..19dda845c3 100644
--- a/README.md
+++ b/README.md
@@ -32,7 +32,7 @@ textfile | Exposes statistics read from local disk. The `--collector.textfile.di
 time | Exposes the current system time. | _any_
 vmstat | Exposes statistics from `/proc/vmstat`. | Linux
 version | Exposes node\_exporter version. | _any_
-
+zfs | Exposes [ZFS](http://open-zfs.org/) performance statistics (ARC-only for now) | [FreeBSD](https://www.freebsd.org/doc/handbook/zfs.html),[Linux](http://zfsonlinux.org/)
 
 ### Disabled by default
 
diff --git a/node_exporter.go b/node_exporter.go
index 4f64663ff1..96e0077456 100644
--- a/node_exporter.go
+++ b/node_exporter.go
@@ -29,7 +29,7 @@ import (
 )
 
 const (
-	defaultCollectors = "conntrack,cpu,diskstats,entropy,filefd,filesystem,loadavg,mdadm,meminfo,netdev,netstat,sockstat,stat,textfile,time,uname,version,vmstat"
+	defaultCollectors = "conntrack,cpu,diskstats,entropy,filefd,filesystem,loadavg,mdadm,meminfo,netdev,netstat,sockstat,stat,textfile,time,uname,version,vmstat,zfs"
 )
 
 var (

From 6c139433729e74ef851b34a2386e5af4a7d72c00 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 01:16:08 +0100
Subject: [PATCH 05/24] Style fixes.

---
 collector/zfs.go         | 29 ++++++++++++++---------------
 collector/zfs_freebsd.go |  2 +-
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/collector/zfs.go b/collector/zfs.go
index e71292128a..7649ddf54d 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -11,32 +11,33 @@ import (
 )
 
 type zfsMetricValue int
+
 const zfsErrorValue = zfsMetricValue(-1)
 
 var zfsNotAvailableError = errors.New("ZFS / ZFS statistics are not available")
 
 type zfsSysctl string
 type zfsSubsystemName string
+
 const (
 	arc = zfsSubsystemName("zfs_arc")
 )
 
-
 //------------------------------------------------------------------------------
 //                                    Metrics
 //------------------------------------------------------------------------------
 
 type zfsMetric struct {
 	subsystem zfsSubsystemName // The Prometheus subsystem name
-	name string // The Prometheus name of the metric
-	sysctl zfsSysctl // The sysctl of the ZFS metric
+	name      string           // The Prometheus name of the metric
+	sysctl    zfsSysctl        // The sysctl of the ZFS metric
 }
 
 func NewZFSMetric(subsystem zfsSubsystemName, sysctl, name string) zfsMetric {
 	return zfsMetric{
-		sysctl: zfsSysctl(sysctl),
+		sysctl:    zfsSysctl(sysctl),
 		subsystem: subsystem,
-		name: name,
+		name:      name,
 	}
 }
 func (m *zfsMetric) BuildFQName() string {
@@ -56,16 +57,16 @@ func init() {
 }
 
 type zfsCollector struct {
-	zfsMetrics []zfsMetric
+	zfsMetrics     []zfsMetric
 	metricProvider zfsMetricProvider
 }
 
 func NewZFSCollector() (Collector, error) {
 	err := zfsInitialize()
 	switch {
-		case err == zfsNotAvailableError:
-			log.Debug(err)
-			break
+	case err == zfsNotAvailableError:
+		log.Debug(err)
+		break
 		return &zfsCollector{}, err
 	}
 
@@ -88,9 +89,9 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 	log.Debug("Preparing metrics update")
 	err = c.metricProvider.PrepareUpdate()
 	switch {
-		case err == zfsNotAvailableError:
-			log.Debug(err)
-			return nil
+	case err == zfsNotAvailableError:
+		log.Debug(err)
+		return nil
 		return err
 	}
 	defer c.metricProvider.InvalidateCache()
@@ -114,7 +115,6 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 			float64(value),
 		)
 
-
 	}
 
 	return err
@@ -131,7 +131,7 @@ type zfsMetricProvider struct {
 
 func NewZFSMetricProvider() zfsMetricProvider {
 	return zfsMetricProvider{
-		values:  make(map[zfsSysctl]zfsMetricValue),
+		values: make(map[zfsSysctl]zfsMetricValue),
 	}
 
 }
@@ -155,4 +155,3 @@ func (p *zfsMetricProvider) Value(s zfsSysctl) (value zfsMetricValue, err error)
 
 	return value, err
 }
-
diff --git a/collector/zfs_freebsd.go b/collector/zfs_freebsd.go
index 73a045b854..370e464b73 100644
--- a/collector/zfs_freebsd.go
+++ b/collector/zfs_freebsd.go
@@ -43,7 +43,7 @@ func zfsInitialize() error {
 }
 
 func (c *zfsMetricProvider) PrepareUpdate() error {
-	if (C.zfsModuleLoaded() == 0) {
+	if C.zfsModuleLoaded() == 0 {
 		return zfsNotAvailableError
 	}
 	return nil

From 4d15f0b68efd850de603723da1f9eb1f839d7abd Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 11:58:12 +0100
Subject: [PATCH 06/24] Comments should end in periods.

---
 collector/zfs.go       | 20 +++++++-------------
 collector/zfs_linux.go |  4 ++--
 2 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/collector/zfs.go b/collector/zfs.go
index 7649ddf54d..4e0fcd007d 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -23,14 +23,12 @@ const (
 	arc = zfsSubsystemName("zfs_arc")
 )
 
-//------------------------------------------------------------------------------
-//                                    Metrics
-//------------------------------------------------------------------------------
+// Metrics
 
 type zfsMetric struct {
-	subsystem zfsSubsystemName // The Prometheus subsystem name
-	name      string           // The Prometheus name of the metric
-	sysctl    zfsSysctl        // The sysctl of the ZFS metric
+	subsystem zfsSubsystemName // The Prometheus subsystem name.
+	name      string           // The Prometheus name of the metric.
+	sysctl    zfsSysctl        // The sysctl of the ZFS metric.
 }
 
 func NewZFSMetric(subsystem zfsSubsystemName, sysctl, name string) zfsMetric {
@@ -48,9 +46,7 @@ func (m *zfsMetric) HelpString() string {
 	return m.name
 }
 
-//------------------------------------------------------------------------------
-//                                 Collector
-//------------------------------------------------------------------------------
+// Collector
 
 func init() {
 	Factories["zfs"] = NewZFSCollector
@@ -120,10 +116,8 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 	return err
 }
 
-//------------------------------------------------------------------------------
-//                                 Metrics Provider
-// Platform-dependend parts implemented in zfs_${platform} files.
-//------------------------------------------------------------------------------
+// Metrics Provider
+// Platform-dependend parts implemented in zfs_${os} files.
 
 type zfsMetricProvider struct {
 	values map[zfsSysctl]zfsMetricValue
diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
index 8d4f2c0bc1..fa349ac46a 100644
--- a/collector/zfs_linux.go
+++ b/collector/zfs_linux.go
@@ -29,7 +29,7 @@ func (p *zfsMetricProvider) PrepareUpdate() (err error) {
 }
 
 func (p *zfsMetricProvider) handleMiss(s zfsSysctl) (value zfsMetricValue, err error) {
-	// all values are fetched in PrepareUpdate()
+	// all values are fetched in PrepareUpdate().
 	return zfsErrorValue, fmt.Errorf("sysctl '%s' found")
 }
 
@@ -50,7 +50,7 @@ func (p *zfsMetricProvider) prepareUpdateArcstats(zfsArcstatsProcpath string) (e
 		parts := strings.Fields(scanner.Text())
 
 		if !parseLine && parts[0] == "name" && parts[1] == "type" && parts[2] == "data" {
-			// Start parsing from here
+			// Start parsing from here.
 			parseLine = true
 			continue
 		}

From fdfe4a69e99893910bdde8d5d3970e224f867176 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 12:00:14 +0100
Subject: [PATCH 07/24] Build system fixes.

---
 collector/zfs.go         | 1 -
 collector/zfs_freebsd.go | 2 --
 collector/zfs_linux.go   | 2 --
 3 files changed, 5 deletions(-)

diff --git a/collector/zfs.go b/collector/zfs.go
index 4e0fcd007d..b3d4eea31c 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -1,6 +1,5 @@
 package collector
 
-// +build !nofilesystem
 // +build !nozfs
 
 import (
diff --git a/collector/zfs_freebsd.go b/collector/zfs_freebsd.go
index 370e464b73..c495cbec55 100644
--- a/collector/zfs_freebsd.go
+++ b/collector/zfs_freebsd.go
@@ -1,7 +1,5 @@
 package collector
 
-// +build freebsd
-
 import (
 	"fmt"
 	"unsafe"
diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
index fa349ac46a..28b6f0f479 100644
--- a/collector/zfs_linux.go
+++ b/collector/zfs_linux.go
@@ -1,7 +1,5 @@
 package collector
 
-// +build linux
-
 import (
 	"bufio"
 	"errors"

From 3ae58bab5532b662ac101e3e76055d56cb2eda0d Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 12:21:47 +0100
Subject: [PATCH 08/24] Remove noop zfsInitialize().

---
 collector/zfs.go         | 8 --------
 collector/zfs_freebsd.go | 4 ----
 collector/zfs_linux.go   | 4 ----
 3 files changed, 16 deletions(-)

diff --git a/collector/zfs.go b/collector/zfs.go
index b3d4eea31c..195f1304c9 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -57,14 +57,6 @@ type zfsCollector struct {
 }
 
 func NewZFSCollector() (Collector, error) {
-	err := zfsInitialize()
-	switch {
-	case err == zfsNotAvailableError:
-		log.Debug(err)
-		break
-		return &zfsCollector{}, err
-	}
-
 	return &zfsCollector{
 		zfsMetrics: []zfsMetric{
 			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.p", "mru_size"),
diff --git a/collector/zfs_freebsd.go b/collector/zfs_freebsd.go
index c495cbec55..20cace26b2 100644
--- a/collector/zfs_freebsd.go
+++ b/collector/zfs_freebsd.go
@@ -36,10 +36,6 @@ int zfsModuleLoaded() {
 */
 import "C"
 
-func zfsInitialize() error {
-	return nil
-}
-
 func (c *zfsMetricProvider) PrepareUpdate() error {
 	if C.zfsModuleLoaded() == 0 {
 		return zfsNotAvailableError
diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
index 28b6f0f479..5f10858718 100644
--- a/collector/zfs_linux.go
+++ b/collector/zfs_linux.go
@@ -13,10 +13,6 @@ import (
 
 const zfsArcstatsProcpath = "spl/kstat/zfs/arcstats"
 
-func zfsInitialize() error {
-	return nil
-}
-
 func (p *zfsMetricProvider) PrepareUpdate() (err error) {
 
 	err = p.prepareUpdateArcstats(zfsArcstatsProcpath)

From ffc3455102d3dc05a71bdb55301cd822ba517234 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 12:22:06 +0100
Subject: [PATCH 09/24] Use struct instantiation instead of constructor for
 zfsMetrics.

---
 collector/zfs.go | 25 +++++++------------------
 1 file changed, 7 insertions(+), 18 deletions(-)

diff --git a/collector/zfs.go b/collector/zfs.go
index 195f1304c9..24b3803aac 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -30,21 +30,10 @@ type zfsMetric struct {
 	sysctl    zfsSysctl        // The sysctl of the ZFS metric.
 }
 
-func NewZFSMetric(subsystem zfsSubsystemName, sysctl, name string) zfsMetric {
-	return zfsMetric{
-		sysctl:    zfsSysctl(sysctl),
-		subsystem: subsystem,
-		name:      name,
-	}
-}
 func (m *zfsMetric) BuildFQName() string {
 	return prometheus.BuildFQName(Namespace, string(m.subsystem), m.name)
 }
 
-func (m *zfsMetric) HelpString() string {
-	return m.name
-}
-
 // Collector
 
 func init() {
@@ -59,13 +48,13 @@ type zfsCollector struct {
 func NewZFSCollector() (Collector, error) {
 	return &zfsCollector{
 		zfsMetrics: []zfsMetric{
-			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.p", "mru_size"),
-			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.size", "size"),
-			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.c_min", "target_min_size"),
-			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.c", "target_size"),
-			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.c_max", "target_max_size"),
-			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.hits", "hits"),
-			NewZFSMetric(arc, "kstat.zfs.misc.arcstats.misses", "misses"),
+			zfsMetric{arc, "mru_size", zfsSysctl("kstat.zfs.misc.arcstats.p")},
+			zfsMetric{arc, "size", zfsSysctl("kstat.zfs.misc.arcstats.size")},
+			zfsMetric{arc, "target_min_size", zfsSysctl("kstat.zfs.misc.arcstats.c_min")},
+			zfsMetric{arc, "target_size", zfsSysctl("kstat.zfs.misc.arcstats.c")},
+			zfsMetric{arc, "target_max_size", zfsSysctl("kstat.zfs.misc.arcstats.c_max")},
+			zfsMetric{arc, "hits", zfsSysctl("kstat.zfs.misc.arcstats.hits")},
+			zfsMetric{arc, "misses", zfsSysctl("kstat.zfs.misc.arcstats.misses")},
 		},
 		metricProvider: NewZFSMetricProvider(),
 	}, nil

From 394305aab29c42d09981a4c147d5d405ee4a2c79 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 12:37:48 +0100
Subject: [PATCH 10/24] Fix unreachable code.

---
 collector/zfs.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/collector/zfs.go b/collector/zfs.go
index 24b3803aac..6c4d7fbd24 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -68,6 +68,7 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 	case err == zfsNotAvailableError:
 		log.Debug(err)
 		return nil
+	case err != nil:
 		return err
 	}
 	defer c.metricProvider.InvalidateCache()

From e828e04811e99a41831ea3878b81ffb00cb7cb96 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 12:41:17 +0100
Subject: [PATCH 11/24] Remove race-condition in access on zfsMetricProvider in
 Update()

Instantiate a zfsMetricProvider per Update() call.
---
 collector/zfs.go | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/collector/zfs.go b/collector/zfs.go
index 6c4d7fbd24..93cbada68f 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -34,6 +34,10 @@ func (m *zfsMetric) BuildFQName() string {
 	return prometheus.BuildFQName(Namespace, string(m.subsystem), m.name)
 }
 
+func (m *zfsMetric) HelpString() string {
+	return m.name
+}
+
 // Collector
 
 func init() {
@@ -42,7 +46,6 @@ func init() {
 
 type zfsCollector struct {
 	zfsMetrics     []zfsMetric
-	metricProvider zfsMetricProvider
 }
 
 func NewZFSCollector() (Collector, error) {
@@ -56,14 +59,15 @@ func NewZFSCollector() (Collector, error) {
 			zfsMetric{arc, "hits", zfsSysctl("kstat.zfs.misc.arcstats.hits")},
 			zfsMetric{arc, "misses", zfsSysctl("kstat.zfs.misc.arcstats.misses")},
 		},
-		metricProvider: NewZFSMetricProvider(),
 	}, nil
 }
 
 func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 
+	metricProvider := NewZFSMetricProvider()
+
 	log.Debug("Preparing metrics update")
-	err = c.metricProvider.PrepareUpdate()
+	err = metricProvider.PrepareUpdate()
 	switch {
 	case err == zfsNotAvailableError:
 		log.Debug(err)
@@ -71,12 +75,11 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 	case err != nil:
 		return err
 	}
-	defer c.metricProvider.InvalidateCache()
 
 	log.Debugf("Fetching %d metrics", len(c.zfsMetrics))
 	for _, metric := range c.zfsMetrics {
 
-		value, err := c.metricProvider.Value(metric.sysctl)
+		value, err := metricProvider.Value(metric.sysctl)
 		if err != nil {
 			return err
 		}
@@ -111,10 +114,6 @@ func NewZFSMetricProvider() zfsMetricProvider {
 
 }
 
-func (p *zfsMetricProvider) InvalidateCache() {
-	p.values = make(map[zfsSysctl]zfsMetricValue)
-}
-
 func (p *zfsMetricProvider) Value(s zfsSysctl) (value zfsMetricValue, err error) {
 
 	var ok bool

From 45e989d29003fc0279ce008a4feb6d7030276e83 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 13:08:06 +0100
Subject: [PATCH 12/24] Extract arcstat procfs file parsing into separate
 method.

---
 collector/zfs_linux.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
index 5f10858718..4cca0593db 100644
--- a/collector/zfs_linux.go
+++ b/collector/zfs_linux.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"strconv"
 	"strings"
+	"io"
 
 	"github.com/prometheus/common/log"
 )
@@ -27,6 +28,7 @@ func (p *zfsMetricProvider) handleMiss(s zfsSysctl) (value zfsMetricValue, err e
 	return zfsErrorValue, fmt.Errorf("sysctl '%s' found")
 }
 
+
 func (p *zfsMetricProvider) prepareUpdateArcstats(zfsArcstatsProcpath string) (err error) {
 
 	file, err := os.Open(procFilePath(zfsArcstatsProcpath))
@@ -36,8 +38,12 @@ func (p *zfsMetricProvider) prepareUpdateArcstats(zfsArcstatsProcpath string) (e
 		return zfsNotAvailableError
 	}
 	defer file.Close()
+	return p.parseArcstatsProcfsFile(file)
+}
+
+func (p *zfsMetricProvider) parseArcstatsProcfsFile(reader io.Reader) (err error) {
 
-	scanner := bufio.NewScanner(file)
+	scanner := bufio.NewScanner(reader)
 
 	parseLine := false
 	for scanner.Scan() {

From 3c33e7ede406dc7782a6663e44e4f5438d4585b8 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 13:09:08 +0100
Subject: [PATCH 13/24] Add unit test for arcstats parsing.

---
 .../fixtures/proc/spl/kstat/zfs/arcstats      | 93 +++++++++++++++++++
 collector/zfs_linux_test.go                   | 36 +++++++
 2 files changed, 129 insertions(+)
 create mode 100644 collector/fixtures/proc/spl/kstat/zfs/arcstats
 create mode 100644 collector/zfs_linux_test.go

diff --git a/collector/fixtures/proc/spl/kstat/zfs/arcstats b/collector/fixtures/proc/spl/kstat/zfs/arcstats
new file mode 100644
index 0000000000..48a73a2c5a
--- /dev/null
+++ b/collector/fixtures/proc/spl/kstat/zfs/arcstats
@@ -0,0 +1,93 @@
+6 1 0x01 91 4368 5266997922 97951858082072
+name                            type data
+hits                            4    8772612
+misses                          4    604635
+demand_data_hits                4    7221032
+demand_data_misses              4    73300
+demand_metadata_hits            4    1464353
+demand_metadata_misses          4    498170
+prefetch_data_hits              4    3615
+prefetch_data_misses            4    17094
+prefetch_metadata_hits          4    83612
+prefetch_metadata_misses        4    16071
+mru_hits                        4    855535
+mru_ghost_hits                  4    21100
+mfu_hits                        4    7829854
+mfu_ghost_hits                  4    821
+deleted                         4    60403
+mutex_miss                      4    2
+evict_skip                      4    2265729
+evict_not_enough                4    680
+evict_l2_cached                 4    0
+evict_l2_eligible               4    8992514560
+evict_l2_ineligible             4    992552448
+evict_l2_skip                   4    0
+hash_elements                   4    42359
+hash_elements_max               4    88245
+hash_collisions                 4    50564
+hash_chains                     4    412
+hash_chain_max                  4    3
+p                               4    516395305
+c                               4    1643208777
+c_min                           4    33554432
+c_max                           4    8367976448
+size                            4    1603939792
+hdr_size                        4    16361080
+data_size                       4    1295836160
+metadata_size                   4    175298560
+other_size                      4    116443992
+anon_size                       4    1917440
+anon_evictable_data             4    0
+anon_evictable_metadata         4    0
+mru_size                        4    402593792
+mru_evictable_data              4    278091264
+mru_evictable_metadata          4    18606592
+mru_ghost_size                  4    999728128
+mru_ghost_evictable_data        4    883765248
+mru_ghost_evictable_metadata    4    115962880
+mfu_size                        4    1066623488
+mfu_evictable_data              4    1017613824
+mfu_evictable_metadata          4    9163776
+mfu_ghost_size                  4    104936448
+mfu_ghost_evictable_data        4    96731136
+mfu_ghost_evictable_metadata    4    8205312
+l2_hits                         4    0
+l2_misses                       4    0
+l2_feeds                        4    0
+l2_rw_clash                     4    0
+l2_read_bytes                   4    0
+l2_write_bytes                  4    0
+l2_writes_sent                  4    0
+l2_writes_done                  4    0
+l2_writes_error                 4    0
+l2_writes_lock_retry            4    0
+l2_evict_lock_retry             4    0
+l2_evict_reading                4    0
+l2_evict_l1cached               4    0
+l2_free_on_write                4    0
+l2_cdata_free_on_write          4    0
+l2_abort_lowmem                 4    0
+l2_cksum_bad                    4    0
+l2_io_error                     4    0
+l2_size                         4    0
+l2_asize                        4    0
+l2_hdr_size                     4    0
+l2_compress_successes           4    0
+l2_compress_zeros               4    0
+l2_compress_failures            4    0
+memory_throttle_count           4    0
+duplicate_buffers               4    0
+duplicate_buffers_size          4    0
+duplicate_reads                 4    0
+memory_direct_count             4    542
+memory_indirect_count           4    3006
+arc_no_grow                     4    0
+arc_tempreserve                 4    0
+arc_loaned_bytes                4    0
+arc_prune                       4    0
+arc_meta_used                   4    308103632
+arc_meta_limit                  4    6275982336
+arc_meta_max                    4    449286096
+arc_meta_min                    4    16777216
+arc_need_free                   4    0
+arc_sys_free                    4    261496832
diff --git a/collector/zfs_linux_test.go b/collector/zfs_linux_test.go
new file mode 100644
index 0000000000..0024709d90
--- /dev/null
+++ b/collector/zfs_linux_test.go
@@ -0,0 +1,36 @@
+package collector
+
+import (
+	"os"
+	"testing"
+)
+
+func TestArcstatsParsing(t *testing.T) {
+
+	arcstatsFile, err := os.Open("fixtures/proc/spl/kstat/zfs/arcstats")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer arcstatsFile.Close()
+
+
+	p := NewZFSMetricProvider()
+	err = p.parseArcstatsProcfsFile(arcstatsFile)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Logf("Parsed values mapped to sysctls:\n%v", p.values)
+
+	value, err := p.Value(zfsSysctl("kstat.zfs.misc.arcstats.hits"))
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if value != zfsMetricValue(8772612) {
+		t.Fatalf("Incorrect value parsed from procfs data")
+	}
+
+}

From 66f4eeb608b09de1ccdc4cd82cbb60eab7dd7e31 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 13:09:28 +0100
Subject: [PATCH 14/24] Fix out-of-bounds on empty lines at the end of the
 procfs file.

---
 collector/zfs_linux.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
index 4cca0593db..20c5ac18ea 100644
--- a/collector/zfs_linux.go
+++ b/collector/zfs_linux.go
@@ -49,7 +49,7 @@ func (p *zfsMetricProvider) parseArcstatsProcfsFile(reader io.Reader) (err error
 	for scanner.Scan() {
 		parts := strings.Fields(scanner.Text())
 
-		if !parseLine && parts[0] == "name" && parts[1] == "type" && parts[2] == "data" {
+		if !parseLine && len(parts) == 3 && parts[0] == "name" && parts[1] == "type" && parts[2] == "data" {
 			// Start parsing from here.
 			parseLine = true
 			continue

From 3a6a8a5d2309cfdb6ca021d444b54da69a7dc3ca Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 14:09:35 +0100
Subject: [PATCH 15/24] Style fixes.

---
 collector/zfs.go            | 16 ++++++++--------
 collector/zfs_linux.go      |  3 +--
 collector/zfs_linux_test.go |  1 -
 3 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/collector/zfs.go b/collector/zfs.go
index 93cbada68f..6e4ea1d078 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -45,19 +45,19 @@ func init() {
 }
 
 type zfsCollector struct {
-	zfsMetrics     []zfsMetric
+	zfsMetrics []zfsMetric
 }
 
 func NewZFSCollector() (Collector, error) {
 	return &zfsCollector{
 		zfsMetrics: []zfsMetric{
-			zfsMetric{arc, "mru_size", zfsSysctl("kstat.zfs.misc.arcstats.p")},
-			zfsMetric{arc, "size", zfsSysctl("kstat.zfs.misc.arcstats.size")},
-			zfsMetric{arc, "target_min_size", zfsSysctl("kstat.zfs.misc.arcstats.c_min")},
-			zfsMetric{arc, "target_size", zfsSysctl("kstat.zfs.misc.arcstats.c")},
-			zfsMetric{arc, "target_max_size", zfsSysctl("kstat.zfs.misc.arcstats.c_max")},
-			zfsMetric{arc, "hits", zfsSysctl("kstat.zfs.misc.arcstats.hits")},
-			zfsMetric{arc, "misses", zfsSysctl("kstat.zfs.misc.arcstats.misses")},
+			{arc, "mru_size", zfsSysctl("kstat.zfs.misc.arcstats.p")},
+			{arc, "size", zfsSysctl("kstat.zfs.misc.arcstats.size")},
+			{arc, "target_min_size", zfsSysctl("kstat.zfs.misc.arcstats.c_min")},
+			{arc, "target_size", zfsSysctl("kstat.zfs.misc.arcstats.c")},
+			{arc, "target_max_size", zfsSysctl("kstat.zfs.misc.arcstats.c_max")},
+			{arc, "hits", zfsSysctl("kstat.zfs.misc.arcstats.hits")},
+			{arc, "misses", zfsSysctl("kstat.zfs.misc.arcstats.misses")},
 		},
 	}, nil
 }
diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
index 20c5ac18ea..660d900344 100644
--- a/collector/zfs_linux.go
+++ b/collector/zfs_linux.go
@@ -4,10 +4,10 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"strconv"
 	"strings"
-	"io"
 
 	"github.com/prometheus/common/log"
 )
@@ -28,7 +28,6 @@ func (p *zfsMetricProvider) handleMiss(s zfsSysctl) (value zfsMetricValue, err e
 	return zfsErrorValue, fmt.Errorf("sysctl '%s' found")
 }
 
-
 func (p *zfsMetricProvider) prepareUpdateArcstats(zfsArcstatsProcpath string) (err error) {
 
 	file, err := os.Open(procFilePath(zfsArcstatsProcpath))
diff --git a/collector/zfs_linux_test.go b/collector/zfs_linux_test.go
index 0024709d90..59379bfe7d 100644
--- a/collector/zfs_linux_test.go
+++ b/collector/zfs_linux_test.go
@@ -13,7 +13,6 @@ func TestArcstatsParsing(t *testing.T) {
 	}
 	defer arcstatsFile.Close()
 
-
 	p := NewZFSMetricProvider()
 	err = p.parseArcstatsProcfsFile(arcstatsFile)
 

From 22bbb8525cb65eb3626e5b37c24745dcad1dd9e9 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 14:15:14 +0100
Subject: [PATCH 16/24] Remove log statement.

---
 collector/zfs_linux_test.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/collector/zfs_linux_test.go b/collector/zfs_linux_test.go
index 59379bfe7d..890e8807d0 100644
--- a/collector/zfs_linux_test.go
+++ b/collector/zfs_linux_test.go
@@ -20,8 +20,6 @@ func TestArcstatsParsing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	t.Logf("Parsed values mapped to sysctls:\n%v", p.values)
-
 	value, err := p.Value(zfsSysctl("kstat.zfs.misc.arcstats.hits"))
 
 	if err != nil {

From 735effcba9b8f825bcff0f280a6b49944e9ff5df Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 14:46:01 +0100
Subject: [PATCH 17/24] Only compile on supported platforms.

---
 collector/zfs.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/collector/zfs.go b/collector/zfs.go
index 6e4ea1d078..e5fb403b43 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -1,5 +1,6 @@
 package collector
 
+// +build linux freebsd
 // +build !nozfs
 
 import (

From 6bdd416f0d25e5c2c56ebf8eb3261f782cc4e478 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Sun, 13 Mar 2016 16:52:44 +0100
Subject: [PATCH 18/24] Restructure implementation without zfsMetricProvider.

---
 collector/zfs.go            | 75 ++++++++++---------------------------
 collector/zfs_freebsd.go    | 24 +++++++-----
 collector/zfs_linux.go      | 52 +++++++++++++++----------
 collector/zfs_linux_test.go | 23 +++++++++---
 4 files changed, 84 insertions(+), 90 deletions(-)

diff --git a/collector/zfs.go b/collector/zfs.go
index e5fb403b43..4d616625c0 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -31,12 +31,22 @@ type zfsMetric struct {
 	sysctl    zfsSysctl        // The sysctl of the ZFS metric.
 }
 
-func (m *zfsMetric) BuildFQName() string {
-	return prometheus.BuildFQName(Namespace, string(m.subsystem), m.name)
+func (m *zfsMetric) ConstMetric(value zfsMetricValue) prometheus.Metric {
+	return prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, string(m.subsystem), m.name),
+			m.name,
+			nil,
+			nil,
+		),
+		prometheus.UntypedValue,
+		float64(value),
+	)
 }
 
-func (m *zfsMetric) HelpString() string {
-	return m.name
+type datasetMetric struct {
+	subsystem zfsSubsystemName
+	name      string
 }
 
 // Collector
@@ -65,10 +75,7 @@ func NewZFSCollector() (Collector, error) {
 
 func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 
-	metricProvider := NewZFSMetricProvider()
-
-	log.Debug("Preparing metrics update")
-	err = metricProvider.PrepareUpdate()
+	err = c.PrepareUpdate()
 	switch {
 	case err == zfsNotAvailableError:
 		log.Debug(err)
@@ -77,56 +84,12 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 		return err
 	}
 
-	log.Debugf("Fetching %d metrics", len(c.zfsMetrics))
-	for _, metric := range c.zfsMetrics {
-
-		value, err := metricProvider.Value(metric.sysctl)
-		if err != nil {
-			return err
-		}
-
-		ch <- prometheus.MustNewConstMetric(
-			prometheus.NewDesc(
-				metric.BuildFQName(),
-				metric.HelpString(),
-				nil,
-				nil,
-			),
-			prometheus.UntypedValue,
-			float64(value),
-		)
+	// Arcstats
 
+	err = c.updateArcstats(ch)
+	if err != nil {
+		return err
 	}
 
 	return err
 }
-
-// Metrics Provider
-// Platform-dependend parts implemented in zfs_${os} files.
-
-type zfsMetricProvider struct {
-	values map[zfsSysctl]zfsMetricValue
-}
-
-func NewZFSMetricProvider() zfsMetricProvider {
-	return zfsMetricProvider{
-		values: make(map[zfsSysctl]zfsMetricValue),
-	}
-
-}
-
-func (p *zfsMetricProvider) Value(s zfsSysctl) (value zfsMetricValue, err error) {
-
-	var ok bool
-	value = zfsErrorValue
-
-	value, ok = p.values[s]
-	if !ok {
-		value, err = p.handleMiss(s)
-		if err != nil {
-			return value, err
-		}
-	}
-
-	return value, err
-}
diff --git a/collector/zfs_freebsd.go b/collector/zfs_freebsd.go
index 20cace26b2..2d6203b5ce 100644
--- a/collector/zfs_freebsd.go
+++ b/collector/zfs_freebsd.go
@@ -3,6 +3,8 @@ package collector
 import (
 	"fmt"
 	"unsafe"
+
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 /*
@@ -36,24 +38,28 @@ int zfsModuleLoaded() {
 */
 import "C"
 
-func (c *zfsMetricProvider) PrepareUpdate() error {
+func (c *zfsCollector) PrepareUpdate() error {
 	if C.zfsModuleLoaded() == 0 {
 		return zfsNotAvailableError
 	}
 	return nil
 }
 
-func (p *zfsMetricProvider) handleMiss(s zfsSysctl) (zfsMetricValue, error) {
+func (c *zfsCollector) updateArcstats(ch chan<- prometheus.Metric) (err error) {
 
-	sysctlCString := C.CString(string(s))
-	defer C.free(unsafe.Pointer(sysctlCString))
+	for _, metric := range c.zfsMetrics {
 
-	value := int(C.zfsIntegerSysctl(sysctlCString))
+		sysctlCString := C.CString(string(metric.sysctl))
+		defer C.free(unsafe.Pointer(sysctlCString))
 
-	if value == -1 {
-		return zfsErrorValue, fmt.Errorf("Could not retrieve sysctl '%s'", s)
-	}
+		value := int(C.zfsIntegerSysctl(sysctlCString))
 
-	return zfsMetricValue(value), nil
+		if value == -1 {
+			return fmt.Errorf("Could not retrieve value for metric '%v'", metric)
+		}
+
+		ch <- metric.ConstMetric(zfsMetricValue(value))
+	}
 
+	return err
 }
diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
index 660d900344..1c464a3e78 100644
--- a/collector/zfs_linux.go
+++ b/collector/zfs_linux.go
@@ -9,38 +9,52 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/log"
 )
 
-const zfsArcstatsProcpath = "spl/kstat/zfs/arcstats"
-
-func (p *zfsMetricProvider) PrepareUpdate() (err error) {
+const (
+	zfsArcstatsProcpath = "spl/kstat/zfs/arcstats"
+)
 
-	err = p.prepareUpdateArcstats(zfsArcstatsProcpath)
+func (c *zfsCollector) PrepareUpdate() (err error) {
+	file, err := c.openArcstatsFile()
 	if err != nil {
-		return
+		file.Close()
 	}
-	return nil
+	return err
 }
 
-func (p *zfsMetricProvider) handleMiss(s zfsSysctl) (value zfsMetricValue, err error) {
-	// all values are fetched in PrepareUpdate().
-	return zfsErrorValue, fmt.Errorf("sysctl '%s' found")
+func (c *zfsCollector) openArcstatsFile() (file *os.File, err error) {
+	file, err = os.Open(procFilePath(zfsArcstatsProcpath))
+	if err != nil {
+		log.Debugf("Cannot open '%s' for reading.Is the kernel module loaded?", procFilePath(zfsArcstatsProcpath))
+		err = zfsNotAvailableError
+	}
+	return
 }
 
-func (p *zfsMetricProvider) prepareUpdateArcstats(zfsArcstatsProcpath string) (err error) {
+func (c *zfsCollector) updateArcstats(ch chan<- prometheus.Metric) (err error) {
 
-	file, err := os.Open(procFilePath(zfsArcstatsProcpath))
-	if err != nil {
-		log.Debugf("Cannot open ZFS arcstats procfs file for reading. " +
-			" Is the kernel module loaded?")
-		return zfsNotAvailableError
-	}
+	file, err := c.openArcstatsFile()
 	defer file.Close()
-	return p.parseArcstatsProcfsFile(file)
+
+	return c.parseArcstatsProcfsFile(file, func(s zfsSysctl, v zfsMetricValue) {
+		// TODO: Find corresponding metric in a more efficient way
+		for _, metric := range c.zfsMetrics {
+			if metric.subsystem != arc {
+				continue
+			}
+			if metric.sysctl != s {
+				continue
+			}
+			ch <- metric.ConstMetric(v)
+		}
+	})
+
 }
 
-func (p *zfsMetricProvider) parseArcstatsProcfsFile(reader io.Reader) (err error) {
+func (c *zfsCollector) parseArcstatsProcfsFile(reader io.Reader, handler func(zfsSysctl, zfsMetricValue)) (err error) {
 
 	scanner := bufio.NewScanner(reader)
 
@@ -65,7 +79,7 @@ func (p *zfsMetricProvider) parseArcstatsProcfsFile(reader io.Reader) (err error
 			return fmt.Errorf("could not parse expected integer value for '%s'", key)
 		}
 		log.Debugf("%s = %d", key, value)
-		p.values[zfsSysctl(key)] = zfsMetricValue(value)
+		handler(zfsSysctl(key), zfsMetricValue(value))
 	}
 	if !parseLine {
 		return errors.New("did not parse a single arcstat metrics")
diff --git a/collector/zfs_linux_test.go b/collector/zfs_linux_test.go
index 890e8807d0..2640ebac2a 100644
--- a/collector/zfs_linux_test.go
+++ b/collector/zfs_linux_test.go
@@ -13,21 +13,32 @@ func TestArcstatsParsing(t *testing.T) {
 	}
 	defer arcstatsFile.Close()
 
-	p := NewZFSMetricProvider()
-	err = p.parseArcstatsProcfsFile(arcstatsFile)
-
+	c := zfsCollector{}
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	value, err := p.Value(zfsSysctl("kstat.zfs.misc.arcstats.hits"))
+	handlerCalled := false
+	err = c.parseArcstatsProcfsFile(arcstatsFile, func(s zfsSysctl, v zfsMetricValue) {
+
+		if s != zfsSysctl("kstat.zfs.misc.arcstats.hits") {
+			return
+		}
+
+		handlerCalled = true
+
+		if v != zfsMetricValue(8772612) {
+			t.Fatalf("Incorrect value parsed from procfs data")
+		}
+
+	})
 
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if value != zfsMetricValue(8772612) {
-		t.Fatalf("Incorrect value parsed from procfs data")
+	if !handlerCalled {
+		t.Fatal("Arcstats parsing handler was not called for some expected sysctls")
 	}
 
 }

From 60aa6974160793acb4bd9ee97e9ac5507db121ea Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Tue, 15 Mar 2016 23:25:05 +0100
Subject: [PATCH 19/24] Rename zfs_arc subsystem.

---
 collector/zfs.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/collector/zfs.go b/collector/zfs.go
index 4d616625c0..ae4560863c 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -20,7 +20,7 @@ type zfsSysctl string
 type zfsSubsystemName string
 
 const (
-	arc = zfsSubsystemName("zfs_arc")
+	arc = zfsSubsystemName("zfsArc")
 )
 
 // Metrics

From 2f99eb647580a225cc92c746cd53cc3cc4aeda7d Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Tue, 15 Mar 2016 23:06:29 +0100
Subject: [PATCH 20/24] Expose all available ARC metrics on FreeBSD and Linux.

Remove predefined list of metrics.
Remove C code for accessing sysctls on FreeBSD, use sysctl(8) instead.
---
 .../freebsd/kstat.zfs.misc.arcstats.txt       | 78 ++++++++++++++++++
 collector/zfs.go                              | 47 ++++++-----
 collector/zfs_freebsd.go                      | 80 ++++++++++++-------
 collector/zfs_freebsd_test.go                 | 44 ++++++++++
 collector/zfs_linux.go                        | 16 +---
 5 files changed, 201 insertions(+), 64 deletions(-)
 create mode 100644 collector/fixtures/sysctl/freebsd/kstat.zfs.misc.arcstats.txt
 create mode 100644 collector/zfs_freebsd_test.go

diff --git a/collector/fixtures/sysctl/freebsd/kstat.zfs.misc.arcstats.txt b/collector/fixtures/sysctl/freebsd/kstat.zfs.misc.arcstats.txt
new file mode 100644
index 0000000000..aaaf650319
--- /dev/null
+++ b/collector/fixtures/sysctl/freebsd/kstat.zfs.misc.arcstats.txt
@@ -0,0 +1,78 @@
+kstat.zfs.misc.arcstats.arc_meta_max: 1503210048
+kstat.zfs.misc.arcstats.arc_meta_limit: 393216000
+kstat.zfs.misc.arcstats.arc_meta_used: 392649848
+kstat.zfs.misc.arcstats.duplicate_reads: 0
+kstat.zfs.misc.arcstats.duplicate_buffers_size: 0
+kstat.zfs.misc.arcstats.duplicate_buffers: 0
+kstat.zfs.misc.arcstats.memory_throttle_count: 0
+kstat.zfs.misc.arcstats.l2_write_buffer_list_null_iter: 0
+kstat.zfs.misc.arcstats.l2_write_buffer_list_iter: 0
+kstat.zfs.misc.arcstats.l2_write_buffer_bytes_scanned: 0
+kstat.zfs.misc.arcstats.l2_write_pios: 0
+kstat.zfs.misc.arcstats.l2_write_buffer_iter: 0
+kstat.zfs.misc.arcstats.l2_write_full: 0
+kstat.zfs.misc.arcstats.l2_write_not_cacheable: 29425
+kstat.zfs.misc.arcstats.l2_write_io_in_progress: 0
+kstat.zfs.misc.arcstats.l2_write_in_l2: 0
+kstat.zfs.misc.arcstats.l2_write_spa_mismatch: 0
+kstat.zfs.misc.arcstats.l2_write_passed_headroom: 0
+kstat.zfs.misc.arcstats.l2_write_trylock_fail: 0
+kstat.zfs.misc.arcstats.l2_compress_failures: 0
+kstat.zfs.misc.arcstats.l2_compress_zeros: 0
+kstat.zfs.misc.arcstats.l2_compress_successes: 0
+kstat.zfs.misc.arcstats.l2_hdr_size: 0
+kstat.zfs.misc.arcstats.l2_asize: 0
+kstat.zfs.misc.arcstats.l2_size: 0
+kstat.zfs.misc.arcstats.l2_io_error: 0
+kstat.zfs.misc.arcstats.l2_cksum_bad: 0
+kstat.zfs.misc.arcstats.l2_abort_lowmem: 0
+kstat.zfs.misc.arcstats.l2_cdata_free_on_write: 0
+kstat.zfs.misc.arcstats.l2_free_on_write: 0
+kstat.zfs.misc.arcstats.l2_evict_reading: 0
+kstat.zfs.misc.arcstats.l2_evict_lock_retry: 0
+kstat.zfs.misc.arcstats.l2_writes_hdr_miss: 0
+kstat.zfs.misc.arcstats.l2_writes_error: 0
+kstat.zfs.misc.arcstats.l2_writes_done: 0
+kstat.zfs.misc.arcstats.l2_writes_sent: 0
+kstat.zfs.misc.arcstats.l2_write_bytes: 0
+kstat.zfs.misc.arcstats.l2_read_bytes: 0
+kstat.zfs.misc.arcstats.l2_rw_clash: 0
+kstat.zfs.misc.arcstats.l2_feeds: 0
+kstat.zfs.misc.arcstats.l2_misses: 0
+kstat.zfs.misc.arcstats.l2_hits: 0
+kstat.zfs.misc.arcstats.other_size: 166832272
+kstat.zfs.misc.arcstats.data_size: 1200779776
+kstat.zfs.misc.arcstats.hdr_size: 27244008
+kstat.zfs.misc.arcstats.size: 1394856056
+kstat.zfs.misc.arcstats.c_max: 1572864000
+kstat.zfs.misc.arcstats.c_min: 196608000
+kstat.zfs.misc.arcstats.c: 1470553736
+kstat.zfs.misc.arcstats.p: 665524427
+kstat.zfs.misc.arcstats.hash_chain_max: 7
+kstat.zfs.misc.arcstats.hash_chains: 14180
+kstat.zfs.misc.arcstats.hash_collisions: 2180398
+kstat.zfs.misc.arcstats.hash_elements_max: 238188
+kstat.zfs.misc.arcstats.hash_elements: 111458
+kstat.zfs.misc.arcstats.evict_l2_ineligible: 60262400
+kstat.zfs.misc.arcstats.evict_l2_eligible: 35702978560
+kstat.zfs.misc.arcstats.evict_l2_cached: 0
+kstat.zfs.misc.arcstats.evict_skip: 21716568
+kstat.zfs.misc.arcstats.mutex_miss: 873
+kstat.zfs.misc.arcstats.recycle_miss: 5018771
+kstat.zfs.misc.arcstats.stolen: 1327563
+kstat.zfs.misc.arcstats.deleted: 1187256
+kstat.zfs.misc.arcstats.allocated: 10150518
+kstat.zfs.misc.arcstats.mfu_ghost_hits: 1408986
+kstat.zfs.misc.arcstats.mfu_hits: 51952454
+kstat.zfs.misc.arcstats.mru_ghost_hits: 696819
+kstat.zfs.misc.arcstats.mru_hits: 11115835
+kstat.zfs.misc.arcstats.prefetch_metadata_misses: 32
+kstat.zfs.misc.arcstats.prefetch_metadata_hits: 2
+kstat.zfs.misc.arcstats.prefetch_data_misses: 0
+kstat.zfs.misc.arcstats.prefetch_data_hits: 0
+kstat.zfs.misc.arcstats.demand_metadata_misses: 9231542
+kstat.zfs.misc.arcstats.demand_metadata_hits: 40650947
+kstat.zfs.misc.arcstats.demand_data_misses: 75230
+kstat.zfs.misc.arcstats.demand_data_hits: 22417340
+kstat.zfs.misc.arcstats.misses: 9306804
+kstat.zfs.misc.arcstats.hits: 63068289
diff --git a/collector/zfs.go b/collector/zfs.go
index ae4560863c..376c05901d 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -5,6 +5,7 @@ package collector
 
 import (
 	"errors"
+	"strings"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/log"
@@ -31,19 +32,6 @@ type zfsMetric struct {
 	sysctl    zfsSysctl        // The sysctl of the ZFS metric.
 }
 
-func (m *zfsMetric) ConstMetric(value zfsMetricValue) prometheus.Metric {
-	return prometheus.MustNewConstMetric(
-		prometheus.NewDesc(
-			prometheus.BuildFQName(Namespace, string(m.subsystem), m.name),
-			m.name,
-			nil,
-			nil,
-		),
-		prometheus.UntypedValue,
-		float64(value),
-	)
-}
-
 type datasetMetric struct {
 	subsystem zfsSubsystemName
 	name      string
@@ -60,17 +48,7 @@ type zfsCollector struct {
 }
 
 func NewZFSCollector() (Collector, error) {
-	return &zfsCollector{
-		zfsMetrics: []zfsMetric{
-			{arc, "mru_size", zfsSysctl("kstat.zfs.misc.arcstats.p")},
-			{arc, "size", zfsSysctl("kstat.zfs.misc.arcstats.size")},
-			{arc, "target_min_size", zfsSysctl("kstat.zfs.misc.arcstats.c_min")},
-			{arc, "target_size", zfsSysctl("kstat.zfs.misc.arcstats.c")},
-			{arc, "target_max_size", zfsSysctl("kstat.zfs.misc.arcstats.c_max")},
-			{arc, "hits", zfsSysctl("kstat.zfs.misc.arcstats.hits")},
-			{arc, "misses", zfsSysctl("kstat.zfs.misc.arcstats.misses")},
-		},
-	}, nil
+	return &zfsCollector{}, nil
 }
 
 func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
@@ -93,3 +71,24 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 
 	return err
 }
+
+func (s zfsSysctl) metricName() string {
+	parts := strings.Split(string(s), ".")
+	return parts[len(parts)-1]
+}
+
+func (c *zfsCollector) ConstSysctlMetric(subsystem zfsSubsystemName, sysctl zfsSysctl, value zfsMetricValue) prometheus.Metric {
+
+	metricName := sysctl.metricName()
+
+	return prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, string(subsystem), metricName),
+			string(sysctl),
+			nil,
+			nil,
+		),
+		prometheus.UntypedValue,
+		float64(value),
+	)
+}
diff --git a/collector/zfs_freebsd.go b/collector/zfs_freebsd.go
index 2d6203b5ce..9cb600d60e 100644
--- a/collector/zfs_freebsd.go
+++ b/collector/zfs_freebsd.go
@@ -1,34 +1,20 @@
 package collector
 
 import (
-	"fmt"
-	"unsafe"
+	"bufio"
+	"io"
+	"os/exec"
+	"strconv"
+	"strings"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/log"
 )
 
 /*
 #cgo LDFLAGS:
-#include <fcntl.h>
-#include <stdlib.h>
 #include <sys/param.h>
 #include <sys/module.h>
-#include <sys/pcpu.h>
-#include <sys/resource.h>
-#include <sys/sysctl.h>
-#include <sys/time.h>
-
-int zfsIntegerSysctl(const char *name) {
-
-	int value;
-	size_t value_size = sizeof(value);
-	if (sysctlbyname(name,  &value, &value_size, NULL, 0) != -1 ||
-	    value_size != sizeof(value)) {
-		return -1;
-	}
-	return value;
-
-}
 
 int zfsModuleLoaded() {
 	int modid = modfind("zfs");
@@ -45,21 +31,59 @@ func (c *zfsCollector) PrepareUpdate() error {
 	return nil
 }
 
+const zfsArcstatsSysctl = "kstat.zfs.misc.arcstats"
+
 func (c *zfsCollector) updateArcstats(ch chan<- prometheus.Metric) (err error) {
 
-	for _, metric := range c.zfsMetrics {
+	cmd := exec.Command("sysctl", zfsArcstatsSysctl)
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return
+	}
+
+	if err = cmd.Start(); err != nil {
+		return
+	}
+
+	err = c.parseArcstatsSysctlOutput(stdout, func(sysctl zfsSysctl, value zfsMetricValue) {
+		ch <- c.ConstSysctlMetric(arc, sysctl, zfsMetricValue(value))
+	})
+	if err != nil {
+		return
+	}
+
+	if err = cmd.Wait(); err != nil {
+		return
+	}
+
+	return err
+}
+
+func (c *zfsCollector) parseArcstatsSysctlOutput(reader io.Reader, handler func(zfsSysctl, zfsMetricValue)) (err error) {
 
-		sysctlCString := C.CString(string(metric.sysctl))
-		defer C.free(unsafe.Pointer(sysctlCString))
+	// Decode values
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
 
-		value := int(C.zfsIntegerSysctl(sysctlCString))
+		fields := strings.Fields(scanner.Text())
+
+		if len(fields) != 2 ||
+			!strings.HasPrefix(fields[0], zfsArcstatsSysctl) ||
+			!strings.HasSuffix(fields[0], ":") {
+
+			log.Debugf("Skipping line of unknown format: %s", scanner.Text())
+			continue
 
-		if value == -1 {
-			return fmt.Errorf("Could not retrieve value for metric '%v'", metric)
 		}
 
-		ch <- metric.ConstMetric(zfsMetricValue(value))
+		sysctl := zfsSysctl(strings.TrimSuffix(fields[0], ":"))
+		value, err := strconv.Atoi(fields[1])
+		if err != nil {
+			return err
+		}
+
+		handler(sysctl, zfsMetricValue(value))
 	}
+	return scanner.Err()
 
-	return err
 }
diff --git a/collector/zfs_freebsd_test.go b/collector/zfs_freebsd_test.go
new file mode 100644
index 0000000000..cad90eab0e
--- /dev/null
+++ b/collector/zfs_freebsd_test.go
@@ -0,0 +1,44 @@
+package collector
+
+import (
+	"os"
+	"testing"
+)
+
+func TestArcstatsParsing(t *testing.T) {
+
+	arcstatsOutput, err := os.Open("fixtures/sysctl/freebsd/kstat.zfs.misc.arcstats.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer arcstatsOutput.Close()
+
+	c := zfsCollector{}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	handlerCalled := false
+	err = c.parseArcstatsSysctlOutput(arcstatsOutput, func(s zfsSysctl, v zfsMetricValue) {
+
+		if s != zfsSysctl("kstat.zfs.misc.arcstats.hits") {
+			return
+		}
+
+		handlerCalled = true
+
+		if v != zfsMetricValue(63068289) {
+			t.Fatalf("Incorrect value parsed from sysctl output")
+		}
+
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !handlerCalled {
+		t.Fatal("Arcstats parsing handler was not called for some expected sysctls")
+	}
+
+}
diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
index 1c464a3e78..49854f0e05 100644
--- a/collector/zfs_linux.go
+++ b/collector/zfs_linux.go
@@ -28,7 +28,7 @@ func (c *zfsCollector) PrepareUpdate() (err error) {
 func (c *zfsCollector) openArcstatsFile() (file *os.File, err error) {
 	file, err = os.Open(procFilePath(zfsArcstatsProcpath))
 	if err != nil {
-		log.Debugf("Cannot open '%s' for reading.Is the kernel module loaded?", procFilePath(zfsArcstatsProcpath))
+		log.Debugf("Cannot open '%s' for reading. Is the kernel module loaded?", procFilePath(zfsArcstatsProcpath))
 		err = zfsNotAvailableError
 	}
 	return
@@ -40,16 +40,7 @@ func (c *zfsCollector) updateArcstats(ch chan<- prometheus.Metric) (err error) {
 	defer file.Close()
 
 	return c.parseArcstatsProcfsFile(file, func(s zfsSysctl, v zfsMetricValue) {
-		// TODO: Find corresponding metric in a more efficient way
-		for _, metric := range c.zfsMetrics {
-			if metric.subsystem != arc {
-				continue
-			}
-			if metric.sysctl != s {
-				continue
-			}
-			ch <- metric.ConstMetric(v)
-		}
+		ch <- c.ConstSysctlMetric(arc, s, v)
 	})
 
 }
@@ -60,6 +51,7 @@ func (c *zfsCollector) parseArcstatsProcfsFile(reader io.Reader, handler func(zf
 
 	parseLine := false
 	for scanner.Scan() {
+
 		parts := strings.Fields(scanner.Text())
 
 		if !parseLine && len(parts) == 3 && parts[0] == "name" && parts[1] == "type" && parts[2] == "data" {
@@ -78,8 +70,8 @@ func (c *zfsCollector) parseArcstatsProcfsFile(reader io.Reader, handler func(zf
 		if err != nil {
 			return fmt.Errorf("could not parse expected integer value for '%s'", key)
 		}
-		log.Debugf("%s = %d", key, value)
 		handler(zfsSysctl(key), zfsMetricValue(value))
+
 	}
 	if !parseLine {
 		return errors.New("did not parse a single arcstat metrics")

From 006cf29bc3d8b5c1371d0596f7e2309607116649 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Tue, 15 Mar 2016 23:22:26 +0100
Subject: [PATCH 21/24] Rename PrepareUpdate() to make intentions behind method
 clear.

---
 collector/zfs.go         | 3 +--
 collector/zfs_freebsd.go | 2 +-
 collector/zfs_linux.go   | 5 ++++-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/collector/zfs.go b/collector/zfs.go
index 376c05901d..6ae03d2f19 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -53,7 +53,7 @@ func NewZFSCollector() (Collector, error) {
 
 func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 
-	err = c.PrepareUpdate()
+	err = c.zfsAvailable()
 	switch {
 	case err == zfsNotAvailableError:
 		log.Debug(err)
@@ -63,7 +63,6 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 	}
 
 	// Arcstats
-
 	err = c.updateArcstats(ch)
 	if err != nil {
 		return err
diff --git a/collector/zfs_freebsd.go b/collector/zfs_freebsd.go
index 9cb600d60e..28cc4c5058 100644
--- a/collector/zfs_freebsd.go
+++ b/collector/zfs_freebsd.go
@@ -24,7 +24,7 @@ int zfsModuleLoaded() {
 */
 import "C"
 
-func (c *zfsCollector) PrepareUpdate() error {
+func (c *zfsCollector) zfsAvailable() error {
 	if C.zfsModuleLoaded() == 0 {
 		return zfsNotAvailableError
 	}
diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
index 49854f0e05..f7627926d2 100644
--- a/collector/zfs_linux.go
+++ b/collector/zfs_linux.go
@@ -17,7 +17,7 @@ const (
 	zfsArcstatsProcpath = "spl/kstat/zfs/arcstats"
 )
 
-func (c *zfsCollector) PrepareUpdate() (err error) {
+func (c *zfsCollector) zfsAvailable() (err error) {
 	file, err := c.openArcstatsFile()
 	if err != nil {
 		file.Close()
@@ -37,6 +37,9 @@ func (c *zfsCollector) openArcstatsFile() (file *os.File, err error) {
 func (c *zfsCollector) updateArcstats(ch chan<- prometheus.Metric) (err error) {
 
 	file, err := c.openArcstatsFile()
+	if err != nil {
+		return err
+	}
 	defer file.Close()
 
 	return c.parseArcstatsProcfsFile(file, func(s zfsSysctl, v zfsMetricValue) {

From 3bb53560205200271c333b97b0c42f77921150ac Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Wed, 16 Mar 2016 22:58:00 +0100
Subject: [PATCH 22/24] Expose zpool metrics on FreeBSD.

---
 collector/fixtures/zfs/zpool_stats_stdout.txt | 12 ++++
 collector/zfs.go                              | 23 +++++++-
 collector/zfs_freebsd.go                      | 36 +++++++++--
 collector/zfs_linux.go                        |  4 ++
 collector/zfs_zpool.go                        | 41 +++++++++++++
 collector/zfs_zpool_test.go                   | 59 +++++++++++++++++++
 6 files changed, 169 insertions(+), 6 deletions(-)
 create mode 100644 collector/fixtures/zfs/zpool_stats_stdout.txt
 create mode 100644 collector/zfs_zpool.go
 create mode 100644 collector/zfs_zpool_test.go

diff --git a/collector/fixtures/zfs/zpool_stats_stdout.txt b/collector/fixtures/zfs/zpool_stats_stdout.txt
new file mode 100644
index 0000000000..104e2d18b3
--- /dev/null
+++ b/collector/fixtures/zfs/zpool_stats_stdout.txt
@@ -0,0 +1,12 @@
+trout	size	4294967296	-
+trout	free	1040117248	-
+trout	allocated	70144	-
+trout	capacity	0%	-
+trout	dedupratio	1.00x	-
+trout	fragmentation	0%	-
+zroot	size	118111600640	-
+zroot	free	3990917120	-
+zroot	allocated	114120683520	-
+zroot	capacity	50%	-
+zroot	dedupratio	1.00x	-
+zroot	fragmentation	67%	-
diff --git a/collector/zfs.go b/collector/zfs.go
index 6ae03d2f19..1b9f020d8c 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -21,7 +21,8 @@ type zfsSysctl string
 type zfsSubsystemName string
 
 const (
-	arc = zfsSubsystemName("zfsArc")
+	arc            = zfsSubsystemName("zfsArc")
+	zpoolSubsystem = zfsSubsystemName("zfsPool")
 )
 
 // Metrics
@@ -68,6 +69,12 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 		return err
 	}
 
+	// Pool stats
+	err = c.updatePoolStats(ch)
+	if err != nil {
+		return err
+	}
+
 	return err
 }
 
@@ -91,3 +98,17 @@ func (c *zfsCollector) ConstSysctlMetric(subsystem zfsSubsystemName, sysctl zfsS
 		float64(value),
 	)
 }
+
+func (c *zfsCollector) ConstZpoolMetric(pool, name string, value float64) prometheus.Metric {
+	return prometheus.MustNewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(Namespace, string(zpoolSubsystem), name),
+			name,
+			[]string{"pool"},
+			nil,
+		),
+		prometheus.UntypedValue,
+		float64(value),
+		pool,
+	)
+}
diff --git a/collector/zfs_freebsd.go b/collector/zfs_freebsd.go
index 28cc4c5058..3fd571c507 100644
--- a/collector/zfs_freebsd.go
+++ b/collector/zfs_freebsd.go
@@ -33,9 +33,8 @@ func (c *zfsCollector) zfsAvailable() error {
 
 const zfsArcstatsSysctl = "kstat.zfs.misc.arcstats"
 
-func (c *zfsCollector) updateArcstats(ch chan<- prometheus.Metric) (err error) {
+func (c *zfsCollector) RunOnStdout(cmd *exec.Cmd, handler func(io.Reader) error) (err error) {
 
-	cmd := exec.Command("sysctl", zfsArcstatsSysctl)
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
 		return
@@ -45,9 +44,7 @@ func (c *zfsCollector) updateArcstats(ch chan<- prometheus.Metric) (err error) {
 		return
 	}
 
-	err = c.parseArcstatsSysctlOutput(stdout, func(sysctl zfsSysctl, value zfsMetricValue) {
-		ch <- c.ConstSysctlMetric(arc, sysctl, zfsMetricValue(value))
-	})
+	err = handler(stdout)
 	if err != nil {
 		return
 	}
@@ -57,6 +54,20 @@ func (c *zfsCollector) updateArcstats(ch chan<- prometheus.Metric) (err error) {
 	}
 
 	return err
+
+}
+
+func (c *zfsCollector) updateArcstats(ch chan<- prometheus.Metric) (err error) {
+
+	cmd := exec.Command("sysctl", zfsArcstatsSysctl)
+
+	err = c.RunOnStdout(cmd, func(stdout io.Reader) error {
+		return c.parseArcstatsSysctlOutput(stdout, func(sysctl zfsSysctl, value zfsMetricValue) {
+			ch <- c.ConstSysctlMetric(arc, sysctl, zfsMetricValue(value))
+		})
+	})
+	return err
+
 }
 
 func (c *zfsCollector) parseArcstatsSysctlOutput(reader io.Reader, handler func(zfsSysctl, zfsMetricValue)) (err error) {
@@ -87,3 +98,18 @@ func (c *zfsCollector) parseArcstatsSysctlOutput(reader io.Reader, handler func(
 	return scanner.Err()
 
 }
+
+func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) (err error) {
+
+	poolProperties := []string{"size", "free", "allocated", "capacity", "dedupratio", "fragmentation"}
+
+	cmd := exec.Command("zpool", "get", "-pH", strings.Join(poolProperties, ","))
+
+	err = c.RunOnStdout(cmd, func(stdout io.Reader) error {
+		return c.parseZpoolOutput(stdout, func(pool, name string, value float64) {
+			ch <- c.ConstZpoolMetric(pool, name, value)
+		})
+	})
+
+	return err
+}
diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
index f7627926d2..44efd47e4d 100644
--- a/collector/zfs_linux.go
+++ b/collector/zfs_linux.go
@@ -82,3 +82,7 @@ func (c *zfsCollector) parseArcstatsProcfsFile(reader io.Reader, handler func(zf
 
 	return scanner.Err()
 }
+
+func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) (err error) {
+	return nil
+}
diff --git a/collector/zfs_zpool.go b/collector/zfs_zpool.go
new file mode 100644
index 0000000000..8ebdba9ac2
--- /dev/null
+++ b/collector/zfs_zpool.go
@@ -0,0 +1,41 @@
+package collector
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+)
+
+// zpool metrics
+
+func (c *zfsCollector) parseZpoolOutput(reader io.Reader, handler func(string, string, float64)) (err error) {
+
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+
+		fields := strings.Fields(scanner.Text())
+		if len(fields) != 4 {
+			return fmt.Errorf("Unexpected output of zpool command")
+		}
+
+		valueString := fields[2]
+		switch {
+		case strings.HasSuffix(fields[2], "%"):
+			percentage := strings.TrimSuffix(fields[2], "%")
+			valueString = "0." + percentage
+		case strings.HasSuffix(fields[2], "x"):
+			valueString = strings.TrimSuffix(fields[2], "x")
+		}
+
+		value, err := strconv.ParseFloat(valueString, 64)
+		if err != nil {
+			return err
+		}
+		handler(fields[0], fields[1], value)
+
+	}
+	return scanner.Err()
+
+}
diff --git a/collector/zfs_zpool_test.go b/collector/zfs_zpool_test.go
new file mode 100644
index 0000000000..fd09babe8b
--- /dev/null
+++ b/collector/zfs_zpool_test.go
@@ -0,0 +1,59 @@
+package collector
+
+import (
+	"os"
+	"testing"
+)
+
+func TestZpoolParsing(t *testing.T) {
+
+	zpoolOutput, err := os.Open("fixtures/zfs/zpool_stats_stdout.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer zpoolOutput.Close()
+
+	c := zfsCollector{}
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pools := make([]string, 2)
+	troutSize := float64(-1)
+	troutDedupratio := float64(-1)
+	zrootCapacity := float64(-1)
+
+	err = c.parseZpoolOutput(zpoolOutput, func(pool, name string, value float64) {
+		pools = append(pools, pool)
+		if pool == "trout" && name == "size" {
+			troutSize = value
+		}
+		if pool == "trout" && name == "dedupratio" {
+			troutDedupratio = value
+		}
+		if pool == "zroot" && name == "capacity" {
+			zrootCapacity = value
+		}
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if pools[0] == "trout" && pools[1] == "zroot" {
+		t.Fatal("Did not parse all pools in fixture")
+	}
+
+	if troutSize != float64(4294967296) {
+		t.Fatal("Unexpected value for pool 'trout's size value")
+	}
+
+	if troutDedupratio != float64(1.0) {
+		t.Fatal("Unexpected value for pool 'trout's dedupratio value")
+	}
+
+	if zrootCapacity != float64(0.5) {
+		t.Fatal("Unexpected value for pool 'zroot's capacity value")
+	}
+
+}

From a102306835b0ee6397ae383e01a76ad35c2c8736 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Wed, 16 Mar 2016 23:46:16 +0100
Subject: [PATCH 23/24] Update ZFS collector description.

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 19dda845c3..de4b1177c9 100644
--- a/README.md
+++ b/README.md
@@ -32,7 +32,7 @@ textfile | Exposes statistics read from local disk. The `--collector.textfile.di
 time | Exposes the current system time. | _any_
 vmstat | Exposes statistics from `/proc/vmstat`. | Linux
 version | Exposes node\_exporter version. | _any_
-zfs | Exposes [ZFS](http://open-zfs.org/) performance statistics (ARC-only for now) | [FreeBSD](https://www.freebsd.org/doc/handbook/zfs.html),[Linux](http://zfsonlinux.org/)
+zfs | Exposes [ZFS](http://open-zfs.org/) performance statistics.<br/> FreeBSD (ARC, zpool), Linux (ARC) | [FreeBSD](https://www.freebsd.org/doc/handbook/zfs.html), [Linux](http://zfsonlinux.org/)
 
 ### Disabled by default
 

From e8d45a242338cd28017c0ef9db90c333af1820b2 Mon Sep 17 00:00:00 2001
From: Christian Schwarz <me@cschwarz.com>
Date: Mon, 28 Mar 2016 10:57:35 +0200
Subject: [PATCH 24/24] Stylistic fixes.

---
 collector/zfs.go         | 7 +------
 collector/zfs_freebsd.go | 9 ++-------
 collector/zfs_linux.go   | 6 +++---
 3 files changed, 6 insertions(+), 16 deletions(-)

diff --git a/collector/zfs.go b/collector/zfs.go
index 1b9f020d8c..9749a0b31b 100644
--- a/collector/zfs.go
+++ b/collector/zfs.go
@@ -70,12 +70,7 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) (err error) {
 	}
 
 	// Pool stats
-	err = c.updatePoolStats(ch)
-	if err != nil {
-		return err
-	}
-
-	return err
+	return c.updatePoolStats(ch)
 }
 
 func (s zfsSysctl) metricName() string {
diff --git a/collector/zfs_freebsd.go b/collector/zfs_freebsd.go
index 3fd571c507..9d344dc8d8 100644
--- a/collector/zfs_freebsd.go
+++ b/collector/zfs_freebsd.go
@@ -49,12 +49,7 @@ func (c *zfsCollector) RunOnStdout(cmd *exec.Cmd, handler func(io.Reader) error)
 		return
 	}
 
-	if err = cmd.Wait(); err != nil {
-		return
-	}
-
-	return err
-
+	return cmd.Wait()
 }
 
 func (c *zfsCollector) updateArcstats(ch chan<- prometheus.Metric) (err error) {
@@ -82,7 +77,7 @@ func (c *zfsCollector) parseArcstatsSysctlOutput(reader io.Reader, handler func(
 			!strings.HasPrefix(fields[0], zfsArcstatsSysctl) ||
 			!strings.HasSuffix(fields[0], ":") {
 
-			log.Debugf("Skipping line of unknown format: %s", scanner.Text())
+			log.Debugf("Skipping line of unknown format: %q", scanner.Text())
 			continue
 
 		}
diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go
index 44efd47e4d..0a1c10e135 100644
--- a/collector/zfs_linux.go
+++ b/collector/zfs_linux.go
@@ -28,7 +28,7 @@ func (c *zfsCollector) zfsAvailable() (err error) {
 func (c *zfsCollector) openArcstatsFile() (file *os.File, err error) {
 	file, err = os.Open(procFilePath(zfsArcstatsProcpath))
 	if err != nil {
-		log.Debugf("Cannot open '%s' for reading. Is the kernel module loaded?", procFilePath(zfsArcstatsProcpath))
+		log.Debugf("Cannot open %q for reading. Is the kernel module loaded?", procFilePath(zfsArcstatsProcpath))
 		err = zfsNotAvailableError
 	}
 	return
@@ -71,13 +71,13 @@ func (c *zfsCollector) parseArcstatsProcfsFile(reader io.Reader, handler func(zf
 
 		value, err := strconv.Atoi(parts[2])
 		if err != nil {
-			return fmt.Errorf("could not parse expected integer value for '%s'", key)
+			return fmt.Errorf("could not parse expected integer value for %q", key)
 		}
 		handler(zfsSysctl(key), zfsMetricValue(value))
 
 	}
 	if !parseLine {
-		return errors.New("did not parse a single arcstat metrics")
+		return errors.New("did not parse a single arcstat metric")
 	}
 
 	return scanner.Err()