Skip to content

Commit

Permalink
Want command to print ZFS Histograms
Browse files Browse the repository at this point in the history 
Histogram option for spa, vdev, and metaslab commands
  • Loading branch information
@sdimitro
sdimitro committed Apr 10, 2020
1 parent 8c14613 commit 47debdd
Show file tree
Hide file tree
Showing 13 changed files with 1,351 additions and 27 deletions.
149 changes: 149 additions & 0 deletions sdb/commands/zfs/histograms.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
#
# Copyright 2020 Delphix
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# pylint: disable=missing-docstring

import argparse
from typing import Iterable

import drgn
import sdb
from sdb.commands.internal import fmt


class ZFSHistogram(sdb.Command):
"""
Print ZFS Histogram and print its median segment size.
NOTE
The median is just an approximation as we can't tell the
exact size of each bucket within a histogram bucket.
EXAMPLES
Dump the histogram of the normal metaslab class of the rpool:
sdb> spa rpool | member spa_normal_class.mc_histogram | zhist
seg-size count
-------- -----
512.0B: 4359 *******************
1.0KB: 3328 ***************
2.0KB: 3800 *****************
4.0KB: 3536 ***************
8.0KB: 3983 *****************
16.0KB: 4876 *********************
32.0KB: 9138 ****************************************
64.0KB: 4508 ********************
128.0KB: 2783 ************
256.0KB: 1952 *********
512.0KB: 1218 *****
1.0MB: 675 ***
2.0MB: 486 **
4.0MB: 267 *
8.0MB: 110
16.0MB: 50
32.0MB: 18
64.0MB: 8
128.0MB: 11
256.0MB: 102
Approx. Median: 339.7MB
"""

names = ["zfs_histogram", "zhist"]

@classmethod
def _init_parser(cls, name: str) -> argparse.ArgumentParser:
parser = super()._init_parser(name)
parser.add_argument("offset", nargs="?", default=0, type=int)
return parser

@staticmethod
def histogram_median(hist: drgn.Object, offset: int = 0) -> int:
"""
Returns the approximated median of a ZFS histogram.
"""
canonical_type = sdb.type_canonicalize(hist.type_)
assert canonical_type.kind == drgn.TypeKind.ARRAY
assert sdb.type_canonicalize(
canonical_type.type).kind == drgn.TypeKind.INT

total_space = 0
for (bucket, value) in enumerate(hist):
total_space += int(value) << (bucket + offset)

if total_space == 0:
return 0

space_left, median = total_space / 2, 0
for (bucket, value) in enumerate(hist):
space_in_bucket = int(value) << (bucket + offset)
if space_left <= space_in_bucket:
median = 1 << (bucket + offset)
#
# Size of segments may vary within one bucket thus we
# attempt to approximate the median by looking at the
# number of segments in the bucket and assuming that
# they are evenly distributed along the bucket's range.
#
bucket_fill = space_left / space_in_bucket
median += round(median * bucket_fill)
break
space_left -= space_in_bucket
return median

@staticmethod
def print_histogram_median(hist: drgn.Object,
offset: int = 0,
indent: int = 0) -> None:
median = ZFSHistogram.histogram_median(hist, offset)
print(f'{" " * indent}Approx. Median: {fmt.size_nicenum(median)}')

@staticmethod
def print_histogram(hist: drgn.Object,
offset: int = 0,
indent: int = 0) -> None:
canonical_type = sdb.type_canonicalize(hist.type_)
assert canonical_type.kind == drgn.TypeKind.ARRAY
assert sdb.type_canonicalize(
canonical_type.type).kind == drgn.TypeKind.INT

max_count = 0
min_bucket = (len(hist) - 1)
max_bucket = 0
for (bucket, value) in enumerate(hist):
count = int(value)
if bucket < min_bucket and count > 0:
min_bucket = bucket
if bucket > max_bucket and count > 0:
max_bucket = bucket
if count > max_count:
max_count = count

HISTOGRAM_WIDTH_MAX = 40
if max_count < HISTOGRAM_WIDTH_MAX:
max_count = HISTOGRAM_WIDTH_MAX

for bucket in range(min_bucket, max_bucket + 1):
count = int(hist[bucket])
stars = round(count * HISTOGRAM_WIDTH_MAX / max_count)
print(f'{" " * indent}{fmt.size_nicenum(2**(bucket+offset)):>8}: '
f'{count:>6} {"*" * stars}')

def _call(self, objs: Iterable[drgn.Object]) -> None:
for obj in objs:
print(f'seg-size count')
print(f'{"-" * 8} {"-" * 5}')
ZFSHistogram.print_histogram(obj, self.args.offset)
ZFSHistogram.print_histogram_median(obj, self.args.offset)
20 changes: 0 additions & 20 deletions sdb/commands/zfs/internal/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,26 +32,6 @@ def enum_lookup(enum_type_name: str, value: int) -> str:
return enum_string[prefix.rfind("_") + 1:]


def print_histogram(histogram: List[int], size: int, offset: int) -> None:
max_data = 0
maxidx = 0
minidx = size - 1

for i in range(0, size):
if histogram[i] > max_data:
max_data = histogram[i]
if histogram[i] > 0 and i > maxidx:
maxidx = i
if histogram[i] > 0 and i < minidx:
minidx = i
if max_data < 40:
max_data = 40

for i in range(minidx, maxidx + 1):
print("%3u: %6u %s" %
(i + offset, histogram[i], "*" * int(histogram[i])))


def nicenum(num: int, suffix: str = "B") -> str:
for unit in ["", "K", "M", "G", "T", "P", "E", "Z"]:
if num < 1024:
Expand Down
12 changes: 8 additions & 4 deletions sdb/commands/zfs/metaslab.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@
from sdb.commands.zfs.internal import (
METASLAB_ACTIVE_MASK, METASLAB_WEIGHT_CLAIM, METASLAB_WEIGHT_PRIMARY,
METASLAB_WEIGHT_SECONDARY, METASLAB_WEIGHT_TYPE, WEIGHT_GET_COUNT,
WEIGHT_GET_INDEX, WEIGHT_IS_SPACEBASED, BTREE_LEAF_SIZE, nicenum,
print_histogram)
WEIGHT_GET_INDEX, WEIGHT_IS_SPACEBASED, BTREE_LEAF_SIZE, nicenum)
from sdb.commands.zfs.histograms import ZFSHistogram


class Metaslab(sdb.Locator, sdb.PrettyPrinter):
Expand Down Expand Up @@ -165,13 +165,17 @@ def pretty_print(self,
indent: int = 0) -> None:
first_time = True
for msp in metaslabs:
if not self.args.histogram and not self.args.weight:
if not self.args.weight:
Metaslab.print_metaslab(msp, first_time, indent)
if self.args.histogram:
spacemap = msp.ms_sm
if spacemap != sdb.get_typed_null(spacemap.type_):
histogram = spacemap.sm_phys.smp_histogram
print_histogram(histogram, 32, spacemap.sm_shift)
ZFSHistogram.print_histogram(histogram,
int(spacemap.sm_shift), indent)
ZFSHistogram.print_histogram_median(histogram,
int(spacemap.sm_shift),
indent)
if self.args.weight:
Metaslab.metaslab_weight_print(msp, first_time, indent)
first_time = False
Expand Down
7 changes: 7 additions & 0 deletions sdb/commands/zfs/spa.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import sdb
from sdb.commands.spl.avl import Avl
from sdb.commands.zfs.vdev import Vdev
from sdb.commands.zfs.histograms import ZFSHistogram


class Spa(sdb.Locator, sdb.PrettyPrinter):
Expand Down Expand Up @@ -68,6 +69,12 @@ def pretty_print(self, spas: Iterable[drgn.Object]) -> None:
for spa in spas:
print("{:18} {}".format(hex(spa),
spa.spa_name.string_().decode("utf-8")))
if self.args.histogram:
ZFSHistogram.print_histogram(spa.spa_normal_class.mc_histogram,
0, 5)
ZFSHistogram.print_histogram_median(
spa.spa_normal_class.mc_histogram, 0, 5)

if self.args.vdevs:
vdevs = sdb.execute_pipeline([spa], [Vdev()])
Vdev(self.arg_string).pretty_print(vdevs, 5)
Expand Down
46 changes: 46 additions & 0 deletions tests/integration/data/regression_output/zfs/spa -H
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
ADDR NAME
------------------------------------------------------------
0xffffa0894e720000 data
512.0B: 32 ********************************
1.0KB: 27 ***************************
2.0KB: 32 ********************************
4.0KB: 7 *******
8.0KB: 1 *
16.0KB: 1 *
32.0KB: 0
64.0KB: 0
128.0KB: 0
256.0KB: 0
512.0KB: 0
1.0MB: 0
2.0MB: 0
4.0MB: 0
8.0MB: 0
16.0MB: 0
32.0MB: 0
64.0MB: 0
128.0MB: 0
256.0MB: 15 ***************
Approx. Median: 384.0MB
0xffffa089413b8000 meta-domain
1.0KB: 18 ******************
2.0KB: 24 ************************
4.0KB: 17 *****************
8.0KB: 21 *********************
16.0KB: 0
32.0KB: 0
64.0KB: 0
128.0KB: 0
256.0KB: 0
512.0KB: 0
1.0MB: 0
2.0MB: 0
4.0MB: 0
8.0MB: 0
16.0MB: 0
32.0MB: 0
64.0MB: 1 *
128.0MB: 4 ****
Approx. Median: 184.0MB
0xffffa08955c44000 rpool
Approx. Median: 0.0B
64 changes: 64 additions & 0 deletions tests/integration/data/regression_output/zfs/spa -vH
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
ADDR NAME
------------------------------------------------------------
0xffffa0894e720000 data
512.0B: 32 ********************************
1.0KB: 27 ***************************
2.0KB: 32 ********************************
4.0KB: 7 *******
8.0KB: 1 *
16.0KB: 1 *
32.0KB: 0
64.0KB: 0
128.0KB: 0
256.0KB: 0
512.0KB: 0
1.0MB: 0
2.0MB: 0
4.0MB: 0
8.0MB: 0
16.0MB: 0
32.0MB: 0
64.0MB: 0
128.0MB: 0
256.0MB: 15 ***************
Approx. Median: 384.0MB
ADDR STATE AUX DESCRIPTION
------------------------------------------------------------
0xffffa089486fc000 HEALTHY NONE root
0xffffa08949ff4000 HEALTHY NONE mirror
0xffffa08948af8000 HEALTHY NONE /dev/sdb1
0xffffa08949ff8000 HEALTHY NONE /dev/sdc1
0xffffa08949e58000 HEALTHY NONE /dev/sdd1
0xffffa089413b8000 meta-domain
1.0KB: 18 ******************
2.0KB: 24 ************************
4.0KB: 17 *****************
8.0KB: 21 *********************
16.0KB: 0
32.0KB: 0
64.0KB: 0
128.0KB: 0
256.0KB: 0
512.0KB: 0
1.0MB: 0
2.0MB: 0
4.0MB: 0
8.0MB: 0
16.0MB: 0
32.0MB: 0
64.0MB: 1 *
128.0MB: 4 ****
Approx. Median: 184.0MB
ADDR STATE AUX DESCRIPTION
------------------------------------------------------------
0xffffa08953aa4000 HEALTHY NONE root
0xffffa08953aa8000 HEALTHY NONE raidz
0xffffa08953aac000 HEALTHY NONE /tmp/dks0
0xffffa08953ab0000 HEALTHY NONE /tmp/dks1
0xffffa08953ab4000 HEALTHY NONE /tmp/dks2
0xffffa08955c44000 rpool
Approx. Median: 0.0B
ADDR STATE AUX DESCRIPTION
------------------------------------------------------------
0xffffa08952efc000 HEALTHY NONE root
0xffffa08953300000 HEALTHY NONE /dev/sda1
Loading

0 comments on commit 47debdd

Please sign in to comment.