Testing

.circleci/config.yml

@@ -0,0 +1,94 @@
+version: 2.1
+
+commands:
+  install_dependencies:
+    parameters:
+      repo:
+        type: string
+      token:
+        type: env_var_name
+      install:
+        type: string
+    steps:
+      # Docker based images by default do not have lsmod or modprobe.
+      # We need to install kmod so that the varnishgather can run in a container.
+      - run:
+          name: "Install dependencies"
+          command: |
+            if command -v apt-get > /dev/null; then
+              apt update -y
+              apt install -y kmod curl
+            else
+              yum update -y
+              yum install -y kmod curl
+            fi
+            curl https://docs.varnish-software.com/scripts/setup.sh | TOKEN=${<<parameters.token>>} REPO=<<parameters.repo>> INSTALL="<<parameters.install>>" bash
+
+varnish: &varnish
+  parameters:
+    platform:
+      type: string
+  docker:
+    - image: <<parameters.platform>>
+  steps:
+    - checkout
+    # Why does the varnishgather exit with a non-zero status code, even if
+    # it is successfull? For now, I'm ignoring the exit code to make progress
+    # on the CircleCi configuration.
+    - run: ./varnishgather || true
+    - run:
+        name: "Move gather"
+        command: |
+          mkdir -p gathers/<<parameters.platform>>
+          mv *.tar.gz gathers/<<parameters.platform>>/
+    - persist_to_workspace:
+        root: ./
+        paths:
+          - gathers
+
+jobs:
+  varnish_41:
+    <<: *varnish
+  varnish_60:
+    <<: *varnish
+  test:
+    docker:
+      - image: cimg/python:3.9.2
+    steps:
+      - checkout
+      - attach_workspace:
+          at: ./gathers
+      - run: python test.py "gathers"
+
+workflows:
+  test:
+    jobs:
+      - varnish_41:
+          pre-steps:
+            - install_dependencies:
+                repo: "41"
+                token: "PROJECT_PACKAGECLOUD_41_TOKEN"
+                install: "varnish-plus"
+          matrix:
+            parameters:
+              platform: [
+                "debian:stretch",
+                "ubuntu:xenial",
+                "centos:7"
+              ]
+      - varnish_60:
+          pre-steps:
+            - install_dependencies:
+                repo: "60"
+                token: "PROJECT_PACKAGECLOUD_60_TOKEN"
+                install: "varnish-plus"
+          matrix:
+            parameters:
+              platform: [
+                "debian:buster", "debian:stretch", "debian:bullseye",
+                "ubuntu:bionic", "ubuntu:focal", "ubuntu:xenial",
+                "centos:7", "centos:8",
+                "rockylinux:8"
+              ]
+      - test:
+          requires: ["varnish_60", "varnish_41"]

test.py

@@ -0,0 +1,208 @@
+import os
+import re
+import inspect
+import unittest
+import argparse
+import tarfile
+from tests.commands import TestRunCommand, TestRunPipeCommand, TestVarnishAdmCommand
+
+class TestCommand(object):
+    def __init__(self, cmd, test_class=TestRunCommand):
+        self.cmd = cmd
+        self.test_class = test_class
+
+    def path(self):
+        return re.sub(r'[\s\\/\.]', '_', self.cmd)
+
+    def matches(self, file):
+        if self.path() in file:
+            return True
+        return False
+
+    def setup_test_case(self, root, log):
+        platform = root.split('/')[-1]
+        testclass = type("platform(%s), command(%s)" % (platform, self.path()), (self.test_class, ),{
+            'command': self.cmd,
+            'command_path': self.path(),
+            'log': log
+        })
+        return unittest.TestLoader().loadTestsFromTestCase(testclass)
+
+class TestSuiteGather(unittest.TestSuite):
+    # For a given varnishgather that was run on a specific platform,
+    #   this test suite will attempt to match each command with the related file.
+    # Each command will have its own test case.
+    # If a command can not be mached with a file, it is up to the test case to
+    #   determine if that is a failure or not.
+    def __init__(self, root, gather, commands, debug=False):
+        super().__init__()
+        self.gather = gather
+        self.tar = tarfile.open(os.path.join(root,gather), 'r:*')
+        self.debug = debug
+        self._debug("Setting up %s" % gather)
+        members = self.tar.getmembers()
+        for c in commands:
+            if type(c) == str:
+                self._debug("Initializing command(%s) with defaults." % c)
+                c = TestCommand(c)
+            else:
+                self._debug("Command(%s) already initalized." % c)
+            exists = None
+
+            for i, member in enumerate(members):
+                log = member.name.split('/')[-1]
+                if c.matches(log):
+                    self._debug("    Command(%s) matched: %s" % (c.path(), log))
+                    exists = members.pop(i)
+                    break
+
+            buf = None
+            if exists is None:
+                self._debug("   Unable to match")
+            else:
+                buf = self.tar.extractfile(exists)
+                buf = buf.readlines()
+
+            self.addTest(c.setup_test_case(root, buf))
+
+    def __del__(self):
+        self._debug("Cleaning up %s" % self.gather)
+        self.tar.close()
+
+    def _debug(self, *msg):
+        if self.debug:
+            print("DEBUG: %s" % msg)
+
+def main():
+    parser = argparse.ArgumentParser(description='Test the output of varnishgather')
+    parser.add_argument('target', type=str)
+    parser.add_argument('--debug', default=False, action="store_true")
+    args = parser.parse_args()
+
+    # A list of commands that are executed during a varnishgather and should be
+    #   tested. Each command will be treated as an individual test suite that
+    #   will default to using the "TestRunCommand" test case.
+    commands = [
+        'varnishd -V',
+        'varnish-agent -V',
+        'vha-agent -V',
+        'broadcaster -V',
+        'nats-server -v',
+        'vcli version',
+        'varnish-controller-brainz -version',
+        'varnish-controller-agent -version',
+        'varnish-controller-api-gw -version',
+        'date',
+        'varnish-controller-ui -version',
+        'dmesg',
+        'hostname',
+        'cat /etc/hostname',
+        'cat /var/log/dmesg',
+        'cat /proc/cpuinfo',
+        'cat /proc/version',
+        'ps aux',
+        'netstat -np',
+        'uptime',
+        'free -m',
+        'ps axo',
+        'vmstat',
+        'mpstat',
+        'iostat',
+        'lsb_release -a',
+        'cat /etc/redhat-release',
+        'sysctl -a',
+        'getenforce',
+        'semodule -l',
+        'semodule -lfull',
+        'ausearch --context varnish --raw',
+        'umask',
+        'systemctl cat',
+        'netstat -s',
+        'ip a',
+        'ip n',
+        'ip r',
+        'ip -s l',
+        'ifconfig',
+        'uname -a',
+        'mount',
+        'mount sort',
+        'df -h',
+        'fdisk -l',
+        'lvdisplay -vm',
+        'vgdisplay -v',
+        'pvdisplay -v',
+        'varnishstat -1',
+        'varnishstat -j',
+        #ldd "$(command -v varnishd)"
+        #varnishadm ${VARNISHADMARG} debug.jemalloc_stats
+        'varnishscoreboard'
+        '/bin/netstat -nlpt',
+        '/bin/netstat -np',
+        'cat /etc/init.d/varnish',
+        'cat /etc/default/varnish',
+        'cat /etc/varnish/nats.conf',
+        'cat /etc/sysconfig/varnish',
+        'cat /etc/varnish/varnish.params',
+        'cat /sys/kernel/mm/transparent_hugepage/enabled',
+        'cat /sys/kernel/mm/redhat_transparent_hugepage/enabled',
+        'cat /proc/user_beancounters',
+        'cat /proc/meminfo',
+        'cat /proc/crypto',
+        'cat /proc/sys/net/ipv4/tcp_tw_reuse',
+        #'cat /proc/$pid/cgroup',
+        'cat /etc/sysconfig/vha-agent',
+        'cat /etc/default/vha-agent',
+        'cat /etc/init.d/vha-agent',
+        'cat /etc/vha-agent/nodes.conf',
+        'cat /etc/varnish/nodes.conf',
+        'cat /var/lib/vha-agent/vha-status',
+        'cat /etc/sysconfig/varnish-agent',
+        'cat /etc/default/varnish-agent',
+        'cat /etc/init.d/varnish-agent',
+        'cat /var/lib/varnish-agent/agent.param',
+        'cat /var/lib/varnish-agent/boot.vcl',
+        'cat /etc/varnish/varnish-agent.params',
+        'cat /etc/hitch/hitch.conf',
+        'cat /etc/varnish/modsec/modsecurity.conf',
+        'cat /etc/init.d/vac',
+        'cat /opt/vac/etc/defaults',
+        'cat /var/opt/vac/log/vac-stderr.log',
+        'cat /var/opt/vac/log/vac.log',
+        'cat /var/log/mongodb/mongodb.log',
+        'cat /etc/sysconfig/vstatdprobe',
+        'cat /etc/default/vstatdprobe',
+        'cat /etc/init.d/vstatdprobe',
+        'cat /etc/sysconfig/vstatd',
+        'cat /etc/default/vstatd',
+        'cat /etc/init.d/vstatd',
+        'cat /etc/varnish/vstatd.params',
+        'cat /etc/varnish/vstatdprobe.params',
+        'cat /etc/hosts',
+        'cat /etc/resolv.conf',
+        'cat /etc/nsswitch.conf',
+        'find /usr/local -name varnish',
+        'find /var/lib/varnish -ls',
+        'find /var/lib/varnish-agent -ls',
+        'lsblk',
+        'lspci -v -nn -k',
+        TestCommand('varnishadm -- vcl.list', test_class=TestVarnishAdmCommand),
+        TestCommand('varnishadm -- param.show', test_class=TestVarnishAdmCommand),
+        TestCommand('varnishadm -- param.show changed', test_class=TestVarnishAdmCommand),
+        TestCommand('varnishadm -- purge.list', test_class=TestVarnishAdmCommand),
+        TestCommand('varnishadm -- ban.list', test_class=TestVarnishAdmCommand),
+        TestCommand('varnishadm -- debug.health', test_class=TestVarnishAdmCommand),
+        TestCommand('varnishadm -- backend.list', test_class=TestVarnishAdmCommand),
+        TestCommand('varnishadm -- panic.show', test_class=TestVarnishAdmCommand)
+    ]
+    runner = unittest.TextTestRunner(verbosity=2)
+    suite = unittest.TestSuite()
+    for root, dirs, files in os.walk(args.target):
+        for file in files:
+            if 'varnishgather' not in file:
+                # todo: this could be a failure condition
+                continue
+            suite.addTest(TestSuiteGather(root,file,commands,debug=args.debug))
+    runner.run(suite)
+
+if __name__ == '__main__':
+    main()

tests/commands.py

@@ -0,0 +1,28 @@
+import unittest
+
+class TestBase(object):
+    def setUp(self):
+        if self.log is None:
+            self.skipTest('Command output log is not present.')
+
+    def test_banner(self):
+        self.assertTrue(len(self.log) >= 3)
+        if self.command not in str(self.log[1]):
+            self.fail("'%s' not in '%s'" % (self.command, self.log[1]))
+
+class TestRunCommand(TestBase, unittest.TestCase):
+    pass
+
+class TestRunPipeCommand(TestBase, unittest.TestCase):
+    pass
+
+class TestVarnishAdmCommand(TestBase, unittest.TestCase):
+    def test_is_it_running(self):
+        if len(self.log) > 5:
+            self.skipTest('Varnish is running.')
+
+        # when varnishadm is not running, we still expect the the varnishadm
+        #   interface to return a stateful message.
+        status = 'Could not get hold of varnishd, is it running?'
+        if status not in str(self.log[-1]):
+            self.fail("'%s' not in '%s'" % (status, self.log[-1]))