From 257d1fc46d8218a822e13ef16466c002bb64e73d Mon Sep 17 00:00:00 2001
From: Zev Weiss <zev@bewilderbeest.net>
Date: Wed, 20 May 2020 17:14:10 -0500
Subject: [PATCH] cli: add top-level retry loop.

We've been hitting some intermittent crashes of the following
form recently:

    Traceback (most recent call last):
      File "/usr/local/bin/packet-networking", line 11, in <module>
        sys.exit(cli())
      File "/usr/local/lib/python3.5/dist-packages/click/core.py", line 722, in __call__
        return self.main(*args, **kwargs)
      File "/usr/local/lib/python3.5/dist-packages/click/core.py", line 697, in main
        rv = self.invoke(ctx)
      File "/usr/local/lib/python3.5/dist-packages/click/core.py", line 895, in invoke
        return ctx.invoke(self.callback, **ctx.params)
      File "/usr/local/lib/python3.5/dist-packages/click/core.py", line 535, in invoke
        return callback(*args, **kwargs)
      File "/usr/local/lib/python3.5/dist-packages/packetnetworking/cli.py", line 107, in cli
        tasks = builder.run(rootfs)
      File "/usr/local/lib/python3.5/dist-packages/packetnetworking/builder.py", line 67, in run
        return builder.run(rootfs_path)
      File "/usr/local/lib/python3.5/dist-packages/packetnetworking/distros/distro_builder.py", line 163, in run
        rendered_tasks = self.render()
      File "/usr/local/lib/python3.5/dist-packages/packetnetworking/distros/distro_builder.py", line 156, in render
        rendered_tasks[path] = template.render(self.context())
      File "/usr/local/lib/python3.5/dist-packages/jinja2/environment.py", line 1008, in render
        return self.environment.handle_exception(exc_info, True)
      File "/usr/local/lib/python3.5/dist-packages/jinja2/environment.py", line 780, in handle_exception
        reraise(exc_type, exc_value, tb)
      File "/usr/local/lib/python3.5/dist-packages/jinja2/_compat.py", line 37, in reraise
        raise value.with_traceback(tb)
      File "<template>", line 13, in top-level template code
    jinja2.exceptions.UndefinedError: 'None' has no attribute 'address'

Current speculation is that this is due to some sort of hegel race;
until the root cause is determined & fixed we're hoping this will keep
things running.
---
 packetnetworking/cli.py | 40 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)

diff --git a/packetnetworking/cli.py b/packetnetworking/cli.py
index 74fbb7b..941d8bf 100644
--- a/packetnetworking/cli.py
+++ b/packetnetworking/cli.py
@@ -1,4 +1,5 @@
 import sys
+import time
 import json
 import click
 import logging
@@ -35,10 +36,18 @@
         + "(otherwise uses ones from /etc/resolv.conf)"
     ),
 )
+@click.option("-n", "--max-attempts", default=10, help="retry up to N times on failure")
 @click.option("-v", "--verbose", count=True, help="Provide more detailed output")
 @click.option("-q", "--quiet", is_flag=True, help="Silences all output")
 def cli(
-    metadata_file, metadata_url, operating_system, rootfs, resolvers, verbose, quiet
+    metadata_file,
+    metadata_url,
+    operating_system,
+    rootfs,
+    resolvers,
+    max_attempts,
+    verbose,
+    quiet,
 ):
     level = logging.WARNING
     if verbose:
@@ -68,6 +77,35 @@ def cli(
             )
         )
 
+    attempt = 1
+    while True:
+        try:
+            try_run(
+                metadata_file,
+                metadata_url,
+                operating_system,
+                rootfs,
+                resolvers,
+                verbose,
+                quiet,
+            )
+            break
+        except Exception as exc:
+            if attempt == max_attempts:
+                raise
+            attempt += 1
+            delay = 2 ** min(attempt, 7)
+            log.error(
+                "Caught unexpected exception ('{}'), retrying in {} seconds...".format(
+                    exc, delay
+                )
+            )
+            time.sleep(delay)
+
+
+def try_run(
+    metadata_file, metadata_url, operating_system, rootfs, resolvers, verbose, quiet
+):
     builder = packetnetworking.Builder()
     if metadata_file:
         builder.set_metadata(json.load(metadata_file))