diff --git a/packetnetworking/cli.py b/packetnetworking/cli.py index 74fbb7b..941d8bf 100644 --- a/packetnetworking/cli.py +++ b/packetnetworking/cli.py @@ -1,4 +1,5 @@ import sys +import time import json import click import logging @@ -35,10 +36,18 @@ + "(otherwise uses ones from /etc/resolv.conf)" ), ) +@click.option("-n", "--max-attempts", default=10, help="retry up to N times on failure") @click.option("-v", "--verbose", count=True, help="Provide more detailed output") @click.option("-q", "--quiet", is_flag=True, help="Silences all output") def cli( - metadata_file, metadata_url, operating_system, rootfs, resolvers, verbose, quiet + metadata_file, + metadata_url, + operating_system, + rootfs, + resolvers, + max_attempts, + verbose, + quiet, ): level = logging.WARNING if verbose: @@ -68,6 +77,35 @@ def cli( ) ) + attempt = 1 + while True: + try: + try_run( + metadata_file, + metadata_url, + operating_system, + rootfs, + resolvers, + verbose, + quiet, + ) + break + except Exception as exc: + if attempt == max_attempts: + raise + attempt += 1 + delay = 2 ** min(attempt, 7) + log.error( + "Caught unexpected exception ('{}'), retrying in {} seconds...".format( + exc, delay + ) + ) + time.sleep(delay) + + +def try_run( + metadata_file, metadata_url, operating_system, rootfs, resolvers, verbose, quiet +): builder = packetnetworking.Builder() if metadata_file: builder.set_metadata(json.load(metadata_file))