Bring in the latest from v31

CHANGELOG.md

@@ -24,7 +24,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
   - <in case of vulnerabilities>
 -->
 
-## [Unreleased](https://github.com/cyverse/atmosphere/compare/v31-0...HEAD)
+## [Unreleased](https://github.com/cyverse/atmosphere/compare/v31-1...HEAD)
 ### Added
   - Include a start date in the resource request api
 
@@ -33,6 +33,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
   - Updated CHANGELOG.md to use the recommendations from [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
   - Change ./manage.py maintenance to be non-interactive
 
+## [v31-1](https://github.com/cyverse/atmosphere/compare/v31-0...v31-1) 2018-04-03
+### Fixed
+  - Worked around nova bug which prevented unshelved instances from getting
+    fixed ips [#599](https://github.com/cyverse/atmosphere/pull/599)
+
 ## [v31-0](https://github.com/cyverse/atmosphere/compare/v30...v31-0) - 2018-03-08
 ### Added
   - Allow volumes and instances to be filtered by provider_id

service/instance.py

@@ -470,28 +470,27 @@ def restore_ip_chain(esh_driver, esh_instance, deploy=True,
     start with: task.apply_async()
     """
     from service.tasks.driver import (
-        wait_for_instance, add_fixed_ip, add_floating_ip,
+        wait_for_instance, restore_port, add_floating_ip,
         deploy_init_to, update_metadata
     )
     init_task = wait_for_instance.s(
         esh_instance.id, esh_driver.__class__, esh_driver.provider,
         esh_driver.identity, "active",
         # TODO: DELETEME below.
         no_tasks=True)
-    # Step 1: Set metadata to initializing
+
     metadata = {'tmp_status': 'initializing'}
     metadata_update_task = update_metadata.si(
         esh_driver.__class__, esh_driver.provider, esh_driver.identity,
         esh_instance.id, metadata, replace_metadata=False)
 
-    # Step 2: Add fixed
-    fixed_ip_task = add_fixed_ip.si(
+    restore_port_task = restore_port.si(
         esh_driver.__class__, esh_driver.provider,
         esh_driver.identity, esh_instance.id, core_identity_uuid)
 
     init_task.link(metadata_update_task)
-    metadata_update_task.link(fixed_ip_task)
-    # Add float and re-deploy OR just add floating IP...
+    metadata_update_task.link(restore_port_task)
+
     if deploy:
         core_identity = CoreIdentity.objects.get(uuid=core_identity_uuid)
         deploy_task = deploy_init_to.si(
@@ -501,16 +500,16 @@ def restore_ip_chain(esh_driver, esh_instance, deploy=True,
             esh_instance.id,
             core_identity,
             redeploy=True)
-        fixed_ip_task.link(deploy_task)
+        restore_port_task.link(deploy_task)
     else:
-        logger.info("Skip deployment, Add floating IP only")
         floating_ip_task = add_floating_ip.si(
             esh_driver.__class__,
             esh_driver.provider,
             esh_driver.identity,
             str(core_identity_uuid),
             esh_instance.id)
-        fixed_ip_task.link(floating_ip_task)
+        restore_port_task.link(floating_ip_task)
+
     return init_task
 
 

service/tasks/driver.py

@@ -168,40 +168,35 @@ def _is_instance_ready(instance, status_query,
         return instance.id
     return True
 
-
-@task(name="add_fixed_ip",
+@task(name="restore_port",
       ignore_result=True,
       default_retry_delay=15,
-      max_retries=15)
-def add_fixed_ip(
-        driverCls,
-        provider,
-        identity,
-        instance_id,
-        core_identity_uuid=None):
-    from service import instance as instance_service
+      max_retries=5,
+      bind=True)
+def restore_port(self,
+                 driverCls,
+                 provider,
+                 identity,
+                 instance_id,
+                 core_identity_uuid=None):
+    from service.instance import _to_network_driver, _get_network_id
     try:
-        celery_logger.debug("add_fixed_ip task started at %s." % datetime.now())
         driver = get_driver(driverCls, provider, identity)
+        core_identity = Identity.objects.get(uuid=core_identity_uuid)
+        network_driver = _to_network_driver(core_identity)
         instance = driver.get_instance(instance_id)
-        if not instance:
-            celery_logger.debug("Instance has been teminated: %s." % instance_id)
-            return None
-        if instance._node.private_ips:
-            # TODO: Attempt to rescue
-            celery_logger.info("Instance has fixed IP: %s" % instance_id)
-            return instance
-
-        network_id = instance_service._get_network_id(driver, instance)
-        fixed_ip = driver._connection.ex_add_fixed_ip(instance, network_id)
-        celery_logger.debug("add_fixed_ip task finished at %s." % datetime.now())
-        return fixed_ip
-    except Exception as exc:
-        if "Not Ready" not in str(exc):
-            # Ignore 'normal' errors.
-            celery_logger.exception(exc)
-        add_fixed_ip.retry(exc=exc)
+        assert instance, "Instance {} no longer exists".format(instance_id)
+
+        ports = network_driver.list_ports(device_id=instance.id)
+        for port in ports:
+            network_driver.delete_port(port)
 
+        network_id = _get_network_id(driver, instance)
+        driver._connection.ex_attach_interface(
+            instance.id, network_id=network_id)
+    except Exception as exc:
+        celery_logger.exception(exc)
+        self.retry(exc=exc)
 
 def current_openstack_identities():
     identities = Identity.objects.filter(
@@ -1197,20 +1192,8 @@ def add_floating_ip(driverCls, provider, identity, core_identity_uuid,
         countdown = min(2**current.request.retries, 128)
         add_floating_ip.retry(exc=floating_ip_err,
                               countdown=countdown)
-    except NeutronBadRequest as bad_request:
-        # NOTE: 'Neutron Bad Request' is a good message to 'catch and fix'
-        # because its a user-supplied problem.
-        # Here we will attempt to 'fix' requests and put the 'add_floating_ip'
-        # task back on the queue after we're done.
-        celery_logger.exception("Neutron did not accept request - %s."
-            % bad_request.message)
-        if 'no fixed ip' in bad_request.message.lower():
-            fixed_ip = add_fixed_ip(driverCls, provider, identity,
-                                    instance_alias)
-            if fixed_ip:
-                celery_logger.debug("Fixed IP %s has been added to Instance %s."
-                             % (fixed_ip, instance_alias))
-        # let the exception bubble-up for a retry..
+    except NeutronBadRequest:
+        # This is an error on our end, we want it to surface
         raise
     except (BaseException, Exception) as exc:
         celery_logger.exception("Error occurred while assigning a floating IP")