diff --git a/CHANGELOG.md b/CHANGELOG.md index e335ac104..b0f4a7873 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) - --> -## [Unreleased](https://github.com/cyverse/atmosphere/compare/v31-0...HEAD) +## [Unreleased](https://github.com/cyverse/atmosphere/compare/v31-1...HEAD) ### Added - Include a start date in the resource request api @@ -33,6 +33,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) - Updated CHANGELOG.md to use the recommendations from [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) - Change ./manage.py maintenance to be non-interactive +## [v31-1](https://github.com/cyverse/atmosphere/compare/v31-0...v31-1) 2018-04-03 +### Fixed + - Worked around nova bug which prevented unshelved instances from getting + fixed ips [#599](https://github.com/cyverse/atmosphere/pull/599) + ## [v31-0](https://github.com/cyverse/atmosphere/compare/v30...v31-0) - 2018-03-08 ### Added - Allow volumes and instances to be filtered by provider_id diff --git a/service/instance.py b/service/instance.py index 6b907547b..7a9016721 100644 --- a/service/instance.py +++ b/service/instance.py @@ -470,7 +470,7 @@ def restore_ip_chain(esh_driver, esh_instance, deploy=True, start with: task.apply_async() """ from service.tasks.driver import ( - wait_for_instance, add_fixed_ip, add_floating_ip, + wait_for_instance, restore_port, add_floating_ip, deploy_init_to, update_metadata ) init_task = wait_for_instance.s( @@ -478,20 +478,19 @@ def restore_ip_chain(esh_driver, esh_instance, deploy=True, esh_driver.identity, "active", # TODO: DELETEME below. no_tasks=True) - # Step 1: Set metadata to initializing + metadata = {'tmp_status': 'initializing'} metadata_update_task = update_metadata.si( esh_driver.__class__, esh_driver.provider, esh_driver.identity, esh_instance.id, metadata, replace_metadata=False) - # Step 2: Add fixed - fixed_ip_task = add_fixed_ip.si( + restore_port_task = restore_port.si( esh_driver.__class__, esh_driver.provider, esh_driver.identity, esh_instance.id, core_identity_uuid) init_task.link(metadata_update_task) - metadata_update_task.link(fixed_ip_task) - # Add float and re-deploy OR just add floating IP... + metadata_update_task.link(restore_port_task) + if deploy: core_identity = CoreIdentity.objects.get(uuid=core_identity_uuid) deploy_task = deploy_init_to.si( @@ -501,16 +500,16 @@ def restore_ip_chain(esh_driver, esh_instance, deploy=True, esh_instance.id, core_identity, redeploy=True) - fixed_ip_task.link(deploy_task) + restore_port_task.link(deploy_task) else: - logger.info("Skip deployment, Add floating IP only") floating_ip_task = add_floating_ip.si( esh_driver.__class__, esh_driver.provider, esh_driver.identity, str(core_identity_uuid), esh_instance.id) - fixed_ip_task.link(floating_ip_task) + restore_port_task.link(floating_ip_task) + return init_task diff --git a/service/tasks/driver.py b/service/tasks/driver.py index 22c956c37..cc22ce83b 100644 --- a/service/tasks/driver.py +++ b/service/tasks/driver.py @@ -168,40 +168,35 @@ def _is_instance_ready(instance, status_query, return instance.id return True - -@task(name="add_fixed_ip", +@task(name="restore_port", ignore_result=True, default_retry_delay=15, - max_retries=15) -def add_fixed_ip( - driverCls, - provider, - identity, - instance_id, - core_identity_uuid=None): - from service import instance as instance_service + max_retries=5, + bind=True) +def restore_port(self, + driverCls, + provider, + identity, + instance_id, + core_identity_uuid=None): + from service.instance import _to_network_driver, _get_network_id try: - celery_logger.debug("add_fixed_ip task started at %s." % datetime.now()) driver = get_driver(driverCls, provider, identity) + core_identity = Identity.objects.get(uuid=core_identity_uuid) + network_driver = _to_network_driver(core_identity) instance = driver.get_instance(instance_id) - if not instance: - celery_logger.debug("Instance has been teminated: %s." % instance_id) - return None - if instance._node.private_ips: - # TODO: Attempt to rescue - celery_logger.info("Instance has fixed IP: %s" % instance_id) - return instance - - network_id = instance_service._get_network_id(driver, instance) - fixed_ip = driver._connection.ex_add_fixed_ip(instance, network_id) - celery_logger.debug("add_fixed_ip task finished at %s." % datetime.now()) - return fixed_ip - except Exception as exc: - if "Not Ready" not in str(exc): - # Ignore 'normal' errors. - celery_logger.exception(exc) - add_fixed_ip.retry(exc=exc) + assert instance, "Instance {} no longer exists".format(instance_id) + + ports = network_driver.list_ports(device_id=instance.id) + for port in ports: + network_driver.delete_port(port) + network_id = _get_network_id(driver, instance) + driver._connection.ex_attach_interface( + instance.id, network_id=network_id) + except Exception as exc: + celery_logger.exception(exc) + self.retry(exc=exc) def current_openstack_identities(): identities = Identity.objects.filter( @@ -1197,20 +1192,8 @@ def add_floating_ip(driverCls, provider, identity, core_identity_uuid, countdown = min(2**current.request.retries, 128) add_floating_ip.retry(exc=floating_ip_err, countdown=countdown) - except NeutronBadRequest as bad_request: - # NOTE: 'Neutron Bad Request' is a good message to 'catch and fix' - # because its a user-supplied problem. - # Here we will attempt to 'fix' requests and put the 'add_floating_ip' - # task back on the queue after we're done. - celery_logger.exception("Neutron did not accept request - %s." - % bad_request.message) - if 'no fixed ip' in bad_request.message.lower(): - fixed_ip = add_fixed_ip(driverCls, provider, identity, - instance_alias) - if fixed_ip: - celery_logger.debug("Fixed IP %s has been added to Instance %s." - % (fixed_ip, instance_alias)) - # let the exception bubble-up for a retry.. + except NeutronBadRequest: + # This is an error on our end, we want it to surface raise except (BaseException, Exception) as exc: celery_logger.exception("Error occurred while assigning a floating IP")