Bump Ruby, for cleanup_memberships() method to support both String and Symbol keys

[bgandon]

The issue was originally reported and analysed in Slack:
https://cloudfoundry.slack.com/archives/C02HPPYQ2/p1630547568034700

Original stack trace

[2021-09-02T01:29:54.320479 #2469] DEBUG -- [req_id cpi-919936]: excon.request DELETE https://openstack-external.lyonsgroup.family:9696/v2.0/ports/3c177bbf-072b-44f2-a075-82a640cf1227 params: {"chunk_size":1048576,"connect_timeout":60,"debug_request":false,"debug_response":true,"idempotent":false,"instrumentor_name":"excon","mock":false,"nonblock":true,"omit_default_port":false,"persistent":false,"read_timeout":60,"retry_errors":["Excon::Error::Timeout","Excon::Error::Socket","Excon::Error::HTTPStatus"],"retry_limit":4,"ssl_verify_peer":true,"ssl_uri_schemes":["https"],"stubs":"global","tcp_nodelay":false,"thread_safe_sockets":true,"versions":"excon/0.68.0 (x86_64-linux) ruby/2.4.9","write_timeout":60,"host":"openstack-external.lyonsgroup.family","query":null,"remote_ip":"174.54.141.197","expects":204,"retries_remaining":4} headers: {"User-Agent":"fog-core/2.1.2","Content-Type":"application/json","Accept":"application/json","X-Auth-Token":"<redacted>","Host":"openstack-external.lyonsgroup.family:9696"} body: null
D, [2021-09-02T01:29:55.450657 #2469] DEBUG -- [req_id cpi-919936]: excon.response HTTP/1.1 204 No Content /v2.0/ports/3c177bbf-072b-44f2-a075-82a640cf1227 params: {"host":"openstack-external.lyonsgroup.family","port":9696,"remote_ip":"174.54.141.197","local_port":46152,"local_address":"10.0.1.6"} headers: {"content-length":"0","x-openstack-request-id":"req-3ff42027-5b8d-480c-9bf7-ca9c7600c4f0","date":"Thu, 02 Sep 2021 01:29:55 GMT","strict-transport-security":"max-age=31536000;"} body: 
E, [2021-09-02T01:29:55.451318 #2469] ERROR -- [req_id cpi-919936]: undefined method `start_with?' for :registry_key:Symbol (NoMethodError)
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/lib/cloud/openstack/loadbalancer_configurator.rb:69:in `block in cleanup_memberships'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/lib/cloud/openstack/loadbalancer_configurator.rb:69:in `select'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/lib/cloud/openstack/loadbalancer_configurator.rb:69:in `cleanup_memberships'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/lib/cloud/openstack/server.rb:53:in `block in destroy'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/lib/cloud/openstack/helpers.rb:38:in `catch_error'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/lib/cloud/openstack/server.rb:53:in `destroy'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/lib/cloud/openstack/server.rb:39:in `rescue in create'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/lib/cloud/openstack/server.rb:21:in `create'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/lib/cloud/openstack/vm_creator.rb:61:in `create_vm_in_az'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/lib/cloud/openstack/vm_creator.rb:31:in `create_vm_single_az'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/lib/cloud/openstack/vm_creator.rb:22:in `perform'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/lib/cloud/openstack/vm_factory.rb:40:in `create_vm'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/lib/cloud/openstack/cloud.rb:154:in `block in create_vm'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/vendor/bundle/ruby/2.4.0/gems/bosh_common-1.3262.24.0/lib/common/thread_formatter.rb:49:in `with_thread_name'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/lib/cloud/openstack/cloud.rb:135:in `create_vm'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/vendor/bundle/ruby/2.4.0/gems/bosh_cpi-2.5.0/lib/bosh/cpi/cli.rb:89:in `public_send'
/var/vcap/data/packages/bosh_openstack_cpi/859634c93ef9620514d5f10f31a44b9e303aea28/vendor/bundle/ruby/2.4.0/gems/bosh_cpi-2.5.0/lib/bosh/cpi/cli.rb:89:in `run'
/var/vcap/packages/bosh_openstack_cpi/bin/openstack_cpi:21:in `block in <top (required)>'
/var/vcap/data/packages/ruby-2.4.9-r0.27.0/ce45aced61af2147fb888ddee30cff154e2a9b2a/lib/ruby/2.4.0/tmpdir.rb:93:in `mktmpdir'
/var/vcap/packages/bosh_openstack_cpi/bin/openstack_cpi:17:in `<top (required)>'
/var/vcap/data/packages/ruby-2.4.9-r0.27.0/ce45aced61af2147fb888ddee30cff154e2a9b2a/lib/ruby/site_ruby/2.4.0/bundler/cli/exec.rb:74:in `load'
/var/vcap/data/packages/ruby-2.4.9-r0.27.0/ce45aced61af2147fb888ddee30cff154e2a9b2a/lib/ruby/site_ruby/2.4.0/bundler/cli/exec.rb:74:in `kernel_load'
/var/vcap/data/packages/ruby-2.4.9-r0.27.0/ce45aced61af2147fb888ddee30cff154e2a9b2a/lib/ruby/site_ruby/2.4.0/bundler/cli/exec.rb:28:in `run'
/var/vcap/data/packages/ruby-2.4.9-r0.27.0/ce45aced61af2147fb888ddee30cff154e2a9b2a/lib/ruby/site_ruby/2.4.0/bundler/cli.rb:424:in `exec'
/var/vcap/data/packages/ruby-2.4.9-r0.27.0/ce45aced61af2147fb888ddee30cff154e2a9b2a/lib/ruby/site_ruby/2.4.0/bundler/vendor/thor/lib/thor/command.rb:27:in `run'
/var/vcap/data/packages/ruby-2.4.9-r0.27.0/ce45aced61af2147fb888ddee30cff154e2a9b2a/lib/ruby/site_ruby/2.4.0/bundler/vendor/thor/lib/thor/invocation.rb:126:in `invoke_command'
/var/vcap/data/packages/ruby-2.4.9-r0.27.0/ce45aced61af2147fb888ddee30cff154e2a9b2a/lib/ruby/site_ruby/2.4.0/bundler/vendor/thor/lib/thor.rb:387:in `dispatch'
/var/vcap/data/packages/ruby-2.4.9-r0.27.0/ce45aced61af2147fb888ddee30cff154e2a9b2a/lib/ruby/site_ruby/2.4.0/bundler/cli.rb:27:in `dispatch'
/var/vcap/data/packages/ruby-2.4.9-r0.27.0/ce45aced61af2147fb888ddee30cff154e2a9b2a/lib/ruby/site_ruby/2.4.0/bundler/vendor/thor/lib/thor/base.rb:466:in `start'
/var/vcap/data/packages/ruby-2.4.9-r0.27.0/ce45aced61af2147fb888ddee30cff154e2a9b2a/lib/ruby/site_ruby/2.4.0/bundler/cli.rb:18:in `start'
/var/vcap/packages/ruby-2.4.9-r0.27.0/bin/bundle:30:in `block in <main>'
/var/vcap/data/packages/ruby-2.4.9-r0.27.0/ce45aced61af2147fb888ddee30cff154e2a9b2a/lib/ruby/site_ruby/2.4.0/bundler/friendly_errors.rb:124:in `with_friendly_errors'
/var/vcap/packages/ruby-2.4.9-r0.27.0/bin/bundle:22:in `<main>'
E, 

Bug analysis

This is an issue in the rescue code when VM instance (called “server” in OpenStack parlance) creation fails here:
https://github.com/cloudfoundry/bosh-openstack-cpi-release/blob/master/src/bosh_openstack_cpi/lib/cloud/openstack/server.rb#L29

The bug happens here:
https://github.com/cloudfoundry/bosh-openstack-cpi-release/blob/master/src/bosh_openstack_cpi/lib/cloud/openstack/loadbalancer_configurator.rb#L69

Only Ruby 2.7 has added support for the .start_with?() method on the Symbol class. But the OpenStack CPI ships with Ruby an old v2.4.9…

The server_metadata (also called server_tags in server.rb) hash is supposed to contain both strings and symbols as keys. So, the cleanup_memberships() method in loadbalancer_configurator.rb should support both.

Bumping the Ruby dependency in the BOSH release to a recent Ruby 2.7+ version would probably fix the issue.

[klakin-pivotal]

Thanks for reporting this. We'll prioritize it appropriately.

Unfortunately, we have a ton of work, so it won't be a very high priority... so if you want this resolved rapidly, developing submitting a PR that resolves the issue is probably your best course of action.

[bgandon]

I would love to!

But bosh vendor-packge requires private write access to the BOSH Release blobstore, unfortunately. Would you share with me the creds to the bosh-openstack-cpi-blobs bucket on AWS S3? As a member of the Bosh team, and reviewer of the BOSH sub-WG for the CFF, this should be fairy possible…

Otherwise, here would be the commands I would run, including the patch to apply:

git clone https://github.com/bosh-packages/ruby-release.git
git clone git@github.com:cloudfoundry/bosh-openstack-cpi-release.git
cd bosh-openstack-cpi-release

bosh vendor-package ruby-2.7.4-r0.63.0 ../ruby-release/

git rm -r packages/ruby-2.4.9-r0.27.0 .final_builds/packages/ruby-2.4.9-*

patch -p1 <<<EOP
diff --git a/jobs/openstack_cpi/spec b/jobs/openstack_cpi/spec
index 426a3a6..1ca9707 100644
--- a/jobs/openstack_cpi/spec
+++ b/jobs/openstack_cpi/spec
@@ -6,7 +6,7 @@ templates:
   cacert.pem.erb: config/cacert.pem
 
 packages:
-- ruby-2.4.9-r0.27.0
+- ruby-2.7.4-r0.63.0
 - bosh_openstack_cpi
 
 properties:
diff --git a/jobs/openstack_cpi/templates/cpi.erb b/jobs/openstack_cpi/templates/cpi.erb
index 053f7a7..377df31 100644
--- a/jobs/openstack_cpi/templates/cpi.erb
+++ b/jobs/openstack_cpi/templates/cpi.erb
@@ -20,12 +20,12 @@ export no_proxy="<%= no_proxy %>"
 BOSH_PACKAGES_DIR=${BOSH_PACKAGES_DIR:-/var/vcap/packages}
 BOSH_JOBS_DIR=${BOSH_JOBS_DIR:-/var/vcap/jobs}
 
-source "${BOSH_PACKAGES_DIR}/ruby-2.4.9-r0.27.0/bosh/runtime.env"
+source "${BOSH_PACKAGES_DIR}/ruby-2.7.4-r0.63.0/bosh/runtime.env"
 export HOME=~
 
 export BUNDLE_GEMFILE=$BOSH_PACKAGES_DIR/bosh_openstack_cpi/Gemfile
 
-bundle_cmd="$BOSH_PACKAGES_DIR/ruby-2.4.9-r0.27.0/bin/bundle"
+bundle_cmd="$BOSH_PACKAGES_DIR/ruby-2.7.4-r0.63.0/bin/bundle"
 
 exec $bundle_cmd exec $BOSH_PACKAGES_DIR/bosh_openstack_cpi/bin/openstack_cpi \
   $BOSH_JOBS_DIR/openstack_cpi/config/cpi.json \
diff --git a/packages/bosh_openstack_cpi/packaging b/packages/bosh_openstack_cpi/packaging
index 6898dd1..219091e 100644
--- a/packages/bosh_openstack_cpi/packaging
+++ b/packages/bosh_openstack_cpi/packaging
@@ -4,13 +4,13 @@ set -e -x
 
 BOSH_PACKAGES_DIR=${BOSH_PACKAGES_DIR:-/var/vcap/packages}
 # shellcheck disable=SC1090
-source "${BOSH_PACKAGES_DIR}/ruby-2.4.9-r0.27.0/bosh/compile.env"
+source "${BOSH_PACKAGES_DIR}/ruby-2.7.4-r0.63.0/bosh/compile.env"
 
 cp -a bosh_openstack_cpi/* "${BOSH_INSTALL_TARGET}"
 
 export BUNDLE_CACHE_PATH="vendor/package"
 export BUNDLE_WITHOUT="development:test"
-bundle_cmd="$BOSH_PACKAGES_DIR/ruby-2.4.9-r0.27.0/bin/bundle"
+bundle_cmd="$BOSH_PACKAGES_DIR/ruby-2.7.4-r0.63.0/bin/bundle"
 
 cd "${BOSH_INSTALL_TARGET}"
 
diff --git a/packages/bosh_openstack_cpi/spec b/packages/bosh_openstack_cpi/spec
index a3e44b6..0182776 100644
--- a/packages/bosh_openstack_cpi/spec
+++ b/packages/bosh_openstack_cpi/spec
@@ -1,7 +1,7 @@
 ---
 name: bosh_openstack_cpi
 dependencies:
-- ruby-2.4.9-r0.27.0
+- ruby-2.7.4-r0.63.0
 files:
 - bosh_openstack_cpi/Gemfile
 - bosh_openstack_cpi/Gemfile.lock
EOP

git add -A .
git commit -m "Bump Ruby to v2.7.8"
git push

I hope this can help!
Don't hesitate to reach out to me on CF Slack.

[rkoster]

Thanks @bgandon