Merge remote-tracking branch 'upstream/startup-locking-support' into …

…rabbitmq-autocluster_startup-locking-support

.travis.yml

@@ -1,7 +1,10 @@
 language: erlang
 
 otp_release:
+  # Primary target
   - 17.5
+  # Secondary target - runing PropEr tests and checking compatibily with modern erlang
+  - 19.1
 
 addons:
   apt:
@@ -16,7 +19,6 @@ cache:
     - $HOME/.cache/bin
 
 install:
-  - pip install --user codecov
   - export PATH=$PATH:$HOME/.local/bin
 
 before_script:
@@ -34,30 +36,60 @@ before_script:
 
 script:
   - make RABBITMQ_CURRENT_FETCH_URL=https://github.com/rabbitmq/rabbitmq-trick-erlang.mk-into-using-proper-url-for-deps
-  - make ct COVER=true
   - |
-    # We want this after ct run, so `rabbit` dependency will be fetched.
+    set -exo pipefail
+    function ensure_plt() {
+    # We want this to happen after ct run, so `rabbit` dependency will be fetched.
     # Then we can add rabbit/rabbit_common to plt, thus getting 0
     # warnings after running dialyzer itself
 
-    # Try to use cached plt if it's valid
-    if [ -r $HOME/.cache/plt/.autocluster.plt ]; then
-      cp -va $HOME/.cache/plt/.autocluster.plt .
-      # Cached PLT can become invalid (i.e. reference some renamed/deleted .beam)
-      dialyzer --check_plt --plt ./.autocluster.plt || rm -f ./.autocluster.plt
-    fi
+        # Try to use cached plt if it's valid
+        if [ -r $HOME/.cache/plt/.autocluster.plt ]; then
+          cp -va $HOME/.cache/plt/.autocluster.plt .
+          # Cached PLT can become invalid (i.e. reference some renamed/deleted .beam)
+          dialyzer --check_plt --plt ./.autocluster.plt || rm -f ./.autocluster.plt
+        fi
+
+        # Build PLT from scratch if there is no cached copy
+        if [ ! -f ./.autocluster.plt ]; then
+          make plt || true # making plt produces some warnings which we don't care about
+          mkdir -p $HOME/.cache/plt/
+          cp -va .autocluster.plt $HOME/.cache/plt/
+        fi
+    }
+
+    function upload_logs() {
+      tar cjvf logs.tar.bz2 logs/
+      echo Uploading logs for further investigation to:
+      curl -sST logs.tar.bz2 chunk.io
+    }
 
-    # Build PLT from scratch if there is no cached copy
-    if [ ! -f ./.autocluster.plt ]; then
-      make plt || true # making plt produces some warnings which we don't care about
-      mkdir -p $HOME/.cache/plt/
-      cp -va .autocluster.plt $HOME/.cache/plt/
+    if [ "$TRAVIS_OTP_RELEASE" == "19.1" ]; then
+      export PROPER_ONLY=true
+    else
+      export PROPER_ONLY=
     fi
-  - make dialyze
 
-after_success:
-  - ./bin/covertool -cover ct.coverdata -appname autocluster
-  - codecov -f coverage.xml -X search
+    case "$PROPER_ONLY" in
+        true)
+            if ! make test-build ct IS_APP=1 CT_SUITES=etcd; then
+                upload_logs
+                exit 1
+            fi
+            ;;
+        *)
+            pip install --user codecov
+            if ! make test-build ct COVER=true IS_APP=1; then
+                upload_logs
+                exit 1
+            fi
+            ensure_plt
+            make dialyze
+            ./bin/covertool -cover ct.coverdata -appname autocluster
+            codecov -f coverage.xml -X search
+            ;;
+    esac
+    set +exo pipefail
 
 before_deploy:
   - make clean

Makefile

@@ -2,7 +2,7 @@ PROJECT = autocluster
 
 DEPS = rabbit_common rabbitmq_aws
 
-TEST_DEPS += rabbit rabbitmq_ct_helpers meck
+TEST_DEPS += rabbit erlsh rabbitmq_ct_helpers meck
 IGNORE_DEPS += rabbitmq_java_client
 
 DEP_PLUGINS = rabbit_common/mk/rabbitmq-plugin.mk

include/autocluster.hrl

@@ -11,6 +11,7 @@
         [{config, backend,               "AUTOCLUSTER_TYPE",       unconfigured, atom,    false}, %% General
          {config, autocluster_failure,   "AUTOCLUSTER_FAILURE",    ignore,       atom,    false},
          {config, startup_delay,         "AUTOCLUSTER_DELAY",      5,            integer, false},
+         {config, lock_wait_time,        "LOCK_WAIT_TIME",         300,          integer, false},
          {config, cluster_cleanup,       "AUTOCLUSTER_CLEANUP",    false,        atom,    false},
          {config, autocluster_log_level, "AUTOCLUSTER_LOG_LEVEL",  info,         atom,    false},
          {config, cleanup_interval,      "CLEANUP_INTERVAL",       60,           integer, false},
@@ -61,3 +62,30 @@
          {config, etcd_node_ttl,         "ETCD_NODE_TTL",          30,           integer, false}]).
 
 -define(CONSUL_CHECK_NOTES, "RabbitMQ Auto-Cluster Plugin TTL Check").
+
+%%--------------------------------------------------------------------
+%% @doc
+%% State that is being passed around and updated by different
+%% initialization steps.
+%% @end
+%%--------------------------------------------------------------------
+-record(startup_state, {backend_name :: atom()
+                       ,backend_module :: module()
+                       ,best_node_to_join :: undefined | node()
+                       ,startup_lock_data = undefined
+                       }).
+
+%%--------------------------------------------------------------------
+%% @doc
+%% Detailed node information neccessary for choosing the best cluster
+%% to join to.
+%% @end
+%%--------------------------------------------------------------------
+-record(augmented_node, {name :: node()
+                        ,uptime :: non_neg_integer()
+                        ,alive :: boolean()
+                        ,clustered_with :: [node()]
+                        ,alive_cluster_nodes :: [node()]
+                        ,partitioned_cluster_nodes :: [node()]
+                        ,other_cluster_nodes :: [node()]
+                        }).