From 2bb400b0e2fcbc2859bf075acd6db7a7846d3103 Mon Sep 17 00:00:00 2001 From: philljj Date: Wed, 27 Dec 2023 15:21:57 -0600 Subject: [PATCH] Add stress test. --- .github/workflows/ubuntu-check-curl.yml | 11 +- .github/workflows/ubuntu-check.yml | 11 +- configure.ac | 29 ++++- examples/mqttexample.c | 2 + examples/mqttnet.c | 152 ++++++++++++++++++------ examples/multithread/multithread.c | 16 ++- scripts/client.test | 79 ++++-------- scripts/firmware.test | 4 +- scripts/include.am | 10 +- scripts/multithread.test | 64 +++------- scripts/nbclient.test | 68 +++-------- scripts/stress.test | 89 ++++++++++++++ scripts/test_common.sh | 40 +++++++ 13 files changed, 373 insertions(+), 202 deletions(-) create mode 100755 scripts/stress.test create mode 100644 scripts/test_common.sh diff --git a/.github/workflows/ubuntu-check-curl.yml b/.github/workflows/ubuntu-check-curl.yml index c5e0e0cc6..6e03adb12 100644 --- a/.github/workflows/ubuntu-check-curl.yml +++ b/.github/workflows/ubuntu-check-curl.yml @@ -2,7 +2,7 @@ name: Ubuntu Build Test with Curl Support on: push: - branches: [ 'master', 'main', 'release/**' ] + branches: [ '*' ] pull_request: branches: [ '*' ] @@ -98,6 +98,15 @@ jobs: - name: wolfmqtt make check run: make check + - name: wolfmqtt configure with libCurl and Stress + env: + WOLFMQTT_NO_EXTERNAL_BROKER_TESTS: 1 + run: ./configure --enable-curl --enable-stress + - name: wolfmqtt make + run: make + - name: wolfmqtt make check + run: make check + # capture logs on failure - name: Show logs on failure if: failure() || cancelled() diff --git a/.github/workflows/ubuntu-check.yml b/.github/workflows/ubuntu-check.yml index 5f6bef603..9c20d6e78 100644 --- a/.github/workflows/ubuntu-check.yml +++ b/.github/workflows/ubuntu-check.yml @@ -2,7 +2,7 @@ name: Ubuntu Build Test on: push: - branches: [ 'master', 'main', 'release/**' ] + branches: [ '*' ] pull_request: branches: [ '*' ] @@ -108,6 +108,15 @@ jobs: - name: make check run: make check + - name: wolfmqtt configure with Stress + env: + WOLFMQTT_NO_EXTERNAL_BROKER_TESTS: 1 + run: ./configure --enable-stress + - name: wolfmqtt make + run: make + - name: wolfmqtt make check + run: make check + # capture logs on failure - name: Show logs on failure if: failure() || cancelled() diff --git a/configure.ac b/configure.ac index 95e3fcfcf..7d54e2dc9 100644 --- a/configure.ac +++ b/configure.ac @@ -263,7 +263,6 @@ if test "x$ENABLED_CURL" = "xyes"; then AC_CHECK_LIB([curl],[curl_easy_init],,[AC_MSG_ERROR([libcurl is required and wasn't found on the system. It can be obtained from https://curl.se/download.html.])]) fi - # MQTT v5.0 AC_ARG_ENABLE([v5], [AS_HELP_STRING([--enable-v5],[Enable MQTT v5.0 support (default: disabled)])], @@ -329,10 +328,37 @@ then AM_CFLAGS="$AM_CFLAGS -DWOLFMQTT_MULTITHREAD" fi +# Stress test convenience build option. +AC_ARG_ENABLE([stress], + [AS_HELP_STRING([--enable-stress],[Enable stress test (default: disabled)])], + [ ENABLED_STRESS=$enableval ], + [ ENABLED_STRESS=no ] + ) + +if test "x$ENABLED_STRESS" = "xyes"; then + if test "x$ENABLED_ALL" = "xyes"; then + AC_MSG_ERROR([--enable-all and --enable-stress are incompatible]) + fi + + if test "x$ENABLED_SN" = "xyes"; then + AC_MSG_ERROR([--enable-sn and --enable-stress are incompatible]) + fi + + ENABLED_TIMEOUT=yes + ENABLED_NONBLOCK=yes + ENABLED_MULTITHREAD=yes + AM_CFLAGS="$AM_CFLAGS -DWOLFMQTT_STRESS" + AM_CFLAGS="$AM_CFLAGS -DWOLFMQTT_NONBLOCK" + AM_CFLAGS="$AM_CFLAGS -DWOLFMQTT_TEST_NONBLOCK" + AM_CFLAGS="$AM_CFLAGS -DWOLFMQTT_TEST_NONBLOCK_TIMES=2" + AM_CFLAGS="$AM_CFLAGS -DWOLFMQTT_MULTITHREAD" + AM_CFLAGS="$AM_CFLAGS -DNUM_PUB_TASKS=5 -DNUM_PUB_PER_TASK=4" +fi AM_CONDITIONAL([HAVE_LIBWOLFSSL], [test "x$ENABLED_TLS" = "xyes"]) AM_CONDITIONAL([HAVE_LIBCURL], [test "x$ENABLED_CURL" = "xyes"]) +AM_CONDITIONAL([BUILD_STRESS], [test "x$ENABLED_STRESS" = "xyes"]) AM_CONDITIONAL([BUILD_EXAMPLES], [test "x$ENABLED_EXAMPLES" = "xyes"]) AM_CONDITIONAL([BUILD_STDINCAP], [test "x$ENABLED_STDINCAP" = "xyes"]) AM_CONDITIONAL([BUILD_SN], [test "x$ENABLED_SN" = "xyes"]) @@ -465,3 +491,4 @@ echo " * STDIN Capture: $ENABLED_STDINCAP" echo " * TLS: $ENABLED_TLS" echo " * CURL: $ENABLED_CURL" echo " * Multi-thread: $ENABLED_MULTITHREAD" +echo " * Stress: $ENABLED_STRESS" diff --git a/examples/mqttexample.c b/examples/mqttexample.c index 405190d7e..f1b0dbe4f 100644 --- a/examples/mqttexample.c +++ b/examples/mqttexample.c @@ -563,6 +563,8 @@ int mqtt_check_timeout(int rc, word32* start_sec, word32 timeout_sec) return rc; } + if (timeout_sec == 0) { timeout_sec = 2; } + elapsed_sec = mqtt_get_timer_seconds(); if (*start_sec < elapsed_sec) { elapsed_sec -= *start_sec; diff --git a/examples/mqttnet.c b/examples/mqttnet.c index 962c4aeb6..e636647f1 100644 --- a/examples/mqttnet.c +++ b/examples/mqttnet.c @@ -33,7 +33,7 @@ typedef struct MulticastCtx { #endif #ifndef WOLFMQTT_TEST_NONBLOCK_TIMES -#define WOLFMQTT_TEST_NONBLOCK_TIMES 1 + #define WOLFMQTT_TEST_NONBLOCK_TIMES 1 #endif /* Private functions */ @@ -386,6 +386,49 @@ static int NetRead(void *context, byte* buf, int buf_len, /* How many times to retry after a timeout. */ #define MQTT_CURL_NUM_RETRY (2) +#if defined(WOLFMQTT_NONBLOCK) && defined(WOLFMQTT_TEST_NONBLOCK) +/* Tells the calling function to either return early with + * MQTT_CODE_CONTINUE, or proceed with a smaller buffer read/write. + * Used for testing nonblocking. */ +static int +mqttcurl_test_nonblock(int * buf_len, int for_recv) +{ + static int testNbAlt = 0; + static int testSmallerBuf = 0; + #if !defined(WOLFMQTT_DEBUG_SOCKET) + (void)for_recv; + #endif + + if (testNbAlt < WOLFMQTT_TEST_NONBLOCK_TIMES) { + testNbAlt++; + #if defined(WOLFMQTT_DEBUG_SOCKET) + PRINTF("mqttcurl_test_nonblock(%d): returning early with CONTINUE", + for_recv); + #endif + return MQTT_CODE_CONTINUE; + } + + testNbAlt = 0; + + if (!testSmallerBuf) { + if (*buf_len > 2) { + *buf_len /= 2; + testSmallerBuf = 1; + #if defined(WOLFMQTT_DEBUG_SOCKET) + PRINTF("mqttcurl_test_nonblock(%d): testing small buff: %d", + for_recv, *buf_len); + #endif + } + } + else { + testSmallerBuf = 0; + } + + return MQTT_CODE_SUCCESS; +} + +#endif /* defined(WOLFMQTT_NONBLOCK) && defined(WOLFMQTT_TEST_NONBLOCK) */ + static int mqttcurl_wait(curl_socket_t sockfd, int for_recv, int timeout_ms, int test_mode) @@ -439,6 +482,10 @@ mqttcurl_wait(curl_socket_t sockfd, int for_recv, int timeout_ms, return MQTT_CODE_ERROR_TIMEOUT; } + #ifndef WOLFMQTT_ENABLE_STDIN_CAP + (void)test_mode; + #endif + return MQTT_CODE_ERROR_NETWORK; } @@ -600,7 +647,7 @@ mqttcurl_connect(SocketContext * sock, const char* host, word16 port, #if 0 /* Set proxy options. * Unused at the moment. */ - if (sock->mqttCtx->use_proxy != NULL) { + if (sock->mqttCtx->use_proxy) { /* Set the proxy hostname or ip address string. Append * ":[port num]" to the string to specify a port. */ res = curl_easy_setopt(sock->curl, CURLOPT_PROXY, @@ -692,10 +739,18 @@ static int NetWrite(void *context, const byte* buf, int buf_len, curl_socket_t sockfd = 0; int wait_rc = 0; - if (context == NULL || buf == NULL || buf_len == 0) { + if (context == NULL || buf == NULL || buf_len <= 0) { return MQTT_CODE_ERROR_BAD_ARG; } +#if defined(WOLFMQTT_NONBLOCK) && defined(WOLFMQTT_TEST_NONBLOCK) + if (sock->mqttCtx->useNonBlockMode) { + if (mqttcurl_test_nonblock(&buf_len, 0)) { + return MQTT_CODE_CONTINUE; + } + } +#endif /* defined(WOLFMQTT_NONBLOCK) && defined(WOLFMQTT_TEST_NONBLOCK) */ + /* get the active socket from libcurl */ res = curl_easy_getinfo(sock->curl, CURLINFO_ACTIVESOCKET, &sockfd); if (res != CURLE_OK) { @@ -767,10 +822,18 @@ static int NetRead(void *context, byte* buf, int buf_len, curl_socket_t sockfd = 0; int wait_rc = 0; - if (context == NULL || buf == NULL || buf_len == 0) { + if (context == NULL || buf == NULL || buf_len <= 0) { return MQTT_CODE_ERROR_BAD_ARG; } +#if defined(WOLFMQTT_NONBLOCK) && defined(WOLFMQTT_TEST_NONBLOCK) + if (sock->mqttCtx->useNonBlockMode) { + if (mqttcurl_test_nonblock(&buf_len, 1)) { + return MQTT_CODE_CONTINUE; + } + } +#endif /* defined(WOLFMQTT_NONBLOCK) && defined(WOLFMQTT_TEST_NONBLOCK) */ + /* get the active socket from libcurl */ res = curl_easy_getinfo(sock->curl, CURLINFO_ACTIVESOCKET, &sockfd); if (res != CURLE_OK) { @@ -858,7 +921,7 @@ static int NetDisconnect(void *context) #else #ifndef WOLFMQTT_NO_TIMEOUT -static void setup_timeout(struct timeval* tv, int timeout_ms) +static void tcp_setup_timeout(struct timeval* tv, int timeout_ms) { tv->tv_sec = timeout_ms / 1000; tv->tv_usec = (timeout_ms % 1000) * 1000; @@ -870,6 +933,23 @@ static void setup_timeout(struct timeval* tv, int timeout_ms) } } +static void tcp_set_fds(SocketContext * sock, fd_set * recvfds, fd_set * errfds) +{ + /* Setup select file descriptors to watch */ + FD_ZERO(errfds); + FD_SET(sock->fd, errfds); + FD_ZERO(recvfds); + FD_SET(sock->fd, recvfds); + #ifdef WOLFMQTT_ENABLE_STDIN_CAP + #ifdef WOLFMQTT_MULTITHREAD + FD_SET(sock->pfd[0], recvfds); + #endif + if (!sock->mqttCtx->test_mode) { + FD_SET(STDIN, recvfds); + } + #endif /* WOLFMQTT_ENABLE_STDIN_CAP */ +} + #ifdef WOLFMQTT_NONBLOCK static void tcp_set_nonblocking(SOCKET_T* sockfd) { @@ -982,7 +1062,7 @@ static int NetConnect(void *context, const char* host, word16 port, struct timeval tv; /* Setup timeout and FD's */ - setup_timeout(&tv, timeout_ms); + tcp_setup_timeout(&tv, timeout_ms); FD_ZERO(&fdset); FD_SET(sock->fd, &fdset); #endif /* !WOLFMQTT_NO_TIMEOUT */ @@ -1110,7 +1190,7 @@ static int SN_NetConnect(void *context, const char* host, word16 port, struct timeval tv; /* Setup timeout and FD's */ - setup_timeout(&tv, timeout_ms); + tcp_setup_timeout(&tv, timeout_ms); FD_ZERO(&fdset); FD_SET(sock->fd, &fdset); #else @@ -1166,8 +1246,9 @@ static int NetWrite(void *context, const byte* buf, int buf_len, } testNbWriteAlt = 0; if (!testSmallerWrite) { - if (buf_len > 2) + if (buf_len > 2) { buf_len /= 2; + } testSmallerWrite = 1; } else { @@ -1178,12 +1259,16 @@ static int NetWrite(void *context, const byte* buf, int buf_len, #ifndef WOLFMQTT_NO_TIMEOUT /* Setup timeout */ - setup_timeout(&tv, timeout_ms); + tcp_setup_timeout(&tv, timeout_ms); (void)setsockopt(sock->fd, SOL_SOCKET, SO_SNDTIMEO, (char *)&tv, sizeof(tv)); #endif rc = (int)SOCK_SEND(sock->fd, buf, buf_len, 0); + #if defined(WOLFMQTT_DEBUG_SOCKET) + PRINTF("info: SOCK_SEND(%d) returned %d, buf_len is %d", + buf_len, rc, buf_len); + #endif if (rc == -1) { { /* Get error */ @@ -1226,6 +1311,8 @@ static int NetRead_ex(void *context, byte* buf, int buf_len, fd_set recvfds; fd_set errfds; struct timeval tv; +#else + (void)timeout_ms; #endif #if defined(WOLFMQTT_NONBLOCK) && defined(WOLFMQTT_TEST_NONBLOCK) static int testNbReadAlt = 0; @@ -1254,8 +1341,9 @@ static int NetRead_ex(void *context, byte* buf, int buf_len, } testNbReadAlt = 0; if (!testSmallerRead) { - if (buf_len > 2) + if (buf_len > 2) { buf_len /= 2; + } testSmallerRead = 1; } else { @@ -1264,27 +1352,6 @@ static int NetRead_ex(void *context, byte* buf, int buf_len, } #endif -#ifndef WOLFMQTT_NO_TIMEOUT - /* Setup timeout */ - setup_timeout(&tv, timeout_ms); - - /* Setup select file descriptors to watch */ - FD_ZERO(&errfds); - FD_SET(sock->fd, &errfds); - FD_ZERO(&recvfds); - FD_SET(sock->fd, &recvfds); - #ifdef WOLFMQTT_ENABLE_STDIN_CAP - #ifdef WOLFMQTT_MULTITHREAD - FD_SET(sock->pfd[0], &recvfds); - #endif - if (!mqttCtx->test_mode) { - FD_SET(STDIN, &recvfds); - } - #endif /* WOLFMQTT_ENABLE_STDIN_CAP */ -#else - (void)timeout_ms; -#endif /* !WOLFMQTT_NO_TIMEOUT */ - /* Loop until buf_len has been read, error or timeout */ while (bytes < buf_len) { int do_read = 0; @@ -1292,9 +1359,13 @@ static int NetRead_ex(void *context, byte* buf, int buf_len, #ifndef WOLFMQTT_NO_TIMEOUT #ifdef WOLFMQTT_NONBLOCK if (mqttCtx->useNonBlockMode) { - #ifdef WOLFMQTT_ENABLE_STDIN_CAP + #ifdef WOLFMQTT_ENABLE_STDIN_CAP /* quick no timeout check if data is available on stdin */ - setup_timeout(&tv, 0); + tcp_setup_timeout(&tv, 0); + + /* Setup select file descriptors to watch */ + tcp_set_fds(sock, &recvfds, &errfds); + rc = select((int)SELECT_FD(sock->fd), &recvfds, NULL, &errfds, &tv); if (rc > 0) { if (FD_ISSET(sock->fd, &recvfds)) { @@ -1304,14 +1375,19 @@ static int NetRead_ex(void *context, byte* buf, int buf_len, return MQTT_CODE_STDIN_WAKE; } } - #else + #else do_read = 1; - #endif + #endif } else - #endif + #endif /* WOLFMQTT_NONBLOCK */ { /* Wait for rx data to be available */ + tcp_setup_timeout(&tv, timeout_ms); + + /* Setup select file descriptors to watch */ + tcp_set_fds(sock, &recvfds, &errfds); + rc = select((int)SELECT_FD(sock->fd), &recvfds, NULL, &errfds, &tv); if (rc > 0) { if (FD_ISSET(sock->fd, &recvfds)) { @@ -1348,6 +1424,10 @@ static int NetRead_ex(void *context, byte* buf, int buf_len, &buf[bytes], buf_len - bytes, flags); + #if defined(WOLFMQTT_DEBUG_SOCKET) + PRINTF("info: SOCK_RECV(%d) returned %d, buf_len - bytes is %d", + bytes, rc, buf_len - bytes); + #endif if (rc <= 0) { rc = -1; goto exit; /* Error */ diff --git a/examples/multithread/multithread.c b/examples/multithread/multithread.c index 7a9b02d42..637ecb93e 100644 --- a/examples/multithread/multithread.c +++ b/examples/multithread/multithread.c @@ -38,13 +38,15 @@ /* Configuration */ /* Number of publish tasks. Each will send a unique message to the broker. */ -#define NUM_PUB_TASKS 5 -#define NUM_PUB_PER_TASK 2 +#if !defined NUM_PUB_TASKS || !defined NUM_PUB_PER_TASK + #define NUM_PUB_TASKS 5 + #define NUM_PUB_PER_TASK 2 +#endif /* Maximum size for network read/write callbacks. There is also a v5 define that describes the max MQTT control packet size, DEFAULT_MAX_PKT_SZ. */ #ifndef MAX_BUFFER_SIZE -#define MAX_BUFFER_SIZE 1024 + #define MAX_BUFFER_SIZE 1024 #endif /* Total size of test message to build */ @@ -841,6 +843,14 @@ int main(int argc, char** argv) if (rc != 0) { return rc; } + #ifdef WOLFMQTT_STRESS + /* Forbid running stress test against anything but localhost. */ + if (XSTRCMP(gMqttCtx.host, "localhost") != 0) { + PRINTF("error: stress build may only run against localhost: host=%s", + gMqttCtx.host); + return -1; + } + #endif #endif #ifdef USE_WINDOWS_API if (SetConsoleCtrlHandler((PHANDLER_ROUTINE)CtrlHandler, diff --git a/scripts/client.test b/scripts/client.test index c2fe1d27b..9ea11a572 100755 --- a/scripts/client.test +++ b/scripts/client.test @@ -2,50 +2,15 @@ # MQTT Client test +name="MQTT Client" +prog="examples/mqttclient/mqttclient" no_pid=-1 broker_pid=$no_pid -do_cleanup() { - if [ $broker_pid != $no_pid ] - then - kill -9 $broker_pid - echo "Killed broker PID $broker_pid" - broker_pid=$no_pid - fi - - if [ $1 -ne 0 ] - then - exit 1 - fi -} - -generate_port() { # function to produce a random port number - if [[ "$OSTYPE" == "linux"* ]]; then - port=$(($(od -An -N2 /dev/urandom) % (65535-49152) + 49152)) - elif [[ "$OSTYPE" == "darwin"* ]]; then - port=$(($(od -An -N2 /dev/random) % (65535-49152) + 49152)) - else - echo "Unknown OS TYPE" - exit 1 - fi - echo -e "Using port $port" -} - -check_broker() { - timeout 10 sh -c 'until nc -v -z $0 $1; do sleep 1; done' localhost $port -} - - -# Check for application -[ ! -x ./examples/mqttclient/mqttclient ] && echo -e "\n\nMQTT Client doesn't exist" && exit 1 - -# Check for TLS support -has_tls=no -./examples/mqttclient/mqttclient -? 2>&1 | grep -- 'Enable TLS' -if [ $? -eq 0 ]; then - has_tls=yes -fi +source scripts/test_common.sh +# Use minimum of 2 seconds +# The check timout will sometimes incorrectly trigger if 1 second is used def_args="-T -C 2000" # Check for mosquitto @@ -74,7 +39,7 @@ then echo "Broker PID is $broker_pid" check_broker - + def_args="${def_args} -h localhost" tls_port_args="-p 18883" port_args="-p ${port}" @@ -86,40 +51,40 @@ fi echo -e "Base args: $def_args $port_args" # Run without TLS and QoS 0-2 -./examples/mqttclient/mqttclient $def_args $port_args -q 0 $1 +./$prog $def_args $port_args -q 0 $1 RESULT=$? -[ $RESULT -ne 0 ] && echo -e "\n\nMQTT Client failed! TLS=Off, QoS=0" && do_cleanup "-1" +[ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=Off, QoS=0" && do_cleanup "-1" -./examples/mqttclient/mqttclient $def_args $port_args -q 1 $1 +./$prog $def_args $port_args -q 1 $1 RESULT=$? -[ $RESULT -ne 0 ] && echo -e "\n\nMQTT Client failed! TLS=Off, QoS=1" && do_cleanup "-1" +[ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=Off, QoS=1" && do_cleanup "-1" -./examples/mqttclient/mqttclient $def_args $port_args -q 2 $1 +./$prog $def_args $port_args -q 2 $1 RESULT=$? -[ $RESULT -ne 0 ] && echo -e "\n\nMQTT Client failed! TLS=Off, QoS=2" && do_cleanup "-1" +[ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=Off, QoS=2" && do_cleanup "-1" if test $has_tls == yes then # Run with TLS and QoS 0-2 - ./examples/mqttclient/mqttclient $def_args $cacert_args $tls_port_args -t -q 0 $1 + ./$prog $def_args $cacert_args $tls_port_args -t -q 0 $1 RESULT=$? - [ $RESULT -ne 0 ] && echo -e "\n\nMQTT Client failed! TLS=On, QoS=0" && do_cleanup "-1" + [ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=On, QoS=0" && do_cleanup "-1" - ./examples/mqttclient/mqttclient $def_args $cacert_args $tls_port_args -t -q 1 $1 + ./$prog $def_args $cacert_args $tls_port_args -t -q 1 $1 RESULT=$? - [ $RESULT -ne 0 ] && echo -e "\n\nMQTT Client failed! TLS=On, QoS=1" && do_cleanup "-1" + [ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=On, QoS=1" && do_cleanup "-1" - ./examples/mqttclient/mqttclient $def_args $cacert_args $tls_port_args -t -q 2 $1 + ./$prog $def_args $cacert_args $tls_port_args -t -q 2 $1 RESULT=$? - [ $RESULT -ne 0 ] && echo -e "\n\nMQTT Client failed! TLS=On, QoS=2" && do_cleanup "-1" + [ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=On, QoS=2" && do_cleanup "-1" - ./examples/mqttclient/mqttclient $def_args $cacert_args $mutual_auth_args $tls_port_args -t -q 0 $1 + ./$prog $def_args $cacert_args $mutual_auth_args $tls_port_args -t -q 0 $1 RESULT=$? - [ $RESULT -ne 0 ] && echo -e "\n\nMQTT Client failed! TLS=On, QoS=0, RSA mutual auth" && do_cleanup "-1" + [ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=On, QoS=0, RSA mutual auth" && do_cleanup "-1" - ./examples/mqttclient/mqttclient $def_args $cacert_args $ecc_mutual_auth_args $tls_port_args -t -q 0 $1 + ./$prog $def_args $cacert_args $ecc_mutual_auth_args $tls_port_args -t -q 0 $1 RESULT=$? - [ $RESULT -ne 0 ] && echo -e "\n\nMQTT Client failed! TLS=On, QoS=0, ECC mutual auth" && do_cleanup "-1" + [ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=On, QoS=0, ECC mutual auth" && do_cleanup "-1" fi # End broker diff --git a/scripts/firmware.test b/scripts/firmware.test index cbcc57f92..8064898be 100755 --- a/scripts/firmware.test +++ b/scripts/firmware.test @@ -83,9 +83,9 @@ then mosquitto $broker_args & broker_pid=$! echo "Broker PID is $broker_pid" - + check_broker - + def_args="${def_args} -h localhost -p ${port}" fi diff --git a/scripts/include.am b/scripts/include.am index 585c047d8..2504880c3 100644 --- a/scripts/include.am +++ b/scripts/include.am @@ -5,6 +5,7 @@ include scripts/broker_test/include.am if BUILD_EXAMPLES +if !BUILD_STRESS dist_noinst_SCRIPTS += scripts/client.test \ scripts/firmware.test \ scripts/azureiothub.test \ @@ -12,8 +13,13 @@ dist_noinst_SCRIPTS += scripts/client.test \ scripts/nbclient.test # WIOT test broker disabled 31MAY2021 # scripts/wiot.test + if BUILD_MULTITHREAD dist_noinst_SCRIPTS += scripts/multithread.test -endif +endif # BUILD_MULTITHREAD -endif +else +# Disable all other tests if checking stress. +dist_noinst_SCRIPTS += scripts/stress.test +endif # BUILD_STRESS +endif # BUILD_EXAMPLES diff --git a/scripts/multithread.test b/scripts/multithread.test index e7d13b7eb..3af2fb1b4 100755 --- a/scripts/multithread.test +++ b/scripts/multithread.test @@ -2,45 +2,15 @@ # MQTT Multithread Client test +name="Multithread Client" +prog="examples/multithread/multithread" no_pid=-1 broker_pid=$no_pid -do_cleanup() { - if [ $broker_pid != $no_pid ] - then - kill -9 $broker_pid - echo "Killed broker PID $broker_pid" - broker_pid=$no_pid - fi - - if [ $1 -ne 0 ] - then - exit 1 - fi -} - -generate_port() { # function to produce a random port number - if [[ "$OSTYPE" == "linux"* ]]; then - port=$(($(od -An -N2 /dev/urandom) % (65535-49152) + 49152)) - elif [[ "$OSTYPE" == "darwin"* ]]; then - port=$(($(od -An -N2 /dev/random) % (65535-49152) + 49152)) - else - echo "Unknown OS TYPE" - exit 1 - fi - echo -e "Using port $port" -} - -# Check for application -[ ! -x ./examples/multithread/multithread ] && echo -e "\n\nMultithread Client doesn't exist" && exit 1 - -# Check for TLS support -has_tls=no -./examples/multithread/multithread -? 2>&1 | grep -- 'Enable TLS' -if [ $? -eq 0 ]; then - has_tls=yes -fi +source scripts/test_common.sh +# Use minimum of 2 seconds +# The check timout will sometimes incorrectly trigger if 1 second is used def_args="-T -C 2000" # Check for mosquitto @@ -78,32 +48,32 @@ fi echo -e "Base args: $def_args $port_args" # Run without TLS and QoS 0-2 -./examples/multithread/multithread $def_args $port_args -q 0 $1 +./$prog $def_args $port_args -q 0 $1 RESULT=$? -[ $RESULT -ne 0 ] && echo -e "\n\nMultithread Client failed! TLS=Off, QoS=0" && do_cleanup "-1" +[ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=Off, QoS=0" && do_cleanup "-1" -./examples/multithread/multithread $def_args $port_args -q 1 $1 +./$prog $def_args $port_args -q 1 $1 RESULT=$? -[ $RESULT -ne 0 ] && echo -e "\n\nMultithread Client failed! TLS=Off, QoS=1" && do_cleanup "-1" +[ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=Off, QoS=1" && do_cleanup "-1" -./examples/multithread/multithread $def_args $port_args -q 2 $1 +./$prog $def_args $port_args -q 2 $1 RESULT=$? -[ $RESULT -ne 0 ] && echo -e "\n\nMultithread Client failed! TLS=Off, QoS=2" && do_cleanup "-1" +[ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=Off, QoS=2" && do_cleanup "-1" if test $has_tls == yes then # Run with TLS and QoS 0-2 - ./examples/multithread/multithread $def_args $cacert_args $tls_port_args -t -q 0 $1 + ./$prog $def_args $cacert_args $tls_port_args -t -q 0 $1 RESULT=$? - [ $RESULT -ne 0 ] && echo -e "\n\nMultithread Client failed! TLS=On, QoS=0" && do_cleanup "-1" + [ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=On, QoS=0" && do_cleanup "-1" - ./examples/multithread/multithread $def_args $cacert_args $tls_port_args -t -q 1 $1 + ./$prog $def_args $cacert_args $tls_port_args -t -q 1 $1 RESULT=$? - [ $RESULT -ne 0 ] && echo -e "\n\nMultithread Client failed! TLS=On, QoS=1" && do_cleanup "-1" + [ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=On, QoS=1" && do_cleanup "-1" - ./examples/multithread/multithread $def_args $cacert_args $tls_port_args -t -q 2 $1 + ./$prog $def_args $cacert_args $tls_port_args -t -q 2 $1 RESULT=$? - [ $RESULT -ne 0 ] && echo -e "\n\nMultithread Client failed! TLS=On, QoS=2" && do_cleanup "-1" + [ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=On, QoS=2" && do_cleanup "-1" fi # End broker diff --git a/scripts/nbclient.test b/scripts/nbclient.test index 87ac7fb25..42072926e 100755 --- a/scripts/nbclient.test +++ b/scripts/nbclient.test @@ -2,48 +2,12 @@ # Non-blocking Client test +name="Non-blocking Client" +prog="examples/nbclient/nbclient" no_pid=-1 broker_pid=$no_pid -do_cleanup() { - if [ $broker_pid != $no_pid ] - then - kill -9 $broker_pid - echo "Killed broker PID $broker_pid" - broker_pid=$no_pid - fi - - if [ $1 -ne 0 ] - then - exit 1 - fi -} - -generate_port() { # function to produce a random port number - if [[ "$OSTYPE" == "linux"* ]]; then - port=$(($(od -An -N2 /dev/urandom) % (65535-49152) + 49152)) - elif [[ "$OSTYPE" == "darwin"* ]]; then - port=$(($(od -An -N2 /dev/random) % (65535-49152) + 49152)) - else - echo "Unknown OS TYPE" - exit 1 - fi - echo -e "Using port $port" -} - -check_broker() { - timeout 10 sh -c 'until nc -v -z $0 $1; do sleep 1; done' localhost $port -} - -# Check for application -[ ! -x ./examples/nbclient/nbclient ] && echo -e "\n\nNon-blocking Client doesn't exist" && exit 1 - -# Check for TLS support -has_tls=no -./examples/nbclient/nbclient -? 2>&1 | grep -- 'Enable TLS' -if [ $? -eq 0 ]; then - has_tls=yes -fi +source scripts/test_common.sh # Use minimum of 2 seconds # The check timout will sometimes incorrectly trigger if 1 second is used @@ -75,7 +39,7 @@ then echo "Broker PID is $broker_pid" check_broker - + def_args="${def_args} -h localhost" tls_port_args="-p 18883" port_args="-p ${port}" @@ -85,32 +49,32 @@ fi echo -e "Base args: $def_args $port_args" # Run without TLS and QoS 0-2 -./examples/nbclient/nbclient $def_args $port_args -q 0 $1 +./$prog $def_args $port_args -q 0 $1 RESULT=$? -[ $RESULT -ne 0 ] && echo -e "\n\nNon-blocking Client failed! TLS=Off, QoS=0" && do_cleanup "-1" +[ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=Off, QoS=0" && do_cleanup "-1" -./examples/nbclient/nbclient $def_args $port_args -q 1 $1 +./$prog $def_args $port_args -q 1 $1 RESULT=$? -[ $RESULT -ne 0 ] && echo -e "\n\nNon-blocking Client failed! TLS=Off, QoS=1" && do_cleanup "-1" +[ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=Off, QoS=1" && do_cleanup "-1" -./examples/nbclient/nbclient $def_args $port_args -q 2 $1 +./$prog $def_args $port_args -q 2 $1 RESULT=$? -[ $RESULT -ne 0 ] && echo -e "\n\nNon-blocking Client failed! TLS=Off, QoS=2" && do_cleanup "-1" +[ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=Off, QoS=2" && do_cleanup "-1" if test $has_tls == yes then # Run with TLS and QoS 0-2 - ./examples/nbclient/nbclient $def_args $cacert_args $tls_port_args -t -q 0 $1 + ./$prog $def_args $cacert_args $tls_port_args -t -q 0 $1 RESULT=$? - [ $RESULT -ne 0 ] && echo -e "\n\nNon-blocking Client failed! TLS=On, QoS=0" && do_cleanup "-1" + [ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=On, QoS=0" && do_cleanup "-1" - ./examples/nbclient/nbclient $def_args $cacert_args $tls_port_args -t -q 1 $1 + ./$prog $def_args $cacert_args $tls_port_args -t -q 1 $1 RESULT=$? - [ $RESULT -ne 0 ] && echo -e "\n\nNon-blocking Client failed! TLS=On, QoS=1" && do_cleanup "-1" + [ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=On, QoS=1" && do_cleanup "-1" - ./examples/nbclient/nbclient $def_args $cacert_args $tls_port_args -t -q 2 $1 + ./$prog $def_args $cacert_args $tls_port_args -t -q 2 $1 RESULT=$? - [ $RESULT -ne 0 ] && echo -e "\n\nNon-blocking Client failed! TLS=On, QoS=2" && do_cleanup "-1" + [ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=On, QoS=2" && do_cleanup "-1" fi # End broker diff --git a/scripts/stress.test b/scripts/stress.test new file mode 100755 index 000000000..f076216a3 --- /dev/null +++ b/scripts/stress.test @@ -0,0 +1,89 @@ +#!/bin/bash + +# MQTT Stress Client test + +name="Stress Client" +prog="examples/multithread/multithread" +timeout_ms=250 +no_pid=-1 +broker_pid=$no_pid + +if [ $# -eq 1 ]; then + timeout_ms=$1 +fi + +# Require mosquitto to run. +if ! command -v mosquitto; then + echo "error: this test requires local mosquitto broker" + exit 1 +fi + +source scripts/test_common.sh + +# This test is local host only! +def_args="-h localhost -T -C $timeout_ms" + +bwrap_path="$(command -v bwrap)" +if [ -n "$bwrap_path" ]; then + # bwrap only if using a local mosquitto instance + if [ "${AM_BWRAPPED-}" != "yes" ]; then + echo "Using bwrap" + export AM_BWRAPPED=yes + exec "$bwrap_path" --unshare-net --dev-bind / / "$0" "$@" + fi + unset AM_BWRAPPED + + broker_args="-c scripts/broker_test/mosquitto.conf" + port=11883 +else + # mosquitto broker custom port non-TLS only + has_tls=no + generate_port + broker_args="-p $port" +fi +mosquitto $broker_args & +broker_pid=$! +echo "Broker PID is $broker_pid" +sleep 0.1 + +tls_port_args="-p 18883" +port_args="-p ${port}" +cacert_args="-A scripts/broker_test/ca-cert.pem" + +echo -e "Base args: $def_args $port_args" + +# Run without TLS and QoS 0-2 +./$prog $def_args $port_args -q 0 +RESULT=$? +[ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=Off, QoS=0" && do_cleanup "-1" + +./$prog $def_args $port_args -q 1 +RESULT=$? +[ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=Off, QoS=1" && do_cleanup "-1" + +./$prog $def_args $port_args -q 2 +RESULT=$? +[ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=Off, QoS=2" && do_cleanup "-1" + +if test $has_tls == yes +then + # Run with TLS and QoS 0-2 + ./$prog $def_args $cacert_args $tls_port_args -t -q 0 + RESULT=$? + [ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=On, QoS=0" && do_cleanup "-1" + + ./$prog $def_args $cacert_args $tls_port_args -t -q 1 + RESULT=$? + [ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=On, QoS=1" && do_cleanup "-1" + + ./$prog $def_args $cacert_args $tls_port_args -t -q 2 + RESULT=$? + [ $RESULT -ne 0 ] && echo -e "\n\n$name failed! TLS=On, QoS=2" && do_cleanup "-1" +fi + +# End broker +do_cleanup "0" + +echo -e "\n\nStress MQTT Client Tests Passed" + +exit 0 diff --git a/scripts/test_common.sh b/scripts/test_common.sh new file mode 100644 index 000000000..e7f7b757f --- /dev/null +++ b/scripts/test_common.sh @@ -0,0 +1,40 @@ +#!/bin/bash +do_cleanup() { + if [ $broker_pid != $no_pid ] + then + kill -9 $broker_pid + echo "Killed broker PID $broker_pid" + broker_pid=$no_pid + fi + + if [ $1 -ne 0 ] + then + exit 1 + fi +} + +generate_port() { # function to produce a random port number + if [[ "$OSTYPE" == "linux"* ]]; then + port=$(($(od -An -N2 /dev/urandom) % (65535-49152) + 49152)) + elif [[ "$OSTYPE" == "darwin"* ]]; then + port=$(($(od -An -N2 /dev/random) % (65535-49152) + 49152)) + else + echo "Unknown OS TYPE" + exit 1 + fi + echo -e "Using port $port" +} + +check_broker() { + timeout 10 sh -c 'until nc -v -z $0 $1; do sleep 1; done' localhost $port +} + +# Check for application +[ ! -x ./$prog ] && echo -e "\n\n$name doesn't exist" && exit 1 + +# Check for TLS support +has_tls=no +./$prog -? 2>&1 | grep -- 'Enable TLS' +if [ $? -eq 0 ]; then + has_tls=yes +fi