From d048249c22133598bb4e0353ab81fef14879af60 Mon Sep 17 00:00:00 2001
From: Brian Atkinson <batkinson@lanl.gov>
Date: Thu, 29 Jun 2023 14:26:35 -0600
Subject: [PATCH] Only print NUMA nodes if requested

After updating the build system to CMake XDD was outputing the NUMA
domain for each worker thread in the header of the output. This address
that issue by printing that info in the event it is requested through
the `-debug` flag.

Since the `GO_DEBUG_USER1` global debug was never used, it was updated
to be used for init debugging and changed to GO_DEBUG_INIT. So if the
user does `-debug init` they will now see what cpus the worker threads
have been pinned to.

As part of the work, I also reorganized the test directory a bit. I made
a separate debug test directory just to test out this functionality as
it really did not make sense for it to be in the fuctional test cases. I
placed all the common test CMake variables in the CMakeLists.txt and
then just added individual test information in the functional and debug
CMakeLists.txt files.

Fixes #13
Signed-off-by: Brian Atkinson <batkinson@lanl.gov>
---
 src/base/worker_thread_init.c                 | 21 +++++------
 src/client/parse_func.c                       |  6 ++--
 src/common/xint_global_data.h                 |  5 +--
 tests/CMakeLists.txt                          | 34 ++++++++++++++++++
 tests/{functional => }/common.sh              |  0
 tests/debug/CMakeLists.txt                    | 12 +++++++
 tests/debug/test_xdd_debug_init.sh            | 36 +++++++++++++++++++
 tests/functional/CMakeLists.txt               | 34 +-----------------
 tests/functional/test_xdd_createnewfiles.sh   |  4 +--
 tests/functional/test_xdd_createnewfiles2.sh  |  4 +--
 tests/functional/test_xdd_heartbeat_byte.sh   |  4 +--
 .../functional/test_xdd_heartbeat_elapsed.sh  |  4 +--
 tests/functional/test_xdd_heartbeat_lf.sh     |  4 +--
 tests/functional/test_xdd_heartbeat_output.sh |  4 +--
 tests/functional/test_xdd_heartbeat_target.sh |  4 +--
 tests/functional/test_xdd_heartbeat_tod.sh    |  4 +--
 tests/functional/test_xdd_lockstep1.sh        |  4 +--
 tests/functional/test_xdd_lockstep2.sh        |  4 +--
 tests/functional/test_xdd_passdelay.sh        |  4 +--
 tests/functional/test_xdd_pretruncate.sh      |  4 +--
 tests/functional/test_xdd_reopen.sh           |  4 +--
 tests/functional/test_xdd_runtime.sh          |  4 +--
 tests/functional/test_xdd_startdelay.sh       |  4 +--
 tests/functional/test_xdd_startoffset.sh      |  4 +--
 tests/functional/test_xdd_syncwrite.sh        |  4 +--
 tests/functional/test_xdd_timelimit.sh        |  4 +--
 26 files changed, 136 insertions(+), 84 deletions(-)
 rename tests/{functional => }/common.sh (100%)
 create mode 100644 tests/debug/CMakeLists.txt
 create mode 100755 tests/debug/test_xdd_debug_init.sh

diff --git a/src/base/worker_thread_init.c b/src/base/worker_thread_init.c
index f0db8c84..c4ea57cb 100644
--- a/src/base/worker_thread_init.c
+++ b/src/base/worker_thread_init.c
@@ -30,16 +30,17 @@ xdd_worker_thread_init(worker_data_t *wdp) {
 	unsigned char	*bufp;		// Generic Buffer pointer
 
 #if HAVE_CPU_SET_T && HAVE_PTHREAD_ATTR_SETAFFINITY_NP
-    // BWS Print the cpuset
-    int i;
-    cpu_set_t cpuset;
-    CPU_ZERO(&cpuset);
-    pthread_getaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
-    printf("Thread %ld bound to NUMA node", (long int) pthread_self());
-    for (i = 0; i< 48; i++)
-        if (CPU_ISSET(i, &cpuset))
-            printf(" %d", i);
-    printf("\n");
+	if (xgp->global_options & GO_DEBUG_INIT) {
+		int i;
+		cpu_set_t cpuset;
+		CPU_ZERO(&cpuset);
+		pthread_getaffinity_np(pthread_self(), sizeof(cpuset), &cpuset);
+		fprintf(stderr, "Thread %ld bound to NUMA node", (long int) pthread_self());
+		for (i = 0; i < CPU_SETSIZE; i++)
+			if (CPU_ISSET(i, &cpuset))
+				fprintf(stderr, " %d", i);
+		fprintf(stderr, "\n");
+	}
 #endif
 
     // Get the Target Data Struct address as well
diff --git a/src/client/parse_func.c b/src/client/parse_func.c
index 259fa6a9..c3e2e727 100644
--- a/src/client/parse_func.c
+++ b/src/client/parse_func.c
@@ -808,9 +808,9 @@ xddfunc_debug(xdd_plan_t *planp, int32_t argc, char *argv[], uint32_t flags)
 			   (strcmp(argv[1], "TL") == 0) ||
 			   (strcmp(argv[1], "tl") == 0)) {
 			xgp->global_options |= GO_DEBUG_TIME_LIMIT;
-	} else if ((strcmp(argv[1], "USER1") == 0) ||
-			   (strcmp(argv[1], "user1") == 0)) {
-			xgp->global_options |= GO_DEBUG_USER1;
+	} else if ((strcmp(argv[1], "INIT") == 0) ||
+			   (strcmp(argv[1], "init") == 0)) {
+			xgp->global_options |= GO_DEBUG_INIT;
 	} else if ((strcmp(argv[1], "ALL") == 0) ||
 			   (strcmp(argv[1], "all") == 0) ||
 			   (strcmp(argv[1], "a") == 0)) {
diff --git a/src/common/xint_global_data.h b/src/common/xint_global_data.h
index 646a27a3..b8eba9d3 100644
--- a/src/common/xint_global_data.h
+++ b/src/common/xint_global_data.h
@@ -51,9 +51,10 @@
 #define GO_DEBUG_TS				0x0400000000000000ULL  /* */
 #define GO_DEBUG_THROTTLE		0x0800000000000000ULL  /* */
 #define GO_DEBUG_TIME_LIMIT		0x1000000000000000ULL  /* */
-#define GO_DEBUG_USER1			0x2000000000000000ULL  /* */
+#define GO_DEBUG_INIT			0x2000000000000000ULL  /* */
 #define GO_DEBUG_RESULTS		0x4000000000000000ULL  /* */
-#define GO_DEBUG_ALL			(GO_DEBUG_IO|GO_DEBUG_E2E|GO_DEBUG_LOCKSTEP|GO_DEBUG_OPEN|GO_DEBUG_TASK|GO_DEBUG_TOT|GO_DEBUG_TS|GO_DEBUG_THROTTLE|GO_DEBUG_USER1|GO_DEBUG_RESULTS)
+#define GO_DEBUG_ALL			(GO_DEBUG_IO | GO_DEBUG_E2E |GO_DEBUG_LOCKSTEP |GO_DEBUG_OPEN |GO_DEBUG_TASK | \
+                                         GO_DEBUG_TOT |GO_DEBUG_TS |GO_DEBUG_THROTTLE |GO_DEBUG_INIT |GO_DEBUG_RESULTS)
 
 struct xdd_global_data {
 	uint64_t		global_options;        				/* I/O Options valid for all targets */
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index cf1140c9..14b5d6e4 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -1 +1,35 @@
+configure_file(common.sh common.sh)
+
+# original tests depended on xdd being installed
+option(TEST_INSTALLED "Use installed xdd for tests" ON)
+if (TEST_INSTALLED)
+  set(XDD_EXEC "$<TARGET_FILE:xdd>")
+else()
+  set(XDD_EXEC "${CMAKE_BINARY_DIR}/bin/xdd")
+endif()
+
+# configurable output directory
+set(TESTDIR "${CMAKE_BINARY_DIR}/test-dir" CACHE PATH "Where to place test files")
+add_custom_target(mk-test-dir ALL
+  COMMAND "${CMAKE_COMMAND}" -E make_directory "${TESTDIR}")
+
+# not configurable
+set(TESTDIR_TESTS "${TESTDIR}/tests")
+add_custom_target(mk-test-tests ALL
+  COMMAND "${CMAKE_COMMAND}" -E make_directory "${TESTDIR_TESTS}"
+  DEPENDS mk-test-dir)
+set(TESTDIR_LOGS "${TESTDIR}/logs")
+add_custom_target(mk-test-logs ALL
+  COMMAND "${CMAKE_COMMAND}" -E make_directory "${TESTDIR_LOGS}"
+  DEPENDS mk-test-dir)
+
+# generate config file
+file(GENERATE
+  OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/test_config"
+  CONTENT "XDDTEST_XDD_EXE=${XDD_EXEC}
+XDDTEST_LOCAL_MOUNT=${TESTDIR}/tests
+XDDTEST_OUTPUT_DIR=${TESTDIR}/logs"
+)
+
 add_subdirectory(functional)
+add_subdirectory(debug)
diff --git a/tests/functional/common.sh b/tests/common.sh
similarity index 100%
rename from tests/functional/common.sh
rename to tests/common.sh
diff --git a/tests/debug/CMakeLists.txt b/tests/debug/CMakeLists.txt
new file mode 100644
index 00000000..37172f4a
--- /dev/null
+++ b/tests/debug/CMakeLists.txt
@@ -0,0 +1,12 @@
+set(DEBUG_TESTS
+    test_xdd_debug_init.sh
+    )
+
+foreach(TEST ${DEBUG_TESTS})
+  configure_file("${TEST}" "${TEST}" COPYONLY)
+  add_test(NAME "${TEST}"
+    COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${TEST}
+    WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+  set_tests_properties("${TEST}" PROPERTIES LABELS "debug")
+  set_tests_properties("${TEST}" PROPERTIES SKIP_RETURN_CODE 255)
+endforeach()
diff --git a/tests/debug/test_xdd_debug_init.sh b/tests/debug/test_xdd_debug_init.sh
new file mode 100755
index 00000000..ad4d74b2
--- /dev/null
+++ b/tests/debug/test_xdd_debug_init.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Debug output test for XDD.
+#
+# Validate XDD header has worker thread NUMA cpus printed if requested
+#
+# Description - Just writes out to /dev/null using XDD with -debug INIT and verifies
+#               that the NUMA cpus are listed if requested through the debug flag
+#
+# Source the test configuration environment
+#
+source ../test_config
+source ../common.sh
+
+initialize_test
+
+echo "Foobar and I am here motherfucker"
+
+${XDDTEST_XDD_EXE} -op write -reqsize 128 -numreqs 1 -targets 1 /dev/null -verbose -debug INIT \
+  2>&1 | grep "bound to NUMA node"
+
+if [[ $? -ne 0 ]]; then
+  # test failed
+  finalize_test 1
+fi
+
+${XDDTEST_XDD_EXE} -op write -reqsize 128 -numreqs 1 -targets 1 /dev/null -verbose \
+  2>&1 | grep "bound to NUMA node"
+
+if [[ $? -ne 1 ]]; then
+  # test failed
+  finalize_test 1
+fi
+
+# test passed
+finalize_test 0
diff --git a/tests/functional/CMakeLists.txt b/tests/functional/CMakeLists.txt
index 789d0ae7..1febdb83 100644
--- a/tests/functional/CMakeLists.txt
+++ b/tests/functional/CMakeLists.txt
@@ -1,5 +1,3 @@
-configure_file(common.sh common.sh)
-
 set(FUNCTIONAL
     test_xdd_createnewfiles2.sh
     test_xdd_heartbeat_byte.sh
@@ -20,40 +18,10 @@ set(FUNCTIONAL
     test_xdd_timelimit.sh
     )
 
-# original tests depended on xdd being installed
-option(TEST_INSTALLED "Use installed xdd for tests" ON)
-if (TEST_INSTALLED)
-  set(XDD_EXEC "$<TARGET_FILE:xdd>")
-else()
-  set(XDD_EXEC "${CMAKE_BINARY_DIR}/bin/xdd")
-endif()
-
-# configurable output directory
-set(TESTDIR "${CMAKE_BINARY_DIR}/test-dir" CACHE PATH "Where to place test files")
-add_custom_target(mk-test-dir ALL
-  COMMAND "${CMAKE_COMMAND}" -E make_directory "${TESTDIR}")
-
-# not configurable
-set(TESTDIR_TESTS "${TESTDIR}/tests")
-add_custom_target(mk-test-tests ALL
-  COMMAND "${CMAKE_COMMAND}" -E make_directory "${TESTDIR_TESTS}"
-  DEPENDS mk-test-dir)
-set(TESTDIR_LOGS "${TESTDIR}/logs")
-add_custom_target(mk-test-logs ALL
-  COMMAND "${CMAKE_COMMAND}" -E make_directory "${TESTDIR_LOGS}"
-  DEPENDS mk-test-dir)
-
-# generate config file
-file(GENERATE
-  OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/test_config"
-  CONTENT "XDDTEST_XDD_EXE=${XDD_EXEC}
-XDDTEST_LOCAL_MOUNT=${TESTDIR}/tests
-XDDTEST_OUTPUT_DIR=${TESTDIR}/logs"
-)
 foreach(TEST ${FUNCTIONAL})
   configure_file("${TEST}" "${TEST}" COPYONLY)
   add_test(NAME "${TEST}"
-    COMMAND sh -c "${CMAKE_CURRENT_BINARY_DIR}/${TEST} 2> ${TESTDIR}/logs/${TEST}.log"
+    COMMAND ${CMAKE_CURRENT_BINARY_DIR}/${TEST}
     WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
   set_tests_properties("${TEST}" PROPERTIES LABELS "functional")
   set_tests_properties("${TEST}" PROPERTIES SKIP_RETURN_CODE 255)
diff --git a/tests/functional/test_xdd_createnewfiles.sh b/tests/functional/test_xdd_createnewfiles.sh
index 60159173..bec7e462 100755
--- a/tests/functional/test_xdd_createnewfiles.sh
+++ b/tests/functional/test_xdd_createnewfiles.sh
@@ -8,8 +8,8 @@
 #
 # Source the test configuration environment
 #
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 
 # Create the test location
diff --git a/tests/functional/test_xdd_createnewfiles2.sh b/tests/functional/test_xdd_createnewfiles2.sh
index 1ec7fe12..1bd61642 100755
--- a/tests/functional/test_xdd_createnewfiles2.sh
+++ b/tests/functional/test_xdd_createnewfiles2.sh
@@ -9,8 +9,8 @@
 #
 # Source the test configuration environment
 #
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 initialize_test
 
diff --git a/tests/functional/test_xdd_heartbeat_byte.sh b/tests/functional/test_xdd_heartbeat_byte.sh
index d8b5eb5e..05e9c296 100755
--- a/tests/functional/test_xdd_heartbeat_byte.sh
+++ b/tests/functional/test_xdd_heartbeat_byte.sh
@@ -6,8 +6,8 @@
 #
 # Source the test configuration environment
 #
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # Pre-test set-up
 
diff --git a/tests/functional/test_xdd_heartbeat_elapsed.sh b/tests/functional/test_xdd_heartbeat_elapsed.sh
index 4f280e77..d4e8485a 100755
--- a/tests/functional/test_xdd_heartbeat_elapsed.sh
+++ b/tests/functional/test_xdd_heartbeat_elapsed.sh
@@ -5,8 +5,8 @@
 # Description: outputs the elapsed time since the start of run
 # Verify heartbeat elapsed time options works
 #
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # Pre-test set-up
 initialize_test
diff --git a/tests/functional/test_xdd_heartbeat_lf.sh b/tests/functional/test_xdd_heartbeat_lf.sh
index decb7ac6..cffd7f24 100755
--- a/tests/functional/test_xdd_heartbeat_lf.sh
+++ b/tests/functional/test_xdd_heartbeat_lf.sh
@@ -6,8 +6,8 @@
 #
 # Source the test configuration environment
 #
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # pre-test set-up
 initialize_test
diff --git a/tests/functional/test_xdd_heartbeat_output.sh b/tests/functional/test_xdd_heartbeat_output.sh
index fc410d28..60a593bc 100755
--- a/tests/functional/test_xdd_heartbeat_output.sh
+++ b/tests/functional/test_xdd_heartbeat_output.sh
@@ -5,8 +5,8 @@
 # Verify -hb output by checking if specified file exists
 #
 # Source test environment
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # Pre-test set-up
 initialize_test
diff --git a/tests/functional/test_xdd_heartbeat_target.sh b/tests/functional/test_xdd_heartbeat_target.sh
index 3162e9ee..ac33b6da 100755
--- a/tests/functional/test_xdd_heartbeat_target.sh
+++ b/tests/functional/test_xdd_heartbeat_target.sh
@@ -5,8 +5,8 @@
 # Verify -tgt by checking if target number displayed matches target number tested
 #
 # Source test environment
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # pre-test set-up
 initialize_test
diff --git a/tests/functional/test_xdd_heartbeat_tod.sh b/tests/functional/test_xdd_heartbeat_tod.sh
index f4fad1dc..afce2113 100755
--- a/tests/functional/test_xdd_heartbeat_tod.sh
+++ b/tests/functional/test_xdd_heartbeat_tod.sh
@@ -5,8 +5,8 @@
 # Validate -hb tod by comparing it to current time
 #
 # Source test configuration environment
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # pre-test set-up
 initialize_test
diff --git a/tests/functional/test_xdd_lockstep1.sh b/tests/functional/test_xdd_lockstep1.sh
index 002064ea..899761e9 100755
--- a/tests/functional/test_xdd_lockstep1.sh
+++ b/tests/functional/test_xdd_lockstep1.sh
@@ -7,8 +7,8 @@
 #
 # Source the test configuration environment
 #
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # Create the test location
 initialize_test
diff --git a/tests/functional/test_xdd_lockstep2.sh b/tests/functional/test_xdd_lockstep2.sh
index a0b073a1..42045f7a 100755
--- a/tests/functional/test_xdd_lockstep2.sh
+++ b/tests/functional/test_xdd_lockstep2.sh
@@ -7,8 +7,8 @@
 #
 # Source the test configuration environment
 #
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # Create the test location
 initialize_test
diff --git a/tests/functional/test_xdd_passdelay.sh b/tests/functional/test_xdd_passdelay.sh
index 281d77f5..81f3c63b 100755
--- a/tests/functional/test_xdd_passdelay.sh
+++ b/tests/functional/test_xdd_passdelay.sh
@@ -9,8 +9,8 @@
 #
 # Source the test configuration environment
 #
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # Pre-test set-up
 initialize_test
diff --git a/tests/functional/test_xdd_pretruncate.sh b/tests/functional/test_xdd_pretruncate.sh
index 6342e51b..059d879c 100755
--- a/tests/functional/test_xdd_pretruncate.sh
+++ b/tests/functional/test_xdd_pretruncate.sh
@@ -9,8 +9,8 @@
 #
 # Source the test configuration environment
 #
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # Pre-test create test directory and file
 initialize_test
diff --git a/tests/functional/test_xdd_reopen.sh b/tests/functional/test_xdd_reopen.sh
index afb6eda7..e3200816 100755
--- a/tests/functional/test_xdd_reopen.sh
+++ b/tests/functional/test_xdd_reopen.sh
@@ -8,8 +8,8 @@
 #
 # Source the test configuration environment
 #
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # Skip test on non-Linux platforms
 if [ `uname` != "Linux" ]; then
diff --git a/tests/functional/test_xdd_runtime.sh b/tests/functional/test_xdd_runtime.sh
index 1c659d8f..58afec8f 100755
--- a/tests/functional/test_xdd_runtime.sh
+++ b/tests/functional/test_xdd_runtime.sh
@@ -8,8 +8,8 @@
 #
 # Source the test configuration environment
 #
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # Perform pre-test
 initialize_test
diff --git a/tests/functional/test_xdd_startdelay.sh b/tests/functional/test_xdd_startdelay.sh
index ea4425c1..b41ee5a7 100755
--- a/tests/functional/test_xdd_startdelay.sh
+++ b/tests/functional/test_xdd_startdelay.sh
@@ -7,8 +7,8 @@
 #
 # Source the test configuration environment
 #
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # Pre-test set-up
 initialize_test
diff --git a/tests/functional/test_xdd_startoffset.sh b/tests/functional/test_xdd_startoffset.sh
index f89b167f..c61d2768 100755
--- a/tests/functional/test_xdd_startoffset.sh
+++ b/tests/functional/test_xdd_startoffset.sh
@@ -7,8 +7,8 @@
 # Validate -startoffset by comparing the size of a file starting from block 0 to its size starting from the nth block
 #
 # Source the test configuration environment
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # Pre-test set-up
 initialize_test
diff --git a/tests/functional/test_xdd_syncwrite.sh b/tests/functional/test_xdd_syncwrite.sh
index eb534996..8d0169d8 100755
--- a/tests/functional/test_xdd_syncwrite.sh
+++ b/tests/functional/test_xdd_syncwrite.sh
@@ -6,8 +6,8 @@
 #
 # Source the test configuration environment
 #
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 #skip test on non-Linux platforms
 if [ `uname` != "Linux" ]; then
diff --git a/tests/functional/test_xdd_timelimit.sh b/tests/functional/test_xdd_timelimit.sh
index a2371ddc..5cc68ed7 100755
--- a/tests/functional/test_xdd_timelimit.sh
+++ b/tests/functional/test_xdd_timelimit.sh
@@ -8,8 +8,8 @@
 #
 # Source the test configuration environment
 #
-source ./test_config
-source ./common.sh
+source ../test_config
+source ../common.sh
 
 # Perform pre-test
 initialize_test