Merge pull request redpanda-data#14775 from michael-redpanda/issues/1…

…4749-14750-14751

dt/audit: Wait for OCSF server to really come up

tests/docker/ducktape-deps/ocsf-server

@@ -26,5 +26,11 @@ mix local.hex --force && mix local.rebar --force
 
 pushd /opt/ocsf-server
 ./build_server.sh
+# The following is required because the server attempts to write logs
+# to the `dist/tmp` directory.  This is fine in docker ducktape as the user
+# who kicks it off is root, but in CDT, the user is not root so the server
+# fails to start causing audit log tests to fail
+mkdir dist/tmp
+chmod a+w dist/tmp
 openssl req -new -newkey rsa:2048 -sha256 -days 365 -nodes -x509 -keyout server.key -out server.crt -subj "/O=Redpanda" -batch
 popd

tests/rptest/services/ocsf_server.py

@@ -13,7 +13,7 @@
 
 from ducktape.cluster.remoteaccount import RemoteCommandError
 from ducktape.services.service import Service
-from rptest.util import wait_until_result
+from rptest.util import wait_until, wait_until_result
 
 SERVER_DIR = '/opt/ocsf-server'
 SCHEMA_DIR = '/opt/ocsf-schema'
@@ -142,6 +142,21 @@ def start_node(self, node):
         self.logger.debug(f'Starting OCSF Server with cmd "{cmd}"')
         node.account.ssh(cmd, allow_fail=False)
 
+        def _wait_for_version():
+            try:
+                _ = self.get_api_version()
+                return True
+            except Exception:
+                # Ignore exceptions as server may be coming up and
+                # connections may time out
+                pass
+            return False
+
+        wait_until(_wait_for_version,
+                   timeout_sec=10,
+                   backoff_sec=1,
+                   err_msg='Failed to get version from server during startup')
+
     def stop_node(self, node):
         cmd = self._stop_cmd()
         self.logger.debug(f'Stopping OCSF server with cmd "{cmd}"')