Add corelight/zeek-community-id package (#43)

The corelight/zeek-community-id package includes a Zeek plugin. Building and loading plugins doesn't work on Windows, so this change consists almost entirely of jiggery-pokery to address that, including a modified cmake submodule at https://github.com/brimsec/cmake.
brimdata · Oct 7, 2020 · 0ef99ec · 0ef99ec
1 parent 1795f70
commit 0ef99ec
Show file tree

Hide file tree

Showing 19 changed files with 243 additions and 157 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -12,7 +12,7 @@
 	url = https://github.com/zeek/btest
 [submodule "cmake"]
 	path = cmake
-	url = https://github.com/zeek/cmake
+	url = https://github.com/brimsec/cmake
 [submodule "src/3rdparty"]
 	path = src/3rdparty
 	url = https://github.com/zeek/zeek-3rdparty

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -78,20 +78,35 @@ get_filename_component(ZEEK_SCRIPT_INSTALL_PATH ${ZEEK_SCRIPT_INSTALL_PATH}
 
 set(BRO_PLUGIN_INSTALL_PATH ${ZEEK_ROOT_DIR}/lib/zeek/plugins CACHE STRING "Installation path for plugins" FORCE)
 
+set(bro_plugin_install_path "${BRO_PLUGIN_INSTALL_PATH}")
+set(cmake_binary_dir "${CMAKE_BINARY_DIR}")
+set(cmake_current_binary_dir "${CMAKE_CURRENT_BINARY_DIR}")
+set(cmake_install_prefix "${CMAKE_INSTALL_PREFIX}")
+set(cmake_source_dir "${CMAKE_SOURCE_DIR}")
+set(zeek_script_install_path "${ZEEK_SCRIPT_INSTALL_PATH}")
+if ( WIN32 )
+    string(REGEX REPLACE "^([A-Za-z]):/(.*)" "/\\1/\\2" bro_plugin_install_path "${bro_plugin_install_path}")
+    string(REGEX REPLACE "^([A-Za-z]):/(.*)" "/\\1/\\2" cmake_binary_dir "${cmake_binary_dir}")
+    string(REGEX REPLACE "^([A-Za-z]):/(.*)" "/\\1/\\2" cmake_current_binary_dir "${cmake_current_binary_dir}")
+    string(REGEX REPLACE "^([A-Za-z]):/(.*)" "/\\1/\\2" cmake_install_prefix "${cmake_install_prefix}")
+    string(REGEX REPLACE "^([A-Za-z]):/(.*)" "/\\1/\\2" cmake_source_dir "${cmake_source_dir}")
+    string(REGEX REPLACE "^([A-Za-z]):/(.*)" "/\\1/\\2" zeek_script_install_path "${zeek_script_install_path}")
+endif ()
+
 configure_file(zeek-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev)
 execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink
                 "${CMAKE_CURRENT_BINARY_DIR}/zeek-wrapper.in"
                 "${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev")
 
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.sh
-     "export ZEEKPATH=`${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev`\n"
-     "export ZEEK_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":${ZEEK_PLUGIN_PATH}\n"
-     "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
+     "export ZEEKPATH=`${cmake_current_binary_dir}/zeek-path-dev`\n"
+     "export ZEEK_PLUGIN_PATH=\"${cmake_current_binary_dir}/src\":${ZEEK_PLUGIN_PATH}\n"
+     "export PATH=\"${cmake_current_binary_dir}/src\":$PATH\n")
 
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.csh
-     "setenv ZEEKPATH `${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev`\n"
-     "setenv ZEEK_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":${ZEEK_PLUGIN_PATH}\n"
-     "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
+     "setenv ZEEKPATH `${cmake_current_binary_dir}/zeek-path-dev`\n"
+     "setenv ZEEK_PLUGIN_PATH \"${cmake_current_binary_dir}/src\":${ZEEK_PLUGIN_PATH}\n"
+     "setenv PATH \"${cmake_current_binary_dir}/src\":$PATH\n")
 
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
 execute_process(COMMAND grep "^#define  *BRO_PLUGIN_API_VERSION"
@@ -247,6 +262,10 @@ if ( CAF_ROOT_DIR )
 endif ()
 
 add_subdirectory(auxil/paraglob)
+if ( WIN32 )
+    cmake_policy(SET CMP0079 NEW)
+    target_link_libraries(paraglob shlwapi)
+endif ()
 set(zeekdeps ${zeekdeps} paraglob)
 
 if ( BROKER_ROOT_DIR )
@@ -396,7 +415,12 @@ endif()
 # Tell the plugin code that we're building as part of the main tree.
 set(ZEEK_PLUGIN_INTERNAL_BUILD true CACHE INTERNAL "" FORCE)
 
-set(DEFAULT_ZEEKPATH .:${ZEEK_SCRIPT_INSTALL_PATH}:${ZEEK_SCRIPT_INSTALL_PATH}/policy:${ZEEK_SCRIPT_INSTALL_PATH}/site)
+set(default_zeekpath ".:${zeek_script_install_path}:${zeek_script_install_path}/policy:${zeek_script_install_path}/site")
+if ( WIN32 )
+    set(DEFAULT_ZEEKPATH ".;${ZEEK_SCRIPT_INSTALL_PATH};${ZEEK_SCRIPT_INSTALL_PATH}/policy;${ZEEK_SCRIPT_INSTALL_PATH}/site")
+else ()
+    set(DEFAULT_ZEEKPATH "${default_zeekpath}")
+endif ()
 
 if ( NOT BINARY_PACKAGING_MODE )
     set(ZEEK_DIST ${CMAKE_SOURCE_DIR})

diff --git a/brim/release b/brim/release
@@ -31,24 +31,37 @@ case $(uname) in
         ;;
     *_NT-*)
         exe=.exe
+        export CMAKE_GENERATOR='MSYS Makefiles'  # For Zeek plugins.
         export MSYS=winsymlinks:nativestrict
         export PATH=/mingw64/bin:$PATH
         go build -o brim/zeekrunner.exe brim/zeekrunner.go
         pacman -S --needed --noconfirm \
-            bison flex mingw-w64-x86_64-ccache \
+            bison flex make mingw-w64-x86_64-ccache \
             mingw-w64-x86_64-cmake mingw-w64-x86_64-gcc \
             mingw-w64-x86_64-libmaxminddb mingw-w64-x86_64-ninja \
             mingw-w64-x86_64-openssl python-pip zip
         install_libpcap /mingw64
-        # Replace the symlink at x509-signed_certificate_timestamp.pac
-        # with a copy of the target file because CMake chokes with "file
-        # INSTALL cannot read symlink" during installation.
-        f=src/file_analysis/analyzer/x509/x509-signed_certificate_timestamp.pac
-        cp $f $f.tmp
-        rm $f
-        mv $f.tmp $f
+        # Replace some symlinks with a copy of the target file because
+        # CMake chokes on them with "file INSTALL cannot read symlink"
+        # during installation.
+        for f in cmake/BroPlugin.cmake src/file_analysis/analyzer/x509/x509-signed_certificate_timestamp.pac; do
+            cp $f $f.tmp
+            rm $f
+            mv $f.tmp $f
+        done
         # Need this until Broker updates to CAF 0.18.0.
         sed -i 's/NOT WIN32/1/' auxil/broker/caf/libcaf_openssl/CMakeLists.txt
+        # Put an ln that can create a symbolic link to "." in the search
+        # path ahead of /usr/bin/ln (which cannot do that) for use by
+        # zeek-plugin-install-package.sh.
+    cat > /mingw64/bin/ln <<'EOF'
+#!/bin/sh
+if [ "$1" = -s ]; then
+    shift
+    exec cmake -E create_symlink "$@"
+fi
+exec /usr/bin/ln "$@"
+EOF
         ;;
     *)
         echo "unknown OS: $(uname)" >&2
@@ -66,40 +79,81 @@ esac
     --enable-static-broker --generator=Ninja --osx-min-version=10.14
 
 ccache -z
-ninja -C build zeek
+ninja -C build
 ccache -s
 
-$sudo ninja -C build scripts/install/strip src/install/strip
+$sudo ninja -C build install/strip
 
 #
-# Install Zeek packages.
+# Install Zeek packages.  We don't use zkg because it doesn't work well
+# on Windows due to symlink issues and because it copies entire plugin
+# build directories to /usr/local/zeek/lib/zeek/plugins/packages/.
 #
 
-$sudo pip install zkg
-PATH=$PWD/build:$PATH zkg autoconfig
-echo '@load packages' | $sudo tee -a /usr/local/zeek/share/zeek/site/local.zeek
-if [ "$OS" = Windows_NT ]; then
-    # Do this again to initialize /usr/local/zeek/share/zeek/site/packages.
-    PATH=$PWD/build:$PATH zkg autoconfig
-    mkdir -p /usr/local/zeek/share/zeek/site/packages/hassh
-    mkdir -p /usr/local/zeek/share/zeek/site/packages/ja3
-    mkdir -p /usr/local/zeek/share/zeek/site/packages/geoip-conn
-fi
-$sudo zkg install --force hassh --version cfa2315257eaa972e86f7fcd694712e0d32762ff
-$sudo zkg install --force ja3 --version 133f2a128b873f9c40e4e65c2b9dc372a801cf24
-$sudo zkg install --force geoip-conn --version f8c66962d15a09052ab64a6413c1eadcb23e4ca2
+zkg_meta() {
+    section=${1:?'section required'}
+    option=${2:?'option required'}
+    python3 <<EOF
+import configparser
+c = configparser.ConfigParser()
+c.read('zkg.meta')
+print(c.get('$section', '$option', fallback=''))
+EOF
+}
+
+install_zeek_package() {
+    github_repo=${1:?'github_repo required'}
+    git_ref=${2:?'git_ref required'}
+    package=${github_repo#*/}
+    mkdir $package
+    (
+        export PATH=/usr/local/zeek/bin:$PATH
+        cd $package
+        wget -qO - https://github.com/$github_repo/tarball/$git_ref |
+            tar -xzf - --strip-components 1
+
+        script_dir=$(zkg_meta package script_dir)
+        $sudo cp -r "$script_dir" /usr/local/zeek/share/zeek/site/$package/
+
+        build_command=$(zkg_meta package build_command)
+        if [ "$build_command" ]; then
+            sh -c "$build_command"
+            $sudo tar -xf build/*.tgz -C /usr/local/zeek/lib/zeek/plugins
+        fi
+
+        test_command=$(zkg_meta package test_command)
+        if [ "$test_command" ]; then
+            # Btest fails without explanation on the GitHub Actions
+            # Windows runners, so skip tests there.
+            if [ "$GITHUB_ACTIONS" != true -o "$OS" != Windows_NT || ]; then
+               sh -c "$test_command"
+            fi
+        fi
+
+        echo "@load $package" | $sudo tee -a /usr/local/zeek/share/zeek/site/local.zeek
+    )
+    rm -r $package
+}
+
+$sudo pip install btest
+
+install_zeek_package brimsec/geoip-conn f8c66962d15a09052ab64a6413c1eadcb23e4ca2
+install_zeek_package corelight/zeek-community-id 181a104b99d9019771ece7e489e46f2268b746d8
+install_zeek_package salesforce/hassh cfa2315257eaa972e86f7fcd694712e0d32762ff
+install_zeek_package salesforce/ja3 133f2a128b873f9c40e4e65c2b9dc372a801cf24
 
 #
 # Create zip file.
 #
 
-mkdir -p zeek/bin zeek/share/zeek
+mkdir -p zeek/bin zeek/lib/zeek zeek/share/zeek
 cp brim/zeekrunner$exe zeek/
 cp /usr/local/zeek/bin/zeek$exe zeek/bin/
+cp -R /usr/local/zeek/lib/zeek/plugins zeek/lib/zeek/
 for d in base policy site; do
     cp -R /usr/local/zeek/share/zeek/$d zeek/share/zeek/
 done
 
-# Can't use --diry with "git describe" on Windows because of the
-# x509-signed_certificate_timestamp.pac shenanigans.
+# Can't use --diry with "git describe" on Windows because of the symlink
+# shenanigans.
 zip -r zeek-$(git describe --always --tags).$(go env GOOS)-$(go env GOARCH).zip zeek
diff --git a/cmake b/cmake
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -402,7 +402,14 @@ if ( NOT "${bro_LINKER_FLAGS}" STREQUAL "" )
     set_target_properties(zeek PROPERTIES LINK_FLAGS "${bro_LINKER_FLAGS}")
 endif ()
 
-install(TARGETS zeek DESTINATION bin)
+if ( WIN32 )
+    set_target_properties(zeek PROPERTIES
+                          ENABLE_EXPORTS 1
+                          LINK_FLAGS "${bro_LINKER_FLAGS} -Wl,--export-all-symbols -Wl,--out-implib,src/libzeek.dll.a"
+)
+endif ()
+
+install(TARGETS zeek RUNTIME DESTINATION bin ARCHIVE DESTINATION lib)
 
 if ( NOT WIN32 )
     # Install wrapper script for Bro-to-Zeek renaming.

diff --git a/src/main.cc b/src/main.cc
@@ -7,8 +7,17 @@
 #include "supervisor/Supervisor.h"
 #include "Net.h"
 
+#ifdef __MINGW32__
+#include <fcntl.h> // For _O_BINARY.
+#endif
+
 int main(int argc, char** argv)
 	{
+#ifdef __MINGW32__
+	_setmode(_fileno(stdout), _O_BINARY);
+	_setmode(_fileno(stderr), _O_BINARY);
+#endif
+
 	auto time_start = current_time(true);
 	auto setup_result = zeek::detail::setup(argc, argv);
 

diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc
@@ -4,8 +4,9 @@
 #include <sstream>
 #include <fstream>
 #include <dirent.h>
-#include <glob.h>
+#ifndef __MINGW32__
 #include <dlfcn.h>
+#endif
 #include <errno.h>
 #include <sys/stat.h>
 
@@ -51,13 +52,13 @@ void Manager::SearchDynamicPlugins(const std::string& dir)
 	if ( dir.empty() )
 		return;
 
-	if ( dir.find(':') != string::npos )
+	if ( dir.find(path_list_separator) != string::npos )
 		{
-		// Split at ":".
+		// Split at path list separator.
 		std::stringstream s(dir);
 		std::string d;
 
-		while ( std::getline(s, d, ':') )
+		while ( std::getline(s, d, path_list_separator) )
 			SearchDynamicPlugins(d);
 
 		return;
@@ -228,38 +229,60 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_
 
 	// Load shared libraries.
 
-	string dypattern = dir + "/lib/*." + HOST_ARCHITECTURE + DYNAMIC_PLUGIN_SUFFIX;
+	string dydir = dir + "/lib";
+	const char *dyext = "." HOST_ARCHITECTURE DYNAMIC_PLUGIN_SUFFIX;
 
-	DBG_LOG(DBG_PLUGINS, "  Searching for shared libraries %s", dypattern.c_str());
+	DBG_LOG(DBG_PLUGINS, "  Searching for shared libraries in %s with extension %s", dydir.c_str(), dyext);
 
-	glob_t gl;
+	DIR* d = opendir(dydir.c_str());
 
-	if ( glob(dypattern.c_str(), 0, 0, &gl) == 0 )
+	if ( ! d )
+		{
+		DBG_LOG(DBG_PLUGINS, "Cannot open directory %s", dydir.c_str());
+		return true;
+		}
+
+	struct dirent *dp;
+
+	while ( (dp = readdir(d)) )
 		{
-		for ( size_t i = 0; i < gl.gl_pathc; i++ )
+		if ( strlen(dp->d_name) >= strlen(dyext)
+		    && streq(dp->d_name + strlen(dp->d_name) - strlen(dyext), dyext) )
 			{
-			const char* path = gl.gl_pathv[i];
+			string path = dydir + "/" + dp->d_name;
 
 			current_plugin = nullptr;
-			current_dir = dir.c_str();
-			current_sopath = path;
-			void* hdl = dlopen(path, RTLD_LAZY | RTLD_GLOBAL);
-
+			current_dir = dydir.c_str();
+			current_sopath = path.c_str();
+
+#ifdef __MINGW32__
+			void* hdl = LoadLibraryA(path.c_str());
+#else
+			void* hdl = dlopen(path.c_str(), RTLD_LAZY | RTLD_GLOBAL);
+#endif
 			if ( ! hdl )
 				{
-				const char* err = dlerror();
-				reporter->FatalError("cannot load plugin library %s: %s", path, err ? err : "<unknown error>");
+				const char* err = nullptr;
+#ifdef __MINGW32__
+				char buf[65535];
+				const int flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS;
+				if ( FormatMessageA(flags, nullptr, GetLastError(), 0, buf, sizeof(buf), nullptr ) )
+					err = buf;
+#else
+				err = dlerror();
+#endif
+				reporter->FatalError("cannot load plugin library %s: %s", path.c_str(), err ? err : "<unknown error>");
 				}
 
 			if ( ! current_plugin )
-				reporter->FatalError("load plugin library %s did not instantiate a plugin", path);
+				reporter->FatalError("load plugin library %s did not instantiate a plugin", path.c_str());
 
 			current_plugin->SetDynamic(true);
 			current_plugin->DoConfigure();
 			DBG_LOG(DBG_PLUGINS, "  InitialzingComponents");
 			current_plugin->InitializeComponents();
 
-			plugins_by_path.insert(std::make_pair(normalize_path(dir), current_plugin));
+			plugins_by_path.insert(std::make_pair(normalize_path(dydir), current_plugin));
 
 			// We execute the pre-script initialization here; this in
 			// fact could be *during* script initialization if we got
@@ -272,21 +295,21 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_
 				reporter->FatalError("inconsistent plugin name: %s vs %s",
 						     current_plugin->Name().c_str(), name.c_str());
 
-			current_dir = nullptr;
-			current_sopath = nullptr;
-			current_plugin = nullptr;
-
-			DBG_LOG(DBG_PLUGINS, "  Loaded %s", path);
+			DBG_LOG(DBG_PLUGINS, "  Loaded %s", path.c_str());
 			}
-
-		globfree(&gl);
 		}
 
-	else
+	closedir(d);
+
+	if ( current_plugin == nullptr )
 		{
 		DBG_LOG(DBG_PLUGINS, "  No shared library found");
 		}
 
+	current_dir = nullptr;
+	current_sopath = nullptr;
+	current_plugin = nullptr;
+
 	// Mark this plugin as activated by clearing the path.
 	m->second.clear();