Reworking of SET parser

.gitignore

@@ -116,6 +116,8 @@ test/tap/tap/cpp-dotenv/cpp-dotenv-*
 test/tap/tests/galera_1_timeout_count
 test/tap/tests/galera_2_timeout_no_count
 test/tap/tests/setparser_test
+test/tap/tests/setparser_test2
+test/tap/tests/setparser_test3
 test/tap/tests/reg_test_3504-change_user_libmariadb_helper
 test/tap/tests/reg_test_3504-change_user_libmysql_helper
 test/tap/tests/generate_set_session_csv

Makefile

@@ -43,7 +43,7 @@ DEBUG=${ALL_DEBUG}
 export MAKE
 export CURVER?=2.6.0
 ifneq (,$(wildcard /etc/os-release))
-	DISTRO := $(shell gawk -F= '/^NAME/{print $$2}' /etc/os-release)
+	DISTRO := $(shell awk -F= '/^NAME/{print $$2}' /etc/os-release)
 else
 	DISTRO := Unknown
 endif
@@ -69,38 +69,38 @@ GROUPCHECK := $(shell getent group proxysql)
 
 
 .PHONY: default
-default: build_deps build_lib build_src
+default: build_src
 
 .PHONY: debug
-debug: build_deps_debug build_lib_debug build_src_debug
+debug: build_src_debug
 
 .PHONY: testaurora
-testaurora: build_deps_debug build_lib_testaurora build_src_testaurora
+testaurora: build_src_testaurora
 	cd test/tap && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA" CC=${CC} CXX=${CXX} ${MAKE}
 	cd test/tap/tests && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA" CC=${CC} CXX=${CXX} ${MAKE} $(MAKECMDGOALS)
 
 .PHONY: testgalera
-testgalera: build_deps_debug build_lib_testgalera build_src_testgalera
+testgalera: build_src_testgalera
 	cd test/tap && OPTZ="${O0} -ggdb -DDEBUG -DTEST_GALERA" CC=${CC} CXX=${CXX} ${MAKE}
 	cd test/tap/tests && OPTZ="${O0} -ggdb -DDEBUG -DTEST_GALERA" CC=${CC} CXX=${CXX} ${MAKE} $(MAKECMDGOALS)
 
 .PHONY: testgrouprep
-testgrouprep: build_deps_debug build_lib_testgrouprep build_src_testgrouprep
+testgrouprep: build_src_testgrouprep
 
 .PHONY: testreadonly
-testreadonly: build_deps_debug build_lib_testreadonly build_src_testreadonly
+testreadonly: build_src_testreadonly
 
 .PHONY: testreplicationlag
-testreplicationlag: build_deps_debug build_lib_testreplicationlag build_src_testreplicationlag
+testreplicationlag: build_src_testreplicationlag
 
 .PHONY: testall
-testall: build_deps_debug build_lib_testall build_src_testall
+testall: build_src_testall
 
 .PHONY: clickhouse
-clickhouse: build_deps_clickhouse build_lib_clickhouse build_src_clickhouse
+clickhouse: build_src_clickhouse
 
 .PHONY: debug_clickhouse
-debug_clickhouse: build_deps_debug_clickhouse build_lib_debug_clickhouse build_src_debug_clickhouse
+debug_clickhouse: build_src_debug_clickhouse
 
 
 
@@ -113,7 +113,7 @@ build_lib: build_deps
 	cd lib && OPTZ="${O2} -ggdb" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_src
-build_src: build_deps build_lib
+build_src: build_lib
 	cd src && OPTZ="${O2} -ggdb" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_deps_debug
@@ -125,47 +125,47 @@ build_lib_debug: build_deps_debug
 	cd lib && OPTZ="${O0} -ggdb -DDEBUG" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_src_testaurora
-build_src_testaurora: build_deps build_lib_testaurora
+build_src_testaurora: build_lib_testaurora
 	cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_lib_testaurora
 build_lib_testaurora: build_deps_debug
 	cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_src_testgalera
-build_src_testgalera: build_deps build_lib_testgalera
+build_src_testgalera: build_lib_testgalera
 	cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_GALERA" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_lib_testgalera
 build_lib_testgalera: build_deps_debug
 	cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_GALERA" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_src_testgrouprep
-build_src_testgrouprep: build_deps build_lib_testgrouprep
+build_src_testgrouprep: build_lib_testgrouprep
 	cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_GROUPREP" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_lib_testgrouprep
 build_lib_testgrouprep: build_deps_debug
 	cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_GROUPREP" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_src_testreadonly
-build_src_testreadonly: build_deps build_lib_testreadonly
+build_src_testreadonly: build_lib_testreadonly
 	cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_READONLY" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_lib_testreadonly
 build_lib_testreadonly: build_deps_debug
 	cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_READONLY" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_src_testreplicationlag
-build_src_testreplicationlag: build_deps build_lib_testreplicationlag
+build_src_testreplicationlag: build_lib_testreplicationlag
 	cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_REPLICATIONLAG" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_lib_testreplicationlag
 build_lib_testreplicationlag: build_deps_debug
 	cd lib && OPTZ="${O0} -ggdb -DDEBUG -DTEST_REPLICATIONLAG" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_src_testall
-build_src_testall: build_deps build_lib_testall
+build_src_testall: build_lib_testall
 	cd src && OPTZ="${O0} -ggdb -DDEBUG -DTEST_AURORA -DTEST_GALERA -DTEST_GROUPREP -DTEST_READONLY -DTEST_REPLICATIONLAG" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_lib_testall
@@ -181,7 +181,7 @@ build_tap_test_debug: build_src_debug
 	cd test/tap && OPTZ="${O0} -ggdb -DDEBUG" CC=${CC} CXX=${CXX} ${MAKE} debug
 
 .PHONY: build_src_debug
-build_src_debug: build_deps build_lib_debug
+build_src_debug: build_lib_debug
 	cd src && OPTZ="${O0} -ggdb -DDEBUG" CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_deps_clickhouse
@@ -201,11 +201,11 @@ build_lib_debug_clickhouse: build_deps_debug_clickhouse
 	cd lib && OPTZ="${O0} -ggdb -DDEBUG" PROXYSQLCLICKHOUSE=1 CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_src_clickhouse
-build_src_clickhouse: build_deps_clickhouse build_lib_clickhouse
+build_src_clickhouse: build_lib_clickhouse
 	cd src && OPTZ="${O2} -ggdb" PROXYSQLCLICKHOUSE=1 CC=${CC} CXX=${CXX} ${MAKE}
 
 .PHONY: build_src_debug_clickhouse
-build_src_debug_clickhouse: build_deps build_lib_debug_clickhouse
+build_src_debug_clickhouse: build_lib_debug_clickhouse
 	cd src && OPTZ="${O0} -ggdb -DDEBUG" PROXYSQLCLICKHOUSE=1 CC=${CC} CXX=${CXX} ${MAKE}
 
 

deps/Makefile

@@ -85,14 +85,10 @@ endif
 libinjection: libinjection/libinjection/src/libinjection.a
 
 
-BIO_MATCH := $(shell ./libssl/verify-bio_st-match.sh \&>/dev/null; echo $$?)
-ifneq ($(BIO_MATCH),0)
-$(error Incompatible OpenSSL version: struct bio_st mismatch!)
-endif
-
 libssl/openssl/libssl.a:
 	cd libssl && rm -rf openssl-openssl-*/ openssl-3*/ || true
 	cd libssl && tar -zxf openssl-*.tar.gz
+	cd libssl && ./verify-bio_st-match.sh
 #	cd libssl/openssl && patch crypto/ec/curve448/curve448.c < ../curve448.c-multiplication-overflow.patch
 #	cd libssl/openssl && patch crypto/asn1/a_time.c < ../a_time.c-multiplication-overflow.patch
 	cd libssl/openssl && ./config no-ssl3 no-tests

include/MySQL_Thread.h

@@ -14,6 +14,8 @@
 
 #include "prometheus_helpers.h"
 
+#include "set_parser.h"
+
 #define MIN_POLL_LEN 8
 #define MIN_POLL_DELETE_RATIO  8
 #define MY_EPOLL_THREAD_MAXEVENTS 128
@@ -214,11 +216,15 @@ class __attribute__((aligned(64))) MySQL_Thread
 		bool query_cache_stores_empty_result;
 	} variables;
 
-  pthread_mutex_t thread_mutex;
-  MySQL_Thread();
-  ~MySQL_Thread();
-  MySQL_Session * create_new_session_and_client_data_stream(int _fd);
-  bool init();
+	pthread_mutex_t thread_mutex;
+
+	// if set_parser_algorithm == 2 , a single thr_SetParser is used
+	SetParser *thr_SetParser;
+
+	MySQL_Thread();
+	~MySQL_Thread();
+	MySQL_Session * create_new_session_and_client_data_stream(int _fd);
+	bool init();
 	void run___get_multiple_idle_connections(int& num_idles);
 	void run___cleanup_mirror_queue();
   	void ProcessAllMyDS_BeforePoll();
@@ -538,6 +544,7 @@ class MySQL_Threads_Handler
 		int query_processor_iterations;
 		int query_processor_regex;
 		int set_query_lock_on_hostgroup;
+		int set_parser_algorithm;
 		int reset_connection_algorithm;
 		int auto_increment_delay_multiplex;
 		int auto_increment_delay_multiplex_timeout_ms;

include/proxysql_structs.h

@@ -815,6 +815,7 @@ __thread int mysql_thread___connect_timeout_server_max;
 __thread int mysql_thread___query_processor_iterations;
 __thread int mysql_thread___query_processor_regex;
 __thread int mysql_thread___set_query_lock_on_hostgroup;
+__thread int mysql_thread___set_parser_algorithm;
 __thread int mysql_thread___reset_connection_algorithm;
 __thread uint32_t mysql_thread___server_capabilities;
 __thread int mysql_thread___auto_increment_delay_multiplex;
@@ -981,6 +982,7 @@ extern __thread int mysql_thread___connect_timeout_server_max;
 extern __thread int mysql_thread___query_processor_iterations;
 extern __thread int mysql_thread___query_processor_regex;
 extern __thread int mysql_thread___set_query_lock_on_hostgroup;
+extern __thread int mysql_thread___set_parser_algorithm;
 extern __thread int mysql_thread___reset_connection_algorithm;
 extern __thread uint32_t mysql_thread___server_capabilities;
 extern __thread int mysql_thread___auto_increment_delay_multiplex;

include/set_parser.h

@@ -4,14 +4,44 @@
 #include <map>
 #include <vector>
 
+#include "re2/re2.h"
+#include "re2/regexp.h"
+
+
 class SetParser {
 	private:
+	// parse1v2 variables used for compile the RE only once
+	bool parse1v2_init;
+	re2::RE2::Options * parse1v2_opt2;
+	re2::RE2 * parse1v2_re;
+	std::string parse1v2_pattern;
 	std::string query;
+#ifdef PARSERDEBUG
+	int verbosity;
+	public:
+	SetParser(std::string q, int verb = 0);
+#else
 	public:
 	SetParser(std::string q);
+#endif
+	// set_query() allows to change the query associated to a SetParser.
+	// This allow to parse multiple queries using just a single SetParser.
+	// At the moment this makes sense only when using parse1v2() because it
+	// allows to compile the regular expression only once
+	void set_query(const std::string& q);
+	// First implementation of the general parser
+	// It uses a single complex RE pattern that is hardcoded
 	std::map<std::string, std::vector<std::string>> parse1();
+	// Second implementation of the general parser .
+	// It uses a RE pattern that is built at runtime .
+	// The final pattern used by parse1v2() is a lot longer than the one used by parse1()
+	// making it very difficult to read, but the code generating it should be clear
+	std::map<std::string, std::vector<std::string>> parse1v2();
+	void generateRE_parse1v2();
+	// First implemenation of the parser for TRANSACTION ISOLATION LEVEL and TRANSACTION READ/WRITE
 	std::map<std::string, std::vector<std::string>> parse2();
 	std::string parse_character_set();
+	~SetParser();
 };
 
 

lib/MySQL_Session.cpp

@@ -6,7 +6,6 @@
 #include "re2/re2.h"
 #include "re2/regexp.h"
 #include "mysqld_error.h"
-#include "set_parser.h"
 
 #include "MySQL_Data_Stream.h"
 #include "query_processor.h"
@@ -6022,7 +6021,15 @@ bool MySQL_Session::handler___status_WAITING_CLIENT_DATA___STATE_SLEEP___MYSQL_C
 				proxy_debug(PROXY_DEBUG_MYSQL_COM, 5, "Parsing SET command %s\n", nq.c_str());
 				proxy_debug(PROXY_DEBUG_MYSQL_QUERY_PROCESSOR, 5, "Parsing SET command = %s\n", nq.c_str());
 				SetParser parser(nq);
-				std::map<std::string, std::vector<std::string>> set = parser.parse1();
+				std::map<std::string, std::vector<std::string>> set = {};
+				if (mysql_thread___set_parser_algorithm == 1) { // legacy behavior
+					set = parser.parse1();
+				} else if (mysql_thread___set_parser_algorithm == 2) { // we use a single SetParser per thread
+					thread->thr_SetParser->set_query(nq); // replace the query
+					set = thread->thr_SetParser->parse1v2(); // use algorithm v2
+				} else {
+					assert(0);
+				}
 				// Flag to be set if any variable within the 'SET' statement fails to be tracked,
 				// due to being unknown or because it's an user defined variable.
 				bool failed_to_parse_var = false;

lib/MySQL_Thread.cpp

@@ -525,6 +525,7 @@ static char * mysql_thread_variables_names[]= {
 	(char *)"query_processor_iterations",
 	(char *)"query_processor_regex",
 	(char *)"set_query_lock_on_hostgroup",
+	(char *)"set_parser_algorithm",
 	(char *)"reset_connection_algorithm",
 	(char *)"auto_increment_delay_multiplex",
 	(char *)"auto_increment_delay_multiplex_timeout_ms",
@@ -1119,6 +1120,7 @@ MySQL_Threads_Handler::MySQL_Threads_Handler() {
 	variables.query_processor_iterations=0;
 	variables.query_processor_regex=1;
 	variables.set_query_lock_on_hostgroup=1;
+	variables.set_parser_algorithm=1; // in 2.6.0 this must become 2
 	variables.reset_connection_algorithm=2;
 	variables.auto_increment_delay_multiplex=5;
 	variables.auto_increment_delay_multiplex_timeout_ms=10000;
@@ -2213,6 +2215,7 @@ char ** MySQL_Threads_Handler::get_variables_list() {
 		VariablesPointers_int["query_processor_regex"]           = make_tuple(&variables.query_processor_regex,            1,           2, false);
 		VariablesPointers_int["query_retries_on_failure"]        = make_tuple(&variables.query_retries_on_failure,         0,        1000, false);
 		VariablesPointers_int["set_query_lock_on_hostgroup"]     = make_tuple(&variables.set_query_lock_on_hostgroup,      0,           1, false);
+		VariablesPointers_int["set_parser_algorithm"]            = make_tuple(&variables.set_parser_algorithm,             1,           2, false);
 
 		// throttle
 		VariablesPointers_int["throttle_connections_per_sec_to_hostgroup"] = make_tuple(&variables.throttle_connections_per_sec_to_hostgroup, 1, 100*1000*1000, false);
@@ -2835,6 +2838,10 @@ MySQL_Thread::~MySQL_Thread() {
 		free(match_regexes);
 		match_regexes=NULL;
 	}
+	if (thr_SetParser != NULL) {
+		delete thr_SetParser;
+		thr_SetParser = NULL;
+	}
 
 }
 
@@ -2941,6 +2948,7 @@ bool MySQL_Thread::init() {
 	mypolls.add(POLLIN, pipefd[0], NULL, 0);
 	assert(i==0);
 
+	thr_SetParser = new SetParser("");
 	match_regexes=(Session_Regex **)malloc(sizeof(Session_Regex *)*4);
 //	match_regexes[0]=new Session_Regex((char *)"^SET (|SESSION |@@|@@session.)SQL_LOG_BIN( *)(:|)=( *)");
 	match_regexes[0] = NULL; // NOTE: historically we used match_regexes[0] for SET SQL_LOG_BIN . Not anymore
@@ -3995,6 +4003,7 @@ void MySQL_Thread::refresh_variables() {
 	mysql_thread___query_processor_iterations=GloMTH->get_variable_int((char *)"query_processor_iterations");
 	mysql_thread___query_processor_regex=GloMTH->get_variable_int((char *)"query_processor_regex");
 	mysql_thread___set_query_lock_on_hostgroup=GloMTH->get_variable_int((char *)"set_query_lock_on_hostgroup");
+	mysql_thread___set_parser_algorithm=GloMTH->get_variable_int((char *)"set_parser_algorithm");
 	mysql_thread___reset_connection_algorithm=GloMTH->get_variable_int((char *)"reset_connection_algorithm");
 	mysql_thread___auto_increment_delay_multiplex=GloMTH->get_variable_int((char *)"auto_increment_delay_multiplex");
 	mysql_thread___auto_increment_delay_multiplex_timeout_ms=GloMTH->get_variable_int((char *)"auto_increment_delay_multiplex_timeout_ms");
@@ -4230,6 +4239,7 @@ MySQL_Thread::MySQL_Thread() {
 		mysql_thread___default_variables[i] = NULL;
 	}
 	shutdown=0;
+	thr_SetParser = NULL;
 }
 
 void MySQL_Thread::register_session_connection_handler(MySQL_Session *_sess, bool _new) {