From 13c364c029b4c73efcf1ee75055af8d2751d71ca Mon Sep 17 00:00:00 2001 From: Mustafa Baser Date: Fri, 16 Sep 2022 16:03:23 +0300 Subject: [PATCH 1/2] feat: jans-linux-setup postgresql support --- .../jans_setup/setup_app/installers/rdbm.py | 18 +- .../jans_setup/setup_app/static.py | 12 + .../setup_app/utils/collect_properties.py | 2 + .../jans_setup/setup_app/utils/db_utils.py | 41 ++-- .../setup_app/utils/properties_utils.py | 77 +++--- .../jans_setup/setup_app/utils/setup_utils.py | 39 +-- .../rdbm/ldap_sql_data_type_mapping.json | 6 + .../jans_setup/static/rdbm/pgsql_index.json | 229 ++++++++++++++++++ .../static/rdbm/sql_data_types.json | 21 ++ ...s-sql.properties => jans-mysql.properties} | 4 +- .../templates/jans-pgsql.properties | 30 +++ 11 files changed, 406 insertions(+), 73 deletions(-) rename jans-linux-setup/jans_setup/templates/{jans-sql.properties => jans-mysql.properties} (90%) create mode 100644 jans-linux-setup/jans_setup/templates/jans-pgsql.properties diff --git a/jans-linux-setup/jans_setup/setup_app/installers/rdbm.py b/jans-linux-setup/jans_setup/setup_app/installers/rdbm.py index 10e2fcdc13b..7acdcd2d8ff 100644 --- a/jans-linux-setup/jans_setup/setup_app/installers/rdbm.py +++ b/jans-linux-setup/jans_setup/setup_app/installers/rdbm.py @@ -26,11 +26,10 @@ def __init__(self): self.install_type = InstallOption.OPTONAL self.install_var = 'rdbm_install' self.register_progess() - self.qchar = '`' if Config.rdbm_type in ('mysql', 'spanner') else '"' self.output_dir = os.path.join(Config.output_dir, Config.rdbm_type) def install(self): - + self.qchar = '`' if Config.rdbm_type in ('mysql', 'spanner') else '"' self.local_install() jans_schema_files = [] self.jans_attributes = [] @@ -179,7 +178,7 @@ def create_tables(self, jans_schema_files): doc_id_type = self.get_sql_col_type('doc_id', sql_tbl_name) if Config.rdbm_type == 'pgsql': sql_cmd = 'CREATE TABLE "{}" (doc_id {} NOT NULL UNIQUE, "objectClass" VARCHAR(48), dn VARCHAR(128), {}, PRIMARY KEY (doc_id));'.format(sql_tbl_name, doc_id_type, ', '.join(sql_tbl_cols)) - if Config.rdbm_type == 'spanner': + elif Config.rdbm_type == 'spanner': sql_cmd = 'CREATE TABLE `{}` (`doc_id` {} NOT NULL, `objectClass` STRING(48), dn STRING(128), {}) PRIMARY KEY (`doc_id`)'.format(sql_tbl_name, doc_id_type, ', '.join(sql_tbl_cols)) else: sql_cmd = 'CREATE TABLE `{}` (`doc_id` {} NOT NULL UNIQUE, `objectClass` VARCHAR(48), dn VARCHAR(128), {}, PRIMARY KEY (`doc_id`));'.format(sql_tbl_name, doc_id_type, ', '.join(sql_tbl_cols)) @@ -235,7 +234,7 @@ def create_indexes(self): if Config.rdbm_type == 'spanner': tables = self.dbUtils.spanner.get_tables() for tblCls in tables: - tbl_fields = sql_indexes.get(tblCls, {}).get('fields', []) + sql_indexes['__common__']['fields'] + tbl_fields = sql_indexes.get(tblCls, {}).get('fields', []) + sql_indexes['__common__']['fields'] tbl_data = self.dbUtils.spanner.exec_sql('SELECT * FROM {} LIMIT 1'.format(tblCls)) @@ -293,7 +292,7 @@ def create_indexes(self): ) self.dbUtils.exec_rdbm_query(sql_cmd) elif Config.rdbm_type == 'pgsql': - sql_cmd ='CREATE INDEX ON "{}" (({}));'.format( + sql_cmd ='CREATE INDEX ON "{}" {};'.format( tblCls, tmp_str.safe_substitute({'field':attr.name}) ) @@ -326,7 +325,7 @@ def create_indexes(self): ) self.dbUtils.exec_rdbm_query(sql_cmd) elif Config.rdbm_type == 'pgsql': - sql_cmd = 'CREATE INDEX ON "{}" ("{}");'.format( + sql_cmd = 'CREATE INDEX ON "{}" {};'.format( tblCls, custom_index ) @@ -362,9 +361,12 @@ def import_ldif(self): self.dbUtils.import_ldif(ldif_files) def rdbmProperties(self): - if Config.rdbm_type in ('sql', 'mysql'): + if Config.rdbm_type in ('pgsql', 'mysql'): Config.rdbm_password_enc = self.obscure(Config.rdbm_password) - self.renderTemplateInOut(Config.jansRDBMProperties, Config.templateFolder, Config.configFolder) + src_temp_fn = os.path.join(Config.templateFolder, 'jans-{}.properties'.format(Config.rdbm_type)) + targtet_fn = os.path.join(Config.configFolder, Config.jansRDBMProperties) + rendered_tmp = self.render_template(src_temp_fn) + self.writeFile(targtet_fn, rendered_tmp) elif Config.rdbm_type == 'spanner': if Config.spanner_emulator_host: diff --git a/jans-linux-setup/jans_setup/setup_app/static.py b/jans-linux-setup/jans_setup/setup_app/static.py index d5bf9cf3653..b3030b8017a 100644 --- a/jans-linux-setup/jans_setup/setup_app/static.py +++ b/jans-linux-setup/jans_setup/setup_app/static.py @@ -21,6 +21,18 @@ class BackendTypes: PGSQL = 4 SPANNER = 5 +class BackendStrings: + LOCAL_OPENDJ = 'Local OpenDj' + REMOTE_OPENDJ = 'Remote OpenDj' + LOCAL_COUCHBASE = 'Local Couchbase' + REMOTE_COUCHBASE = 'Remote Couchbase' + LOCAL_MYSQL = 'Local MySQL' + REMOTE_MYSQL = 'Remote MySQL' + CLOUD_SPANNER = 'Cloud Spanner' + SAPNNER_EMULATOR = 'Spanner Emulator' + LOCAL_PGSQL = 'Local PgSQL' + REMOTE_PGSQL = 'Remote PgSQL' + class AppType: APPLICATION = 1 SERVICE = 2 diff --git a/jans-linux-setup/jans_setup/setup_app/utils/collect_properties.py b/jans-linux-setup/jans_setup/setup_app/utils/collect_properties.py index 507e119801a..6e43b0cffe3 100644 --- a/jans-linux-setup/jans_setup/setup_app/utils/collect_properties.py +++ b/jans-linux-setup/jans_setup/setup_app/utils/collect_properties.py @@ -79,6 +79,8 @@ def collect(self): Config.rdbm_password_enc = jans_sql_prop['auth.userPassword'] Config.rdbm_password = self.unobscure(Config.rdbm_password_enc) Config.rdbm_db = jans_sql_prop['db.schema.name'] + if Config.rdbm_type == 'postgresql': + Config.rdbm_type = 'pgsql' if not Config.persistence_type in ('couchbase', 'ldap') and Config.get('jansSpannerProperties') and os.path.exists(Config.jansSpannerProperties): Config.rdbm_type = 'spanner' diff --git a/jans-linux-setup/jans_setup/setup_app/utils/db_utils.py b/jans-linux-setup/jans_setup/setup_app/utils/db_utils.py index 90250a4569e..2f658b327ae 100644 --- a/jans-linux-setup/jans_setup/setup_app/utils/db_utils.py +++ b/jans-linux-setup/jans_setup/setup_app/utils/db_utils.py @@ -84,7 +84,7 @@ def bind(self, use_ssl=True, force=False): for group in Config.mapping_locations: if Config.mapping_locations[group] == 'rdbm': if Config.rdbm_type in ('mysql', 'pgsql'): - base.logIt("Making MySql Conncetion") + base.logIt("Making {} Conncetion".format(Config.rdbm_type)) result = self.mysqlconnection() if not result[0]: print("{}FATAL: {}{}".format(colors.FAIL, result[1], colors.ENDC)) @@ -131,7 +131,7 @@ def sqlconnection(self, log=True): @property def json_dialects_instance(self): - return sqlalchemy.dialects.mysql.json.JSON if Config.rdbm_type == 'mysql' else sqlalchemy.dialects.postgresql.json.JSON + return sqlalchemy.dialects.mysql.json.JSON if Config.rdbm_type == 'mysql' else sqlalchemy.dialects.postgresql.json.JSONB def mysqlconnection(self, log=True): return self.sqlconnection(log) @@ -151,7 +151,7 @@ def read_jans_schema(self, others=[]): for attr in attribDataTypes.listAttributes: if not attr in self.sql_data_types: - self.sql_data_types[attr] = { 'mysql': {'type': 'JSON'}, 'spanner': {'type': 'ARRAY'} } + self.sql_data_types[attr] = { 'mysql': {'type': 'JSON'}, 'pgsql': {'type': 'JSONB'}, 'spanner': {'type': 'ARRAY'} } def in_subtable(self, table, attr): if table in self.sub_tables[Config.rdbm_type]: @@ -355,7 +355,7 @@ def dn_exists_rdbm(self, dn, table): return result return sqlalchemy_table = self.Base.classes[table].__table__ - return self.session.query(sqlalchemy_table).filter(sqlalchemy_table).filter(sqlalchemy_table.columns.dn == dn).first() + return self.session.query(sqlalchemy_table).filter(sqlalchemy_table.columns.dn == dn).first() def spanner_to_dict(self, data): @@ -577,14 +577,19 @@ def add_client2script(self, script_inum, client_id): else: jansConfProperty = {'v': []} - for oxconfigprop in jansConfProperty['v']: - if oxconfigprop.get('value1') == 'allowed_clients' and not client_id in oxconfigprop['value2']: + ox_configuration_property_list = jansConfProperty['v'] if Config.rdbm_type == 'mysql' else jansConfProperty + + for i, oxconfigprop in enumerate(ox_configuration_property_list[:]): + if isinstance(oxconfigprop, str): + oxconfigprop = json.loads(oxconfigprop) + if oxconfigprop.get('value1') == 'allowed_clients' and client_id not in oxconfigprop['value2']: oxconfigprop['value2'] = self.add2strlist(client_id, oxconfigprop['value2']) + ox_configuration_property_list[i] = json.dumps(oxconfigprop) break else: - jansConfProperty['v'].append({'value1': 'allowed_clients', 'value2': client_id}) + ox_configuration_property_list.append(json.dumps({'value1': 'allowed_clients', 'value2': client_id})) - sqlalchemyObj.jansConfProperty = jansConfProperty + sqlalchemyObj.jansConfProperty = jansConfProperty if BackendTypes.MYSQL else ox_configuration_property_list self.session.commit() @@ -683,11 +688,13 @@ def rdm_automapper(self, force=False): self.Base = sqlalchemy.ext.automap.automap_base(metadata=self.metadata) self.Base.prepare() + # fix JSON type for mariadb - for tbl in self.Base.classes: - for col in tbl.__table__.columns: - if isinstance(col.type, sqlalchemy.dialects.mysql.LONGTEXT) and col.comment.lower() == 'json': - col.type = sqlalchemy.dialects.mysql.json.JSON() + if Config.rdbm_type == 'mysql': + for tbl in self.Base.classes: + for col in tbl.__table__.columns: + if isinstance(col.type, sqlalchemy.dialects.mysql.LONGTEXT) and col.comment.lower() == 'json': + col.type = sqlalchemy.dialects.mysql.json.JSON() base.logIt("Reflected tables {}".format(list(self.metadata.tables.keys()))) @@ -730,7 +737,7 @@ def get_rdbm_val(self, key, val, rdbm_type=None): data_type = self.get_attr_sql_data_type(key) - if data_type in ('SMALLINT', 'BOOL'): + if data_type in ('SMALLINT', 'BOOL', 'BOOLEAN'): if val[0].lower() in ('1', 'on', 'true', 'yes', 'ok'): return 1 if data_type == 'SMALLINT' else True return 0 if data_type == 'SMALLINT' else False @@ -751,7 +758,7 @@ def get_rdbm_val(self, key, val, rdbm_type=None): return json_data - if data_type == 'ARRAY': + if data_type in ('ARRAY', 'JSONB'): return val return val[0] @@ -835,6 +842,10 @@ def import_ldif(self, ldif_files, bucket=None, force=None): cur_val = copy.deepcopy(getattr(sqlalchObj, attribute)) for val_ in new_val: cur_val['v'].append(val_) + if Config.rdbm_type == 'mysql': + cur_val['v'].append(val_) + else: + cur_val.append(val_) setattr(sqlalchObj, attribute, cur_val) else: setattr(sqlalchObj, attribute, new_val[0]) @@ -888,7 +899,7 @@ def import_ldif(self, ldif_files, bucket=None, force=None): for col in sqlalchCls.__table__.columns: if isinstance(col.type, self.json_dialects_instance) and not col.name in vals: - vals[col.name] = {'v': []} + vals[col.name] = {'v': []} if Config.rdbm_type == 'mysql' else [] sqlalchObj = sqlalchCls() diff --git a/jans-linux-setup/jans_setup/setup_app/utils/properties_utils.py b/jans-linux-setup/jans_setup/setup_app/utils/properties_utils.py index 62656629727..ad7fc06a810 100644 --- a/jans-linux-setup/jans_setup/setup_app/utils/properties_utils.py +++ b/jans-linux-setup/jans_setup/setup_app/utils/properties_utils.py @@ -8,6 +8,7 @@ import ssl import re import pymysql +import psycopg2 import inspect import ldap3 @@ -15,7 +16,7 @@ from setup_app.messages import msg from setup_app.utils import base from setup_app.utils.cbm import CBM -from setup_app.static import InstallTypes, colors +from setup_app.static import InstallTypes, colors, BackendStrings from setup_app.config import Config from setup_app.utils.setup_utils import SetupUtils @@ -659,16 +660,18 @@ def prompt_for_rdbm(self): def prompt_for_backend(self): print('Chose Backend Type:') - backend_types = ['Local OpenDj', - 'Remote OpenDj', - 'Local MySQL', - 'Remote MySQL', - ] + backend_types = [ + BackendStrings.LOCAL_OPENDJ, + BackendStrings.LOCAL_MYSQL, + BackendStrings.REMOTE_MYSQL, + BackendStrings.LOCAL_PGSQL, + BackendStrings.REMOTE_PGSQL, + ] if not os.path.exists(os.path.join(Config.install_dir, 'package')): - backend_types += ['Remote Couchbase', 'Cloud Spanner'] + backend_types += [BackendStrings.REMOTE_COUCHBASE, BackendStrings.CLOUD_SPANNER] if 'couchbase' in self.getBackendTypes(): - backend_types.insert(2, 'Local Couchbase') + backend_types.insert(2, BackendStrings.LOCAL_COUCHBASE) nlist = [] for i, btype in enumerate(backend_types): @@ -697,7 +700,7 @@ def prompt_for_backend(self): backend_type_str = backend_types[int(choice)-1] - if backend_type_str == 'Local OpenDj': + if backend_type_str == BackendStrings.LOCAL_OPENDJ: Config.opendj_install = InstallTypes.LOCAL ldapPass = Config.ldapPass or Config.admin_password or self.getPW(special='.*=!%&+/-') @@ -712,7 +715,7 @@ def prompt_for_backend(self): Config.ldapPass = ldapPass - elif backend_type_str == 'Remote OpenDj': + elif backend_type_str == BackendStrings.REMOTE_OPENDJ: Config.opendj_install = InstallTypes.REMOTE while True: ldapHost = self.getPrompt(" LDAP hostname") @@ -726,7 +729,7 @@ def prompt_for_backend(self): Config.ldapPass = ldapPass Config.ldap_hostname = ldapHost - elif backend_type_str == 'Local Couchbase': + elif backend_type_str == BackendStrings.LOCAL_COUCHBASE: Config.opendj_install = InstallTypes.NONE Config.cb_install = InstallTypes.LOCAL Config.isCouchbaseUserAdmin = True @@ -742,7 +745,7 @@ def prompt_for_backend(self): Config.cb_password = cbPass Config.mapping_locations = { group: 'couchbase' for group in Config.couchbaseBucketDict } - elif backend_type_str == 'Remote Couchbase': + elif backend_type_str == BackendStrings.REMOTE_COUCHBASE: Config.opendj_install = InstallTypes.NONE Config.cb_install = InstallTypes.REMOTE @@ -756,38 +759,50 @@ def prompt_for_backend(self): Config.mapping_locations = { group: 'couchbase' for group in Config.couchbaseBucketDict } - elif backend_type_str == 'Local MySQL': + elif backend_type_str in (BackendStrings.LOCAL_MYSQL, BackendStrings.LOCAL_PGSQL): Config.opendj_install = InstallTypes.NONE Config.rdbm_install = True Config.rdbm_install_type = InstallTypes.LOCAL - Config.rdbm_type = 'mysql' - Config.rdbm_host = 'localhost' - Config.rdbm_user = 'jans' - Config.rdbm_password = self.getPW(special='.*=+-()[]{}') - Config.rdbm_port = 3306 - Config.rdbm_db = 'jansdb' - - elif backend_type_str == 'Remote MySQL': + if backend_type_str == BackendStrings.LOCAL_MYSQL: + Config.rdbm_port = 3306 + Config.rdbm_type = 'mysql' + else: + Config.rdbm_port = 5432 + Config.rdbm_type = 'pgsql' + if not Config.rdbm_host: + + if not Config.rdbm_password: + Config.rdbm_password = self.getPW(special='.*=+-()[]{}') + + elif backend_type_str in (BackendStrings.REMOTE_MYSQL, BackendStrings.REMOTE_PGSQL): Config.opendj_install = InstallTypes.NONE Config.rdbm_install = True Config.rdbm_install_type = InstallTypes.REMOTE - Config.rdbm_type = 'mysql' + if backend_type_str == BackendStrings.REMOTE_MYSQL: + Config.rdbm_port = 3306 + Config.rdbm_type = 'mysql' + else: + Config.rdbm_port = 5432 + Config.rdbm_type = 'pgsql' while True: - Config.rdbm_host = self.getPrompt(" Mysql host", Config.get('rdbm_host')) - Config.rdbm_port = self.getPrompt(" Mysql port", 3306, itype=int, indent=1) - Config.rdbm_user = self.getPrompt(" Mysql user", Config.get('rdbm_user')) - Config.rdbm_password = self.getPrompt(" Mysql password") - Config.rdbm_db = self.getPrompt(" Mysql database", Config.get('rdbm_db')) + Config.rdbm_host = self.getPrompt(" {} host".format(Config.rdbm_type.upper()), Config.get('rdbm_host')) + Config.rdbm_port = self.getPrompt(" {} port".format(Config.rdbm_type.upper()), Config.rdbm_port, itype=int, indent=1) + Config.rdbm_user = self.getPrompt(" {} user".format(Config.rdbm_type.upper()), Config.get('rdbm_user')) + Config.rdbm_password = self.getPrompt(" {} password".format(Config.rdbm_type.upper())) + Config.rdbm_db = self.getPrompt(" {} database".format(Config.rdbm_type.upper()), Config.get('rdbm_db')) try: - pymysql.connect(host=Config.rdbm_host, user=Config.rdbm_user, password=Config.rdbm_password, database=Config.rdbm_db, port=Config.rdbm_port) - print(" {}MySQL connection was successful{}".format(colors.OKGREEN, colors.ENDC)) + if Config.rdbm_type == 'mysql': + pymysql.connect(host=Config.rdbm_host, user=Config.rdbm_user, password=Config.rdbm_password, database=Config.rdbm_db, port=Config.rdbm_port) + else: + psycopg2.connect(dbname=Config.rdbm_db, user=Config.rdbm_user, password=Config.rdbm_password, host=Config.rdbm_host, port=Config.rdbm_port) + print(" {}{} connection was successful{}".format(colors.OKGREEN, Config.rdbm_type.upper(), colors.ENDC)) break except Exception as e: - print(" {}Can't connect to MySQL: {}{}".format(colors.DANGER, e, colors.ENDC)) + print(" {}Can't connect to {}: {}{}".format(colors.DANGER,Config.rdbm_type.upper(), e, colors.ENDC)) - elif backend_type_str == 'Cloud Spanner': + elif backend_type_str == BackendStrings.CLOUD_SPANNER: Config.opendj_install = InstallTypes.NONE Config.rdbm_type = 'spanner' Config.rdbm_install = True diff --git a/jans-linux-setup/jans_setup/setup_app/utils/setup_utils.py b/jans-linux-setup/jans_setup/setup_app/utils/setup_utils.py index 86c557062ed..7e09fa747ea 100644 --- a/jans-linux-setup/jans_setup/setup_app/utils/setup_utils.py +++ b/jans-linux-setup/jans_setup/setup_app/utils/setup_utils.py @@ -216,18 +216,18 @@ def replaceInText(self, text, pattern, update): def applyChangesInFiles(self, changes): self.logIt("Applying changes to %s files..." % changes['name']) for change in changes['files']: - file = change['path'] + cfile = change['path'] - text = self.readFile(file) - file_backup = '%s.bak' % file + text = self.readFile(cfile) + file_backup = '%s.bak' % cfile self.writeFile(file_backup, text) self.logIt("Created backup of %s file %s..." % (changes['name'], file_backup)) for replace in change['replace']: text = self.replaceInText(text, replace['pattern'], replace['update']) - self.writeFile(file, text) - self.logIt("Wrote updated %s file %s..." % (changes['name'], file)) + self.writeFile(cfile, text) + self.logIt("Wrote updated %s file %s..." % (changes['name'], cfile)) def copyFile(self, inFile, destFolder, backup=True): @@ -387,24 +387,15 @@ def fomatWithDict(self, text, dictionary): def renderTemplateInOut(self, file_path, template_folder, output_dir, pystring=False): fn = os.path.basename(file_path) - in_fp = os.path.join(template_folder, fn) + in_fp = os.path.join(template_folder, fn) + out_fp = os.path.join(output_dir, fn) self.logIt("Rendering template %s" % in_fp) - template_text = self.readFile(in_fp) # Create output folder if needed if not os.path.exists(output_dir): os.makedirs(output_dir) - format_dict = self.merge_dicts(Config.__dict__, Config.templateRenderingDict) - for k in format_dict: - if isinstance(format_dict[k], bool): - format_dict[k] = str(format_dict[k]).lower() - - if pystring: - rendered_text = Template(template_text).substitute(format_dict) - else: - rendered_text = self.fomatWithDict(template_text, format_dict) - out_fp = os.path.join(output_dir, fn) + rendered_text = self.render_template(in_fp, pystring) self.writeFile(out_fp, rendered_text) @@ -486,6 +477,20 @@ def in_ignoredirs(p): self.logIt("Writing rendered template {}".format(full_output_file)) full_output_file.write_text(rendered_text) + def render_template(self, tmp_fn, pystring=False): + template_text = self.readFile(tmp_fn) + format_dict = self.merge_dicts(Config.__dict__, Config.templateRenderingDict) + for k in format_dict: + if isinstance(format_dict[k], bool): + format_dict[k] = str(format_dict[k]).lower() + + if pystring: + rendered_text = Template(template_text).substitute(format_dict) + else: + rendered_text = self.fomatWithDict(template_text, format_dict) + + return rendered_text + def add_yacron_job(self, command, schedule, name=None, args={}): import ruamel.yaml diff --git a/jans-linux-setup/jans_setup/static/rdbm/ldap_sql_data_type_mapping.json b/jans-linux-setup/jans_setup/static/rdbm/ldap_sql_data_type_mapping.json index 70836894173..b15d7f471ad 100644 --- a/jans-linux-setup/jans_setup/static/rdbm/ldap_sql_data_type_mapping.json +++ b/jans-linux-setup/jans_setup/static/rdbm/ldap_sql_data_type_mapping.json @@ -113,6 +113,9 @@ "mysql": { "type": "SMALLINT" }, + "pgsql": { + "type": "BOOLEAN" + }, "spanner": { "type": "BOOL" } @@ -132,6 +135,9 @@ "mysql": { "type": "JSON" }, + "pgsql": { + "type": "JSONB" + }, "spanner": { "type": "ARRAY" } diff --git a/jans-linux-setup/jans_setup/static/rdbm/pgsql_index.json b/jans-linux-setup/jans_setup/static/rdbm/pgsql_index.json index 8763042f29b..b0502e03760 100644 --- a/jans-linux-setup/jans_setup/static/rdbm/pgsql_index.json +++ b/jans-linux-setup/jans_setup/static/rdbm/pgsql_index.json @@ -1,6 +1,235 @@ { + "jansPerson": { + "fields": [ + "inum", + "uid", + "mail", + "displayName", + "givenName", + "ppid", + "sn" + ], + "custom": [ + "(lower(\"uid\"))", + "(lower(\"mail\"))" + ] + }, + "jansClnt": { + "fields": [ + "inum", + "displayName", + "description", + "jansRegistrationAccessTkn", + "jansClntSecretExpAt" + ], + "custom": [ + "(\"del\", \"exp\")" + ] + }, + "jansPar": { + "fields": [ + "jansId" + ], + "custom": [ + "(\"del\", \"exp\")" + ] + }, + "jansUmaPCT": { + "fields": [ + "tknCde" + ], + "custom": [ + "(\"del\", \"exp\")" + ] + }, + "jansScope": { + "fields": [ + "jansId", + "displayName", + "description" + ], + "custom": [ + "(\"del\", \"exp\")" + ] + }, + "jansU2fReq": { + "fields": [ + "creationDate" + ], + "custom": [ + "(\"del\", \"exp\")" + ] + }, + "jansMetric": { + "fields": [ + "jansAppTyp", + "jansMetricTyp", + "jansStartDate", + "jansEndDate" + ], + "custom": [ + "(\"del\", \"exp\")" + ] + }, + "jansToken": { + "fields": [], + "custom": [ + "(\"del\", \"exp\")" + ] + }, + "jansDeviceRegistration": { + "fields": [ + "jansApp", + "personInum", + "jansStatus", + "jansDeviceHashCode", + "jansDeviceKeyHandle", + "creationDate" + ], + "custom": [ + "(\"del\", \"exp\")" + ] + }, + "jansCache": { + "fields": [], + "custom": [ + "(\"del\", \"exp\")" + ] + }, + "jansSessId": { + "fields": [ + "jansUsrDN", + "sid" + ], + "custom": [ + "(\"del\", \"exp\")" + ] + }, + "jansUmaResource": { + "fields": [ + "jansId", + "displayName", + "jansUmaScope" + ], + "custom": [ + "(\"del\", \"exp\")" + ] + }, + "jansClntAuthz": { + "fields": [ + "jansClntId", + "jansUsrId" + ], + "custom": [ + "(\"del\", \"exp\")" + ] + }, + "jansPerson_oxExternalUid": { + "fields": [ + "jansExternalUid" + ], + "custom": [] + }, + "jansGrp": { + "fields": [ + "inum", + "displayName", + "description" + ], + "custom": [] + }, + "token": { + "fields": [ + "authzCode", + "grtId", + "ssnId", + "tknCde" + ], + "custom": [ + "(\"del\", \"exp\")" + ] + }, + "cache": { + "fields": [], + "custom": [ + "(\"del\", \"exp\")" + ] + }, + "jansAttr": { + "fields": [ + "inum", + "displayName", + "description", + "jansAttrName", + "jansAttrOrigin" + ], + "custom": [] + }, + "jansCustomScr": { + "fields": [ + "inum", + "jansScrTyp" + ], + "custom": [] + }, + "jansUmaResourcePermission": { + "fields": [ + "jansTicket" + ], + "custom": [] + }, + "pairwiseIdentifier": { + "fields": [ + "jansClntId", + "jansUsrId", + "jansSectorIdentifier" + ], + "custom": [] + }, + "jansCibaReq": { + "fields": [], + "custom": [ + "(\"jansStatus\", \"exp\")" + ] + }, + "jansStatEntry": { + "fields": [ + "jansId" + ], + "custom": [] + }, + "jansPassResetReq": { + "fields": [ + "creationDate" + ], + "custom": [] + }, + "jansFido2AuthnEntry": { + "fields": [ + "creationDate", + "jansStatus", + "personInum" + ], + "custom": [] + }, + "jansFido2RegistrationEntry": { + "fields": [ + "creationDate", + "jansStatus", + "personInum" + ], + "custom": [] + }, + "jansInumMap": { + "fields": [ + "inum", + "jansStatus" + ], + "custom": [] + }, "__common__": { "JSON": [ + "USING GIN (JSONB_PATH_QUERY_ARRAY(\"$field\", '$[*]'))" ], "fields": [ "uid" diff --git a/jans-linux-setup/jans_setup/static/rdbm/sql_data_types.json b/jans-linux-setup/jans_setup/static/rdbm/sql_data_types.json index 83bb42fc3b8..8f32c9ecff4 100644 --- a/jans-linux-setup/jans_setup/static/rdbm/sql_data_types.json +++ b/jans-linux-setup/jans_setup/static/rdbm/sql_data_types.json @@ -134,6 +134,9 @@ "mysql": { "type": "JSON" }, + "pgsql": { + "type": "JSONB" + }, "spanner": { "type": "ARRAY" } @@ -150,6 +153,9 @@ "mysql": { "type": "JSON" }, + "pgsql": { + "type": "JSONB" + }, "spanner": { "type": "ARRAY" } @@ -168,6 +174,9 @@ "mysql": { "type": "JSON" }, + "pgsql": { + "type": "JSONB" + }, "spanner": { "type": "ARRAY" } @@ -205,6 +214,9 @@ "mysql": { "type": "JSON" }, + "pgsql": { + "type": "JSONB" + }, "spanner": { "type": "ARRAY" } @@ -548,6 +560,9 @@ "mysql": { "type": "JSON" }, + "pgsql": { + "type": "JSONB" + }, "spanner": { "type": "ARRAY" } @@ -740,6 +755,9 @@ "mysql": { "type": "JSON" }, + "pgsql": { + "type": "JSONB" + }, "spanner": { "type": "ARRAY" } @@ -748,6 +766,9 @@ "mysql": { "type": "MEDIUMTEXT" }, + "pgsql": { + "type": "TEXT" + }, "spanner": { "type": "STRING(MAX)" } diff --git a/jans-linux-setup/jans_setup/templates/jans-sql.properties b/jans-linux-setup/jans_setup/templates/jans-mysql.properties similarity index 90% rename from jans-linux-setup/jans_setup/templates/jans-sql.properties rename to jans-linux-setup/jans_setup/templates/jans-mysql.properties index 3886f393992..690b94b4c42 100644 --- a/jans-linux-setup/jans_setup/templates/jans-sql.properties +++ b/jans-linux-setup/jans_setup/templates/jans-mysql.properties @@ -1,6 +1,6 @@ -db.schema.name=%(rdbm_db)s +db.schema.name=jansdb -connection.uri=jdbc:%(rdbm_type)s://%(rdbm_host)s:%(rdbm_port)s/%(rdbm_db)s +connection.uri=jdbc:mysql://%(rdbm_host)s:%(rdbm_port)s/%(rdbm_db)s?enabledTLSProtocols=TLSv1.2 connection.driver-property.serverTimezone=%(server_time_zone)s # Prefix connection.driver-property.key=value will be coverterd to key=value JDBC driver properties diff --git a/jans-linux-setup/jans_setup/templates/jans-pgsql.properties b/jans-linux-setup/jans_setup/templates/jans-pgsql.properties new file mode 100644 index 00000000000..e932372cc9b --- /dev/null +++ b/jans-linux-setup/jans_setup/templates/jans-pgsql.properties @@ -0,0 +1,30 @@ +db.schema.name=public + +connection.uri=jdbc:postgresql://%(rdbm_host)s:%(rdbm_port)s/%(rdbm_db)s + +# Prefix connection.driver-property.key=value will be coverterd to key=value JDBC driver properties +#connection.driver-property.driverProperty=driverPropertyValu + + +auth.userName=%(rdbm_user)s +auth.userPassword=%(rdbm_password_enc)s + +# Password hash method +password.encryption.method=SSHA-256 + +# Connection pool size +connection.pool.max-total=40 +connection.pool.max-idle=15 +connection.pool.min-idle=5 + +# Max time needed to create connection pool in milliseconds +connection.pool.create-max-wait-time-millis=20000 + +# Max wait 20 seconds +connection.pool.max-wait-time-millis=20000 + +# Allow to evict connection in pool after 30 minutes +connection.pool.min-evictable-idle-time-millis=1800000 + +binaryAttributes=objectGUID +certificateAttributes=userCertificate From c59c4cdacddab7ff81dc83a6d394a5239eacf052 Mon Sep 17 00:00:00 2001 From: Yuriy Movchan Date: Fri, 16 Sep 2022 18:20:25 +0300 Subject: [PATCH 2/2] feat: add PostgreSQL support --- .../io/jans/orm/impl/BaseEntryManager.java | 2 +- .../impl/CouchbaseOperationServiceImpl.java | 2 +- .../spanner/impl/SpannerFilterConverter.java | 2 +- ...Templates.java => MySQLJsonTemplates.java} | 6 +- .../dsl/template/PostgreSQLJsonTemplates.java | 38 +++ .../orm/sql/dsl/types/PostgreSQLJsonType.java | 49 +++ .../io/jans/orm/sql/impl/SqlEntryManager.java | 92 +++-- .../jans/orm/sql/impl/SqlFilterConverter.java | 323 +++++++++++------- .../java/io/jans/orm/sql/impl/SqlOps.java | 5 +- .../io/jans/orm/sql/model/JsonString.java | 25 ++ .../sql/operation/SqlOperationService.java | 15 +- .../orm/sql/operation/SupportedDbType.java | 39 +++ .../operation/impl/SqlConnectionProvider.java | 67 ++-- .../impl/SqlOperationServiceImpl.java | 168 ++++++--- ...FilterConverterCheckExcludeFilterTest.java | 14 +- .../sql/impl/test/SqlFilterConverterTest.java | 112 +++--- 16 files changed, 665 insertions(+), 294 deletions(-) rename jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/template/{SqlJsonMySQLTemplates.java => MySQLJsonTemplates.java} (75%) create mode 100644 jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/template/PostgreSQLJsonTemplates.java create mode 100644 jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/types/PostgreSQLJsonType.java create mode 100644 jans-orm/sql/src/main/java/io/jans/orm/sql/model/JsonString.java create mode 100644 jans-orm/sql/src/main/java/io/jans/orm/sql/operation/SupportedDbType.java diff --git a/jans-orm/core/src/main/java/io/jans/orm/impl/BaseEntryManager.java b/jans-orm/core/src/main/java/io/jans/orm/impl/BaseEntryManager.java index 2a327b0ff7d..2b4371e07ff 100644 --- a/jans-orm/core/src/main/java/io/jans/orm/impl/BaseEntryManager.java +++ b/jans-orm/core/src/main/java/io/jans/orm/impl/BaseEntryManager.java @@ -214,7 +214,7 @@ public int countEntries(Object entry) { @SuppressWarnings("unchecked") protected Void merge(Object entry, boolean isSchemaUpdate, boolean isConfigurationUpdate, AttributeModificationType schemaModificationType) { if (entry == null) { - throw new MappingException("Entry to persist is null"); + throw new MappingException("Entry for check if exists is null"); } Class entryClass = entry.getClass(); diff --git a/jans-orm/couchbase/src/main/java/io/jans/orm/couchbase/operation/impl/CouchbaseOperationServiceImpl.java b/jans-orm/couchbase/src/main/java/io/jans/orm/couchbase/operation/impl/CouchbaseOperationServiceImpl.java index af15f9c3446..aa1cd8efe9d 100644 --- a/jans-orm/couchbase/src/main/java/io/jans/orm/couchbase/operation/impl/CouchbaseOperationServiceImpl.java +++ b/jans-orm/couchbase/src/main/java/io/jans/orm/couchbase/operation/impl/CouchbaseOperationServiceImpl.java @@ -476,7 +476,7 @@ private PagedResult searchImpl(BucketMapping bucketMapping, Stri } String[] select = attributes; - if (select == null) { + if (ArrayHelper.isEmpty(select)) { select = new String[] { "jans_doc.*", CouchbaseOperationService.DN }; } else if ((select.length == 1) && StringHelper.isEmpty(select[0])) { // Compatibility with base persistence layer when application pass filter new String[] { "" } diff --git a/jans-orm/spanner/src/main/java/io/jans/orm/cloud/spanner/impl/SpannerFilterConverter.java b/jans-orm/spanner/src/main/java/io/jans/orm/cloud/spanner/impl/SpannerFilterConverter.java index b69519efe1b..f6713230df3 100644 --- a/jans-orm/spanner/src/main/java/io/jans/orm/cloud/spanner/impl/SpannerFilterConverter.java +++ b/jans-orm/spanner/src/main/java/io/jans/orm/cloud/spanner/impl/SpannerFilterConverter.java @@ -338,7 +338,7 @@ private StructField getStructField(TableMapping tableMapping, String attributeNa structField = childTableMapping.getColumTypes().get(attributeNameLower); } } - + if (structField == null) { throw new SearchException(String.format("Unknown column name '%s' in table/child table '%s'", attributeName, tableMapping.getTableName())); } diff --git a/jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/template/SqlJsonMySQLTemplates.java b/jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/template/MySQLJsonTemplates.java similarity index 75% rename from jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/template/SqlJsonMySQLTemplates.java rename to jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/template/MySQLJsonTemplates.java index 0eb2187daba..73a02daa49c 100644 --- a/jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/template/SqlJsonMySQLTemplates.java +++ b/jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/template/MySQLJsonTemplates.java @@ -10,18 +10,18 @@ * * @author Yuriy Movchan Date: 01/27/2021 */ -public class SqlJsonMySQLTemplates extends MySQLTemplates { +public class MySQLJsonTemplates extends MySQLTemplates { public static Builder builder() { return new Builder() { @Override protected SQLTemplates build(char escape, boolean quote) { - return new SqlJsonMySQLTemplates(escape, quote); + return new MySQLJsonTemplates(escape, quote); } }; } - public SqlJsonMySQLTemplates(char escape, boolean quote) { + public MySQLJsonTemplates(char escape, boolean quote) { super(escape, quote); add(SqlOps.JSON_CONTAINS, "JSON_CONTAINS({0}->{2}, CAST({1} AS JSON))"); diff --git a/jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/template/PostgreSQLJsonTemplates.java b/jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/template/PostgreSQLJsonTemplates.java new file mode 100644 index 00000000000..3a8def5d972 --- /dev/null +++ b/jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/template/PostgreSQLJsonTemplates.java @@ -0,0 +1,38 @@ +package io.jans.orm.sql.dsl.template; + + +import com.querydsl.sql.PostgreSQLTemplates; +import com.querydsl.sql.SQLTemplates; + +import io.jans.orm.sql.dsl.types.PostgreSQLJsonType; +import io.jans.orm.sql.impl.SqlOps; + +/** + * PostgreSQL DSL templates for JSON support + * + * @author Yuriy Movchan Date: 09/01/2022 + */ +public class PostgreSQLJsonTemplates extends PostgreSQLTemplates { + + public static Builder builder() { + return new Builder() { + @Override + protected SQLTemplates build(char escape, boolean quote) { + return new PostgreSQLJsonTemplates(escape, quote); + } + }; + } + + public PostgreSQLJsonTemplates(char escape, boolean quote) { + super(escape, quote); + + addCustomType(new PostgreSQLJsonType()); + + add(SqlOps.PGSQL_JSON_CONTAINS, "{0} @> {1}::jsonb"); +// add(SqlOps.PGSQL_JSON_PATH_QUERY_ARRAY, "jsonb_array_length(jsonb_path_query_array({0}, CONCAT('$[*] ? (@', {1}, {2}, ')')::jsonpath)) > 0"); + add(SqlOps.PGSQL_JSON_PATH_QUERY_ARRAY, "jsonb_path_query_array({0}, CONCAT('$[*] ? (@', {1}, {2}, ')')::jsonpath)"); + + add(SqlOps.PGSQL_JSON_NOT_EMPTY_ARRAY, "jsonb_array_length({0}) > 0"); + } + +} diff --git a/jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/types/PostgreSQLJsonType.java b/jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/types/PostgreSQLJsonType.java new file mode 100644 index 00000000000..2bc457aa5c7 --- /dev/null +++ b/jans-orm/sql/src/main/java/io/jans/orm/sql/dsl/types/PostgreSQLJsonType.java @@ -0,0 +1,49 @@ +package io.jans.orm.sql.dsl.types; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; + +import org.postgresql.util.PGobject; + +import com.querydsl.sql.types.AbstractType; + +import io.jans.orm.sql.model.JsonString; + +/** + * PostgreSQL JSON column type support + * + * @author Yuriy Movchan Date: 09/01/2022 + */ + +public class PostgreSQLJsonType extends AbstractType { + + public PostgreSQLJsonType() { + super(Types.JAVA_OBJECT); + } + + public PostgreSQLJsonType(int type) { + super(type); + } + + @Override + public JsonString getValue(ResultSet rs, int startIndex) throws SQLException { + return new JsonString(rs.getString(startIndex)); + } + + @Override + public Class getReturnedClass() { + return JsonString.class; + } + + @Override + public void setValue(PreparedStatement st, int startIndex, JsonString value) + throws SQLException { + final PGobject jsonObject = new PGobject(); + jsonObject.setType("json"); + jsonObject.setValue(value.getValue()); + + st.setObject(startIndex, jsonObject); + } +} diff --git a/jans-orm/sql/src/main/java/io/jans/orm/sql/impl/SqlEntryManager.java b/jans-orm/sql/src/main/java/io/jans/orm/sql/impl/SqlEntryManager.java index 6b17bca0f9f..904fc6005e7 100644 --- a/jans-orm/sql/src/main/java/io/jans/orm/sql/impl/SqlEntryManager.java +++ b/jans-orm/sql/src/main/java/io/jans/orm/sql/impl/SqlEntryManager.java @@ -16,7 +16,6 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.function.Function; import javax.inject.Inject; @@ -350,20 +349,21 @@ protected int removeImpl(String dn, Class entryClass, Filter filter, int List propertiesAnnotations = getEntryPropertyAnnotations(entryClass); Map propertiesAnnotationsMap = prepareEntryPropertiesTypes(entryClass, propertiesAnnotations); - ParsedKey keyWithInum = toSQLKey(dn); + String key = toSQLKey(dn).getKey(); + ConvertedExpression convertedExpression; try { - convertedExpression = toSqlFilterWithEmptyAlias(searchFilter, propertiesAnnotationsMap); + convertedExpression = toSqlFilterWithEmptyAlias(key, getBaseObjectClass(entryClass, objectClasses), searchFilter, propertiesAnnotationsMap); } catch (SearchException ex) { throw new EntryDeleteException(String.format("Failed to convert filter '%s' to expression", searchFilter), ex); } - + try { - int processed = (int) getOperationService().delete(keyWithInum.getKey(), getBaseObjectClass(entryClass, objectClasses), convertedExpression, count); + int processed = (int) getOperationService().delete(key, getBaseObjectClass(entryClass, objectClasses), convertedExpression, count); return processed; } catch (Exception ex) { - throw new EntryDeleteException(String.format("Failed to delete entries with key: '%s', expression: '%s'", keyWithInum.getKey(), convertedExpression), ex); + throw new EntryDeleteException(String.format("Failed to delete entries with key: '%s', expression: '%s'", key, convertedExpression), ex); } } @@ -470,10 +470,12 @@ protected PagedResult findEntriesImpl(String baseDN, Class ent // Prepare properties types to allow build filter properly Map propertiesAnnotationsMap = prepareEntryPropertiesTypes(entryClass, propertiesAnnotations); - ParsedKey keyWithInum = toSQLKey(baseDN); + + String key = toSQLKey(baseDN).getKey(); + ConvertedExpression convertedExpression; try { - convertedExpression = toSqlFilter(searchFilter, propertiesAnnotationsMap); + convertedExpression = toSqlFilter(key, getBaseObjectClass(entryClass, objectClasses), searchFilter, propertiesAnnotationsMap); } catch (SearchException ex) { throw new EntryPersistenceException(String.format("Failed to convert filter '%s' to expression", searchFilter)); } @@ -484,18 +486,18 @@ protected PagedResult findEntriesImpl(String baseDN, Class ent if (batchOperation != null) { batchOperationWraper = new SqlBatchOperationWraper(batchOperation, this, entryClass, propertiesAnnotations); } - searchResult = searchImpl(keyWithInum.getKey(), getBaseObjectClass(entryClass, objectClasses), convertedExpression, scope, currentLdapReturnAttributes, + searchResult = searchImpl(key, getBaseObjectClass(entryClass, objectClasses), convertedExpression, scope, currentLdapReturnAttributes, defaultSort, batchOperationWraper, returnDataType, start, count, chunkSize); if (searchResult == null) { - throw new EntryPersistenceException(String.format("Failed to find entries with key: '%s', expression: '%s'", keyWithInum.getKey(), convertedExpression)); + throw new EntryPersistenceException(String.format("Failed to find entries with key: '%s', expression: '%s'", key, convertedExpression)); } return searchResult; } catch (SearchException ex) { - throw new EntryPersistenceException(String.format("Failed to find entries with key: '%s'", keyWithInum.getKey()), ex); + throw new EntryPersistenceException(String.format("Failed to find entries with key: '%s'", key), ex); } catch (Exception ex) { - throw new EntryPersistenceException(String.format("Failed to find entries with key: '%s', expression: '%s'", keyWithInum.getKey(), convertedExpression), ex); + throw new EntryPersistenceException(String.format("Failed to find entries with key: '%s', expression: '%s'", key, convertedExpression), ex); } } @@ -516,17 +518,18 @@ protected boolean contains(String baseDN, String[] objectClasses, Class e // Prepare properties types to allow build filter properly Map propertiesAnnotationsMap = prepareEntryPropertiesTypes(entryClass, propertiesAnnotations); + String key = toSQLKey(baseDN).getKey(); + ConvertedExpression convertedExpression; try { - convertedExpression = toSqlFilter(searchFilter, propertiesAnnotationsMap); + convertedExpression = toSqlFilter(key, getBaseObjectClass(entryClass, objectClasses), searchFilter, propertiesAnnotationsMap); } catch (SearchException ex) { throw new EntryPersistenceException(String.format("Failed to convert filter '%s' to expression", searchFilter)); } PagedResult searchResult = null; try { - ParsedKey keyWithInum = toSQLKey(baseDN); - searchResult = searchImpl(keyWithInum.getKey(), getBaseObjectClass(entryClass, objectClasses), convertedExpression, SearchScope.SUB, ldapReturnAttributes, null, + searchResult = searchImpl(key, getBaseObjectClass(entryClass, objectClasses), convertedExpression, SearchScope.SUB, ldapReturnAttributes, null, null, SearchReturnDataType.SEARCH, 0, 1, 0); if (searchResult == null) { throw new EntryPersistenceException(String.format("Failed to find entry with baseDN: '%s', filter: '%s'", baseDN, searchFilter)); @@ -608,15 +611,17 @@ public boolean authenticate(String baseDN, Class entryClass, String userN // Prepare properties types to allow build filter properly Map propertiesAnnotationsMap = prepareEntryPropertiesTypes(entryClass, propertiesAnnotations); + String key = toSQLKey(baseDN).getKey(); + ConvertedExpression convertedExpression; try { - convertedExpression = toSqlFilter(searchFilter, propertiesAnnotationsMap); + convertedExpression = toSqlFilter(key, getBaseObjectClass(entryClass, objectClasses), searchFilter, propertiesAnnotationsMap); } catch (SearchException ex) { throw new EntryPersistenceException(String.format("Failed to convert filter '%s' to expression", searchFilter)); } try { - PagedResult searchResult = searchImpl(toSQLKey(baseDN).getKey(), getBaseObjectClass(entryClass, objectClasses), convertedExpression, + PagedResult searchResult = searchImpl(key, getBaseObjectClass(entryClass, objectClasses), convertedExpression, SearchScope.SUB, SqlOperationService.UID_ARRAY, null, null, SearchReturnDataType.SEARCH, 0, 1, 1); if ((searchResult == null) || (searchResult.getEntriesCount() != 1)) { return false; @@ -687,16 +692,18 @@ public int countEntries(String baseDN, Class entryClass, Filter filter, S // Prepare properties types to allow build filter properly Map propertiesAnnotationsMap = prepareEntryPropertiesTypes(entryClass, propertiesAnnotations); + String key = toSQLKey(baseDN).getKey(); + ConvertedExpression convertedExpression; try { - convertedExpression = toSqlFilter(searchFilter, propertiesAnnotationsMap); + convertedExpression = toSqlFilter(key, getBaseObjectClass(entryClass, objectClasses), searchFilter, propertiesAnnotationsMap); } catch (SearchException ex) { throw new EntryPersistenceException(String.format("Failed to convert filter '%s' to expression", searchFilter)); } PagedResult searchResult; try { - searchResult = searchImpl(toSQLKey(baseDN).getKey(), getBaseObjectClass(entryClass, objectClasses), convertedExpression, scope, null, null, + searchResult = searchImpl(key, getBaseObjectClass(entryClass, objectClasses), convertedExpression, scope, null, null, null, SearchReturnDataType.COUNT, 0, 0, 0); } catch (Exception ex) { throw new EntryPersistenceException( @@ -783,21 +790,27 @@ public List exportEntry(String dn, String objectClass) { } } - private ConvertedExpression toSqlFilter(Filter genericFilter, Map propertiesAnnotationsMap) throws SearchException { - return filterConverter.convertToSqlFilter(excludeObjectClassFilters(genericFilter), propertiesAnnotationsMap); - } + private ConvertedExpression toSqlFilter(String key, String objectClass, Filter genericFilter, Map propertiesAnnotationsMap) throws SearchException { + TableMapping tableMapping = getTableMapping(key, objectClass); - private ConvertedExpression toSqlFilterWithEmptyAlias(Filter genericFilter, Map propertiesAnnotationsMap) throws SearchException { - return filterConverter.convertToSqlFilter(excludeObjectClassFilters(genericFilter), propertiesAnnotationsMap, true); + return filterConverter.convertToSqlFilter(tableMapping, excludeObjectClassFilters(genericFilter), propertiesAnnotationsMap); } - private ConvertedExpression toSqlFilter(Filter genericFilter, Map propertiesAnnotationsMap, Function processor) throws SearchException { - return filterConverter.convertToSqlFilter(excludeObjectClassFilters(genericFilter), propertiesAnnotationsMap, processor); - } - private ConvertedExpression toSqlFilterWithEmptyAlias(Filter genericFilter, Map propertiesAnnotationsMap, Function processor) throws SearchException { - return filterConverter.convertToSqlFilter(excludeObjectClassFilters(genericFilter), propertiesAnnotationsMap, processor, true); + private ConvertedExpression toSqlFilterWithEmptyAlias(String key, String objectClass, Filter genericFilter, Map propertiesAnnotationsMap) throws SearchException { + TableMapping tableMapping = getTableMapping(key, objectClass); + + return filterConverter.convertToSqlFilter(tableMapping, excludeObjectClassFilters(genericFilter), propertiesAnnotationsMap, true); } + private TableMapping getTableMapping(String key, String objectClass) { + TableMapping tableMapping = getOperationService().getTabeMapping(key, objectClass); + if (tableMapping == null) { + throw new MappingException(String.format("Failed to get table mapping by key '%s' and objectClass '%s'", key, objectClass)); + } + + return tableMapping; + } + private ParsedKey toSQLKey(String dn) { return KEY_CONVERTER.convertToKey(dn); } @@ -963,21 +976,6 @@ protected boolean isSupportForceUpdate() { return true; } - @Override - protected List getAttributeDataFromLocalizedString(String ldapAttributeName, LocalizedString localizedString) { - List listAttributes = new ArrayList<>(); - - localizedString.getLanguageTags().forEach(languageTag -> { - String value = localizedString.getValue(languageTag); - String key = localizedString.addLdapLanguageTag(ldapAttributeName, languageTag); - AttributeData attributeData = new AttributeData(key, value); - - listAttributes.add(attributeData); - }); - - return listAttributes; - } - private String getBaseObjectClass(String[] objectClasses) { if (ArrayHelper.isEmpty(objectClasses)) { throw new MappingException("Object class isn't defined!"); @@ -986,7 +984,7 @@ private String getBaseObjectClass(String[] objectClasses) { if (StringHelper.isEmpty(objectClasses[0])) { throw new MappingException("First object class is invalid!"); } - + return objectClasses[0]; } @@ -994,8 +992,8 @@ private String getBaseObjectClass(Class entryClass, String[] objectClasses) { if (ArrayHelper.isEmpty(objectClasses)) { throw new MappingException(String.format("Object class isn't defined in bean '%s'!", entryClass)); } - + return objectClasses[0]; - } + } } diff --git a/jans-orm/sql/src/main/java/io/jans/orm/sql/impl/SqlFilterConverter.java b/jans-orm/sql/src/main/java/io/jans/orm/sql/impl/SqlFilterConverter.java index ee513740a3c..2db6ed8e18a 100644 --- a/jans-orm/sql/src/main/java/io/jans/orm/sql/impl/SqlFilterConverter.java +++ b/jans-orm/sql/src/main/java/io/jans/orm/sql/impl/SqlFilterConverter.java @@ -6,7 +6,6 @@ package io.jans.orm.sql.impl; -import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -31,12 +30,15 @@ import io.jans.orm.annotation.AttributeEnum; import io.jans.orm.exception.operation.SearchException; import io.jans.orm.ldap.impl.LdapFilterConverter; +import io.jans.orm.model.AttributeType; import io.jans.orm.reflect.property.PropertyAnnotation; import io.jans.orm.reflect.util.ReflectHelper; import io.jans.orm.search.filter.Filter; import io.jans.orm.search.filter.FilterType; import io.jans.orm.sql.model.ConvertedExpression; +import io.jans.orm.sql.model.TableMapping; import io.jans.orm.sql.operation.SqlOperationService; +import io.jans.orm.sql.operation.SupportedDbType; import io.jans.orm.util.ArrayHelper; import io.jans.orm.util.StringHelper; @@ -48,44 +50,42 @@ public class SqlFilterConverter { private static final Logger LOG = LoggerFactory.getLogger(SqlFilterConverter.class); - - private static final String SQL_DATA_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS"; + private static final LdapFilterConverter ldapFilterConverter = new LdapFilterConverter(); private static final ObjectMapper JSON_OBJECT_MAPPER = new ObjectMapper(); private SqlOperationService operationService; + private SupportedDbType dbType; private Path stringDocAlias = ExpressionUtils.path(String.class, "doc"); private Path booleanDocAlias = ExpressionUtils.path(Boolean.class, "doc"); private Path integerDocAlias = ExpressionUtils.path(Integer.class, "doc"); private Path longDocAlias = ExpressionUtils.path(Long.class, "doc"); + private Path dateDocAlias = ExpressionUtils.path(Date.class, "doc"); private Path objectDocAlias = ExpressionUtils.path(Object.class, "doc"); + public SqlFilterConverter(SqlOperationService operationService) { this.operationService = operationService; + this.dbType = operationService.getConnectionProvider().getDbType(); } - public ConvertedExpression convertToSqlFilter(Filter genericFilter, Map propertiesAnnotationsMap) throws SearchException { - return convertToSqlFilter(genericFilter, propertiesAnnotationsMap, false); - } - - public ConvertedExpression convertToSqlFilter(Filter genericFilter, Map propertiesAnnotationsMap, boolean skipAlias) throws SearchException { - return convertToSqlFilter(genericFilter, propertiesAnnotationsMap, null, skipAlias); + public ConvertedExpression convertToSqlFilter(TableMapping tableMapping, Filter genericFilter, Map propertiesAnnotationsMap) throws SearchException { + return convertToSqlFilter(tableMapping, genericFilter, propertiesAnnotationsMap, false); } - public ConvertedExpression convertToSqlFilter(Filter genericFilter, Map propertiesAnnotationsMap, Function processor) throws SearchException { - return convertToSqlFilter(genericFilter, propertiesAnnotationsMap, processor, false); + public ConvertedExpression convertToSqlFilter(TableMapping tableMapping, Filter genericFilter, Map propertiesAnnotationsMap, boolean skipAlias) throws SearchException { + return convertToSqlFilter(tableMapping, genericFilter, propertiesAnnotationsMap, null, skipAlias); } - @SuppressWarnings({ "unchecked", "rawtypes" }) - public ConvertedExpression convertToSqlFilter(Filter genericFilter, Map propertiesAnnotationsMap, Function processor, boolean skipAlias) throws SearchException { + public ConvertedExpression convertToSqlFilter(TableMapping tableMapping, Filter genericFilter, Map propertiesAnnotationsMap, Function processor, boolean skipAlias) throws SearchException { Map> jsonAttributes = new HashMap<>(); - ConvertedExpression convertedExpression = convertToSqlFilterImpl(genericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias); + ConvertedExpression convertedExpression = convertToSqlFilterImpl(tableMapping, genericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias); return convertedExpression; } - private ConvertedExpression convertToSqlFilterImpl(Filter genericFilter, Map propertiesAnnotationsMap, + private ConvertedExpression convertToSqlFilterImpl(TableMapping tableMapping, Filter genericFilter, Map propertiesAnnotationsMap, Map> jsonAttributes, Function processor, boolean skipAlias) throws SearchException { if (genericFilter == null) { return null; @@ -114,7 +114,7 @@ private ConvertedExpression convertToSqlFilterImpl(Filter genericFilter, Map operation = ExpressionUtils.predicate(SqlOps.JSON_CONTAINS, expression, - buildTypedExpression(currentGenericFilter, true), Expressions.constant("$.v")); - - return ConvertedExpression.build(operation, jsonAttributes); + if (SupportedDbType.POSTGRESQL == this.dbType) { + Operation operation = ExpressionUtils.predicate(SqlOps.PGSQL_JSON_CONTAINS, columnExpression, + buildTypedArrayExpression(tableMapping, currentGenericFilter)); + + return ConvertedExpression.build(operation, jsonAttributes); + } else { + Operation operation = ExpressionUtils.predicate(SqlOps.JSON_CONTAINS, columnExpression, + buildTypedArrayExpression(tableMapping, currentGenericFilter), Expressions.constant("$.v")); + + return ConvertedExpression.build(operation, jsonAttributes); + } } - return ConvertedExpression.build(ExpressionUtils.eq(expression, buildTypedExpression(currentGenericFilter)), jsonAttributes); + return ConvertedExpression.build(ExpressionUtils.eq(columnExpression, buildTypedExpression(tableMapping, currentGenericFilter)), jsonAttributes); } if (FilterType.LESS_OR_EQUAL == type) { if (multiValued) { - if (currentGenericFilter.getMultiValuedCount() > 1) { - Collection expressions = new ArrayList<>(currentGenericFilter.getMultiValuedCount()); - for (int i = 0; i < currentGenericFilter.getMultiValuedCount(); i++) { - Operation operation = ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, - buildTypedPath(currentGenericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias), Expressions.constant("$.v[" + i + "]")); - Predicate predicate = Expressions.asComparable(operation).loe(buildTypedExpression(currentGenericFilter)); - - expressions.add(predicate); - } - - Expression expression = ExpressionUtils.anyOf(expressions); - - return ConvertedExpression.build(expression, jsonAttributes); - } - - Operation operation = ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, - buildTypedPath(currentGenericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias), Expressions.constant("$.v[0]")); - Expression expression = Expressions.asComparable(operation).loe(buildTypedExpression(currentGenericFilter)); - - return ConvertedExpression.build(expression, jsonAttributes); + if (SupportedDbType.POSTGRESQL == this.dbType) { + return buildPostgreSqlMultivaluedComparisionExpression(tableMapping, jsonAttributes, + currentGenericFilter, type, columnExpression); + } else { + if (currentGenericFilter.getMultiValuedCount() > 1) { + Collection expressions = new ArrayList<>(currentGenericFilter.getMultiValuedCount()); + for (int i = 0; i < currentGenericFilter.getMultiValuedCount(); i++) { + Operation operation = ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, + columnExpression, Expressions.constant("$.v[" + i + "]")); + Predicate predicate = Expressions.asComparable(operation).loe(buildTypedExpression(tableMapping, currentGenericFilter)); + + expressions.add(predicate); + } + + Expression expression = ExpressionUtils.anyOf(expressions); + + return ConvertedExpression.build(expression, jsonAttributes); + } + + Operation operation = ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, + columnExpression, Expressions.constant("$.v[0]")); + Expression expression = Expressions.asComparable(operation).loe(buildTypedExpression(tableMapping, currentGenericFilter)); + + return ConvertedExpression.build(expression, jsonAttributes); + } } else { - return ConvertedExpression.build(Expressions.asComparable(buildTypedPath(currentGenericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias)).loe(buildTypedExpression(currentGenericFilter)), jsonAttributes); + return ConvertedExpression.build(Expressions.asComparable(columnExpression).loe(buildTypedExpression(tableMapping, currentGenericFilter)), jsonAttributes); } } if (FilterType.GREATER_OR_EQUAL == type) { if (multiValued) { - if (currentGenericFilter.getMultiValuedCount() > 1) { - Collection expressions = new ArrayList<>(currentGenericFilter.getMultiValuedCount()); - for (int i = 0; i < currentGenericFilter.getMultiValuedCount(); i++) { - Operation operation = ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, - buildTypedPath(currentGenericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias), Expressions.constant("$.v[" + i + "]")); - Predicate predicate = Expressions.asComparable(operation).goe(buildTypedExpression(currentGenericFilter)); - - expressions.add(predicate); - } - Expression expression = ExpressionUtils.anyOf(expressions); - - return ConvertedExpression.build(expression, jsonAttributes); - } - - Operation operation = ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, - buildTypedPath(currentGenericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias), Expressions.constant("$.v[0]")); - Expression expression = Expressions.asComparable(operation).goe(buildTypedExpression(currentGenericFilter)); - - return ConvertedExpression.build(expression, jsonAttributes); + if (SupportedDbType.POSTGRESQL == this.dbType) { + return buildPostgreSqlMultivaluedComparisionExpression(tableMapping, jsonAttributes, + currentGenericFilter, type, columnExpression); + } else { + if (currentGenericFilter.getMultiValuedCount() > 1) { + Collection expressions = new ArrayList<>(currentGenericFilter.getMultiValuedCount()); + for (int i = 0; i < currentGenericFilter.getMultiValuedCount(); i++) { + Operation operation = ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, + columnExpression, Expressions.constant("$.v[" + i + "]")); + Predicate predicate = Expressions.asComparable(operation).goe(buildTypedExpression(tableMapping, currentGenericFilter)); + + expressions.add(predicate); + } + Expression expression = ExpressionUtils.anyOf(expressions); + + return ConvertedExpression.build(expression, jsonAttributes); + } + + Operation operation = ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, + columnExpression, Expressions.constant("$.v[0]")); + Expression expression = Expressions.asComparable(operation).goe(buildTypedExpression(tableMapping, currentGenericFilter)); + + return ConvertedExpression.build(expression, jsonAttributes); + } } else { - return ConvertedExpression.build(Expressions.asComparable(buildTypedPath(currentGenericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias)).goe(buildTypedExpression(currentGenericFilter)), jsonAttributes); + return ConvertedExpression.build(Expressions.asComparable(columnExpression).goe(buildTypedExpression(tableMapping, currentGenericFilter)), jsonAttributes); } } if (FilterType.PRESENCE == type) { Expression expression; if (multiValued) { - if (currentGenericFilter.getMultiValuedCount() > 1) { - Collection expressions = new ArrayList<>(currentGenericFilter.getMultiValuedCount()); - for (int i = 0; i < currentGenericFilter.getMultiValuedCount(); i++) { - Predicate predicate = ExpressionUtils.isNotNull(ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, - buildTypedPath(currentGenericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias), Expressions.constant("$.v[" + i + "]"))); - expressions.add(predicate); - } - Predicate predicate = ExpressionUtils.anyOf(expressions); - - return ConvertedExpression.build(predicate, jsonAttributes); - } - - expression = ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, - buildTypedPath(currentGenericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias), Expressions.constant("$.v[0]")); + if (SupportedDbType.POSTGRESQL == this.dbType) { + Operation operation = ExpressionUtils.predicate(SqlOps.PGSQL_JSON_NOT_EMPTY_ARRAY, + columnExpression); + return ConvertedExpression.build(operation, jsonAttributes); + } else { + if (currentGenericFilter.getMultiValuedCount() > 1) { + Collection expressions = new ArrayList<>(currentGenericFilter.getMultiValuedCount()); + for (int i = 0; i < currentGenericFilter.getMultiValuedCount(); i++) { + Predicate predicate = ExpressionUtils.isNotNull(ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, + columnExpression, Expressions.constant("$.v[" + i + "]"))); + expressions.add(predicate); + } + Predicate predicate = ExpressionUtils.anyOf(expressions); + + return ConvertedExpression.build(predicate, jsonAttributes); + } + + expression = ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, + columnExpression, Expressions.constant("$.v[0]")); + } } else { - expression = buildTypedPath(currentGenericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias); + expression = columnExpression; } return ConvertedExpression.build(ExpressionUtils.isNotNull(expression), jsonAttributes); @@ -292,36 +315,51 @@ private ConvertedExpression convertToSqlFilterImpl(Filter genericFilter, Map 1) { - Collection expressions = new ArrayList<>(currentGenericFilter.getMultiValuedCount()); - for (int i = 0; i < currentGenericFilter.getMultiValuedCount(); i++) { - Operation operation = ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, - buildTypedPath(currentGenericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias), Expressions.constant("$.v[" + i + "]")); - Predicate predicate = Expressions.booleanOperation(Ops.LIKE, operation, Expressions.constant(like.toString())); - - expressions.add(predicate); - } - Predicate predicate = ExpressionUtils.anyOf(expressions); - - return ConvertedExpression.build(predicate, jsonAttributes); - } - - expression = ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, - buildTypedPath(currentGenericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias), Expressions.constant("$.v[0]")); + if (SupportedDbType.POSTGRESQL == this.dbType) { + return buildPostgreSqlMultivaluedComparisionExpression(tableMapping, jsonAttributes, + currentGenericFilter, type, columnExpression); + } else { + if (currentGenericFilter.getMultiValuedCount() > 1) { + Collection expressions = new ArrayList<>(currentGenericFilter.getMultiValuedCount()); + for (int i = 0; i < currentGenericFilter.getMultiValuedCount(); i++) { + Operation operation = ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, + columnExpression, Expressions.constant("$.v[" + i + "]")); + Predicate predicate = Expressions.booleanOperation(Ops.LIKE, operation, Expressions.constant(like.toString())); + + expressions.add(predicate); + } + Predicate predicate = ExpressionUtils.anyOf(expressions); + + return ConvertedExpression.build(predicate, jsonAttributes); + } + + expression = ExpressionUtils.predicate(SqlOps.JSON_EXTRACT, + columnExpression, Expressions.constant("$.v[0]")); + } } else { - expression = buildTypedPath(currentGenericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias); + expression = columnExpression; } return ConvertedExpression.build(Expressions.booleanOperation(Ops.LIKE, expression, Expressions.constant(like.toString())), jsonAttributes); } if (FilterType.LOWERCASE == type) { - return ConvertedExpression.build(ExpressionUtils.toLower(buildTypedPath(currentGenericFilter, propertiesAnnotationsMap, jsonAttributes, processor, skipAlias)), jsonAttributes); + return ConvertedExpression.build(ExpressionUtils.toLower(columnExpression), jsonAttributes); } throw new SearchException(String.format("Unknown filter type '%s'", type)); } + private ConvertedExpression buildPostgreSqlMultivaluedComparisionExpression(TableMapping tableMapping, + Map> jsonAttributes, Filter currentGenericFilter, FilterType type, + Expression columnExpression) throws SearchException { + Operation operation = ExpressionUtils.predicate(SqlOps.PGSQL_JSON_NOT_EMPTY_ARRAY, + ExpressionUtils.predicate(SqlOps.PGSQL_JSON_PATH_QUERY_ARRAY, + columnExpression, Expressions.constant(type.getSign()), + Expressions.constant(prepareTypedArrayExpressionValue(tableMapping, currentGenericFilter)))); + return ConvertedExpression.build(operation, jsonAttributes); + } + protected Boolean isMultiValue(Filter currentGenericFilter, Map propertiesAnnotationsMap) { Boolean isMultiValuedDetected = determineMultiValuedByType(currentGenericFilter.getAttributeName(), propertiesAnnotationsMap); if (Boolean.TRUE.equals(currentGenericFilter.getMultiValued()) || Boolean.TRUE.equals(isMultiValuedDetected)) { @@ -331,6 +369,21 @@ protected Boolean isMultiValue(Filter currentGenericFilter, Map propertiesAnnotationsMap, + private Expression buildTypedPath(TableMapping tableMapping, Filter genericFilter, Map propertiesAnnotationsMap, Map> jsonAttributes, Function processor, boolean skipAlias) throws SearchException { boolean hasSubFilters = ArrayHelper.isNotEmpty(genericFilter.getFilters()); if (hasSubFilters) { - return convertToSqlFilterImpl(genericFilter.getFilters()[0], propertiesAnnotationsMap, jsonAttributes, processor, skipAlias).expression(); + return convertToSqlFilterImpl(tableMapping, genericFilter.getFilters()[0], propertiesAnnotationsMap, jsonAttributes, processor, skipAlias).expression(); } String internalAttribute = toInternalAttribute(genericFilter); - return buildTypedPath(genericFilter, internalAttribute, skipAlias); + return buildTypedPath(tableMapping, genericFilter, internalAttribute, skipAlias); } - private Expression buildTypedPath(Filter filter, String attributeName, boolean skipAlias) { + private Expression buildTypedPath(TableMapping tableMapping, Filter filter, String attributeName, boolean skipAlias) throws SearchException { + AttributeType attributeType = getAttributeType(tableMapping, filter.getAttributeName()); + if (attributeType == null) { + if (tableMapping != null) { + throw new SearchException(String.format(String.format("Failed to find attribute type for '%s'", filter.getAttributeName()))); + } + } + + if ((attributeType != null) && SqlOperationService.TIMESTAMP.equals(attributeType.getType())) { + if (skipAlias) { + return Expressions.dateTimePath(Date.class, attributeName); + } else { + return Expressions.dateTimePath(Date.class, dateDocAlias, attributeName); + } + } if (filter.getAssertionValue() instanceof String) { if (skipAlias) { return Expressions.stringPath(attributeName); diff --git a/jans-orm/sql/src/main/java/io/jans/orm/sql/impl/SqlOps.java b/jans-orm/sql/src/main/java/io/jans/orm/sql/impl/SqlOps.java index e4e9f3e64dd..ab82728c233 100644 --- a/jans-orm/sql/src/main/java/io/jans/orm/sql/impl/SqlOps.java +++ b/jans-orm/sql/src/main/java/io/jans/orm/sql/impl/SqlOps.java @@ -5,7 +5,10 @@ public enum SqlOps implements Operator { JSON_CONTAINS(Object.class), - JSON_EXTRACT(Object.class); + JSON_EXTRACT(Object.class), + PGSQL_JSON_CONTAINS(Object.class), + PGSQL_JSON_PATH_QUERY_ARRAY(Object.class), + PGSQL_JSON_NOT_EMPTY_ARRAY(Object.class); private final Class type; diff --git a/jans-orm/sql/src/main/java/io/jans/orm/sql/model/JsonString.java b/jans-orm/sql/src/main/java/io/jans/orm/sql/model/JsonString.java new file mode 100644 index 00000000000..cdf51abd1f8 --- /dev/null +++ b/jans-orm/sql/src/main/java/io/jans/orm/sql/model/JsonString.java @@ -0,0 +1,25 @@ +package io.jans.orm.sql.model; + +/** + * PostgreSQL JSON column type support + * + * @author Yuriy Movchan Date: 09/01/2022 + */ + +public class JsonString { + + private String value; + + public JsonString(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + +} diff --git a/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/SqlOperationService.java b/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/SqlOperationService.java index 47bf6211d34..9b9dfdc69e9 100644 --- a/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/SqlOperationService.java +++ b/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/SqlOperationService.java @@ -9,6 +9,7 @@ import java.sql.Connection; import java.sql.DatabaseMetaData; import java.util.Collection; +import java.util.Date; import java.util.List; import com.querydsl.core.types.OrderSpecifier; @@ -28,6 +29,7 @@ import io.jans.orm.sql.impl.SqlBatchOperationWraper; import io.jans.orm.sql.model.ConvertedExpression; import io.jans.orm.sql.model.SearchReturnDataType; +import io.jans.orm.sql.model.TableMapping; import io.jans.orm.sql.operation.impl.SqlConnectionProvider; /** @@ -37,6 +39,11 @@ */ public interface SqlOperationService extends PersistenceOperationService { + String JSON_TYPE_NAME = "json"; + String JSONB_TYPE_NAME = "jsonb"; + String LONGTEXT_TYPE_NAME = "longtext"; + String TIMESTAMP = "timestamp"; + static String DN = "dn"; static String UID = "uid"; static String[] UID_ARRAY = new String[] { "uid" }; @@ -44,10 +51,11 @@ public interface SqlOperationService extends PersistenceOperationService { static String OBJECT_CLASS = "objectClass"; static String DOC_ALIAS = "doc"; + static String DOC_INNER_ALIAS = "doc_inner"; static String ID = "id"; static String DOC_ID = "doc_id"; - public static final String SQL_DATA_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; + public static final String SQL_DATA_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS"; public static final Object[] NO_OBJECTS = new Object[0]; SqlConnectionProvider getConnectionProvider(); @@ -92,4 +100,9 @@ PagedResult search(String key, String objectClass, ConvertedExpre boolean isJsonColumn(String tableName, String attributeType); + TableMapping getTabeMapping(String key, String objectClass); + + String encodeTime(Date date); + Date decodeTime(String date, boolean silent); + } diff --git a/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/SupportedDbType.java b/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/SupportedDbType.java new file mode 100644 index 00000000000..67326872e99 --- /dev/null +++ b/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/SupportedDbType.java @@ -0,0 +1,39 @@ +package io.jans.orm.sql.operation; + +import java.util.HashMap; +import java.util.Map; + +/** + * Supported SQL DB types + * + * @author Yuriy Movchan Date: 09/16/2022 + */ +public enum SupportedDbType { + + MYSQL("mysql"), + POSTGRESQL("postgresql"); + + private String dbType; + + private static Map MAP_BY_VALUES = new HashMap(); + + static { + for (SupportedDbType enumType : values()) { + MAP_BY_VALUES.put(enumType.dbType, enumType); + } + } + + SupportedDbType(String dbType) { + this.dbType = dbType; + } + + public static SupportedDbType resolveDbType(String dbType) { + return MAP_BY_VALUES.get(dbType); + } + + @Override + public String toString() { + return dbType; + } + +} diff --git a/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/impl/SqlConnectionProvider.java b/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/impl/SqlConnectionProvider.java index e708111b696..4a718ceed38 100644 --- a/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/impl/SqlConnectionProvider.java +++ b/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/impl/SqlConnectionProvider.java @@ -30,7 +30,6 @@ import org.slf4j.LoggerFactory; import com.querydsl.sql.Configuration; -import com.querydsl.sql.MySQLTemplates; import com.querydsl.sql.SQLQueryFactory; import com.querydsl.sql.SQLTemplates; import com.querydsl.sql.SQLTemplatesRegistry; @@ -40,9 +39,12 @@ import io.jans.orm.exception.operation.ConnectionException; import io.jans.orm.model.AttributeType; import io.jans.orm.operation.auth.PasswordEncryptionMethod; -import io.jans.orm.sql.dsl.template.SqlJsonMySQLTemplates; +import io.jans.orm.sql.dsl.template.MySQLJsonTemplates; +import io.jans.orm.sql.dsl.template.PostgreSQLJsonTemplates; import io.jans.orm.sql.model.ResultCode; import io.jans.orm.sql.model.TableMapping; +import io.jans.orm.sql.operation.SqlOperationService; +import io.jans.orm.sql.operation.SupportedDbType; import io.jans.orm.util.ArrayHelper; import io.jans.orm.util.PropertiesHelper; import io.jans.orm.util.StringHelper; @@ -54,12 +56,9 @@ */ public class SqlConnectionProvider { - protected static final String JSON_TYPE_NAME = "json"; - protected static final String LONGTEXT_TYPE_NAME = "longtext"; - private static final Logger LOG = LoggerFactory.getLogger(SqlConnectionProvider.class); - private static final String QUERY_ENGINE_TYPE = + private static final String MYSQL_QUERY_ENGINE_TYPE = "SELECT TABLE_NAME, ENGINE FROM information_schema.tables WHERE table_schema = ?"; private static final String DRIVER_PROPERTIES_PREFIX = "connection.driver-property"; @@ -78,7 +77,7 @@ public class SqlConnectionProvider { private PasswordEncryptionMethod passwordEncryptionMethod; - private String dbType; + private SupportedDbType dbType; private String dbVersion; private boolean mariaDb = false; @@ -195,7 +194,13 @@ protected void init() throws Exception { try (Connection con = this.poolingDataSource.getConnection()) { DatabaseMetaData databaseMetaData = con.getMetaData(); - this.dbType = databaseMetaData.getDatabaseProductName().toLowerCase(); + String dbTypeString = databaseMetaData.getDatabaseProductName(); + this.dbType = SupportedDbType.resolveDbType(dbTypeString.toLowerCase()); + + if (this.dbType == null) { + throw new ConnectionException(String.format("Database type '%s' is not supported", dbTypeString)); + } + this.dbVersion = databaseMetaData.getDatabaseProductVersion().toLowerCase(); if ((this.dbVersion != null) && this.dbVersion.toLowerCase().contains("mariadb")) { this.mariaDb = true; @@ -212,26 +217,28 @@ protected void init() throws Exception { private void loadTableMetaData(DatabaseMetaData databaseMetaData, Connection con) throws SQLException { long takes = System.currentTimeMillis(); - LOG.info("Detecting engine types..."); - PreparedStatement preparedStatement = con.prepareStatement(QUERY_ENGINE_TYPE); - preparedStatement.setString(1, schemaName); - - try (ResultSet tableEnginesResultSet = preparedStatement.executeQuery()) { - while (tableEnginesResultSet.next()) { - String tableName = tableEnginesResultSet.getString("TABLE_NAME"); - String engineName = tableEnginesResultSet.getString("ENGINE"); + if (SupportedDbType.MYSQL == dbType) { + LOG.info("Detecting engine types..."); + PreparedStatement preparedStatement = con.prepareStatement(MYSQL_QUERY_ENGINE_TYPE); + preparedStatement.setString(1, schemaName); - tableEnginesMap.put(tableName, engineName); + try (ResultSet tableEnginesResultSet = preparedStatement.executeQuery()) { + while (tableEnginesResultSet.next()) { + String tableName = tableEnginesResultSet.getString("TABLE_NAME"); + String engineName = tableEnginesResultSet.getString("ENGINE"); + + tableEnginesMap.put(tableName, engineName); + } } - } - + } + LOG.info("Scanning DB metadata..."); try (ResultSet tableResultSet = databaseMetaData.getTables(null, schemaName, null, new String[]{"TABLE"})) { while (tableResultSet.next()) { String tableName = tableResultSet.getString("TABLE_NAME"); Map tableColumns = new HashMap<>(); -// String engineType = tableEnginesMap.get(tableName); + // String engineType = tableEnginesMap.get(tableName); LOG.debug("Found table: '{}'.", tableName); try (ResultSet columnResultSet = databaseMetaData.getColumns(null, schemaName, tableName, null)) { @@ -240,11 +247,15 @@ private void loadTableMetaData(DatabaseMetaData databaseMetaData, Connection con String columnTypeName = columnResultSet.getString("TYPE_NAME").toLowerCase(); String remark = columnResultSet.getString("REMARKS"); - if (mariaDb && LONGTEXT_TYPE_NAME.equalsIgnoreCase(columnTypeName) && JSON_TYPE_NAME.equalsIgnoreCase(remark)) { - columnTypeName = JSON_TYPE_NAME; + if (mariaDb && SqlOperationService.LONGTEXT_TYPE_NAME.equalsIgnoreCase(columnTypeName) && SqlOperationService.JSON_TYPE_NAME.equalsIgnoreCase(remark)) { + columnTypeName = SqlOperationService.JSON_TYPE_NAME; + } + + if (SqlOperationService.JSONB_TYPE_NAME.equalsIgnoreCase(columnTypeName)) { + columnTypeName = SqlOperationService.JSONB_TYPE_NAME; } - boolean multiValued = SqlConnectionProvider.JSON_TYPE_NAME.equals(columnTypeName); + boolean multiValued = SqlOperationService.JSON_TYPE_NAME.equals(columnTypeName) || SqlOperationService.JSONB_TYPE_NAME.equals(columnTypeName); AttributeType attributeType = new AttributeType(columnName, columnTypeName, multiValued); tableColumns.put(columnName, attributeType); @@ -264,8 +275,10 @@ private void initDsl() throws SQLException { try (Connection con = poolingDataSource.getConnection()) { DatabaseMetaData databaseMetaData = con.getMetaData(); SQLTemplates.Builder sqlBuilder = templatesRegistry.getBuilder(databaseMetaData); - if (sqlBuilder instanceof MySQLTemplates.Builder) { - sqlBuilder = SqlJsonMySQLTemplates.builder(); + if (SupportedDbType.MYSQL == dbType) { + sqlBuilder = MySQLJsonTemplates.builder(); + } else if (SupportedDbType.POSTGRESQL == dbType) { + sqlBuilder = PostgreSQLJsonTemplates.builder().quote(); } this.sqlTemplates = sqlBuilder.printSchema().build(); Configuration configuration = new Configuration(sqlTemplates); @@ -446,4 +459,8 @@ public boolean isMariaDb() { return mariaDb; } + public SupportedDbType getDbType() { + return dbType; + } + } diff --git a/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/impl/SqlOperationServiceImpl.java b/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/impl/SqlOperationServiceImpl.java index c3482f169b6..c75547a49e1 100644 --- a/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/impl/SqlOperationServiceImpl.java +++ b/jans-orm/sql/src/main/java/io/jans/orm/sql/operation/impl/SqlOperationServiceImpl.java @@ -12,11 +12,14 @@ import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.sql.Timestamp; +import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.Duration; import java.time.Instant; +import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Collection; +import java.util.Date; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -32,6 +35,7 @@ import com.querydsl.core.types.OrderSpecifier; import com.querydsl.core.types.Path; import com.querydsl.core.types.Predicate; +import com.querydsl.core.types.SubQueryExpression; import com.querydsl.core.types.dsl.Expressions; import com.querydsl.core.types.dsl.Wildcard; import com.querydsl.sql.RelationalPathBase; @@ -61,9 +65,11 @@ import io.jans.orm.sql.impl.SqlBatchOperationWraper; import io.jans.orm.sql.model.ConvertedExpression; import io.jans.orm.sql.model.JsonAttributeValue; +import io.jans.orm.sql.model.JsonString; import io.jans.orm.sql.model.SearchReturnDataType; import io.jans.orm.sql.model.TableMapping; import io.jans.orm.sql.operation.SqlOperationService; +import io.jans.orm.sql.operation.SupportedDbType; import io.jans.orm.sql.operation.watch.OperationDurationUtil; import io.jans.orm.util.ArrayHelper; import io.jans.orm.util.StringHelper; @@ -93,6 +99,7 @@ public class SqlOperationServiceImpl implements SqlOperationService { private String schemaName; private Path docAlias = ExpressionUtils.path(String.class, DOC_ALIAS); + private Path docInnerAlias = ExpressionUtils.path(String.class, DOC_INNER_ALIAS); @SuppressWarnings("unused") private SqlOperationServiceImpl() { @@ -182,13 +189,18 @@ private boolean addEntryImpl(TableMapping tableMapping, String key, Collection tableRelationalPath = buildTableRelationalPath(tableMapping); - SQLDeleteClause sqlDeleteQuery = this.sqlQueryFactory.delete(tableRelationalPath); + if ((count > 0) && (SupportedDbType.POSTGRESQL == connectionProvider.getDbType())) { + // Workaround because PostgreSQL not supports limit in delete request - Predicate exp = (Predicate) expression.expression(); - sqlDeleteQuery.where(exp); + // Inner query + RelationalPathBase innerTableRelationalPath = new RelationalPathBase<>(Object.class, DOC_INNER_ALIAS, this.schemaName, tableMapping.getTableName()); + SubQueryExpression sqlSelectQuery = this.sqlQueryFactory.select(Expressions.path(String.class, docInnerAlias, DOC_ID)).from(innerTableRelationalPath) + .where(exp).limit(count); - if (count > 0) { - sqlDeleteQuery = sqlDeleteQuery.limit(count); - } + Predicate deleteExp = ExpressionUtils.in(Expressions.stringPath(DOC_ID), sqlSelectQuery); + sqlDeleteQuery = this.sqlQueryFactory.delete(tableRelationalPath).where(deleteExp); + } else { + sqlDeleteQuery = this.sqlQueryFactory.delete(tableRelationalPath).where(exp); + if (count > 0) { + sqlDeleteQuery = sqlDeleteQuery.limit(count); + } + } long rowDeleted = sqlDeleteQuery.execute(); @@ -623,11 +650,11 @@ private List getAttributeDataList(TableMapping tableMapping, Resu attributeValueObjects = new Object[] { attributeObject }; } else if (attributeObject instanceof String) { Object value = attributeObject.toString(); - try { - SimpleDateFormat jsonDateFormat = new SimpleDateFormat(SQL_DATA_FORMAT); - value = jsonDateFormat.parse(attributeObject.toString()); - } catch (Exception ex) { + Date dateValue = decodeTime(attributeObject.toString(), true); + if (dateValue != null) { + value = dateValue; } + attributeValueObjects = new Object[] { value }; } else if (attributeObject instanceof Timestamp) { attributeValueObjects = new Object[] { @@ -715,6 +742,13 @@ public DatabaseMetaData getMetadata() { return connectionProvider.getDatabaseMetaData(); } + @Override + public TableMapping getTabeMapping(String key, String objectClass) { + TableMapping tableMapping = connectionProvider.getTableMappingByKey(key, objectClass); + + return tableMapping; + } + @Override public void setPersistenceExtension(PersistenceExtension persistenceExtension) { this.persistenceExtension = persistenceExtension; @@ -839,24 +873,73 @@ public String[] fromInternalAttributes(String[] internalAttributeNames) { // return resultAttributeNames; } - private String convertValueToDbJson(Object propertyValue) { + @Override + public String encodeTime(Date date) { + if (date == null) { + return null; + } + + SimpleDateFormat jsonDateFormat = new SimpleDateFormat(SqlOperationService.SQL_DATA_FORMAT); + return jsonDateFormat.format(date); + } + + @Override + public Date decodeTime(String date, boolean silent) { + if (StringHelper.isEmpty(date)) { + return null; + } + + // Add ending Z if necessary + String dateZ = date.endsWith("Z") ? date : date + "Z"; + try { + SimpleDateFormat jsonDateFormat = new SimpleDateFormat(SqlOperationService.SQL_DATA_FORMAT); + return jsonDateFormat.parse(date); + } catch (ParseException ex) { + try { + return new Date(Instant.parse(dateZ).toEpochMilli()); + } catch (DateTimeParseException ex2) { + if (!silent) { + LOG.error("Failed to decode generalized time '{}'", date, ex2); + } + + return null; + } + } + } + + private Object convertValueToDbJson(Object propertyValue) { try { -// String value = JSON_OBJECT_MAPPER.writeValueAsString(propertyValue); - - JsonAttributeValue attributeValue; - if (propertyValue == null) { - attributeValue = new JsonAttributeValue(); - } if (propertyValue instanceof List) { - attributeValue = new JsonAttributeValue(((List) propertyValue).toArray()); - } else if (propertyValue.getClass().isArray()) { - attributeValue = new JsonAttributeValue((Object[]) propertyValue); + if (SupportedDbType.POSTGRESQL == connectionProvider.getDbType()) { + Object[] attributeValue; + if (propertyValue == null) { + attributeValue = new Object[0]; + } if (propertyValue instanceof List) { + attributeValue = ((List) propertyValue).toArray(); + } else if (propertyValue.getClass().isArray()) { + attributeValue = (Object[]) propertyValue; + } else { + attributeValue = new Object[] { propertyValue }; + } + + String value = JSON_OBJECT_MAPPER.writeValueAsString(attributeValue); + + return new JsonString(value); } else { - attributeValue = new JsonAttributeValue(new Object[] { propertyValue }); - } + JsonAttributeValue attributeValue; + if (propertyValue == null) { + attributeValue = new JsonAttributeValue(); + } if (propertyValue instanceof List) { + attributeValue = new JsonAttributeValue(((List) propertyValue).toArray()); + } else if (propertyValue.getClass().isArray()) { + attributeValue = new JsonAttributeValue((Object[]) propertyValue); + } else { + attributeValue = new JsonAttributeValue(new Object[] { propertyValue }); + } - String value = JSON_OBJECT_MAPPER.writeValueAsString(attributeValue); + String value = JSON_OBJECT_MAPPER.writeValueAsString(attributeValue); - return value; + return value; + } } catch (Exception ex) { LOG.error("Failed to convert '{}' to json value:", propertyValue, ex); throw new MappingException(String.format("Failed to convert '%s' to json value", propertyValue)); @@ -866,15 +949,20 @@ private String convertValueToDbJson(Object propertyValue) { private Object[] convertDbJsonToValue(String jsonValue) { try { // Object[] values = JSON_OBJECT_MAPPER.readValue(jsonValue, Object[].class); + if (SupportedDbType.POSTGRESQL == connectionProvider.getDbType()) { + Object[] values = JSON_OBJECT_MAPPER.readValue(jsonValue, Object[].class); - JsonAttributeValue attributeValue = JSON_OBJECT_MAPPER.readValue(jsonValue, JsonAttributeValue.class); - - Object[] values = null; - if (attributeValue != null) { - values = attributeValue.getValues(); + return values; + } else { + JsonAttributeValue attributeValue = JSON_OBJECT_MAPPER.readValue(jsonValue, JsonAttributeValue.class); + + Object[] values = null; + if (attributeValue != null) { + values = attributeValue.getValues(); + } + + return values; } - - return values; } catch (Exception ex) { LOG.error("Failed to convert json value '{}' to array:", jsonValue, ex); throw new MappingException(String.format("Failed to convert json value '%s' to array", jsonValue)); @@ -886,7 +974,7 @@ public boolean isJsonColumn(String tableName, String columnTypeName) { return false; } - if (mariaDb && SqlConnectionProvider.LONGTEXT_TYPE_NAME.equals(columnTypeName)) { + if (mariaDb && SqlOperationService.LONGTEXT_TYPE_NAME.equals(columnTypeName)) { return true; } @@ -895,8 +983,12 @@ public boolean isJsonColumn(String tableName, String columnTypeName) { // return "longtext".equals(columnTypeName); // } - return SqlConnectionProvider.JSON_TYPE_NAME.equals(columnTypeName); + return SqlOperationService.JSON_TYPE_NAME.equals(columnTypeName) || SqlOperationService.JSONB_TYPE_NAME.equals(columnTypeName); } + private AttributeType getAttributeType(Map columTypes, AttributeData attribute) { + return columTypes.get(attribute.getName().toLowerCase()); + } + } diff --git a/jans-orm/sql/src/test/java/io/jans/orm/sql/impl/test/SqlFilterConverterCheckExcludeFilterTest.java b/jans-orm/sql/src/test/java/io/jans/orm/sql/impl/test/SqlFilterConverterCheckExcludeFilterTest.java index da788eb21fc..dba37ea1c19 100644 --- a/jans-orm/sql/src/test/java/io/jans/orm/sql/impl/test/SqlFilterConverterCheckExcludeFilterTest.java +++ b/jans-orm/sql/src/test/java/io/jans/orm/sql/impl/test/SqlFilterConverterCheckExcludeFilterTest.java @@ -22,9 +22,11 @@ import io.jans.orm.exception.operation.SearchException; import io.jans.orm.search.filter.Filter; import io.jans.orm.search.filter.FilterProcessor; -import io.jans.orm.sql.dsl.template.SqlJsonMySQLTemplates; +import io.jans.orm.sql.dsl.template.MySQLJsonTemplates; import io.jans.orm.sql.impl.SqlFilterConverter; import io.jans.orm.sql.model.ConvertedExpression; +import io.jans.orm.sql.operation.impl.SqlConnectionProvider; +import io.jans.orm.sql.operation.impl.SqlOperationServiceImpl; @SuppressWarnings({ "rawtypes", "unchecked"}) public class SqlFilterConverterCheckExcludeFilterTest { @@ -40,7 +42,7 @@ public class SqlFilterConverterCheckExcludeFilterTest { @BeforeClass public void init() { - this.simpleConverter = new SqlFilterConverter(null); + this.simpleConverter = new SqlFilterConverter(new SqlOperationServiceImpl(null, new SqlConnectionProvider(null))); this.filterProcessor = new FilterProcessor(); this.tablePath = ExpressionUtils.path(Object.class, "table"); this.docAlias = ExpressionUtils.path(Object.class, "doc"); @@ -48,7 +50,7 @@ public void init() { this.allPath = Expressions.stringPath(docAlias, "*"); // this.sqlTemplates = MySQLTemplates.builder().printSchema().build(); - this.sqlTemplates = SqlJsonMySQLTemplates.builder().printSchema().build(); + this.sqlTemplates = MySQLJsonTemplates.builder().printSchema().build(); this.configuration = new Configuration(sqlTemplates); } @@ -63,21 +65,21 @@ public void checkObjectClassExcludeFilter() throws SearchException { Filter orFilter = Filter.createANDFilter(filterEq1, filterEq2, filterEq3, andFilter, filterEq4); Filter filter1 = Filter.createANDFilter(filterEq3, orFilter); - ConvertedExpression expression1 = simpleConverter.convertToSqlFilter(filter1, null, null); + ConvertedExpression expression1 = simpleConverter.convertToSqlFilter(null, filter1, null); String query1 = toSelectSQL(expression1); assertEquals(query1, "select doc.`*` from `table` as doc where doc.objectClass = 'jansPerson' and (doc.uid = 'test' and lower(doc.uid) = 'test' and doc.objectClass = 'jansPerson' and (doc.uid = 'test' and lower(doc.uid) = 'test' and doc.objectClass = 'jansPerson' and JSON_CONTAINS(doc.added->'$.v', CAST('[\"2020-12-16T14:58:18.398\"]' AS JSON))) and JSON_CONTAINS(doc.added->'$.v', CAST('[\"2020-12-16T14:58:18.398\"]' AS JSON)))"); Filter filter2 = filterProcessor.excludeFilter(filter1, filterEq3); - ConvertedExpression expression2 = simpleConverter.convertToSqlFilter(filter2, null, null); + ConvertedExpression expression2 = simpleConverter.convertToSqlFilter(null, filter2, null); String query2 = toSelectSQL(expression2); assertEquals(query2, "select doc.`*` from `table` as doc where doc.uid = 'test' and lower(doc.uid) = 'test' and (doc.uid = 'test' and lower(doc.uid) = 'test' and JSON_CONTAINS(doc.added->'$.v', CAST('[\"2020-12-16T14:58:18.398\"]' AS JSON))) and JSON_CONTAINS(doc.added->'$.v', CAST('[\"2020-12-16T14:58:18.398\"]' AS JSON))"); Filter filter3 = filterProcessor.excludeFilter(filter1, Filter.createEqualityFilter("objectClass", null)); - ConvertedExpression expression3 = simpleConverter.convertToSqlFilter(filter3, null, null); + ConvertedExpression expression3 = simpleConverter.convertToSqlFilter(null, filter3, null); String query3 = toSelectSQL(expression3); assertEquals(query3, "select doc.`*` from `table` as doc where doc.uid = 'test' and lower(doc.uid) = 'test' and (doc.uid = 'test' and lower(doc.uid) = 'test' and JSON_CONTAINS(doc.added->'$.v', CAST('[\"2020-12-16T14:58:18.398\"]' AS JSON))) and JSON_CONTAINS(doc.added->'$.v', CAST('[\"2020-12-16T14:58:18.398\"]' AS JSON))"); diff --git a/jans-orm/sql/src/test/java/io/jans/orm/sql/impl/test/SqlFilterConverterTest.java b/jans-orm/sql/src/test/java/io/jans/orm/sql/impl/test/SqlFilterConverterTest.java index b4e4f21c1a5..d11811c9dad 100644 --- a/jans-orm/sql/src/test/java/io/jans/orm/sql/impl/test/SqlFilterConverterTest.java +++ b/jans-orm/sql/src/test/java/io/jans/orm/sql/impl/test/SqlFilterConverterTest.java @@ -21,9 +21,11 @@ import io.jans.orm.exception.operation.SearchException; import io.jans.orm.search.filter.Filter; -import io.jans.orm.sql.dsl.template.SqlJsonMySQLTemplates; +import io.jans.orm.sql.dsl.template.MySQLJsonTemplates; import io.jans.orm.sql.impl.SqlFilterConverter; import io.jans.orm.sql.model.ConvertedExpression; +import io.jans.orm.sql.operation.impl.SqlConnectionProvider; +import io.jans.orm.sql.operation.impl.SqlOperationServiceImpl; @SuppressWarnings({ "rawtypes", "unchecked"}) public class SqlFilterConverterTest { @@ -38,14 +40,14 @@ public class SqlFilterConverterTest { @BeforeClass public void init() { - this.simpleConverter = new SqlFilterConverter(null); + this.simpleConverter = new SqlFilterConverter(new SqlOperationServiceImpl(null, new SqlConnectionProvider(null))); this.tablePath = ExpressionUtils.path(Object.class, "table"); this.docAlias = ExpressionUtils.path(Object.class, "doc"); this.tableAlieasPath = Expressions.as(tablePath, docAlias); this.allPath = Expressions.stringPath(docAlias, "*"); // this.sqlTemplates = MySQLTemplates.builder().printSchema().build(); - this.sqlTemplates = SqlJsonMySQLTemplates.builder().printSchema().build(); + this.sqlTemplates = MySQLJsonTemplates.builder().printSchema().build(); this.configuration = new Configuration(sqlTemplates); } @@ -53,52 +55,52 @@ public void init() { public void checkEqFilters() throws SearchException { // EQ -- String Filter filterEq1 = Filter.createEqualityFilter("uid", "test"); - ConvertedExpression expressionEq1 = simpleConverter.convertToSqlFilter(filterEq1, null, null); + ConvertedExpression expressionEq1 = simpleConverter.convertToSqlFilter(null, filterEq1, null); String queryEq1 = toSelectSQL(expressionEq1); assertEquals(queryEq1, "select doc.`*` from `table` as doc where doc.uid = 'test'"); // EQ -- Integer Filter filterEq2 = Filter.createEqualityFilter("age", 23); - ConvertedExpression expressionEq2 = simpleConverter.convertToSqlFilter(filterEq2, null, null); + ConvertedExpression expressionEq2 = simpleConverter.convertToSqlFilter(null, filterEq2, null); String queryEq2 = toSelectSQL(expressionEq2); assertEquals(queryEq2, "select doc.`*` from `table` as doc where doc.age = 23"); // EQ -- Long Filter filterEq3 = Filter.createEqualityFilter("age", 23L); - ConvertedExpression expressionEq3 = simpleConverter.convertToSqlFilter(filterEq3, null, null); + ConvertedExpression expressionEq3 = simpleConverter.convertToSqlFilter(null, filterEq3, null); String queryEq3 = toSelectSQL(expressionEq3); assertEquals(queryEq3, "select doc.`*` from `table` as doc where doc.age = 23"); // EQ -- Date Filter filterEq4 = Filter.createEqualityFilter("added", getUtcDateFromMillis(1608130698398L)); - ConvertedExpression expressionEq4 = simpleConverter.convertToSqlFilter(filterEq4, null, null); + ConvertedExpression expressionEq4 = simpleConverter.convertToSqlFilter(null, filterEq4, null); String queryEq4 = toSelectSQL(expressionEq4); - assertEquals(queryEq4, "select doc.`*` from `table` as doc where doc.added = '2020-12-16T14:58:18.398'"); + assertEquals(queryEq4, "select doc.`*` from `table` as doc where doc.added = (timestamp '2020-12-16 17:58:18')"); } @Test public void checkMultivaluedEqFilters() throws SearchException { // EQ -- String Filter filterEq1 = Filter.createEqualityFilter("uid", "test").multiValued(); - ConvertedExpression expressionEq1 = simpleConverter.convertToSqlFilter(filterEq1, null, null); + ConvertedExpression expressionEq1 = simpleConverter.convertToSqlFilter(null, filterEq1, null); String queryEq1 = toSelectSQL(expressionEq1); assertEquals(queryEq1, "select doc.`*` from `table` as doc where JSON_CONTAINS(doc.uid->'$.v', CAST('[\"test\"]' AS JSON))"); // EQ -- Integer Filter filterEq2 = Filter.createEqualityFilter("age", 23).multiValued(); - ConvertedExpression expressionEq2 = simpleConverter.convertToSqlFilter(filterEq2, null, null); + ConvertedExpression expressionEq2 = simpleConverter.convertToSqlFilter(null, filterEq2, null); String queryEq2 = toSelectSQL(expressionEq2); assertEquals(queryEq2, "select doc.`*` from `table` as doc where JSON_CONTAINS(doc.age->'$.v', CAST('[23]' AS JSON))"); // EQ -- Long Filter filterEq3 = Filter.createEqualityFilter("age", 23L).multiValued(); - ConvertedExpression expressionEq3 = simpleConverter.convertToSqlFilter(filterEq3, null, null); + ConvertedExpression expressionEq3 = simpleConverter.convertToSqlFilter(null, filterEq3, null); String queryEq3 = toSelectSQL(expressionEq3); assertEquals(queryEq3, "select doc.`*` from `table` as doc where JSON_CONTAINS(doc.age->'$.v', CAST('[23]' AS JSON))"); @@ -107,7 +109,7 @@ public void checkMultivaluedEqFilters() throws SearchException { // EQ -- Date Filter filterEq4 = Filter.createEqualityFilter("added", getUtcDateFromMillis(1608130698398L)).multiValued(); - ConvertedExpression expressionEq4 = simpleConverter.convertToSqlFilter(filterEq4, null, null); + ConvertedExpression expressionEq4 = simpleConverter.convertToSqlFilter(null, filterEq4, null); String queryEq4 = toSelectSQL(expressionEq4); assertEquals(queryEq4, "select doc.`*` from `table` as doc where JSON_CONTAINS(doc.added->'$.v', CAST('[\"2020-12-16T14:58:18.398\"]' AS JSON))"); @@ -117,66 +119,66 @@ public void checkMultivaluedEqFilters() throws SearchException { public void checkLeFilters() throws SearchException { // LE -- String Filter filterLe1 = Filter.createLessOrEqualFilter("uid", "test"); - ConvertedExpression expressionLe1 = simpleConverter.convertToSqlFilter(filterLe1, null, null); + ConvertedExpression expressionLe1 = simpleConverter.convertToSqlFilter(null, filterLe1, null); String queryLe1 = toSelectSQL(expressionLe1); assertEquals(queryLe1, "select doc.`*` from `table` as doc where doc.uid <= 'test'"); // LE -- Integer Filter filterLe2 = Filter.createLessOrEqualFilter("age", 23); - ConvertedExpression expressionLe2 = simpleConverter.convertToSqlFilter(filterLe2, null, null); + ConvertedExpression expressionLe2 = simpleConverter.convertToSqlFilter(null, filterLe2, null); String queryLe2 = toSelectSQL(expressionLe2); assertEquals(queryLe2, "select doc.`*` from `table` as doc where doc.age <= 23"); // LE -- Long Filter filterLe3 = Filter.createLessOrEqualFilter("age", 23L); - ConvertedExpression expressionLe3 = simpleConverter.convertToSqlFilter(filterLe3, null, null); + ConvertedExpression expressionLe3 = simpleConverter.convertToSqlFilter(null, filterLe3, null); String queryLe3 = toSelectSQL(expressionLe3); assertEquals(queryLe3, "select doc.`*` from `table` as doc where doc.age <= 23"); // LE -- Date Filter filterLe4 = Filter.createLessOrEqualFilter("added", getUtcDateFromMillis(1608130698398L)); - ConvertedExpression expressionLe4 = simpleConverter.convertToSqlFilter(filterLe4, null, null); + ConvertedExpression expressionLe4 = simpleConverter.convertToSqlFilter(null, filterLe4, null); String queryLe4 = toSelectSQL(expressionLe4); - assertEquals(queryLe4, "select doc.`*` from `table` as doc where doc.added <= '2020-12-16T14:58:18.398'"); + assertEquals(queryLe4, "select doc.`*` from `table` as doc where doc.added <= (timestamp '2020-12-16 17:58:18')"); } @Test public void checkMultivaluedLeFilters() throws SearchException { // LE -- String Filter filterLe1 = Filter.createLessOrEqualFilter("uid", "test").multiValued(); - ConvertedExpression expressionLe1 = simpleConverter.convertToSqlFilter(filterLe1, null, null); + ConvertedExpression expressionLe1 = simpleConverter.convertToSqlFilter(null, filterLe1, null); String queryLe1 = toSelectSQL(expressionLe1); assertEquals(queryLe1, "select doc.`*` from `table` as doc where doc.uid->'$.v[0]' <= '[\"test\"]'"); // LE -- Integer Filter filterLe2 = Filter.createLessOrEqualFilter("age", 23).multiValued(); - ConvertedExpression expressionLe2 = simpleConverter.convertToSqlFilter(filterLe2, null, null); + ConvertedExpression expressionLe2 = simpleConverter.convertToSqlFilter(null, filterLe2, null); String queryLe2 = toSelectSQL(expressionLe2); assertEquals(queryLe2, "select doc.`*` from `table` as doc where doc.age->'$.v[0]' <= '[23]'"); // LE -- Long Filter filterLe3 = Filter.createLessOrEqualFilter("age", 23L).multiValued(); - ConvertedExpression expressionLe3 = simpleConverter.convertToSqlFilter(filterLe3, null, null); + ConvertedExpression expressionLe3 = simpleConverter.convertToSqlFilter(null, filterLe3, null); String queryLe3 = toSelectSQL(expressionLe3); assertEquals(queryLe3, "select doc.`*` from `table` as doc where doc.age->'$.v[0]' <= '[23]'"); // LE -- Date Filter filterLe4 = Filter.createLessOrEqualFilter("added", getUtcDateFromMillis(1608130698398L)).multiValued(); - ConvertedExpression expressionLe4 = simpleConverter.convertToSqlFilter(filterLe4, null, null); + ConvertedExpression expressionLe4 = simpleConverter.convertToSqlFilter(null, filterLe4, null); String queryLe4 = toSelectSQL(expressionLe4); assertEquals(queryLe4, "select doc.`*` from `table` as doc where doc.added->'$.v[0]' <= '[\"2020-12-16T14:58:18.398\"]'"); // LE -- Date Filter filterLe5 = Filter.createLessOrEqualFilter("added", getUtcDateFromMillis(1608130698398L)).multiValued(3); - ConvertedExpression expressionLe5 = simpleConverter.convertToSqlFilter(filterLe5, null, null); + ConvertedExpression expressionLe5 = simpleConverter.convertToSqlFilter(null, filterLe5, null); String queryLe5 = toSelectSQL(expressionLe5); assertEquals(queryLe5, "select doc.`*` from `table` as doc where doc.added->'$.v[0]' <= '[\"2020-12-16T14:58:18.398\"]' or doc.added->'$.v[1]' <= '[\"2020-12-16T14:58:18.398\"]' or doc.added->'$.v[2]' <= '[\"2020-12-16T14:58:18.398\"]'"); @@ -186,66 +188,66 @@ public void checkMultivaluedLeFilters() throws SearchException { public void checkGeFilters() throws SearchException { // LE -- String Filter filterGe1 = Filter.createGreaterOrEqualFilter("uid", "test"); - ConvertedExpression expressionGe1 = simpleConverter.convertToSqlFilter(filterGe1, null, null); + ConvertedExpression expressionGe1 = simpleConverter.convertToSqlFilter(null, filterGe1, null); String queryGe1 = toSelectSQL(expressionGe1); assertEquals(queryGe1, "select doc.`*` from `table` as doc where doc.uid >= 'test'"); // LE -- Integer Filter filterGe2 = Filter.createGreaterOrEqualFilter("age", 23); - ConvertedExpression expressionGe2 = simpleConverter.convertToSqlFilter(filterGe2, null, null); + ConvertedExpression expressionGe2 = simpleConverter.convertToSqlFilter(null, filterGe2, null); String queryGe2 = toSelectSQL(expressionGe2); assertEquals(queryGe2, "select doc.`*` from `table` as doc where doc.age >= 23"); // LE -- Long Filter filterGe3 = Filter.createGreaterOrEqualFilter("age", 23L); - ConvertedExpression expressionGe3 = simpleConverter.convertToSqlFilter(filterGe3, null, null); + ConvertedExpression expressionGe3 = simpleConverter.convertToSqlFilter(null, filterGe3, null); String queryGe3 = toSelectSQL(expressionGe3); assertEquals(queryGe3, "select doc.`*` from `table` as doc where doc.age >= 23"); // LE -- Date Filter filterGe4 = Filter.createGreaterOrEqualFilter("added", getUtcDateFromMillis(1608130698398L)); - ConvertedExpression expressionGe4 = simpleConverter.convertToSqlFilter(filterGe4, null, null); + ConvertedExpression expressionGe4 = simpleConverter.convertToSqlFilter(null, filterGe4, null); String queryGe4 = toSelectSQL(expressionGe4); - assertEquals(queryGe4, "select doc.`*` from `table` as doc where doc.added >= '2020-12-16T14:58:18.398'"); + assertEquals(queryGe4, "select doc.`*` from `table` as doc where doc.added >= (timestamp '2020-12-16 17:58:18')"); } @Test public void checkMultivaluedGeFilters() throws SearchException { // GE -- String Filter filterGe1 = Filter.createGreaterOrEqualFilter("uid", "test").multiValued(); - ConvertedExpression expressionGe1 = simpleConverter.convertToSqlFilter(filterGe1, null, null); + ConvertedExpression expressionGe1 = simpleConverter.convertToSqlFilter(null, filterGe1, null); String queryGe1 = toSelectSQL(expressionGe1); assertEquals(queryGe1, "select doc.`*` from `table` as doc where doc.uid->'$.v[0]' >= '[\"test\"]'"); // GE -- Integer Filter filterGe2 = Filter.createGreaterOrEqualFilter("age", 23).multiValued(); - ConvertedExpression expressionGe2 = simpleConverter.convertToSqlFilter(filterGe2, null, null); + ConvertedExpression expressionGe2 = simpleConverter.convertToSqlFilter(null, filterGe2, null); String queryGe2 = toSelectSQL(expressionGe2); assertEquals(queryGe2, "select doc.`*` from `table` as doc where doc.age->'$.v[0]' >= '[23]'"); // GE -- Long Filter filterGe3 = Filter.createGreaterOrEqualFilter("age", 23L).multiValued(); - ConvertedExpression expressionGe3 = simpleConverter.convertToSqlFilter(filterGe3, null, null); + ConvertedExpression expressionGe3 = simpleConverter.convertToSqlFilter(null, filterGe3, null); String queryGe3 = toSelectSQL(expressionGe3); assertEquals(queryGe3, "select doc.`*` from `table` as doc where doc.age->'$.v[0]' >= '[23]'"); // GE -- Date Filter filterGe4 = Filter.createGreaterOrEqualFilter("added", getUtcDateFromMillis(1608130698398L)).multiValued(); - ConvertedExpression expressionGe4 = simpleConverter.convertToSqlFilter(filterGe4, null, null); + ConvertedExpression expressionGe4 = simpleConverter.convertToSqlFilter(null, filterGe4, null); String queryGe4 = toSelectSQL(expressionGe4); assertEquals(queryGe4, "select doc.`*` from `table` as doc where doc.added->'$.v[0]' >= '[\"2020-12-16T14:58:18.398\"]'"); // GE -- Date Filter filterGe5 = Filter.createGreaterOrEqualFilter("added", getUtcDateFromMillis(1608130698398L)).multiValued(3); - ConvertedExpression expressionGe5 = simpleConverter.convertToSqlFilter(filterGe5, null, null); + ConvertedExpression expressionGe5 = simpleConverter.convertToSqlFilter(null, filterGe5, null); String queryGe5 = toSelectSQL(expressionGe5); assertEquals(queryGe5, "select doc.`*` from `table` as doc where doc.added->'$.v[0]' >= '[\"2020-12-16T14:58:18.398\"]' or doc.added->'$.v[1]' >= '[\"2020-12-16T14:58:18.398\"]' or doc.added->'$.v[2]' >= '[\"2020-12-16T14:58:18.398\"]'"); @@ -255,7 +257,7 @@ public void checkMultivaluedGeFilters() throws SearchException { public void checkPresenceFilters() throws SearchException { // Presence -- String Filter filterPresence = Filter.createPresenceFilter("uid"); - ConvertedExpression expressionPresence = simpleConverter.convertToSqlFilter(filterPresence, null, null); + ConvertedExpression expressionPresence = simpleConverter.convertToSqlFilter(null, filterPresence, null); String queryPresence = toSelectSQL(expressionPresence); assertEquals(queryPresence, "select doc.`*` from `table` as doc where doc.uid is not null"); @@ -265,14 +267,14 @@ public void checkPresenceFilters() throws SearchException { public void checkMultivaluedPresenceFilters() throws SearchException { // Presence -- String Filter filterPresence1 = Filter.createPresenceFilter("uid").multiValued(); - ConvertedExpression expressionPresence1 = simpleConverter.convertToSqlFilter(filterPresence1, null, null); + ConvertedExpression expressionPresence1 = simpleConverter.convertToSqlFilter(null, filterPresence1, null); String queryPresence1 = toSelectSQL(expressionPresence1); assertEquals(queryPresence1, "select doc.`*` from `table` as doc where doc.uid->'$.v[0]' is not null"); // Presence -- String -- Multivalued = 3 Filter filterPresence2 = Filter.createPresenceFilter("uid").multiValued(3); - ConvertedExpression expressionPresence2 = simpleConverter.convertToSqlFilter(filterPresence2, null, null); + ConvertedExpression expressionPresence2 = simpleConverter.convertToSqlFilter(null, filterPresence2, null); String queryPresence2 = toSelectSQL(expressionPresence2); assertEquals(queryPresence2, "select doc.`*` from `table` as doc where doc.uid->'$.v[0]' is not null or doc.uid->'$.v[1]' is not null or doc.uid->'$.v[2]' is not null"); @@ -281,19 +283,19 @@ public void checkMultivaluedPresenceFilters() throws SearchException { @Test public void checkSubFilters() throws SearchException { Filter filterSub1 = Filter.createSubstringFilter("uid", null, new String[] { "test" }, null); - ConvertedExpression expressionSub1 = simpleConverter.convertToSqlFilter(filterSub1, null, null); + ConvertedExpression expressionSub1 = simpleConverter.convertToSqlFilter(null, filterSub1, null); String querySub1 = toSelectSQL(expressionSub1); assertEquals(querySub1, "select doc.`*` from `table` as doc where doc.uid like '%test%'"); Filter filterSub2 = Filter.createSubstringFilter("uid", "a", new String[] { "test" }, null); - ConvertedExpression expressionSub2 = simpleConverter.convertToSqlFilter(filterSub2, null, null); + ConvertedExpression expressionSub2 = simpleConverter.convertToSqlFilter(null, filterSub2, null); String querySub2 = toSelectSQL(expressionSub2); assertEquals(querySub2, "select doc.`*` from `table` as doc where doc.uid like 'a%test%'"); Filter filterSub3 = Filter.createSubstringFilter("uid", null, new String[] { "test" }, "z"); - ConvertedExpression expressionSub3 = simpleConverter.convertToSqlFilter(filterSub3, null, null); + ConvertedExpression expressionSub3 = simpleConverter.convertToSqlFilter(null, filterSub3, null); String querySub3 = toSelectSQL(expressionSub3); assertEquals(querySub3, "select doc.`*` from `table` as doc where doc.uid like '%test%z'"); @@ -302,25 +304,25 @@ public void checkSubFilters() throws SearchException { @Test public void checkMultivaluedSubFilters() throws SearchException { Filter filterSub1 = Filter.createSubstringFilter("uid", null, new String[] { "test" }, null).multiValued(); - ConvertedExpression expressionSub1 = simpleConverter.convertToSqlFilter(filterSub1, null, null); + ConvertedExpression expressionSub1 = simpleConverter.convertToSqlFilter(null, filterSub1, null); String querySub1 = toSelectSQL(expressionSub1); assertEquals(querySub1, "select doc.`*` from `table` as doc where doc.uid->'$.v[0]' like '%test%'"); Filter filterSub2 = Filter.createSubstringFilter("uid", "a", new String[] { "test" }, null).multiValued(); - ConvertedExpression expressionSub2 = simpleConverter.convertToSqlFilter(filterSub2, null, null); + ConvertedExpression expressionSub2 = simpleConverter.convertToSqlFilter(null, filterSub2, null); String querySub2 = toSelectSQL(expressionSub2); assertEquals(querySub2, "select doc.`*` from `table` as doc where doc.uid->'$.v[0]' like 'a%test%'"); Filter filterSub3 = Filter.createSubstringFilter("uid", null, new String[] { "test" }, "z").multiValued(); - ConvertedExpression expressionSub3 = simpleConverter.convertToSqlFilter(filterSub3, null, null); + ConvertedExpression expressionSub3 = simpleConverter.convertToSqlFilter(null, filterSub3, null); String querySub3 = toSelectSQL(expressionSub3); assertEquals(querySub3, "select doc.`*` from `table` as doc where doc.uid->'$.v[0]' like '%test%z'"); Filter filterSub4 = Filter.createSubstringFilter("uid", null, new String[] { "test" }, "z").multiValued(3); - ConvertedExpression expressionSub4 = simpleConverter.convertToSqlFilter(filterSub4, null, null); + ConvertedExpression expressionSub4 = simpleConverter.convertToSqlFilter(null, filterSub4, null); String querySub4 = toSelectSQL(expressionSub4); assertEquals(querySub4, "select doc.`*` from `table` as doc where doc.uid->'$.v[0]' like '%test%z' or doc.uid->'$.v[1]' like '%test%z' or doc.uid->'$.v[2]' like '%test%z'"); @@ -329,25 +331,25 @@ public void checkMultivaluedSubFilters() throws SearchException { @Test public void checkMultivaluedSubWithLowerFilters() throws SearchException { Filter filterSub1 = Filter.createSubstringFilter(Filter.createLowercaseFilter("uid"), null, new String[] { "test" }, null).multiValued(); - ConvertedExpression expressionSub1 = simpleConverter.convertToSqlFilter(filterSub1, null, null); + ConvertedExpression expressionSub1 = simpleConverter.convertToSqlFilter(null, filterSub1, null); String querySub1 = toSelectSQL(expressionSub1); assertEquals(querySub1, "select doc.`*` from `table` as doc where lower(doc.uid)->'$.v[0]' like '%test%'"); Filter filterSub2 = Filter.createSubstringFilter(Filter.createLowercaseFilter("uid"), "a", new String[] { "test" }, null).multiValued(); - ConvertedExpression expressionSub2 = simpleConverter.convertToSqlFilter(filterSub2, null, null); + ConvertedExpression expressionSub2 = simpleConverter.convertToSqlFilter(null, filterSub2, null); String querySub2 = toSelectSQL(expressionSub2); assertEquals(querySub2, "select doc.`*` from `table` as doc where lower(doc.uid)->'$.v[0]' like 'a%test%'"); Filter filterSub3 = Filter.createSubstringFilter(Filter.createLowercaseFilter("uid"), null, new String[] { "test" }, "z").multiValued(); - ConvertedExpression expressionSub3 = simpleConverter.convertToSqlFilter(filterSub3, null, null); + ConvertedExpression expressionSub3 = simpleConverter.convertToSqlFilter(null, filterSub3, null); String querySub3 = toSelectSQL(expressionSub3); assertEquals(querySub3, "select doc.`*` from `table` as doc where lower(doc.uid)->'$.v[0]' like '%test%z'"); Filter filterSub4 = Filter.createSubstringFilter(Filter.createLowercaseFilter("uid"), null, new String[] { "test" }, "z").multiValued(3); - ConvertedExpression expressionSub4 = simpleConverter.convertToSqlFilter(filterSub4, null, null); + ConvertedExpression expressionSub4 = simpleConverter.convertToSqlFilter(null, filterSub4, null); String querySub4 = toSelectSQL(expressionSub4); assertEquals(querySub4, "select doc.`*` from `table` as doc where lower(doc.uid)->'$.v[0]' like '%test%z' or lower(doc.uid)->'$.v[1]' like '%test%z' or lower(doc.uid)->'$.v[2]' like '%test%z'"); @@ -357,7 +359,7 @@ public void checkMultivaluedSubWithLowerFilters() throws SearchException { public void checkLowerFilters() throws SearchException { Filter userUidFilter1 = Filter.createEqualityFilter(Filter.createLowercaseFilter("uid"), "test"); - ConvertedExpression expressionUserUid1 = simpleConverter.convertToSqlFilter(userUidFilter1, null, null); + ConvertedExpression expressionUserUid1 = simpleConverter.convertToSqlFilter(null, userUidFilter1, null); String queryUserUid1 = toSelectSQL(expressionUserUid1); assertEquals(queryUserUid1, "select doc.`*` from `table` as doc where lower(doc.uid) = 'test'"); @@ -367,7 +369,7 @@ public void checkLowerFilters() throws SearchException { public void checkMultivaluedLowerFilters() throws SearchException { Filter userUidFilter = Filter.createEqualityFilter(Filter.createLowercaseFilter("uid"), "test").multiValued(); - ConvertedExpression expressionUserUid = simpleConverter.convertToSqlFilter(userUidFilter, null, null); + ConvertedExpression expressionUserUid = simpleConverter.convertToSqlFilter(null, userUidFilter, null); String queryUserUid = toSelectSQL(expressionUserUid); assertEquals(queryUserUid, "select doc.`*` from `table` as doc where JSON_CONTAINS(lower(doc.uid)->'$.v', CAST('[\"test\"]' AS JSON))"); @@ -377,14 +379,14 @@ public void checkMultivaluedLowerFilters() throws SearchException { public void checkNotFilters() throws SearchException { Filter notFilter1 = Filter.createNOTFilter(Filter.createLessOrEqualFilter("age", 23)); - ConvertedExpression expressionNot1 = simpleConverter.convertToSqlFilter(notFilter1, null, null); + ConvertedExpression expressionNot1 = simpleConverter.convertToSqlFilter(null, notFilter1, null); String queryUserUid1 = toSelectSQL(expressionNot1); assertEquals(queryUserUid1, "select doc.`*` from `table` as doc where not doc.age <= 23"); Filter notFilter2 = Filter.createNOTFilter(Filter.createANDFilter(Filter.createLessOrEqualFilter("age", 23), Filter.createGreaterOrEqualFilter("age", 25))); - ConvertedExpression expressionNot2 = simpleConverter.convertToSqlFilter(notFilter2, null, null); + ConvertedExpression expressionNot2 = simpleConverter.convertToSqlFilter(null, notFilter2, null); String queryUserUid2 = toSelectSQL(expressionNot2); assertEquals(queryUserUid2, "select doc.`*` from `table` as doc where not (doc.age <= 23 and doc.age >= 25)"); @@ -396,7 +398,7 @@ public void checkAndFilters() throws SearchException { Filter filterPresence1 = Filter.createPresenceFilter("mail"); Filter filterLe1 = Filter.createLessOrEqualFilter("age", 23); Filter filterAnd1 = Filter.createANDFilter(filterPresence1, filterEq1, filterLe1); - ConvertedExpression expressionAnd1 = simpleConverter.convertToSqlFilter(filterAnd1, null, null); + ConvertedExpression expressionAnd1 = simpleConverter.convertToSqlFilter(null, filterAnd1, null); String queryAnd1 = toSelectSQL(expressionAnd1); assertEquals(queryAnd1, "select doc.`*` from `table` as doc where doc.mail is not null and doc.uid = 'test' and doc.age <= 23"); @@ -408,7 +410,7 @@ public void checkOrFilters() throws SearchException { Filter filterPresence1 = Filter.createPresenceFilter("mail"); Filter filterLe1 = Filter.createLessOrEqualFilter("age", 23); Filter filterOr1 = Filter.createORFilter(filterPresence1, filterEq1, filterLe1); - ConvertedExpression expressionAnd1 = simpleConverter.convertToSqlFilter(filterOr1, null, null); + ConvertedExpression expressionAnd1 = simpleConverter.convertToSqlFilter(null, filterOr1, null); String queryAnd1 = toSelectSQL(expressionAnd1); assertEquals(queryAnd1, "select doc.`*` from `table` as doc where doc.mail is not null or doc.uid = 'test' or doc.age <= 23"); @@ -421,13 +423,13 @@ public void checkOrJoinFilters() throws SearchException { Filter filterEq2 = Filter.createEqualityFilter("uid", "test2"); Filter filterEq3 = Filter.createEqualityFilter("uid", "test3"); Filter filterOr1 = Filter.createORFilter(filterEq1, filterEq2, filterEq3).multiValued(false); - ConvertedExpression expressionOr1 = simpleConverter.convertToSqlFilter(filterOr1, null, null); + ConvertedExpression expressionOr1 = simpleConverter.convertToSqlFilter(null, filterOr1, null); String queryOr1 = toSelectSQL(expressionOr1); assertEquals(queryOr1, "select doc.`*` from `table` as doc where doc.uid in ('test', 'test2', 'test3')"); Filter filterOr2 = Filter.createORFilter(filterEq1, filterEq2, filterEq3); - ConvertedExpression expressionOr2 = simpleConverter.convertToSqlFilter(filterOr2, null, null); + ConvertedExpression expressionOr2 = simpleConverter.convertToSqlFilter(null, filterOr2, null); String queryOr2 = toSelectSQL(expressionOr2); assertEquals(queryOr2, "select doc.`*` from `table` as doc where doc.uid = 'test' or doc.uid = 'test2' or doc.uid = 'test3'"); @@ -452,7 +454,7 @@ public void checkOrWithLowerCaseFilter() throws SearchException { Filter typeFilter = Filter.createEqualityFilter("jansScrTyp", "person_authentication"); Filter filter = Filter.createANDFilter(searchFilter, typeFilter); - ConvertedExpression expression = simpleConverter.convertToSqlFilter(filter, null, null); + ConvertedExpression expression = simpleConverter.convertToSqlFilter(null, filter, null); String query = toSelectSQL(expression); assertEquals(query, "select doc.`*` from `table` as doc where (lower(doc.description) like '%test_value%' or lower(doc.displayName) like '%test_value%') and doc.jansScrTyp = 'person_authentication'"); }