From bec76abeb2d3c9473daa079244b274ce684c2ed6 Mon Sep 17 00:00:00 2001 From: nkumar2 Date: Thu, 14 Nov 2024 10:08:36 +0000 Subject: [PATCH 1/2] add hikari max pool size connection property configuration in property files --- ebi_eva_internal_pyutils/spring_properties.py | 4 +++- tests/internal/test_spring_prop.py | 9 +++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ebi_eva_internal_pyutils/spring_properties.py b/ebi_eva_internal_pyutils/spring_properties.py index 882f171..3938791 100644 --- a/ebi_eva_internal_pyutils/spring_properties.py +++ b/ebi_eva_internal_pyutils/spring_properties.py @@ -83,7 +83,7 @@ def _count_stats_properties(self): 'eva.count-stats.password': counts_password } - def _common_properties(self, *, read_preference='primary', chunk_size=100): + def _common_properties(self, *, read_preference='primary', chunk_size=100, max_pool_size=2): """Properties common to all Spring pipelines""" props = { 'spring.datasource.driver-class-name': 'org.postgresql.Driver', @@ -98,6 +98,8 @@ def _common_properties(self, *, read_preference='primary', chunk_size=100): 'spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults': 'false', 'spring.jpa.database-platform': 'org.hibernate.dialect.PostgreSQL9Dialect', 'parameters.chunkSize': chunk_size, + + 'spring.datasource.hikari.maximum-pool-size': max_pool_size } merge = {**self._mongo_properties(), **self._count_stats_properties(), **props} return merge diff --git a/tests/internal/test_spring_prop.py b/tests/internal/test_spring_prop.py index 7e23462..0baeb0a 100644 --- a/tests/internal/test_spring_prop.py +++ b/tests/internal/test_spring_prop.py @@ -26,6 +26,7 @@ def test_get_remapping_extraction_properties(self): spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults=false spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL9Dialect +spring.datasource.hikari.maximum-pool-size=2 spring.datasource.url=jdbc:postgresql://host1.example.com:5432/accjtdb spring.datasource.username=accuser spring.datasource.password=accpassword @@ -65,6 +66,7 @@ def test_get_remapping_ingestion_properties(self): spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults=false spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL9Dialect +spring.datasource.hikari.maximum-pool-size=2 spring.datasource.url=jdbc:postgresql://host1.example.com:5432/accjtdb spring.datasource.username=accuser spring.datasource.password=accpassword @@ -103,6 +105,7 @@ def test_get_clustering_properties(self): spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults=false spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL9Dialect +spring.datasource.hikari.maximum-pool-size=2 spring.datasource.url=jdbc:postgresql://host1.example.com:5432/accjtdb spring.datasource.username=accuser spring.datasource.password=accpassword @@ -152,6 +155,7 @@ def test_get_accessioning_properties(self): spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults=false spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL9Dialect +spring.datasource.hikari.maximum-pool-size=2 spring.datasource.url=jdbc:postgresql://host1.example.com:5432/accjtdb spring.datasource.username=accuser spring.datasource.password=accpassword @@ -206,6 +210,7 @@ def test_get_variant_load_properties(self): spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults=false spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL9Dialect +spring.datasource.hikari.maximum-pool-size=2 spring.profiles.active=production,mongo spring.profiles.include=variant-writer-mongo,variant-annotation-mongo spring.data.mongodb.authentication-mechanism=SCRAM-SHA-1 @@ -272,6 +277,7 @@ def test_get_variant_load_properties_no_mongo_cred(self): spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults=false spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL9Dialect +spring.datasource.hikari.maximum-pool-size=2 spring.profiles.active=production,mongo spring.profiles.include=variant-writer-mongo,variant-annotation-mongo spring.data.mongodb.authentication-mechanism=SCRAM-SHA-1 @@ -337,6 +343,7 @@ def test_get_accession_import_properties(self): spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults=false spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL9Dialect +spring.datasource.hikari.maximum-pool-size=2 spring.profiles.active=production,mongo spring.profiles.include=variant-writer-mongo,variant-annotation-mongo spring.data.mongodb.authentication-mechanism=SCRAM-SHA-1 @@ -385,6 +392,7 @@ def test_get_accessioning_properties_with_none(self): spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults=false spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL9Dialect +spring.datasource.hikari.maximum-pool-size=2 spring.datasource.url=jdbc:postgresql://host1.example.com:5432/accjtdb spring.datasource.username=accuser spring.datasource.password=accpassword @@ -437,6 +445,7 @@ def test_get_release_properties(self): spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults=false spring.jpa.database-platform=org.hibernate.dialect.PostgreSQL9Dialect +spring.datasource.hikari.maximum-pool-size=2 spring.datasource.url=jdbc:postgresql://host1.example.com:5432/accjtdb spring.datasource.username=accuser spring.datasource.password=accpassword From 19c33f152e022d00e658451efdd187ef3634ec66 Mon Sep 17 00:00:00 2001 From: nitin-ebi <79518737+nitin-ebi@users.noreply.github.com> Date: Fri, 15 Nov 2024 09:20:48 +0000 Subject: [PATCH 2/2] Update ebi_eva_internal_pyutils/spring_properties.py Co-authored-by: April Shen --- ebi_eva_internal_pyutils/spring_properties.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ebi_eva_internal_pyutils/spring_properties.py b/ebi_eva_internal_pyutils/spring_properties.py index 3938791..05f3c18 100644 --- a/ebi_eva_internal_pyutils/spring_properties.py +++ b/ebi_eva_internal_pyutils/spring_properties.py @@ -98,7 +98,6 @@ def _common_properties(self, *, read_preference='primary', chunk_size=100, max_p 'spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults': 'false', 'spring.jpa.database-platform': 'org.hibernate.dialect.PostgreSQL9Dialect', 'parameters.chunkSize': chunk_size, - 'spring.datasource.hikari.maximum-pool-size': max_pool_size } merge = {**self._mongo_properties(), **self._count_stats_properties(), **props}