From 86cce0500634995694745e47bb325b885c4dba18 Mon Sep 17 00:00:00 2001
From: David Lutterkort <lutter@watzmann.net>
Date: Thu, 4 Apr 2024 12:32:57 -0700
Subject: [PATCH] store: Avoid 'too many bind params' error in
 FindDerivedEntityQuery

The FindDerivedQuery would contain a clause `id not in (<lots of values>)`
where each entry in the list is a separate bind variable. If we have more
than 65k derived entities already loaded into memory, this would require
more bind variables than what Postgres supports. With this change, we pass
those values as one array and therefore only need one bind variable.
---
 store/postgres/src/relational.rs         |  2 ++
 store/postgres/src/relational_queries.rs | 22 +++++++++-------------
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/store/postgres/src/relational.rs b/store/postgres/src/relational.rs
index b9a6b409058..54a9c340024 100644
--- a/store/postgres/src/relational.rs
+++ b/store/postgres/src/relational.rs
@@ -520,6 +520,8 @@ impl Layout {
         excluded_keys: &Vec<EntityKey>,
     ) -> Result<BTreeMap<EntityKey, Entity>, StoreError> {
         let table = self.table_for_entity(&derived_query.entity_type)?;
+        let ids = excluded_keys.iter().map(|key| &key.entity_id).cloned();
+        let excluded_keys = IdList::try_from_iter(derived_query.entity_type.id_type()?, ids)?;
         let query = FindDerivedQuery::new(table, derived_query, block, excluded_keys);
 
         let mut entities = BTreeMap::new();
diff --git a/store/postgres/src/relational_queries.rs b/store/postgres/src/relational_queries.rs
index 80e775adda8..0cec85ff5a2 100644
--- a/store/postgres/src/relational_queries.rs
+++ b/store/postgres/src/relational_queries.rs
@@ -2170,7 +2170,7 @@ impl<'a, Conn> RunQueryDsl<Conn> for FindManyQuery<'a> {}
 pub struct FindDerivedQuery<'a> {
     table: &'a Table,
     derived_query: &'a DerivedEntityQuery,
-    excluded_keys: &'a Vec<EntityKey>,
+    excluded_keys: IdList,
     br_column: BlockRangeColumn<'a>,
 }
 
@@ -2179,7 +2179,7 @@ impl<'a> FindDerivedQuery<'a> {
         table: &'a Table,
         derived_query: &'a DerivedEntityQuery,
         block: BlockNumber,
-        excluded_keys: &'a Vec<EntityKey>,
+        excluded_keys: IdList,
     ) -> Self {
         let br_column = BlockRangeColumn::new(table, "e.", block);
         Self {
@@ -2211,18 +2211,14 @@ impl<'a> QueryFragment<Pg> for FindDerivedQuery<'a> {
         out.push_sql("  from ");
         out.push_sql(self.table.qualified_name.as_str());
         out.push_sql(" e\n where ");
-
+        // This clause with an empty array would filter out everything
         if self.excluded_keys.len() > 0 {
-            let primary_key = self.table.primary_key();
-            out.push_identifier(primary_key.name.as_str())?;
-            out.push_sql(" not in (");
-            for (i, value) in self.excluded_keys.iter().enumerate() {
-                if i > 0 {
-                    out.push_sql(", ");
-                }
-
-                value.entity_id.push_bind_param(&mut out)?;
-            }
+            out.push_identifier(&self.table.primary_key().name)?;
+            // For truly gigantic `excluded_keys` lists, this will be slow, and
+            // we should rewrite this query to use a CTE or a temp table to hold
+            // the excluded keys.
+            out.push_sql(" != any(");
+            self.excluded_keys.push_bind_param(&mut out)?;
             out.push_sql(") and ");
         }
         out.push_identifier(entity_field.to_snake_case().as_str())?;