From 1000c810369e11b79836275c5ed9f82a2140b15e Mon Sep 17 00:00:00 2001 From: SHAHROKH DAIJAVAD Date: Thu, 30 May 2024 11:04:30 -0700 Subject: [PATCH] Update filter_local.py --- transforms/universal/filter/spark/src/filter_local.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/transforms/universal/filter/spark/src/filter_local.py b/transforms/universal/filter/spark/src/filter_local.py index 10e31f23e..1050b478b 100644 --- a/transforms/universal/filter/spark/src/filter_local.py +++ b/transforms/universal/filter/spark/src/filter_local.py @@ -36,11 +36,6 @@ "ibmkenlm_docq_perplex_score < 230", ] -filter_criteria = [ - "date_acquired BETWEEN CAST(CAST('2023-07-04 05:00:00' AS timestamp) AS bigint) AND CAST(CAST('2023-07-08 05:00:00' AS timestamp) AS bigint)", - "title LIKE 'https://%'", -] - filter_logical_operator = "AND" filter_columns_to_drop = ["extra", "cluster"]