feat: implement excludes for namespaces

api/src/main/java/marquez/MarquezApp.java

@@ -26,6 +26,8 @@
 import lombok.NonNull;
 import lombok.extern.slf4j.Slf4j;
 import marquez.api.filter.JobRedirectFilter;
+import marquez.api.filter.exclusions.ExclusionsConfig;
+import marquez.api.filter.exclusions.ExclusionsFilter;
 import marquez.cli.DbMigrationCommand;
 import marquez.cli.DbRetentionCommand;
 import marquez.cli.MetadataCommand;
@@ -141,7 +143,13 @@ public void run(@NonNull MarquezConfig config, @NonNull Environment env) {
     }
 
     // set namespaceFilter
-    Utils.setNamespaceFilter(config.getNamespaceFilter());
+    if (config.hasExcludingPatterns()) {
+      ExclusionsConfig exclusions = config.getExclude();
+      if (exclusions.namespaces.onRead) {
+        ExclusionsFilter.setNamespacesReadFilter(exclusions.namespaces.patterns);
+      }
+      ;
+    }
   }
 
   private boolean isSentryEnabled(MarquezConfig config) {

api/src/main/java/marquez/MarquezConfig.java

@@ -12,6 +12,7 @@
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
+import marquez.api.filter.exclusions.ExclusionsConfig;
 import marquez.db.FlywayFactory;
 import marquez.graphql.GraphqlConfig;
 import marquez.jobs.DbRetentionConfig;
@@ -49,11 +50,16 @@ public class MarquezConfig extends Configuration {
   private DbRetentionConfig dbRetention; // OPTIONAL
 
   @Getter
-  @JsonProperty("namespaceFilter")
-  private String namespaceFilter; // OPTIONAL
+  @JsonProperty("exclude")
+  private ExclusionsConfig exclude; // OPTIONAL
 
   /** Returns {@code true} if a data retention policy has been configured. */
   public boolean hasDbRetentionPolicy() {
     return (dbRetention != null);
   }
+
+  /** Returns {@code true} if an exclude pattern has been configured. */
+  public boolean hasExcludingPatterns() {
+    return (exclude != null);
+  }
 }

api/src/main/java/marquez/api/NamespaceResource.java

@@ -27,7 +27,7 @@
 import lombok.NonNull;
 import lombok.Value;
 import marquez.api.exceptions.NamespaceNotFoundException;
-import marquez.common.Utils;
+import marquez.api.filter.exclusions.ExclusionsFilter;
 import marquez.common.models.NamespaceName;
 import marquez.service.ServiceFactory;
 import marquez.service.models.Namespace;
@@ -75,16 +75,14 @@ public Response get(@PathParam("namespace") NamespaceName name) {
   public Response list(
       @QueryParam("limit") @DefaultValue("100") @Min(value = 0) int limit,
       @QueryParam("offset") @DefaultValue("0") @Min(value = 0) int offset) {
-    final String namespaceFilter = Utils.getNamespaceFilter();
+    final String namespaceFilter = ExclusionsFilter.getNamespacesReadFilter();
     final List<Namespace> allNamespaces = namespaceService.findAll(limit, offset);
-    // If the key: namespaceFilter is in the config file, exclude the corresponding namespaces
+
     if (namespaceFilter != null) {
       final List<Namespace> FilterNamespaces =
           namespaceService.findAllFilter(namespaceFilter, limit, offset);
       return Response.ok(new Namespaces(FilterNamespaces)).build();
-    }
-    // Else return all namespace
-    else {
+    } else {
       return Response.ok(new Namespaces(allNamespaces)).build();
     }
   }

api/src/main/java/marquez/api/filter/exclusions/ExclusionsConfig.java

@@ -0,0 +1,15 @@
+package marquez.api.filter.exclusions;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.Getter;
+
+public class ExclusionsConfig {
+  @Getter @JsonProperty public NamespaceExclusion namespaces;
+
+  public static class NamespaceExclusion {
+    @Getter @JsonProperty public boolean onRead;
+    @Getter @JsonProperty public boolean onWrite;
+    @Getter @JsonProperty public String patterns;
+  }
+  ;
+}

api/src/main/java/marquez/api/filter/exclusions/ExclusionsFilter.java

@@ -0,0 +1,13 @@
+package marquez.api.filter.exclusions;
+
+public class ExclusionsFilter {
+  private static String namespacesFilter;
+
+  public static void setNamespacesReadFilter(String filter) {
+    namespacesFilter = filter;
+  }
+
+  public static String getNamespacesReadFilter() {
+    return namespacesFilter;
+  }
+}

api/src/main/java/marquez/common/Utils.java

@@ -434,14 +434,4 @@ private <T> void setFields(
       }
     }
   }
-
-  private static String namespaceFilter;
-
-  public static void setNamespaceFilter(String filter) {
-    namespaceFilter = filter;
-  }
-
-  public static String getNamespaceFilter() {
-    return namespaceFilter;
-  }
 }

api/src/main/java/marquez/db/NamespaceDao.java

@@ -79,12 +79,7 @@ default Namespace upsertNamespaceMeta(
   List<Namespace> findAll(int limit, int offset);
 
   @SqlQuery(
-      """
-          SELECT * FROM namespaces
-          WHERE name !~ :excluded
-          ORDER BY name
-          LIMIT :limit OFFSET :offset
-      """)
+      "SELECT * FROM namespaces WHERE name !~ :excluded ORDER BY name LIMIT :limit OFFSET :offset")
   List<Namespace> findAllFilter(String excluded, int limit, int offset);
 
   @SqlQuery("UPDATE namespaces SET is_hidden=false WHERE name = :name RETURNING *")

api/src/test/java/marquez/db/NamespaceDaoTest.java

@@ -5,11 +5,14 @@
 
 package marquez.db;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
+import java.util.List;
 import marquez.common.models.NamespaceName;
 import marquez.common.models.OwnerName;
 import marquez.jdbi.MarquezJdbiExternalPostgresExtension;
+import marquez.service.models.Namespace;
 import marquez.service.models.NamespaceMeta;
 import org.jdbi.v3.core.Jdbi;
 import org.junit.jupiter.api.BeforeAll;
@@ -34,4 +37,20 @@ void testWriteAndReadNamespace() {
 
     assertTrue(namespaceDao.exists(namespaceName.getValue()));
   }
+
+  @Test
+  void testFindAllFilter() {
+    var namespaceName1 = NamespaceName.of("postgres://localhost:5432");
+    var namespaceMeta1 = new NamespaceMeta(new OwnerName("marquez"), null);
+    namespaceDao.upsertNamespaceMeta(namespaceName1, namespaceMeta1);
+
+    var namespaceName2 = NamespaceName.of("s3://bucket");
+    var namespaceMeta2 = new NamespaceMeta(new OwnerName("yannick"), null);
+    namespaceDao.upsertNamespaceMeta(namespaceName2, namespaceMeta2);
+
+    List<Namespace> namespaces = namespaceDao.findAllFilter("s3.*", 10, 0);
+
+    assertEquals(1, namespaces.size());
+    assertEquals(namespaceName1, namespaces.get(0).getName());
+  }
 }