From e7e81f5ce5ca50c8fff25f99bc25b54aeed3f252 Mon Sep 17 00:00:00 2001 From: Thorsten Marx Date: Fri, 28 Jun 2024 21:47:02 +0200 Subject: [PATCH] #217 added most query methods --- .../github/thmarx/cms/api/db/ContentNode.java | 3 +- .../thmarx/cms/api/utils/FileUtils.java | 24 +- .../thmarx/cms/filesystem/FileSystem.java | 22 +- .../thmarx/cms/filesystem/MetaData.java | 27 ++ .../filesystem/metadata/AbstractMetaData.java | 40 +++ .../metadata/memory/MemoryMetaData.java | 38 ++- .../metadata/persistent/LuceneIndex.java | 65 +++-- .../metadata/persistent/LuceneQuery.java | 243 ++++++++++++++++++ .../persistent/PersistentMetaData.java | 64 ++++- .../metadata/persistent/utils/FlattenMap.java | 22 ++ .../thmarx/cms/filesystem/query/Filter.java | 3 - .../thmarx/cms/filesystem/query/Query.java | 3 +- .../cms/filesystem/query/QueryUtil.java | 4 +- .../filesystem/PresistentFileSystemTest.java | 34 ++- .../persistent/utils/FlattenMapTest.java | 24 +- .../src/test/resources/content/index.md | 2 +- 16 files changed, 561 insertions(+), 57 deletions(-) create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/AbstractMetaData.java create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneQuery.java diff --git a/cms-api/src/main/java/com/github/thmarx/cms/api/db/ContentNode.java b/cms-api/src/main/java/com/github/thmarx/cms/api/db/ContentNode.java index d84e1d57..5ccc4002 100644 --- a/cms-api/src/main/java/com/github/thmarx/cms/api/db/ContentNode.java +++ b/cms-api/src/main/java/com/github/thmarx/cms/api/db/ContentNode.java @@ -28,6 +28,7 @@ import com.github.thmarx.cms.api.feature.features.SitePropertiesFeature; import com.github.thmarx.cms.api.utils.MapUtil; import com.github.thmarx.cms.api.utils.SectionUtil; +import java.io.Serializable; import java.time.Instant; import java.time.LocalDate; import java.util.Date; @@ -40,7 +41,7 @@ * @author t.marx */ public record ContentNode(String uri, String name, Map data, - boolean directory, Map children, LocalDate lastmodified) { + boolean directory, Map children, LocalDate lastmodified) implements Serializable { public ContentNode(String uri, String name, Map data, boolean directory, Map children) { this(uri, name, data, directory, children, LocalDate.now()); diff --git a/cms-api/src/main/java/com/github/thmarx/cms/api/utils/FileUtils.java b/cms-api/src/main/java/com/github/thmarx/cms/api/utils/FileUtils.java index 895a0d7f..0d24c441 100644 --- a/cms-api/src/main/java/com/github/thmarx/cms/api/utils/FileUtils.java +++ b/cms-api/src/main/java/com/github/thmarx/cms/api/utils/FileUtils.java @@ -1,5 +1,27 @@ package com.github.thmarx.cms.api.utils; +/*- + * #%L + * cms-api + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -18,4 +40,4 @@ public static void deleteFolder(Path pathToBeDeleted) throws IOException { .map(Path::toFile) .forEach(File::delete); } -} \ No newline at end of file +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileSystem.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileSystem.java index 361bf50b..00a8a070 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileSystem.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileSystem.java @@ -25,6 +25,7 @@ import com.github.thmarx.cms.api.ModuleFileSystem; import com.github.thmarx.cms.api.Constants; import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.api.db.ContentQuery; import com.github.thmarx.cms.api.db.DBFileSystem; import com.github.thmarx.cms.api.eventbus.EventBus; import com.github.thmarx.cms.api.eventbus.events.ContentChangedEvent; @@ -33,6 +34,7 @@ import com.github.thmarx.cms.api.eventbus.events.ReIndexContentMetaDataEvent; import com.github.thmarx.cms.api.eventbus.events.TemplateChangedEvent; import com.github.thmarx.cms.api.utils.PathUtil; +import com.github.thmarx.cms.filesystem.metadata.AbstractMetaData; import com.github.thmarx.cms.filesystem.metadata.persistent.PersistentMetaData; import com.github.thmarx.cms.filesystem.query.Query; import java.io.IOException; @@ -79,22 +81,12 @@ public Path base () { return hostBaseDirectory; } - public Query query(final BiFunction nodeMapper) { - return new Query(new ArrayList<>(metaData.nodes().values()), metaData, nodeMapper); + public ContentQuery query(final BiFunction nodeMapper) { + return metaData.query(nodeMapper); } - public Query query(final String startURI, final BiFunction nodeMapper) { - - final String uri; - if (startURI.startsWith("/")) { - uri = startURI.substring(1); - } else { - uri = startURI; - } - - var nodes = metaData.nodes().values().stream().filter(node -> node.uri().startsWith(uri)).toList(); - - return new Query(nodes, metaData, nodeMapper); + public ContentQuery query(final String startURI, final BiFunction nodeMapper) { + return metaData.query(startURI, nodeMapper); } public boolean isVisible(final String uri) { @@ -103,7 +95,7 @@ public boolean isVisible(final String uri) { return false; } var n = node.get(); - return MemoryMetaData.isVisible(n); + return AbstractMetaData.isVisible(n); } @Override diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/MetaData.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/MetaData.java index b3fc22df..1b5de75d 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/MetaData.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/MetaData.java @@ -1,12 +1,36 @@ package com.github.thmarx.cms.filesystem; +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.api.db.ContentQuery; import com.github.thmarx.cms.filesystem.index.IndexProviding; import java.io.IOException; import java.time.LocalDate; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.BiFunction; /** * @@ -36,4 +60,7 @@ public enum Type { Map nodes(); Map tree(); + + ContentQuery query(final BiFunction nodeMapper); + ContentQuery query(final String startURI, final BiFunction nodeMapper); } diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/AbstractMetaData.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/AbstractMetaData.java new file mode 100644 index 00000000..5659fdb3 --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/AbstractMetaData.java @@ -0,0 +1,40 @@ +package com.github.thmarx.cms.filesystem.metadata; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +import com.github.thmarx.cms.api.db.ContentNode; + +/** + * + * @author t.marx + */ +public class AbstractMetaData { + public static boolean isVisible (ContentNode node) { + return node != null + // check if some parent is hidden + && !node.uri().startsWith(".") && !node.uri().contains("/.") + && node.isPublished() + && !node.isHidden() + && !node.isSection(); + } +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryMetaData.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryMetaData.java index 23510736..12129267 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryMetaData.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryMetaData.java @@ -24,12 +24,16 @@ import com.github.thmarx.cms.api.Constants; import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.api.db.ContentQuery; import com.github.thmarx.cms.filesystem.MetaData; import com.github.thmarx.cms.filesystem.index.IndexProviding; import com.github.thmarx.cms.filesystem.index.SecondaryIndex; +import com.github.thmarx.cms.filesystem.metadata.AbstractMetaData; +import com.github.thmarx.cms.filesystem.query.Query; import com.google.common.base.Strings; import java.io.IOException; import java.time.LocalDate; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -38,6 +42,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -46,7 +51,7 @@ * * @author t.marx */ -public class MemoryMetaData implements IndexProviding, MetaData { +public class MemoryMetaData extends AbstractMetaData implements IndexProviding, MetaData { private final ConcurrentMap nodes = new ConcurrentHashMap<>(); @@ -75,10 +80,12 @@ public void clear() { secondaryIndexes.clear(); } + @Override public ConcurrentMap nodes() { return new ConcurrentHashMap<>(nodes); } + @Override public ConcurrentMap tree() { return new ConcurrentHashMap<>(tree); } @@ -146,15 +153,6 @@ protected ContentNode mapToIndex(ContentNode node) { } } - public static boolean isVisible (ContentNode node) { - return node != null - // check if some parent is hidden - && !node.uri().startsWith(".") && !node.uri().contains("/.") - && node.isPublished() - && !node.isHidden() - && !node.isSection(); - } - @Override public Optional findFolder(String uri) { return getFolder(uri); @@ -225,6 +223,26 @@ public void open() throws IOException { @Override public void close() throws IOException { } + + @Override + public ContentQuery query(final BiFunction nodeMapper) { + return new Query(new ArrayList<>(nodes.values()), this, nodeMapper); + } + + @Override + public ContentQuery query(final String startURI, final BiFunction nodeMapper) { + + final String uri; + if (startURI.startsWith("/")) { + uri = startURI.substring(1); + } else { + uri = startURI; + } + + var filtered = nodes().values().stream().filter(node -> node.uri().startsWith(uri)).toList(); + + return new Query(filtered, this, nodeMapper); + } } diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneIndex.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneIndex.java index da45f139..92378cf4 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneIndex.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneIndex.java @@ -1,10 +1,35 @@ package com.github.thmarx.cms.filesystem.metadata.persistent; +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ import com.github.thmarx.cms.api.utils.FileUtils; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.Optional; +import lombok.extern.slf4j.Slf4j; import org.apache.lucene.analysis.core.KeywordAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexWriter; @@ -22,8 +47,9 @@ * * @author t.marx */ +@Slf4j public class LuceneIndex implements AutoCloseable { - + private Directory directory; private IndexWriter writer = null; @@ -34,45 +60,54 @@ public class LuceneIndex implements AutoCloseable { public void close() throws Exception { if (nrt_manager != null) { nrt_manager.close(); - + writer.commit(); writer.close(); directory.close(); } } - + public void commit() throws IOException { writer.flush(); writer.commit(); nrt_manager.maybeRefresh(); } - - void add (Document document) throws IOException { + + void add(Document document) throws IOException { writer.addDocument(document); commit(); } - - void update (Term term, Document document) throws IOException { + + void update(Term term, Document document) throws IOException { writer.updateDocument(term, document); commit(); } - - void delete (Query query) throws IOException { + + void delete(Query query) throws IOException { writer.deleteDocuments(query); commit(); } - - Optional query (Query query) throws IOException { + + List query(Query query) throws IOException { IndexSearcher searcher = nrt_manager.acquire(); try { - + var topDocs = searcher.search(query, Integer.MAX_VALUE); + + List result = new ArrayList<>(); + for (var scoreDoc : topDocs.scoreDocs) { + result.add(searcher.storedFields().document(scoreDoc.doc)); + } + + return result; + } catch (IOException e) { + log.error("", e); } finally { nrt_manager.release(searcher); } - return Optional.empty(); + return Collections.emptyList(); } - - public void open (Path path) throws IOException { + + public void open(Path path) throws IOException { if (Files.exists(path)) { FileUtils.deleteFolder(path); } diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneQuery.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneQuery.java new file mode 100644 index 00000000..0d643ba6 --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneQuery.java @@ -0,0 +1,243 @@ +package com.github.thmarx.cms.filesystem.metadata.persistent; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ +import com.github.thmarx.cms.api.Constants; +import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.api.db.ContentQuery; +import com.github.thmarx.cms.api.db.Page; +import com.github.thmarx.cms.filesystem.MetaData; +import com.github.thmarx.cms.filesystem.metadata.AbstractMetaData; +import com.github.thmarx.cms.filesystem.query.ExcerptMapperFunction; +import com.github.thmarx.cms.filesystem.query.QueryUtil; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.lucene.document.IntField; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; + +/** + * + * @author t.marx + */ +@Slf4j +@RequiredArgsConstructor +public class LuceneQuery implements ContentQuery { + + private final LuceneIndex index; + private final MetaData metaData; + private final ExcerptMapperFunction nodeMapper; + + private String contentType = Constants.DEFAULT_CONTENT_TYPE; + + private final BooleanQuery.Builder queryBuilder = new BooleanQuery.Builder(); + + @Override + public ContentQuery excerpt(long excerptLength) { + nodeMapper.setExcerpt((int) excerptLength); + return this; + } + + @Override + public Page page(long page, long size) { + return null; + } + + @Override + public List get() { + queryBuilder.add(new TermQuery(new Term("content.type", contentType)), BooleanClause.Occur.MUST); + + try { + var result = index.query(queryBuilder.build()); + + return result.stream() + .map(document -> document.get("_uri")) + .map(metaData::byUri) + .filter(Optional::isPresent) + .map(Optional::get) + .filter(node -> !node.isDirectory()) + .filter(AbstractMetaData::isVisible) + .map(nodeMapper).toList(); + } catch (IOException ex) { + Logger.getLogger(LuceneQuery.class.getName()).log(Level.SEVERE, null, ex); + } + + return Collections.emptyList(); + } + + @Override + public Map> groupby(String field) { + return Collections.emptyMap(); + } + + @Override + public Sort orderby(String field) { + return null; + } + + @Override + public ContentQuery json() { + this.contentType = Constants.ContentTypes.JSON; + return this; + } + + @Override + public ContentQuery html() { + this.contentType = Constants.ContentTypes.HTML; + return this; + } + + @Override + public ContentQuery contentType(String contentType) { + this.contentType = contentType; + return this; + } + + @Override + public ContentQuery where(String field, Object value) { + return where(field, QueryUtil.Operator.EQ, value); + } + + @Override + public ContentQuery where(String field, String operator, Object value) { + if (QueryUtil.isDefaultOperation(operator)) { + return where(field, QueryUtil.operator4String(operator), value); + } + throw new IllegalArgumentException("unknown operator " + operator); + } + + @Override + public ContentQuery whereContains(String field, Object value) { + return where(field, QueryUtil.Operator.CONTAINS, value); + } + + @Override + public ContentQuery whereNotContains(String field, Object value) { + return where(field, QueryUtil.Operator.CONTAINS_NOT, value); + } + + @Override + public ContentQuery whereIn(String field, Object... value) { + return where(field, QueryUtil.Operator.IN, value); + } + + @Override + public ContentQuery whereIn(String field, List value) { + return where(field, QueryUtil.Operator.IN, value); + } + + @Override + public ContentQuery whereNotIn(String field, Object... value) { + return where(field, QueryUtil.Operator.NOT_IN, value); + } + + @Override + public ContentQuery whereNotIn(String field, List value) { + return where(field, QueryUtil.Operator.NOT_IN, value); + } + + private ContentQuery where(final String field, final QueryUtil.Operator operator, final Object value) { + + switch (operator) { + case EQ -> + eq(field, value, BooleanClause.Occur.MUST); + case NOT_EQ -> + eq(field, value, BooleanClause.Occur.MUST_NOT); + case CONTAINS -> + contains(field, value, BooleanClause.Occur.MUST); + case CONTAINS_NOT -> + contains(field, value, BooleanClause.Occur.MUST_NOT); + case IN -> + in(field, value, BooleanClause.Occur.MUST); + case NOT_IN -> + in(field, value, BooleanClause.Occur.MUST_NOT); + } + + return this; + } + + private void eq(String field, Object value, BooleanClause.Occur occur) { + var query = toQuery(field, value); + if (query != null) { + queryBuilder.add(query, occur); + } + } + + private void contains(String field, Object value, BooleanClause.Occur occur) { + var query = toQuery(field, value); + if (query != null) { + queryBuilder.add(query, occur); + } + } + + private void in(String field, Object value, BooleanClause.Occur occur) { + if (value == null) { + log.warn("value is null"); + return; + } + if (!(value instanceof List || value.getClass().isArray())) { + log.warn("value is not of type list"); + return; + } + + BooleanQuery.Builder inBuilder = new BooleanQuery.Builder(); + + List listValues = Collections.emptyList(); + if (value instanceof List) { + listValues = (List) value; + } else if (value.getClass().isArray()) { + listValues = Arrays.asList((Object[]) value); + } + + listValues.forEach(item -> { + inBuilder.add(toQuery(field, item), BooleanClause.Occur.SHOULD); + }); + + var inQuery = inBuilder.build(); + if (!inQuery.clauses().isEmpty()) { + queryBuilder.add(inQuery, occur); + } + } + + private Query toQuery(final String field, final Object value) { + if (value instanceof String stringValue) { + return new TermQuery(new Term(field, stringValue)); + } else if (value instanceof Boolean booleanValue) { + return IntField.newExactQuery( + field, + booleanValue ? 1 : 0 + ); + } + return null; + } +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/PersistentMetaData.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/PersistentMetaData.java index b2a7caaa..ac44db0c 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/PersistentMetaData.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/PersistentMetaData.java @@ -1,11 +1,35 @@ package com.github.thmarx.cms.filesystem.metadata.persistent; +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ import com.github.thmarx.cms.api.Constants; import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.api.db.ContentQuery; import com.github.thmarx.cms.filesystem.MetaData; import com.github.thmarx.cms.filesystem.index.SecondaryIndex; import com.github.thmarx.cms.filesystem.metadata.persistent.utils.FlattenMap; import com.github.thmarx.cms.filesystem.metadata.memory.MemoryMetaData; +import com.github.thmarx.cms.filesystem.query.ExcerptMapperFunction; +import com.github.thmarx.cms.filesystem.query.Query; import com.google.common.base.Strings; import com.google.gson.Gson; import java.io.IOException; @@ -18,9 +42,8 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiFunction; import java.util.function.Function; -import java.util.logging.Level; -import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; import lombok.RequiredArgsConstructor; @@ -32,7 +55,6 @@ import org.apache.lucene.document.IntField; import org.apache.lucene.document.LongField; import org.apache.lucene.document.StringField; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.h2.mvstore.MVMap; import org.h2.mvstore.MVStore; @@ -100,10 +122,15 @@ public void addFile(String uri, Map data, LocalDate lastModified } Document document = new Document(); - document.add(new StringField("_uri", uri, Field.Store.NO)); + document.add(new StringField("_uri", uri, Field.Store.YES)); //document.add(new StringField("_source", GSON.toJson(node), Field.Store.NO)); addData(document, data); + +// if (document.getField("content.type") != null) { + document.add(new StringField("content.type", node.contentType(), Field.Store.NO)); +// } + try { this.index.add(document); } catch (IOException ex) { @@ -142,6 +169,14 @@ private void addValue(Document document, String name, Object value) { document.add(new FloatField(name, floatValue, Field.Store.NO)); case Double doubleValue -> document.add(new DoubleField(name, doubleValue, Field.Store.NO)); + case Boolean booleanValue -> + document.add( + new IntField( + name, + booleanValue ? 1 : 0, + Field.Store.NO + ) + ); case List listValue -> handleList(document, name, listValue); default -> { @@ -266,4 +301,25 @@ public SecondaryIndex getOrCreateIndex(String field, Function ContentQuery query(final BiFunction nodeMapper) { + return new LuceneQuery<>(this.index, this, new ExcerptMapperFunction<>(nodeMapper)); +// return new Query(new ArrayList<>(nodes.values()), this, nodeMapper); + } + + @Override + public ContentQuery query(final String startURI, final BiFunction nodeMapper) { + + final String uri; + if (startURI.startsWith("/")) { + uri = startURI.substring(1); + } else { + uri = startURI; + } + + var filtered = nodes().values().stream().filter(node -> node.uri().startsWith(uri)).toList(); + + return new Query(filtered, this, nodeMapper); + } + } diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/utils/FlattenMap.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/utils/FlattenMap.java index 339b302e..42df776e 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/utils/FlattenMap.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/utils/FlattenMap.java @@ -1,5 +1,27 @@ package com.github.thmarx.cms.filesystem.metadata.persistent.utils; +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + import java.util.HashMap; import java.util.Map; diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Filter.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Filter.java index 38e9f6fa..f5dd7394 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Filter.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Filter.java @@ -22,9 +22,6 @@ * #L% */ -import com.github.thmarx.cms.api.db.ContentNode; -import java.util.function.Predicate; - /** * * @author thmar diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Query.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Query.java index 66034fea..1fa60d95 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Query.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Query.java @@ -31,6 +31,7 @@ import static com.github.thmarx.cms.filesystem.query.QueryUtil.filteredWithIndex; import static com.github.thmarx.cms.filesystem.query.QueryUtil.sorted; import com.github.thmarx.cms.api.utils.NodeUtil; +import com.github.thmarx.cms.filesystem.metadata.AbstractMetaData; import java.util.Collection; import java.util.List; import java.util.Map; @@ -151,7 +152,7 @@ public List get() { return context.getNodes() .filter(NodeUtil.contentTypeFiler(context.getContentType())) .filter(node -> !node.isDirectory()) - .filter(MemoryMetaData::isVisible) + .filter(AbstractMetaData::isVisible) .map(context.getNodeMapper()) .toList(); } diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/QueryUtil.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/QueryUtil.java index 8fa5c809..74471c87 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/QueryUtil.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/QueryUtil.java @@ -58,7 +58,7 @@ public static enum Operator { LTE; } - private static Map filters = new HashMap<>(); + private final static Map filters = new HashMap<>(); static { filters.put(Operator.EQ, (node_value, value) -> Objects.equals(node_value, value)); @@ -89,7 +89,7 @@ public static enum Operator { }); } - private static List operations = List.of( + private static final List operations = List.of( "=" , "!=", ">", ">=", "<", "<=", "in", "not in", "contains", "not contains" ); diff --git a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/PresistentFileSystemTest.java b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/PresistentFileSystemTest.java index 5a6e3bc8..7bb35b52 100644 --- a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/PresistentFileSystemTest.java +++ b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/PresistentFileSystemTest.java @@ -22,6 +22,7 @@ * #L% */ import com.github.thmarx.cms.api.eventbus.EventBus; +import com.github.thmarx.cms.api.utils.FileUtils; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -40,6 +41,7 @@ public class PresistentFileSystemTest { static FileSystem fileSystem; + @BeforeAll static void setup() throws IOException { @@ -56,8 +58,12 @@ static void setup() throws IOException { } @AfterAll - static void shutdown () { + static void shutdown () throws IOException { fileSystem.shutdown(); + + if (Files.exists(Path.of("src/test/resources/data"))) { + FileUtils.deleteFolder(Path.of("src/test/resources/data")); + } } @Test @@ -71,11 +77,33 @@ public void test_seconday_index() throws IOException { @Test public void test_query() throws IOException { - var nodes = fileSystem.query((node, i) -> node).where("featured", true).get(); - Assertions.assertThat(nodes).hasSize(2); } + + @Test + public void test_query_in() throws IOException { + var nodes = fileSystem.query((node, i) -> node).whereIn("name", "test1", "test2").get(); + Assertions.assertThat(nodes).hasSize(2); + } + + @Test + public void test_query_not_in() throws IOException { + var nodes = fileSystem.query((node, i) -> node).whereNotIn("name", "test1", "test2").get(); + Assertions.assertThat(nodes).hasSize(1); + } + @Test + public void test_query_contains() throws IOException { + var nodes = fileSystem.query((node, i) -> node).whereContains("taxonomy.tags", "eins").get(); + Assertions.assertThat(nodes).hasSize(1); + } + + @Test + public void test_query_contains_not() throws IOException { + var nodes = fileSystem.query((node, i) -> node).whereNotContains("taxonomy.tags", "eins").get(); + Assertions.assertThat(nodes).hasSize(1); + } + @Test public void test_query_with_start_uri() throws IOException { diff --git a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/persistent/utils/FlattenMapTest.java b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/persistent/utils/FlattenMapTest.java index 8dfde067..2de6f391 100644 --- a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/persistent/utils/FlattenMapTest.java +++ b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/persistent/utils/FlattenMapTest.java @@ -1,5 +1,27 @@ package com.github.thmarx.cms.filesystem.persistent.utils; +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 - 2024 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + import com.github.thmarx.cms.filesystem.metadata.persistent.utils.FlattenMap; import org.junit.jupiter.api.Test; @@ -67,4 +89,4 @@ public void testFlattenMapWithSingleLevelMap() { // Überprüfen, ob die flache Map korrekt ist Assertions.assertThat(actualFlatMap).isEqualTo(expectedFlatMap); } -} \ No newline at end of file +} diff --git a/cms-filesystem/src/test/resources/content/index.md b/cms-filesystem/src/test/resources/content/index.md index 77ec3cb9..605899c6 100644 --- a/cms-filesystem/src/test/resources/content/index.md +++ b/cms-filesystem/src/test/resources/content/index.md @@ -1 +1 @@ -featured: true +featured: true \ No newline at end of file