From 1120858a3821b02836db8f71c3e0ee7db7f4d385 Mon Sep 17 00:00:00 2001 From: Thorsten Marx Date: Fri, 28 Jun 2024 12:43:27 +0200 Subject: [PATCH] #217 simple persistence --- .../thmarx/cms/api/utils/FileUtils.java | 21 ++ cms-filesystem/.gitignore | 1 + cms-filesystem/pom.xml | 14 + .../thmarx/cms/filesystem/FileSystem.java | 23 +- .../thmarx/cms/filesystem/MetaData.java | 205 +------------ .../cms/filesystem/index/SecondaryIndex.java | 2 - .../metadata/memory/MemoryMetaData.java | 230 +++++++++++++++ .../metadata/persistent/LuceneIndex.java | 91 ++++++ .../persistent/PersistentMetaData.java | 269 ++++++++++++++++++ .../metadata/persistent/utils/FlattenMap.java | 27 ++ .../thmarx/cms/filesystem/query/Query.java | 6 +- .../filesystem/PresistentFileSystemTest.java | 88 ++++++ .../persistent/utils/FlattenMapTest.java | 70 +++++ .../com/github/thmarx/cms/SectionsTest.java | 2 +- .../thmarx/cms/utils/NodeUtilNGTest.java | 2 +- pom.xml | 18 +- 16 files changed, 866 insertions(+), 203 deletions(-) create mode 100644 cms-api/src/main/java/com/github/thmarx/cms/api/utils/FileUtils.java create mode 100644 cms-filesystem/.gitignore create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryMetaData.java create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneIndex.java create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/PersistentMetaData.java create mode 100644 cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/utils/FlattenMap.java create mode 100644 cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/PresistentFileSystemTest.java create mode 100644 cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/persistent/utils/FlattenMapTest.java diff --git a/cms-api/src/main/java/com/github/thmarx/cms/api/utils/FileUtils.java b/cms-api/src/main/java/com/github/thmarx/cms/api/utils/FileUtils.java new file mode 100644 index 00000000..895a0d7f --- /dev/null +++ b/cms-api/src/main/java/com/github/thmarx/cms/api/utils/FileUtils.java @@ -0,0 +1,21 @@ +package com.github.thmarx.cms.api.utils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; + +/** + * + * @author thmar + */ +public class FileUtils { + + public static void deleteFolder(Path pathToBeDeleted) throws IOException { + Files.walk(pathToBeDeleted) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } +} \ No newline at end of file diff --git a/cms-filesystem/.gitignore b/cms-filesystem/.gitignore new file mode 100644 index 00000000..fb254a98 --- /dev/null +++ b/cms-filesystem/.gitignore @@ -0,0 +1 @@ +src/test/resources/data/ \ No newline at end of file diff --git a/cms-filesystem/pom.xml b/cms-filesystem/pom.xml index 91b2b592..9c464861 100644 --- a/cms-filesystem/pom.xml +++ b/cms-filesystem/pom.xml @@ -10,6 +10,20 @@ jar + + + org.apache.lucene + lucene-core + + + org.apache.lucene + lucene-analysis-common + + + com.h2database + h2-mvstore + + com.github.thmarx.cms cms-api diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileSystem.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileSystem.java index 81e8a820..361bf50b 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileSystem.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/FileSystem.java @@ -21,6 +21,7 @@ * . * #L% */ +import com.github.thmarx.cms.filesystem.metadata.memory.MemoryMetaData; import com.github.thmarx.cms.api.ModuleFileSystem; import com.github.thmarx.cms.api.Constants; import com.github.thmarx.cms.api.db.ContentNode; @@ -32,6 +33,7 @@ import com.github.thmarx.cms.api.eventbus.events.ReIndexContentMetaDataEvent; import com.github.thmarx.cms.api.eventbus.events.TemplateChangedEvent; import com.github.thmarx.cms.api.utils.PathUtil; +import com.github.thmarx.cms.filesystem.metadata.persistent.PersistentMetaData; import com.github.thmarx.cms.filesystem.query.Query; import java.io.IOException; import java.nio.charset.Charset; @@ -70,7 +72,7 @@ public class FileSystem implements ModuleFileSystem, DBFileSystem { private Path contentBase; @Getter - private final MetaData metaData = new MetaData(); + private MetaData metaData; @Override public Path base () { @@ -101,7 +103,7 @@ public boolean isVisible(final String uri) { return false; } var n = node.get(); - return MetaData.isVisible(n); + return MemoryMetaData.isVisible(n); } @Override @@ -147,7 +149,7 @@ public List loadLines(final Path file, final Charset charset) throws IOE public List listDirectories(final Path base, final String start) { var startPath = base.resolve(start); - String folder = PathUtil.toRelativePath(startPath, contentBase).toString(); + String folder = PathUtil.toRelativePath(startPath, contentBase); List nodes = new ArrayList<>(); @@ -191,7 +193,7 @@ public List listDirectories(final Path base, final String start) { public List listContent(final Path base, final String start) { var startPath = base.resolve(start); - String folder = PathUtil.toRelativePath(startPath, contentBase).toString(); + String folder = PathUtil.toRelativePath(startPath, contentBase); if ("".equals(folder)) { return metaData.listChildren(""); @@ -202,7 +204,7 @@ public List listContent(final Path base, final String start) { } public List listSections(final Path contentFile) { - String folder = PathUtil.toRelativePath(contentFile, contentBase).toString(); + String folder = PathUtil.toRelativePath(contentFile, contentBase); String filename = contentFile.getFileName().toString(); filename = filename.substring(0, filename.length() - 3); @@ -267,7 +269,18 @@ private void addOrUpdateMetaData(Path file) { } public void init() throws IOException { + init(MetaData.Type.MEMORY); + } + + public void init(MetaData.Type metaDataType) throws IOException { log.debug("init filesystem"); + + if (MetaData.Type.PERSISTENT.equals(metaDataType)) { + this.metaData = new PersistentMetaData(this.hostBaseDirectory); + this.metaData.open(); + } else { + this.metaData = new MemoryMetaData(); + } this.contentBase = resolve("content/"); var templateBase = resolve("templates/"); diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/MetaData.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/MetaData.java index bce8569e..b3fc22df 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/MetaData.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/MetaData.java @@ -1,214 +1,39 @@ package com.github.thmarx.cms.filesystem; -/*- - * #%L - * cms-server - * %% - * Copyright (C) 2023 Marx-Software - * %% - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public - * License along with this program. If not, see - * . - * #L% - */ - -import com.github.thmarx.cms.api.Constants; import com.github.thmarx.cms.api.db.ContentNode; import com.github.thmarx.cms.filesystem.index.IndexProviding; -import com.github.thmarx.cms.filesystem.index.SecondaryIndex; -import com.google.common.base.Strings; +import java.io.IOException; import java.time.LocalDate; -import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicReference; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** * * @author t.marx */ -public class MetaData implements IndexProviding { - - private ConcurrentMap nodes = new ConcurrentHashMap<>(); - - private ConcurrentMap tree = new ConcurrentHashMap<>(); +public interface MetaData extends IndexProviding { - private ConcurrentMap> secondaryIndexes = new ConcurrentHashMap<>(); - - @Override - public SecondaryIndex getOrCreateIndex (final String field, Function indexFunction) { - - if (!secondaryIndexes.containsKey(field)) { - var index = SecondaryIndex.builder() - .indexFunction(indexFunction) - .build(); - index.addAll(nodes.values()); - secondaryIndexes.put(field, index); - } - - return secondaryIndexes.get(field); - } - - void clear() { - nodes.clear(); - tree.clear(); - secondaryIndexes.clear(); + public enum Type { + MEMORY, PERSISTENT } - ConcurrentMap nodes() { - return new ConcurrentHashMap<>(nodes); - } + void open () throws IOException; + void close () throws IOException; - ConcurrentMap tree() { - return new ConcurrentHashMap<>(tree); - } - - public void createDirectory(final String uri) { - if (Strings.isNullOrEmpty(uri)) { - return; - } - var parts = uri.split(Constants.SPLIT_PATH_PATTERN); - ContentNode n = new ContentNode(uri, parts[parts.length - 1], Map.of(), true); - - Optional parentFolder; - if (parts.length == 1) { - parentFolder = getFolder(uri); - } else { - var parentPath = Arrays.copyOfRange(parts, 0, parts.length - 1); - var parentUri = String.join("/", parentPath); - parentFolder = getFolder(parentUri); - } - - if (parentFolder.isPresent()) { - parentFolder.get().children().put(n.name(), n); - } else { - tree.put(n.name(), n); - } - } + void addFile(final String uri, final Map data, final LocalDate lastModified); - public List listChildren(String uri) { - if ("".equals(uri)) { - return tree.values().stream() - .filter(node -> !node.isHidden()) - .map(this::mapToIndex) - .filter(node -> node != null) - .filter(MetaData::isVisible) - .collect(Collectors.toList()); + Optional byUri(final String uri); - } else { - Optional findFolder = findFolder(uri); - if (findFolder.isPresent()) { - return findFolder.get().children().values() - .stream() - .filter(node -> !node.isHidden()) - .map(this::mapToIndex) - .filter(node -> node != null) - .filter(MetaData::isVisible) - .collect(Collectors.toList()); - } - } - return Collections.emptyList(); - } + void createDirectory(final String uri); - protected ContentNode mapToIndex(ContentNode node) { - if (node.isDirectory()) { - var tempNode = node.children().entrySet().stream().filter((entry) - -> entry.getKey().equals("index.md") - ).findFirst(); - if (tempNode.isPresent()) { - return tempNode.get().getValue(); - } - return null; - } else { - return node; - } - } + Optional findFolder(String uri); - public static boolean isVisible (ContentNode node) { - return node != null - // check if some parent is hidden - && !node.uri().startsWith(".") && !node.uri().contains("/.") - && node.isPublished() - && !node.isHidden() - && !node.isSection(); - } + List listChildren(String uri); - public Optional findFolder(String uri) { - return getFolder(uri); - } - - private Optional getFolder(String uri) { - var parts = uri.split(Constants.SPLIT_PATH_PATTERN); - - final AtomicReference folder = new AtomicReference<>(null); - Stream.of(parts).forEach(part -> { - if (part.endsWith(".md")) { - return; - } - if (folder.get() == null) { - folder.set(tree.get(part)); - } else { - folder.set(folder.get().children().get(part)); - } - }); - return Optional.ofNullable(folder.get()); - } - - public void addFile(final String uri, final Map data, final LocalDate lastModified) { - - var parts = uri.split(Constants.SPLIT_PATH_PATTERN); - final ContentNode node = new ContentNode(uri, parts[parts.length - 1], data, lastModified); - - nodes.put(uri, node); - - var folder = getFolder(uri); - if (folder.isPresent()) { - folder.get().children().put(node.name(), node); - } else { - tree.put(node.name(), node); - } - - secondaryIndexes.values().forEach(index -> index.add(node)); - } - - public Optional byUri(final String uri) { - if (!nodes.containsKey(uri)) { - return Optional.empty(); - } - return Optional.of(nodes.get(uri)); - } - - void remove(String uri) { - var node = nodes.remove(uri); - - var folder = getFolder(uri); - var parts = uri.split(Constants.SPLIT_PATH_PATTERN); - var name = parts[parts.length - 1]; - if (folder.isPresent()) { - folder.get().children().remove(name); - } else { - tree.remove(name); - } - - secondaryIndexes.values().forEach(index -> index.remove(node)); - } - + void clear (); + Map nodes(); + + Map tree(); } diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/index/SecondaryIndex.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/index/SecondaryIndex.java index 5dacb5e3..200f6705 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/index/SecondaryIndex.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/index/SecondaryIndex.java @@ -25,11 +25,9 @@ import com.github.thmarx.cms.api.db.ContentNode; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.NavigableMap; import java.util.TreeMap; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.Function; import lombok.Builder; diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryMetaData.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryMetaData.java new file mode 100644 index 00000000..23510736 --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/memory/MemoryMetaData.java @@ -0,0 +1,230 @@ +package com.github.thmarx.cms.filesystem.metadata.memory; + +/*- + * #%L + * cms-server + * %% + * Copyright (C) 2023 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ + +import com.github.thmarx.cms.api.Constants; +import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.filesystem.MetaData; +import com.github.thmarx.cms.filesystem.index.IndexProviding; +import com.github.thmarx.cms.filesystem.index.SecondaryIndex; +import com.google.common.base.Strings; +import java.io.IOException; +import java.time.LocalDate; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * + * @author t.marx + */ +public class MemoryMetaData implements IndexProviding, MetaData { + + private final ConcurrentMap nodes = new ConcurrentHashMap<>(); + + private final ConcurrentMap tree = new ConcurrentHashMap<>(); + + private final ConcurrentMap> secondaryIndexes = new ConcurrentHashMap<>(); + + @Override + public SecondaryIndex getOrCreateIndex (final String field, Function indexFunction) { + + if (!secondaryIndexes.containsKey(field)) { + var index = SecondaryIndex.builder() + .indexFunction(indexFunction) + .build(); + index.addAll(nodes.values()); + secondaryIndexes.put(field, index); + } + + return secondaryIndexes.get(field); + } + + @Override + public void clear() { + nodes.clear(); + tree.clear(); + secondaryIndexes.clear(); + } + + public ConcurrentMap nodes() { + return new ConcurrentHashMap<>(nodes); + } + + public ConcurrentMap tree() { + return new ConcurrentHashMap<>(tree); + } + + @Override + public void createDirectory(final String uri) { + if (Strings.isNullOrEmpty(uri)) { + return; + } + var parts = uri.split(Constants.SPLIT_PATH_PATTERN); + ContentNode n = new ContentNode(uri, parts[parts.length - 1], Map.of(), true); + + Optional parentFolder; + if (parts.length == 1) { + parentFolder = getFolder(uri); + } else { + var parentPath = Arrays.copyOfRange(parts, 0, parts.length - 1); + var parentUri = String.join("/", parentPath); + parentFolder = getFolder(parentUri); + } + + if (parentFolder.isPresent()) { + parentFolder.get().children().put(n.name(), n); + } else { + tree.put(n.name(), n); + } + } + + @Override + public List listChildren(String uri) { + if ("".equals(uri)) { + return tree.values().stream() + .filter(node -> !node.isHidden()) + .map(this::mapToIndex) + .filter(node -> node != null) + .filter(MemoryMetaData::isVisible) + .collect(Collectors.toList()); + + } else { + Optional findFolder = findFolder(uri); + if (findFolder.isPresent()) { + return findFolder.get().children().values() + .stream() + .filter(node -> !node.isHidden()) + .map(this::mapToIndex) + .filter(node -> node != null) + .filter(MemoryMetaData::isVisible) + .collect(Collectors.toList()); + } + } + return Collections.emptyList(); + } + + protected ContentNode mapToIndex(ContentNode node) { + if (node.isDirectory()) { + var tempNode = node.children().entrySet().stream().filter((entry) + -> entry.getKey().equals("index.md") + ).findFirst(); + if (tempNode.isPresent()) { + return tempNode.get().getValue(); + } + return null; + } else { + return node; + } + } + + public static boolean isVisible (ContentNode node) { + return node != null + // check if some parent is hidden + && !node.uri().startsWith(".") && !node.uri().contains("/.") + && node.isPublished() + && !node.isHidden() + && !node.isSection(); + } + + @Override + public Optional findFolder(String uri) { + return getFolder(uri); + } + + private Optional getFolder(String uri) { + var parts = uri.split(Constants.SPLIT_PATH_PATTERN); + + final AtomicReference folder = new AtomicReference<>(null); + Stream.of(parts).forEach(part -> { + if (part.endsWith(".md")) { + return; + } + if (folder.get() == null) { + folder.set(tree.get(part)); + } else { + folder.set(folder.get().children().get(part)); + } + }); + return Optional.ofNullable(folder.get()); + } + + @Override + public void addFile(final String uri, final Map data, final LocalDate lastModified) { + + var parts = uri.split(Constants.SPLIT_PATH_PATTERN); + final ContentNode node = new ContentNode(uri, parts[parts.length - 1], data, lastModified); + + nodes.put(uri, node); + + var folder = getFolder(uri); + if (folder.isPresent()) { + folder.get().children().put(node.name(), node); + } else { + tree.put(node.name(), node); + } + + secondaryIndexes.values().forEach(index -> index.add(node)); + } + + @Override + public Optional byUri(final String uri) { + if (!nodes.containsKey(uri)) { + return Optional.empty(); + } + return Optional.of(nodes.get(uri)); + } + + void remove(String uri) { + var node = nodes.remove(uri); + + var folder = getFolder(uri); + var parts = uri.split(Constants.SPLIT_PATH_PATTERN); + var name = parts[parts.length - 1]; + if (folder.isPresent()) { + folder.get().children().remove(name); + } else { + tree.remove(name); + } + + secondaryIndexes.values().forEach(index -> index.remove(node)); + } + + @Override + public void open() throws IOException { + } + + @Override + public void close() throws IOException { + } + + +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneIndex.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneIndex.java new file mode 100644 index 00000000..da45f139 --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/LuceneIndex.java @@ -0,0 +1,91 @@ +package com.github.thmarx.cms.filesystem.metadata.persistent; + +import com.github.thmarx.cms.api.utils.FileUtils; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; +import org.apache.lucene.analysis.core.KeywordAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SearcherFactory; +import org.apache.lucene.search.SearcherManager; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.NRTCachingDirectory; + +/** + * + * @author t.marx + */ +public class LuceneIndex implements AutoCloseable { + + private Directory directory; + private IndexWriter writer = null; + + private SearcherManager nrt_manager; + private NRTCachingDirectory nrt_index; + + @Override + public void close() throws Exception { + if (nrt_manager != null) { + nrt_manager.close(); + + writer.commit(); + writer.close(); + directory.close(); + } + } + + public void commit() throws IOException { + writer.flush(); + writer.commit(); + nrt_manager.maybeRefresh(); + } + + void add (Document document) throws IOException { + writer.addDocument(document); + commit(); + } + + void update (Term term, Document document) throws IOException { + writer.updateDocument(term, document); + commit(); + } + + void delete (Query query) throws IOException { + writer.deleteDocuments(query); + commit(); + } + + Optional query (Query query) throws IOException { + IndexSearcher searcher = nrt_manager.acquire(); + try { + + } finally { + nrt_manager.release(searcher); + } + return Optional.empty(); + } + + public void open (Path path) throws IOException { + if (Files.exists(path)) { + FileUtils.deleteFolder(path); + } + Files.createDirectories(path); + + this.directory = FSDirectory.open(path); + IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new KeywordAnalyzer()); + indexWriterConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND); + indexWriterConfig.setCommitOnClose(true); + nrt_index = new NRTCachingDirectory(directory, 5.0, 60.0); + writer = new IndexWriter(nrt_index, indexWriterConfig); + + final SearcherFactory sf = new SearcherFactory(); + nrt_manager = new SearcherManager(writer, true, true, sf); + } +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/PersistentMetaData.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/PersistentMetaData.java new file mode 100644 index 00000000..b2a7caaa --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/PersistentMetaData.java @@ -0,0 +1,269 @@ +package com.github.thmarx.cms.filesystem.metadata.persistent; + +import com.github.thmarx.cms.api.Constants; +import com.github.thmarx.cms.api.db.ContentNode; +import com.github.thmarx.cms.filesystem.MetaData; +import com.github.thmarx.cms.filesystem.index.SecondaryIndex; +import com.github.thmarx.cms.filesystem.metadata.persistent.utils.FlattenMap; +import com.github.thmarx.cms.filesystem.metadata.memory.MemoryMetaData; +import com.google.common.base.Strings; +import com.google.gson.Gson; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDate; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoubleField; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FloatField; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.LongField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.h2.mvstore.MVMap; +import org.h2.mvstore.MVStore; + +/** + * + * @author t.marx + */ +@Slf4j +@RequiredArgsConstructor +public class PersistentMetaData implements AutoCloseable, MetaData { + + private final Path hostPath; + + private LuceneIndex index; + private MVStore store; + + private static final Gson GSON = new Gson(); + + MVMap nodes; + MVMap tree; + + @Override + public void open() throws IOException { + + Files.createDirectories(hostPath.resolve("data/store")); + Files.createDirectories(hostPath.resolve("data/index")); + + index = new LuceneIndex(); + index.open(hostPath.resolve("data/index")); + + store = MVStore.open(hostPath.resolve("data/store/data").toString()); + + nodes = store.openMap("nodes"); + tree = store.openMap("tree"); + } + + @Override + public void close() throws IOException { + try { + if (index != null) { + index.close(); + } + if (store != null) { + store.close(); + } + } catch (Exception ex) { + throw new IOException(ex); + } + } + + @Override + public void addFile(String uri, Map data, LocalDate lastModified) { + + var parts = uri.split(Constants.SPLIT_PATH_PATTERN); + final ContentNode node = new ContentNode(uri, parts[parts.length - 1], data, lastModified); + + nodes.put(uri, node); + + var folder = getFolder(uri); + if (folder.isPresent()) { + folder.get().children().put(node.name(), node); + } else { + tree.put(node.name(), node); + } + + Document document = new Document(); + document.add(new StringField("_uri", uri, Field.Store.NO)); + //document.add(new StringField("_source", GSON.toJson(node), Field.Store.NO)); + + addData(document, data); + try { + this.index.add(document); + } catch (IOException ex) { + log.error("", ex); + } + } + + private void addData(final Document document, Map data) { + var flatten = FlattenMap.flattenMap(data); + + flatten.entrySet().forEach(entry -> { + + switch (entry.getValue()) { + case List listValue -> + handleList(document, entry.getKey(), listValue); + default -> { + addValue(document, entry.getKey(), entry.getValue()); + } + } + }); + } + + private void handleList(Document document, String name, List list) { + list.forEach(item -> addValue(document, name, item)); + } + + private void addValue(Document document, String name, Object value) { + switch (value) { + case String stringValue -> + document.add(new StringField(name, stringValue, Field.Store.NO)); + case Integer intValue -> + document.add(new IntField(name, intValue, Field.Store.NO)); + case Long longValue -> + document.add(new LongField(name, longValue, Field.Store.NO)); + case Float floatValue -> + document.add(new FloatField(name, floatValue, Field.Store.NO)); + case Double doubleValue -> + document.add(new DoubleField(name, doubleValue, Field.Store.NO)); + case List listValue -> + handleList(document, name, listValue); + default -> { + } + } + } + + @Override + public Optional byUri(String uri) { + if (!nodes.containsKey(uri)) { + return Optional.empty(); + } + return Optional.of(nodes.get(uri)); + } + + @Override + public void createDirectory(String uri) { + if (Strings.isNullOrEmpty(uri)) { + return; + } + var parts = uri.split(Constants.SPLIT_PATH_PATTERN); + ContentNode n = new ContentNode(uri, parts[parts.length - 1], Map.of(), true); + + Optional parentFolder; + if (parts.length == 1) { + parentFolder = getFolder(uri); + } else { + var parentPath = Arrays.copyOfRange(parts, 0, parts.length - 1); + var parentUri = String.join("/", parentPath); + parentFolder = getFolder(parentUri); + } + + if (parentFolder.isPresent()) { + parentFolder.get().children().put(n.name(), n); + } else { + tree.put(n.name(), n); + } + } + + @Override + public Optional findFolder(String uri) { + return getFolder(uri); + } + + private Optional getFolder(String uri) { + var parts = uri.split(Constants.SPLIT_PATH_PATTERN); + + final AtomicReference folder = new AtomicReference<>(null); + Stream.of(parts).forEach(part -> { + if (part.endsWith(".md")) { + return; + } + if (folder.get() == null) { + folder.set(tree.get(part)); + } else { + folder.set(folder.get().children().get(part)); + } + }); + return Optional.ofNullable(folder.get()); + } + + @Override + public List listChildren(String uri) { + if ("".equals(uri)) { + return tree.values().stream() + .filter(node -> !node.isHidden()) + .map(this::mapToIndex) + .filter(node -> node != null) + .filter(MemoryMetaData::isVisible) + .collect(Collectors.toList()); + + } else { + Optional findFolder = findFolder(uri); + if (findFolder.isPresent()) { + return findFolder.get().children().values() + .stream() + .filter(node -> !node.isHidden()) + .map(this::mapToIndex) + .filter(node -> node != null) + .filter(MemoryMetaData::isVisible) + .collect(Collectors.toList()); + } + } + return Collections.emptyList(); + } + + protected ContentNode mapToIndex(ContentNode node) { + if (node.isDirectory()) { + var tempNode = node.children().entrySet().stream().filter((entry) + -> entry.getKey().equals("index.md") + ).findFirst(); + if (tempNode.isPresent()) { + return tempNode.get().getValue(); + } + return null; + } else { + return node; + } + } + + @Override + public void clear() { + try { + index.delete(new MatchAllDocsQuery()); + } catch (IOException ex) { + log.error("", ex); + } + } + + @Override + public Map nodes() { + return nodes; + } + + @Override + public Map tree() { + return tree; + } + + @Override + public SecondaryIndex getOrCreateIndex(String field, Function indexFunction) { + throw new UnsupportedOperationException("Not supported yet."); // Generated from nbfs://nbhost/SystemFileSystem/Templates/Classes/Code/GeneratedMethodBody + } + +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/utils/FlattenMap.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/utils/FlattenMap.java new file mode 100644 index 00000000..339b302e --- /dev/null +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/metadata/persistent/utils/FlattenMap.java @@ -0,0 +1,27 @@ +package com.github.thmarx.cms.filesystem.metadata.persistent.utils; + +import java.util.HashMap; +import java.util.Map; + +public class FlattenMap { + + public static Map flattenMap(Map map) { + Map result = new HashMap<>(); + flattenMap("", map, result); + return result; + } + + private static void flattenMap(String prefix, Map map, Map result) { + for (Map.Entry entry : map.entrySet()) { + String key = prefix.isEmpty() ? entry.getKey() : prefix + "." + entry.getKey(); + Object value = entry.getValue(); + + if (value instanceof Map) { + // Rekursion für verschachtelte Maps + flattenMap(key, (Map) value, result); + } else { + result.put(key, value); + } + } + } +} diff --git a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Query.java b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Query.java index 9b6157ea..66034fea 100644 --- a/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Query.java +++ b/cms-filesystem/src/main/java/com/github/thmarx/cms/filesystem/query/Query.java @@ -25,7 +25,7 @@ import com.github.thmarx.cms.api.db.ContentQuery; import com.github.thmarx.cms.api.db.ContentNode; import com.github.thmarx.cms.api.db.Page; -import com.github.thmarx.cms.filesystem.MetaData; +import com.github.thmarx.cms.filesystem.metadata.memory.MemoryMetaData; import com.github.thmarx.cms.filesystem.index.IndexProviding; import static com.github.thmarx.cms.filesystem.query.QueryUtil.filtered; import static com.github.thmarx.cms.filesystem.query.QueryUtil.filteredWithIndex; @@ -151,7 +151,7 @@ public List get() { return context.getNodes() .filter(NodeUtil.contentTypeFiler(context.getContentType())) .filter(node -> !node.isDirectory()) - .filter(MetaData::isVisible) + .filter(MemoryMetaData::isVisible) .map(context.getNodeMapper()) .toList(); } @@ -179,7 +179,7 @@ public Page page(final long page, final long size) { var filteredNodes = context.getNodes() .filter(NodeUtil.contentTypeFiler(context.getContentType())) .filter(node -> !node.isDirectory()) - .filter(MetaData::isVisible) + .filter(MemoryMetaData::isVisible) .toList(); var total = filteredNodes.size(); diff --git a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/PresistentFileSystemTest.java b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/PresistentFileSystemTest.java new file mode 100644 index 00000000..5a6e3bc8 --- /dev/null +++ b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/PresistentFileSystemTest.java @@ -0,0 +1,88 @@ +package com.github.thmarx.cms.filesystem; + +/*- + * #%L + * cms-filesystem + * %% + * Copyright (C) 2023 Marx-Software + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * . + * #L% + */ +import com.github.thmarx.cms.api.eventbus.EventBus; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.yaml.snakeyaml.Yaml; + +/** + * + * @author t.marx + */ +public class PresistentFileSystemTest { + + static FileSystem fileSystem; + + @BeforeAll + static void setup() throws IOException { + + var eventBus = Mockito.mock(EventBus.class); + + fileSystem = new FileSystem(Path.of("src/test/resources"), eventBus, (file) -> { + try { + return new Yaml().load(Files.readString(file)); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + fileSystem.init(MetaData.Type.PERSISTENT); + } + + @AfterAll + static void shutdown () { + fileSystem.shutdown(); + } + + @Test + public void test_seconday_index() throws IOException { + +// var dimension = fileSystem.createDimension("featured", (ContentNode node) -> node.data().containsKey("featured") ? (Boolean) node.data().get("featured") : false, Boolean.class); + +// Assertions.assertThat(dimension.filter(Boolean.TRUE)).hasSize(2); +// Assertions.assertThat(dimension.filter(Boolean.FALSE)).hasSize(1); + } + + @Test + public void test_query() throws IOException { + + var nodes = fileSystem.query((node, i) -> node).where("featured", true).get(); + + Assertions.assertThat(nodes).hasSize(2); + } + + @Test + public void test_query_with_start_uri() throws IOException { + + var nodes = fileSystem.query("/test", (node, i) -> node).where("featured", true).get(); + + Assertions.assertThat(nodes).hasSize(1); + Assertions.assertThat(nodes.getFirst().uri()).isEqualTo("test/test1.md"); + } +} diff --git a/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/persistent/utils/FlattenMapTest.java b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/persistent/utils/FlattenMapTest.java new file mode 100644 index 00000000..8dfde067 --- /dev/null +++ b/cms-filesystem/src/test/java/com/github/thmarx/cms/filesystem/persistent/utils/FlattenMapTest.java @@ -0,0 +1,70 @@ +package com.github.thmarx.cms.filesystem.persistent.utils; + +import com.github.thmarx.cms.filesystem.metadata.persistent.utils.FlattenMap; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import org.assertj.core.api.Assertions; + + +public class FlattenMapTest { + + @Test + public void testFlattenMap() { + // Beispielinput + Map nestedMap = new HashMap<>(); + Map nestedLevel1 = new HashMap<>(); + Map nestedLevel2 = new HashMap<>(); + + nestedLevel2.put("key3", "value3"); + nestedLevel1.put("key2", nestedLevel2); + nestedMap.put("key1", nestedLevel1); + nestedMap.put("key4", "value4"); + + // Erwartete flache Map + Map expectedFlatMap = new HashMap<>(); + expectedFlatMap.put("key1.key2.key3", "value3"); + expectedFlatMap.put("key4", "value4"); + + // Flache Map erzeugen + Map actualFlatMap = FlattenMap.flattenMap(nestedMap); + + // Überprüfen, ob die flache Map korrekt ist + Assertions.assertThat(actualFlatMap).isEqualTo(expectedFlatMap); + } + + @Test + public void testFlattenMapWithEmptyMap() { + // Leere Map + Map emptyMap = new HashMap<>(); + + // Erwartete flache Map + Map expectedFlatMap = new HashMap<>(); + + // Flache Map erzeugen + Map actualFlatMap = FlattenMap.flattenMap(emptyMap); + + // Überprüfen, ob die flache Map korrekt ist + Assertions.assertThat(actualFlatMap).isEqualTo(expectedFlatMap); + } + + @Test + public void testFlattenMapWithSingleLevelMap() { + // Einfache Map ohne Verschachtelung + Map singleLevelMap = new HashMap<>(); + singleLevelMap.put("key1", "value1"); + singleLevelMap.put("key2", "value2"); + + // Erwartete flache Map + Map expectedFlatMap = new HashMap<>(); + expectedFlatMap.put("key1", "value1"); + expectedFlatMap.put("key2", "value2"); + + // Flache Map erzeugen + Map actualFlatMap = FlattenMap.flattenMap(singleLevelMap); + + // Überprüfen, ob die flache Map korrekt ist + Assertions.assertThat(actualFlatMap).isEqualTo(expectedFlatMap); + } +} \ No newline at end of file diff --git a/cms-server/src/test/java/com/github/thmarx/cms/SectionsTest.java b/cms-server/src/test/java/com/github/thmarx/cms/SectionsTest.java index 651042be..af1b36d5 100644 --- a/cms-server/src/test/java/com/github/thmarx/cms/SectionsTest.java +++ b/cms-server/src/test/java/com/github/thmarx/cms/SectionsTest.java @@ -27,7 +27,7 @@ import com.github.thmarx.cms.api.configuration.Configuration; import com.github.thmarx.cms.api.db.ContentNode; import com.github.thmarx.cms.eventbus.DefaultEventBus; -import com.github.thmarx.cms.filesystem.MetaData; +import com.github.thmarx.cms.filesystem.metadata.memory.MemoryMetaData; import com.github.thmarx.cms.api.markdown.MarkdownRenderer; import com.github.thmarx.cms.api.template.TemplateEngine; import com.github.thmarx.cms.content.Section; diff --git a/cms-server/src/test/java/com/github/thmarx/cms/utils/NodeUtilNGTest.java b/cms-server/src/test/java/com/github/thmarx/cms/utils/NodeUtilNGTest.java index 0e923b3e..36d1f158 100644 --- a/cms-server/src/test/java/com/github/thmarx/cms/utils/NodeUtilNGTest.java +++ b/cms-server/src/test/java/com/github/thmarx/cms/utils/NodeUtilNGTest.java @@ -25,7 +25,7 @@ import com.github.thmarx.cms.api.utils.NodeUtil; import com.github.thmarx.cms.api.Constants; import com.github.thmarx.cms.api.db.ContentNode; -import com.github.thmarx.cms.filesystem.MetaData; +import com.github.thmarx.cms.filesystem.metadata.memory.MemoryMetaData; import java.util.Map; import org.assertj.core.api.Assertions; import org.junit.jupiter.api.Test; diff --git a/pom.xml b/pom.xml index 1c60464d..c19398fb 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ cms-content cms-extensions cms-auth - integration-tests + integration-tests 2023 @@ -130,6 +130,22 @@ 7.0.0 + + org.apache.lucene + lucene-core + 9.11.1 + + + org.apache.lucene + lucene-analysis-common + 9.11.1 + + + com.h2database + h2-mvstore + 2.2.224 + + org.graalvm.polyglot polyglot