From 6e6f7ca1cb3406f2c1b4d4bf6d333474be146ac1 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Tue, 5 Jul 2022 20:03:43 +0200 Subject: [PATCH 01/17] make index "optimization" explicit action fixes #3982 --- .../indexer/configuration/Configuration.java | 10 ---- .../configuration/RuntimeEnvironment.java | 8 --- .../opengrok/indexer/index/IndexDatabase.java | 20 +++++--- .../org/opengrok/indexer/index/Indexer.java | 51 ++++++------------- .../analysis/executables/JarAnalyzerTest.java | 4 +- .../configuration/RuntimeEnvironmentTest.java | 8 --- .../indexer/index/IndexCheckTest.java | 2 +- .../index/IndexDatabaseSymlinksTest.java | 4 +- .../indexer/index/IndexDatabaseTest.java | 2 +- .../opengrok/indexer/index/IndexerTest.java | 4 +- .../indexer/search/SearchEngineTest.java | 4 +- .../SearchAndContextFormatterTest.java | 4 +- .../SearchAndContextFormatterTest2.java | 4 +- .../indexer/web/SearchHelperTest.java | 4 +- .../java/org/opengrok/web/PageConfigTest.java | 2 +- .../api/v1/controller/FileControllerTest.java | 2 +- .../v1/controller/ProjectsControllerTest.java | 4 +- ...ggesterControllerProjectsDisabledTest.java | 4 +- .../controller/SuggesterControllerTest.java | 4 +- 19 files changed, 52 insertions(+), 93 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java index 23b9df56109..67a23de5929 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/Configuration.java @@ -167,7 +167,6 @@ public final class Configuration { private String reviewPattern; private String webappLAF; private RemoteSCM remoteScmSupported; - private boolean optimizeDatabase; private boolean quickContextScan; private LuceneLockName luceneLocking = LuceneLockName.OFF; @@ -557,7 +556,6 @@ public Configuration() { setMessageLimit(500); setNavigateWindowEnabled(false); setNestingMaximum(1); - setOptimizeDatabase(true); setPluginDirectory(null); setPluginStack(new AuthorizationStack(AuthControlFlag.REQUIRED, "default stack")); setPrintProgress(false); @@ -1088,14 +1086,6 @@ public void setRemoteScmSupported(RemoteSCM remoteScmSupported) { this.remoteScmSupported = remoteScmSupported; } - public boolean isOptimizeDatabase() { - return optimizeDatabase; - } - - public void setOptimizeDatabase(boolean optimizeDatabase) { - this.optimizeDatabase = optimizeDatabase; - } - public LuceneLockName getLuceneLocking() { return luceneLocking; } diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java index 82677bbc666..aeaf83f5ce2 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java @@ -1085,14 +1085,6 @@ public void setRemoteScmSupported(Configuration.RemoteSCM remoteScmSupported) { syncWriteConfiguration(remoteScmSupported, Configuration::setRemoteScmSupported); } - public boolean isOptimizeDatabase() { - return syncReadConfiguration(Configuration::isOptimizeDatabase); - } - - public void setOptimizeDatabase(boolean optimizeDatabase) { - syncWriteConfiguration(optimizeDatabase, Configuration::setOptimizeDatabase); - } - public LuceneLockName getLuceneLocking() { return syncReadConfiguration(Configuration::getLuceneLocking); } diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java index 6f14d6feaa3..69c4de210bc 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java @@ -720,9 +720,6 @@ public void update() throws IOException { } if (!isInterrupted() && isDirty()) { - if (env.isOptimizeDatabase()) { - optimize(); - } env.setIndexTimestamp(); } } @@ -808,11 +805,11 @@ void indexDownUsingHistory(File sourceRoot, IndexDownArgs args) throws IOExcepti } /** - * Optimize all index databases. + * Reduce segment counts of all index databases. * * @throws IOException if an error occurs */ - static CountDownLatch optimizeAll() throws IOException { + static void optimizeAll() throws IOException { List dbs = new ArrayList<>(); RuntimeEnvironment env = RuntimeEnvironment.getInstance(); IndexerParallelizer parallelizer = env.getIndexerParallelizer(); @@ -830,17 +827,24 @@ static CountDownLatch optimizeAll() throws IOException { if (db.isDirty()) { parallelizer.getFixedExecutor().submit(() -> { try { - db.update(); + db.optimize(); } catch (Throwable e) { LOGGER.log(Level.SEVERE, - "Problem updating lucene index database: ", e); + "Problem reducing segment count of Lucene index database: ", e); } finally { latch.countDown(); } }); } } - return latch; + + try { + LOGGER.info("Waiting for the Lucene segment count reduction to finish"); + latch.await(); + } catch (InterruptedException exp) { + LOGGER.log(Level.WARNING, "Received interrupt while waiting" + + " for index segment count reduction to finish", exp); + } } /** diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java index acb6d96751a..f30b4f7e179 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java @@ -116,7 +116,7 @@ public final class Indexer { private static Configuration cfg = null; private static boolean checkIndex = false; private static boolean runIndex = true; - private static boolean optimizedChanged = false; + private static boolean optimize = false; private static boolean addProjects = false; private static boolean searchRepositories = false; private static boolean bareConfig = false; @@ -385,9 +385,13 @@ public static void main(String[] argv) { env.setDefaultProjectsFromNames(defaultProjects); // And now index it all. - if (runIndex || (optimizedChanged && env.isOptimizeDatabase())) { + if (runIndex) { IndexChangedListener progress = new DefaultIndexChangedListener(); - getInstance().doIndexerExecution(update, subFiles, progress); + getInstance().doIndexerExecution(subFiles, progress); + } + + if (optimize) { + IndexDatabase.optimizeAll(); } writeConfigToFile(env, configFilename); @@ -672,16 +676,10 @@ public static String[] parseOptions(String[] argv) throws ParseException { "Maximum depth of nested repositories. Default is 1.").execute(v -> cfg.setNestingMaximum((Integer) v)); - parser.on("-O", "--optimize", "=on|off", ON_OFF, Boolean.class, - "Turn on/off the optimization of the index database as part of the", - "indexing step. Default is on."). - execute(v -> { - boolean oldval = cfg.isOptimizeDatabase(); - cfg.setOptimizeDatabase((Boolean) v); - if (oldval != cfg.isOptimizeDatabase()) { - optimizedChanged = true; - } - } + parser.on("-O", "--optimize", + "Reduce the number of segments in each index database. This might ", + "(or might not) bring some improved performance."). + execute(v -> { optimize = true; } ); parser.on("-o", "--ctagOpts", "=path", @@ -1075,17 +1073,13 @@ private void addProjects(File[] files, Map projects) { /** * This is the second phase of the indexer which generates Lucene index * by passing source code files through ctags, generating xrefs - * and storing data from the source files in the index (along with history, - * if any). + * and storing data from the source files in the index (along with history, if any). * - * @param update if set to true, index database is updated, otherwise optimized * @param subFiles index just some subdirectories * @param progress object to receive notifications as indexer progress is made * @throws IOException if I/O exception occurred */ - public void doIndexerExecution(final boolean update, List subFiles, - IndexChangedListener progress) - throws IOException { + public void doIndexerExecution(List subFiles, IndexChangedListener progress) throws IOException { Statistics elapsed = new Statistics(); LOGGER.info("Starting indexing"); @@ -1093,13 +1087,7 @@ public void doIndexerExecution(final boolean update, List subFiles, IndexerParallelizer parallelizer = env.getIndexerParallelizer(); final CountDownLatch latch; if (subFiles == null || subFiles.isEmpty()) { - if (update) { - latch = IndexDatabase.updateAll(progress); - } else if (env.isOptimizeDatabase()) { - latch = IndexDatabase.optimizeAll(); - } else { - latch = new CountDownLatch(0); - } + latch = IndexDatabase.updateAll(progress); } else { List dbs = new ArrayList<>(); @@ -1131,19 +1119,12 @@ public void doIndexerExecution(final boolean update, List subFiles, latch = new CountDownLatch(dbs.size()); for (final IndexDatabase db : dbs) { - final boolean optimize = env.isOptimizeDatabase(); db.addIndexChangedListener(progress); parallelizer.getFixedExecutor().submit(() -> { try { - if (update) { - db.update(); - } else if (optimize) { - db.optimize(); - } + db.update(); } catch (Throwable e) { - LOGGER.log(Level.SEVERE, "An error occurred while " - + (update ? "updating" : "optimizing") - + " index", e); + LOGGER.log(Level.SEVERE, "An error occurred while updating index", e); } finally { latch.countDown(); } diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java index 56795831517..a54aac3ac2a 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.analysis.executables; @@ -79,7 +79,7 @@ public static void setUpClass() throws Exception { false, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - Indexer.getInstance().doIndexerExecution(true, null, progress); + Indexer.getInstance().doIndexerExecution(null, progress); configFile = File.createTempFile("configuration", ".xml"); env.writeConfiguration(configFile); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/configuration/RuntimeEnvironmentTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/configuration/RuntimeEnvironmentTest.java index 5ecebae1898..3095a6ce1a0 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/configuration/RuntimeEnvironmentTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/configuration/RuntimeEnvironmentTest.java @@ -401,14 +401,6 @@ void testRemoteScmSupported() { assertEquals(Configuration.RemoteSCM.UIONLY, instance.getRemoteScmSupported()); } - @Test - void testOptimizeDatabase() { - RuntimeEnvironment instance = RuntimeEnvironment.getInstance(); - assertTrue(instance.isOptimizeDatabase()); - instance.setOptimizeDatabase(false); - assertFalse(instance.isOptimizeDatabase()); - } - @Test void testUsingLuceneLocking() { RuntimeEnvironment instance = RuntimeEnvironment.getInstance(); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java index f9630c4c96c..a7a00566157 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java @@ -97,7 +97,7 @@ private void testIndexVersion(boolean projectsEnabled, List subFiles) th configuration.setProjectsEnabled(projectsEnabled); Indexer.getInstance().prepareIndexer(env, true, true, false, null, null); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); IndexCheck.check(configuration, subFiles); } diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java index 21c37624872..fd38a821dfb 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2010, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2010, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.index; @@ -208,7 +208,7 @@ private static void runIndexer() throws IndexerException, IOException { Indexer indexer = Indexer.getInstance(); indexer.prepareIndexer(env, true, true, false, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - indexer.doIndexerExecution(true, null, null); + indexer.doIndexerExecution(null, null); } private void assertSymlinkAsExpected(String message, File expectedCanonical, Path symlink) diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java index 1cb247660c9..11ae967184e 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java @@ -152,7 +152,7 @@ public void setUpClass() throws Exception { env.setDefaultProjectsFromNames(new TreeSet<>(Arrays.asList("/c"))); - indexer.doIndexerExecution(true, null, null); + indexer.doIndexerExecution(null, null); env.clearFileCollector(); } diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java index a1c44181e75..4c77d198fa0 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2017, 2019, Chris Fraire . * Portions Copyright (c) 2020, Ric Harris . */ @@ -105,7 +105,7 @@ void testXrefGeneration() throws Exception { env.setHistoryEnabled(false); Indexer.getInstance().prepareIndexer(env, true, true, false, null, null); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); // There should be certain number of xref files produced. List result = null; diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java index f1ba5ae5284..de25a678746 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.search; @@ -69,7 +69,7 @@ public static void setUpClass() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, false, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); configFile = File.createTempFile("configuration", ".xml"); env.writeConfiguration(configFile); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java index c25b915a85e..344073dbd2d 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.search.context; @@ -79,7 +79,7 @@ public static void setUpClass() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, false, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); configFile = File.createTempFile("configuration", ".xml"); env.writeConfiguration(configFile); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java index b503d34e890..cddcf2b101c 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.search.context; @@ -128,7 +128,7 @@ public static void setUpClass() throws Exception { assertNotNull(proj1, "symlink1 project"); proj1.setTabSize(TABSIZE); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); configFile = File.createTempFile("configuration", ".xml"); env.writeConfiguration(configFile); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java index e9fdc041a88..7d99b25c7a0 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2020, Chris Fraire . */ package org.opengrok.indexer.web; @@ -74,7 +74,7 @@ private void reindex() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, false, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); } private SearchHelper getSearchHelper(String searchTerm) { diff --git a/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java b/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java index 81995ba34ca..dea9546d713 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java @@ -336,7 +336,7 @@ void testGetLatestRevisionViaIndex() throws Exception { false, // don't create dictionary null, // subFiles - needed when refreshing history partially null); // repositories - needed when refreshing history partially - indexer.doIndexerExecution(true, null, null); + indexer.doIndexerExecution(null, null); DummyHttpServletRequest req1 = new DummyHttpServletRequest() { @Override diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java index c41399237e7..8106311f3fb 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java @@ -73,7 +73,7 @@ public void setUp() throws Exception { false, // don't create dictionary null, // subFiles - needed when refreshing history partially null); // repositories - needed when refreshing history partially - Indexer.getInstance().doIndexerExecution(true, Collections.singletonList("/git"), null); + Indexer.getInstance().doIndexerExecution(Collections.singletonList("/git"), null); } @AfterEach diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java index 862437306c2..b409c513d71 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java @@ -317,7 +317,7 @@ void testDelete() throws Exception { false, // don't create dictionary subFiles, // subFiles - needed when refreshing history partially repos); // repositories - needed when refreshing history partially - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); for (String proj : projectsToDelete) { deleteProject(proj); @@ -588,7 +588,7 @@ void testListFiles() throws IOException, IndexerException { false, // don't create dictionary new ArrayList<>(), // subFiles - needed when refreshing history partially new ArrayList<>()); // repositories - needed when refreshing history partially - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); List filesFromRequest = target("projects") .path(projectName) diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java index b1aba0e0550..33203479986 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2019, 2020, Chris Fraire . */ package org.opengrok.web.api.v1.controller; @@ -70,7 +70,7 @@ public static void setUpClass() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, false, null, null); env.setDefaultProjectsFromNames(Collections.singleton("__all__")); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); env.getSuggesterConfig().setRebuildCronConfig(null); } diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java index 6da19340841..ffebe9f5ad8 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2019, 2020, Chris Fraire . */ package org.opengrok.web.api.v1.controller; @@ -109,7 +109,7 @@ public static void setUpClass() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, false, null, null); env.setDefaultProjectsFromNames(Collections.singleton("__all__")); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); env.getSuggesterConfig().setRebuildCronConfig(null); } From 8c50368ba06496ffcabc554c25d64dc4b7042675 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Tue, 5 Jul 2022 20:08:37 +0200 Subject: [PATCH 02/17] finish the rename --- .../opengrok/indexer/index/IndexDatabase.java | 20 +++++++++---------- .../org/opengrok/indexer/index/Indexer.java | 10 +++++----- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java index 69c4de210bc..a4edaf3e986 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java @@ -809,7 +809,7 @@ void indexDownUsingHistory(File sourceRoot, IndexDownArgs args) throws IOExcepti * * @throws IOException if an error occurs */ - static void optimizeAll() throws IOException { + static void reduceSegmentCountAll() throws IOException { List dbs = new ArrayList<>(); RuntimeEnvironment env = RuntimeEnvironment.getInstance(); IndexerParallelizer parallelizer = env.getIndexerParallelizer(); @@ -827,7 +827,7 @@ static void optimizeAll() throws IOException { if (db.isDirty()) { parallelizer.getFixedExecutor().submit(() -> { try { - db.optimize(); + db.reduceSegmentCount(); } catch (Throwable e) { LOGGER.log(Level.SEVERE, "Problem reducing segment count of Lucene index database: ", e); @@ -848,13 +848,13 @@ static void optimizeAll() throws IOException { } /** - * Optimize the index database. + * Reduce number of segments in the index database. * @throws IOException I/O exception */ - public void optimize() throws IOException { + public void reduceSegmentCount() throws IOException { synchronized (lock) { if (running) { - LOGGER.warning("Optimize terminated... Someone else is updating / optimizing it!"); + LOGGER.warning("Segment count reduction terminated... Someone else is running the operation!"); return; } running = true; @@ -865,15 +865,15 @@ public void optimize() throws IOException { try { Statistics elapsed = new Statistics(); String projectDetail = this.project != null ? " for project " + project.getName() : ""; - LOGGER.log(Level.INFO, "Optimizing the index{0}", projectDetail); + LOGGER.log(Level.INFO, "Reducing number of segments in the index{0}", projectDetail); Analyzer analyzer = new StandardAnalyzer(); IndexWriterConfig conf = new IndexWriterConfig(analyzer); conf.setOpenMode(OpenMode.CREATE_OR_APPEND); wrt = new IndexWriter(indexDirectory, conf); - wrt.forceMerge(1); // this is deprecated and not needed anymore - elapsed.report(LOGGER, String.format("Done optimizing index%s", projectDetail), - "indexer.db.optimize"); + wrt.forceMerge(1); + elapsed.report(LOGGER, String.format("Done reducing number of segments in index%s", projectDetail), + "indexer.db.reduceSegments"); synchronized (lock) { if (dirtyFile.exists() && !dirtyFile.delete()) { LOGGER.log(Level.FINE, "Failed to remove \"dirty-file\": {0}", @@ -883,7 +883,7 @@ public void optimize() throws IOException { } } catch (IOException e) { writerException = e; - LOGGER.log(Level.SEVERE, "ERROR: optimizing index", e); + LOGGER.log(Level.SEVERE, "ERROR: reducing number of segments index", e); } finally { if (wrt != null) { try { diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java index f30b4f7e179..c95fbdfb514 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java @@ -116,7 +116,7 @@ public final class Indexer { private static Configuration cfg = null; private static boolean checkIndex = false; private static boolean runIndex = true; - private static boolean optimize = false; + private static boolean reduceSegmentCount = false; private static boolean addProjects = false; private static boolean searchRepositories = false; private static boolean bareConfig = false; @@ -390,8 +390,8 @@ public static void main(String[] argv) { getInstance().doIndexerExecution(subFiles, progress); } - if (optimize) { - IndexDatabase.optimizeAll(); + if (reduceSegmentCount) { + IndexDatabase.reduceSegmentCountAll(); } writeConfigToFile(env, configFilename); @@ -676,10 +676,10 @@ public static String[] parseOptions(String[] argv) throws ParseException { "Maximum depth of nested repositories. Default is 1.").execute(v -> cfg.setNestingMaximum((Integer) v)); - parser.on("-O", "--optimize", + parser.on("--reduceSegmentCount", "Reduce the number of segments in each index database. This might ", "(or might not) bring some improved performance."). - execute(v -> { optimize = true; } + execute(v -> { reduceSegmentCount = true; } ); parser.on("-o", "--ctagOpts", "=path", From 840f203902ec86e2aefddd47c63ae4130bb05b74 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Tue, 5 Jul 2022 20:15:31 +0200 Subject: [PATCH 03/17] fix style --- .../src/main/java/org/opengrok/indexer/index/Indexer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java index c95fbdfb514..99a6eebcb83 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java @@ -679,8 +679,7 @@ public static String[] parseOptions(String[] argv) throws ParseException { parser.on("--reduceSegmentCount", "Reduce the number of segments in each index database. This might ", "(or might not) bring some improved performance."). - execute(v -> { reduceSegmentCount = true; } - ); + execute(v -> reduceSegmentCount = true); parser.on("-o", "--ctagOpts", "=path", "File with extra command line options for ctags."). From e3326f3a7180d217d72d3c3ec57e1a8f0d50c3a1 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Tue, 5 Jul 2022 22:30:10 +0200 Subject: [PATCH 04/17] refactor dirty file removal --- .../opengrok/indexer/index/IndexDatabase.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java index a4edaf3e986..4d634f61aed 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java @@ -874,13 +874,7 @@ public void reduceSegmentCount() throws IOException { wrt.forceMerge(1); elapsed.report(LOGGER, String.format("Done reducing number of segments in index%s", projectDetail), "indexer.db.reduceSegments"); - synchronized (lock) { - if (dirtyFile.exists() && !dirtyFile.delete()) { - LOGGER.log(Level.FINE, "Failed to remove \"dirty-file\": {0}", - dirtyFile.getAbsolutePath()); - } - dirty = false; - } + unsetDirty(); } catch (IOException e) { writerException = e; LOGGER.log(Level.SEVERE, "ERROR: reducing number of segments index", e); @@ -929,6 +923,15 @@ private void setDirty() { } } + private void unsetDirty() { + synchronized (lock) { + if (dirtyFile.exists() && !dirtyFile.delete()) { + LOGGER.log(Level.FINE, "Failed to remove \"dirty-file\": {0}", dirtyFile.getAbsolutePath()); + } + dirty = false; + } + } + private File whatXrefFile(String path, boolean compress) { String xrefPath = compress ? TandemPath.join(path, ".gz") : path; return new File(xrefDir, xrefPath); From 3bb9b725fe5fa961dbea16bb5cda1ef81e72d667 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Tue, 5 Jul 2022 22:51:30 +0200 Subject: [PATCH 05/17] fix handling of the dirty flag --- .../org/opengrok/indexer/index/IndexDatabase.java | 12 +++++++++++- .../java/org/opengrok/indexer/index/Indexer.java | 8 ++++++-- .../analysis/executables/JarAnalyzerTest.java | 2 +- .../org/opengrok/indexer/index/IndexCheckTest.java | 2 +- .../indexer/index/IndexDatabaseSymlinksTest.java | 2 +- .../opengrok/indexer/index/IndexDatabaseTest.java | 2 +- .../java/org/opengrok/indexer/index/IndexerTest.java | 2 +- .../opengrok/indexer/search/SearchEngineTest.java | 2 +- .../context/SearchAndContextFormatterTest.java | 2 +- .../context/SearchAndContextFormatterTest2.java | 2 +- .../org/opengrok/indexer/web/SearchHelperTest.java | 2 +- .../test/java/org/opengrok/web/PageConfigTest.java | 2 +- .../web/api/v1/controller/FileControllerTest.java | 2 +- .../api/v1/controller/ProjectsControllerTest.java | 4 ++-- .../SuggesterControllerProjectsDisabledTest.java | 2 +- .../api/v1/controller/SuggesterControllerTest.java | 2 +- 16 files changed, 32 insertions(+), 18 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java index 4d634f61aed..a0ca743ab2b 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java @@ -161,6 +161,7 @@ public class IndexDatabase { private File dirtyFile; private final Object lock = new Object(); private boolean dirty; + private boolean clearDirtyOnUpdate; private boolean running; private boolean isCountingDeltas; private boolean isWithDirectoryCounts; @@ -720,6 +721,9 @@ public void update() throws IOException { } if (!isInterrupted() && isDirty()) { + if (clearDirtyOnUpdate) { + unsetDirty(); + } env.setIndexTimestamp(); } } @@ -874,7 +878,9 @@ public void reduceSegmentCount() throws IOException { wrt.forceMerge(1); elapsed.report(LOGGER, String.format("Done reducing number of segments in index%s", projectDetail), "indexer.db.reduceSegments"); - unsetDirty(); + if (isDirty()) { + unsetDirty(); + } } catch (IOException e) { writerException = e; LOGGER.log(Level.SEVERE, "ERROR: reducing number of segments index", e); @@ -932,6 +938,10 @@ private void unsetDirty() { } } + void setClearDirtyOnUpdate() { + clearDirtyOnUpdate = true; + } + private File whatXrefFile(String path, boolean compress) { String xrefPath = compress ? TandemPath.join(path, ".gz") : path; return new File(xrefDir, xrefPath); diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java index 99a6eebcb83..782fb6397a5 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java @@ -387,7 +387,7 @@ public static void main(String[] argv) { // And now index it all. if (runIndex) { IndexChangedListener progress = new DefaultIndexChangedListener(); - getInstance().doIndexerExecution(subFiles, progress); + getInstance().doIndexerExecution(!reduceSegmentCount, subFiles, progress); } if (reduceSegmentCount) { @@ -1074,11 +1074,12 @@ private void addProjects(File[] files, Map projects) { * by passing source code files through ctags, generating xrefs * and storing data from the source files in the index (along with history, if any). * + * @param clearDirty clear dirty flag of index database(s) in the end * @param subFiles index just some subdirectories * @param progress object to receive notifications as indexer progress is made * @throws IOException if I/O exception occurred */ - public void doIndexerExecution(List subFiles, IndexChangedListener progress) throws IOException { + public void doIndexerExecution(boolean clearDirty, List subFiles, IndexChangedListener progress) throws IOException { Statistics elapsed = new Statistics(); LOGGER.info("Starting indexing"); @@ -1101,6 +1102,9 @@ public void doIndexerExecution(List subFiles, IndexChangedListener progr } else { db = new IndexDatabase(project); } + if (clearDirty) { + db.setClearDirtyOnUpdate(); + } int idx = dbs.indexOf(db); if (idx != -1) { db = dbs.get(idx); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java index a54aac3ac2a..ff71c8db206 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java @@ -79,7 +79,7 @@ public static void setUpClass() throws Exception { false, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - Indexer.getInstance().doIndexerExecution(null, progress); + Indexer.getInstance().doIndexerExecution(true, null, progress); configFile = File.createTempFile("configuration", ".xml"); env.writeConfiguration(configFile); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java index a7a00566157..f9630c4c96c 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java @@ -97,7 +97,7 @@ private void testIndexVersion(boolean projectsEnabled, List subFiles) th configuration.setProjectsEnabled(projectsEnabled); Indexer.getInstance().prepareIndexer(env, true, true, false, null, null); - Indexer.getInstance().doIndexerExecution(null, null); + Indexer.getInstance().doIndexerExecution(true, null, null); IndexCheck.check(configuration, subFiles); } diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java index fd38a821dfb..4ab32f88148 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java @@ -208,7 +208,7 @@ private static void runIndexer() throws IndexerException, IOException { Indexer indexer = Indexer.getInstance(); indexer.prepareIndexer(env, true, true, false, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - indexer.doIndexerExecution(null, null); + indexer.doIndexerExecution(true, null, null); } private void assertSymlinkAsExpected(String message, File expectedCanonical, Path symlink) diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java index 11ae967184e..1cb247660c9 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java @@ -152,7 +152,7 @@ public void setUpClass() throws Exception { env.setDefaultProjectsFromNames(new TreeSet<>(Arrays.asList("/c"))); - indexer.doIndexerExecution(null, null); + indexer.doIndexerExecution(true, null, null); env.clearFileCollector(); } diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java index 4c77d198fa0..816197e9546 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java @@ -105,7 +105,7 @@ void testXrefGeneration() throws Exception { env.setHistoryEnabled(false); Indexer.getInstance().prepareIndexer(env, true, true, false, null, null); - Indexer.getInstance().doIndexerExecution(null, null); + Indexer.getInstance().doIndexerExecution(true, null, null); // There should be certain number of xref files produced. List result = null; diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java index de25a678746..7abf0214133 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java @@ -69,7 +69,7 @@ public static void setUpClass() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, false, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - Indexer.getInstance().doIndexerExecution(null, null); + Indexer.getInstance().doIndexerExecution(true, null, null); configFile = File.createTempFile("configuration", ".xml"); env.writeConfiguration(configFile); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java index 344073dbd2d..aa7919319bc 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java @@ -79,7 +79,7 @@ public static void setUpClass() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, false, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - Indexer.getInstance().doIndexerExecution(null, null); + Indexer.getInstance().doIndexerExecution(true, null, null); configFile = File.createTempFile("configuration", ".xml"); env.writeConfiguration(configFile); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java index cddcf2b101c..bef44cdac82 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java @@ -128,7 +128,7 @@ public static void setUpClass() throws Exception { assertNotNull(proj1, "symlink1 project"); proj1.setTabSize(TABSIZE); - Indexer.getInstance().doIndexerExecution(null, null); + Indexer.getInstance().doIndexerExecution(true, null, null); configFile = File.createTempFile("configuration", ".xml"); env.writeConfiguration(configFile); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java index 7d99b25c7a0..e57f9356d72 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java @@ -74,7 +74,7 @@ private void reindex() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, false, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - Indexer.getInstance().doIndexerExecution(null, null); + Indexer.getInstance().doIndexerExecution(true, null, null); } private SearchHelper getSearchHelper(String searchTerm) { diff --git a/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java b/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java index dea9546d713..81995ba34ca 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java @@ -336,7 +336,7 @@ void testGetLatestRevisionViaIndex() throws Exception { false, // don't create dictionary null, // subFiles - needed when refreshing history partially null); // repositories - needed when refreshing history partially - indexer.doIndexerExecution(null, null); + indexer.doIndexerExecution(true, null, null); DummyHttpServletRequest req1 = new DummyHttpServletRequest() { @Override diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java index 8106311f3fb..c41399237e7 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java @@ -73,7 +73,7 @@ public void setUp() throws Exception { false, // don't create dictionary null, // subFiles - needed when refreshing history partially null); // repositories - needed when refreshing history partially - Indexer.getInstance().doIndexerExecution(Collections.singletonList("/git"), null); + Indexer.getInstance().doIndexerExecution(true, Collections.singletonList("/git"), null); } @AfterEach diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java index b409c513d71..862437306c2 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java @@ -317,7 +317,7 @@ void testDelete() throws Exception { false, // don't create dictionary subFiles, // subFiles - needed when refreshing history partially repos); // repositories - needed when refreshing history partially - Indexer.getInstance().doIndexerExecution(null, null); + Indexer.getInstance().doIndexerExecution(true, null, null); for (String proj : projectsToDelete) { deleteProject(proj); @@ -588,7 +588,7 @@ void testListFiles() throws IOException, IndexerException { false, // don't create dictionary new ArrayList<>(), // subFiles - needed when refreshing history partially new ArrayList<>()); // repositories - needed when refreshing history partially - Indexer.getInstance().doIndexerExecution(null, null); + Indexer.getInstance().doIndexerExecution(true, null, null); List filesFromRequest = target("projects") .path(projectName) diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java index 33203479986..81dbb81a119 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java @@ -70,7 +70,7 @@ public static void setUpClass() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, false, null, null); env.setDefaultProjectsFromNames(Collections.singleton("__all__")); - Indexer.getInstance().doIndexerExecution(null, null); + Indexer.getInstance().doIndexerExecution(true, null, null); env.getSuggesterConfig().setRebuildCronConfig(null); } diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java index ffebe9f5ad8..1ec299a3c96 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java @@ -109,7 +109,7 @@ public static void setUpClass() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, false, null, null); env.setDefaultProjectsFromNames(Collections.singleton("__all__")); - Indexer.getInstance().doIndexerExecution(null, null); + Indexer.getInstance().doIndexerExecution(true, null, null); env.getSuggesterConfig().setRebuildCronConfig(null); } From e73563af309c77cffca364c506bca09ba583f20a Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Wed, 6 Jul 2022 22:34:21 +0200 Subject: [PATCH 06/17] make sure IndexDatabase.updateAll() clears the dirty file --- .../main/java/org/opengrok/indexer/index/IndexDatabase.java | 6 +++++- .../src/main/java/org/opengrok/indexer/index/Indexer.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java index a0ca743ab2b..5b9917094e4 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java @@ -219,10 +219,11 @@ public IndexDatabase(Project project, IndexDownArgsFactory factory) throws IOExc /** * Update the index database for all the projects. * + * @param clearDirty clear dirty flag of index database(s) in the end * @param listener where to signal the changes to the database * @throws IOException if an error occurs */ - static CountDownLatch updateAll(IndexChangedListener listener) throws IOException { + static CountDownLatch updateAll(boolean clearDirty, IndexChangedListener listener) throws IOException { RuntimeEnvironment env = RuntimeEnvironment.getInstance(); List dbs = new ArrayList<>(); @@ -238,6 +239,9 @@ static CountDownLatch updateAll(IndexChangedListener listener) throws IOExceptio IndexerParallelizer parallelizer = RuntimeEnvironment.getInstance().getIndexerParallelizer(); CountDownLatch latch = new CountDownLatch(dbs.size()); for (IndexDatabase d : dbs) { + if (clearDirty) { + d.setClearDirtyOnUpdate(); + } final IndexDatabase db = d; if (listener != null) { db.addIndexChangedListener(listener); diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java index 782fb6397a5..44aa7b9b68c 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java @@ -1087,7 +1087,7 @@ public void doIndexerExecution(boolean clearDirty, List subFiles, IndexC IndexerParallelizer parallelizer = env.getIndexerParallelizer(); final CountDownLatch latch; if (subFiles == null || subFiles.isEmpty()) { - latch = IndexDatabase.updateAll(progress); + latch = IndexDatabase.updateAll(clearDirty, progress); } else { List dbs = new ArrayList<>(); From 8f1a8952145dfc920d42c72386ef6aa3f815a55e Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Thu, 7 Jul 2022 00:36:12 +0200 Subject: [PATCH 07/17] no need to check whether the index is dirty --- .../opengrok/indexer/index/IndexDatabase.java | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java index 5b9917094e4..66381be0e31 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java @@ -832,18 +832,16 @@ static void reduceSegmentCountAll() throws IOException { CountDownLatch latch = new CountDownLatch(dbs.size()); for (IndexDatabase d : dbs) { final IndexDatabase db = d; - if (db.isDirty()) { - parallelizer.getFixedExecutor().submit(() -> { - try { - db.reduceSegmentCount(); - } catch (Throwable e) { - LOGGER.log(Level.SEVERE, - "Problem reducing segment count of Lucene index database: ", e); - } finally { - latch.countDown(); - } - }); - } + parallelizer.getFixedExecutor().submit(() -> { + try { + db.reduceSegmentCount(); + } catch (Throwable e) { + LOGGER.log(Level.SEVERE, + "Problem reducing segment count of Lucene index database: ", e); + } finally { + latch.countDown(); + } + }); } try { From 0fc9e1cb106a3cc5bccf922384a213a90056f3ac Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Thu, 7 Jul 2022 00:36:56 +0200 Subject: [PATCH 08/17] use iterator correctly --- .../opengrok/indexer/index/IndexDatabase.java | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java index 66381be0e31..ab0b3514678 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java @@ -1812,17 +1812,11 @@ public Set getFiles() throws IOException { terms = MultiTerms.getTerms(ireader, QueryBuilder.U); iter = terms.iterator(); // init uid iterator } - while (iter != null && iter.term() != null) { - String value = iter.term().utf8ToString(); - if (value.isEmpty()) { - iter.next(); - continue; - } - - files.add(Util.uid2url(value)); - BytesRef next = iter.next(); - if (next == null) { - iter = null; + BytesRef term; + while (iter != null && (term = iter.next()) != null) { + String value = term.utf8ToString(); + if (!value.isEmpty()) { + files.add(Util.uid2url(value)); } } } finally { @@ -1897,17 +1891,13 @@ public void listTokens(int freq) throws IOException { terms = MultiTerms.getTerms(ireader, QueryBuilder.DEFS); iter = terms.iterator(); // init uid iterator } - while (iter != null && iter.term() != null) { - if (iter.docFreq() > 16 && iter.term().utf8ToString().length() > freq) { - LOGGER.warning(iter.term().utf8ToString()); - } - BytesRef next = iter.next(); - if (next == null) { - iter = null; + BytesRef term; + while (iter != null && (term = iter.next()) != null) { + if (iter.docFreq() > 16 && term.utf8ToString().length() > freq) { + LOGGER.log(Level.WARNING, "{0}", term.utf8ToString()); } } } finally { - if (ireader != null) { try { ireader.close(); From 8f2325edf1a7a66f80036fd3f1a2002f606220b1 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Thu, 7 Jul 2022 22:21:17 +0200 Subject: [PATCH 09/17] revert unnecessary year bumps --- .../org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java | 2 +- .../src/test/java/org/opengrok/indexer/index/IndexerTest.java | 2 +- .../test/java/org/opengrok/indexer/search/SearchEngineTest.java | 2 +- .../indexer/search/context/SearchAndContextFormatterTest.java | 2 +- .../indexer/search/context/SearchAndContextFormatterTest2.java | 2 +- .../test/java/org/opengrok/indexer/web/SearchHelperTest.java | 2 +- .../v1/controller/SuggesterControllerProjectsDisabledTest.java | 2 +- .../opengrok/web/api/v1/controller/SuggesterControllerTest.java | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java index 4ab32f88148..21c37624872 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2010, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2010, 2021, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.index; diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java index 816197e9546..a1c44181e75 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2017, 2019, Chris Fraire . * Portions Copyright (c) 2020, Ric Harris . */ diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java index 7abf0214133..f1ba5ae5284 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.search; diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java index aa7919319bc..c25b915a85e 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.search.context; diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java index bef44cdac82..b503d34e890 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.search.context; diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java index e57f9356d72..e9fdc041a88 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2020, Chris Fraire . */ package org.opengrok.indexer.web; diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java index 81dbb81a119..b1aba0e0550 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2019, 2020, Chris Fraire . */ package org.opengrok.web.api.v1.controller; diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java index 1ec299a3c96..6da19340841 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2019, 2020, Chris Fraire . */ package org.opengrok.web.api.v1.controller; From c62fab9920a25deae7afdd2cece82c87ec356213 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Thu, 7 Jul 2022 22:22:02 +0200 Subject: [PATCH 10/17] revert year bump that is unnecessary --- .../opengrok/indexer/analysis/executables/JarAnalyzerTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java index ff71c8db206..56795831517 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.analysis.executables; From 93bbb3d6ccc2d61aef7dc8edec42554a45826f5e Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Thu, 7 Jul 2022 22:52:51 +0200 Subject: [PATCH 11/17] improve the comment --- .../src/main/java/org/opengrok/indexer/index/Indexer.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java index 44aa7b9b68c..8a8bd23b673 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java @@ -677,8 +677,9 @@ public static String[] parseOptions(String[] argv) throws ParseException { cfg.setNestingMaximum((Integer) v)); parser.on("--reduceSegmentCount", - "Reduce the number of segments in each index database. This might ", - "(or might not) bring some improved performance."). + "Reduce the number of segments in each index database to 1. This might ", + "(or might not) bring some improved performance. Anyhow, this operation", + "takes non-trivial time to complete."). execute(v -> reduceSegmentCount = true); parser.on("-o", "--ctagOpts", "=path", From 4ef0b1366c6ce937f6b050d75debdf02a07a73db Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 8 Jul 2022 00:12:15 +0200 Subject: [PATCH 12/17] remove the list tokens (create dictionary) functionality it does not seem to be used --- .../opengrok/indexer/index/IndexDatabase.java | 55 ------------------- .../org/opengrok/indexer/index/Indexer.java | 23 +++----- .../analysis/executables/JarAnalyzerTest.java | 4 +- .../indexer/index/IndexCheckTest.java | 2 +- .../index/IndexDatabaseSymlinksTest.java | 4 +- .../indexer/index/IndexDatabaseTest.java | 14 ++--- .../indexer/index/IndexerRepoTest.java | 4 +- .../opengrok/indexer/index/IndexerTest.java | 14 ++--- .../indexer/search/SearchEngineTest.java | 4 +- .../SearchAndContextFormatterTest.java | 4 +- .../SearchAndContextFormatterTest2.java | 4 +- .../indexer/web/SearchHelperTest.java | 4 +- .../test/java/org/opengrok/web/DiffTest.java | 4 +- .../java/org/opengrok/web/PageConfigTest.java | 2 +- .../controller/AnnotationControllerTest.java | 4 +- .../api/v1/controller/FileControllerTest.java | 3 +- .../v1/controller/HistoryControllerTest.java | 2 +- .../v1/controller/ProjectsControllerTest.java | 4 +- .../RepositoriesControllerTest.java | 4 +- ...ggesterControllerProjectsDisabledTest.java | 4 +- .../controller/SuggesterControllerTest.java | 4 +- 21 files changed, 52 insertions(+), 115 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java index ab0b3514678..197e8d3c177 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java @@ -1853,61 +1853,6 @@ public int getNumFiles() throws IOException { } } - static void listFrequentTokens(List subFiles) throws IOException { - final int limit = 4; - - RuntimeEnvironment env = RuntimeEnvironment.getInstance(); - if (env.hasProjects()) { - if (subFiles == null || subFiles.isEmpty()) { - for (Project project : env.getProjectList()) { - IndexDatabase db = new IndexDatabase(project); - db.listTokens(limit); - } - } else { - for (String path : subFiles) { - Project project = Project.getProject(path); - if (project == null) { - LOGGER.log(Level.WARNING, "Could not find a project for \"{0}\"", path); - } else { - IndexDatabase db = new IndexDatabase(project); - db.listTokens(limit); - } - } - } - } else { - IndexDatabase db = new IndexDatabase(); - db.listTokens(limit); - } - } - - public void listTokens(int freq) throws IOException { - IndexReader ireader = null; - TermsEnum iter = null; - Terms terms; - - try { - ireader = DirectoryReader.open(indexDirectory); - if (ireader.numDocs() > 0) { - terms = MultiTerms.getTerms(ireader, QueryBuilder.DEFS); - iter = terms.iterator(); // init uid iterator - } - BytesRef term; - while (iter != null && (term = iter.next()) != null) { - if (iter.docFreq() > 16 && term.utf8ToString().length() > freq) { - LOGGER.log(Level.WARNING, "{0}", term.utf8ToString()); - } - } - } finally { - if (ireader != null) { - try { - ireader.close(); - } catch (IOException e) { - LOGGER.log(Level.WARNING, "An error occurred while closing index reader", e); - } - } - } - } - /** * Get an indexReader for the Index database where a given file. * diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java index 8a8bd23b673..35e11803e4b 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java @@ -166,8 +166,6 @@ public static void main(String[] argv) { List subFiles = RuntimeEnvironment.getInstance().getSubFiles(); Set subFilesArgs = new HashSet<>(); - boolean createDict = false; - try { argv = parseOptions(argv); @@ -375,7 +373,7 @@ public static void main(String[] argv) { collect(Collectors.toSet()); } getInstance().prepareIndexer(env, searchPaths, addProjects, - createDict, runIndex, subFiles, new ArrayList<>(repositories)); + runIndex, subFiles, new ArrayList<>(repositories)); // Set updated configuration in RuntimeEnvironment. This is called so that repositories discovered // in prepareIndexer() are stored in the Configuration used by RuntimeEnvironment. @@ -964,13 +962,12 @@ public static void writeConfigToFile(RuntimeEnvironment env, String filename) th public void prepareIndexer(RuntimeEnvironment env, boolean searchRepositories, boolean addProjects, - boolean createDict, List subFiles, List repositories) throws IndexerException, IOException { prepareIndexer(env, searchRepositories ? Collections.singleton(env.getSourceRootPath()) : Collections.emptySet(), - addProjects, createDict, true, subFiles, repositories); + addProjects, true, subFiles, repositories); } /** @@ -983,7 +980,6 @@ public void prepareIndexer(RuntimeEnvironment env, * @param env runtime environment * @param searchPaths list of paths in which to search for repositories * @param addProjects if true, add projects - * @param createDict if true, create dictionary * @param createHistoryCache create history cache flag * @param subFiles list of directories * @param repositories list of repositories @@ -991,12 +987,11 @@ public void prepareIndexer(RuntimeEnvironment env, * @throws IOException I/O exception */ public void prepareIndexer(RuntimeEnvironment env, - Set searchPaths, - boolean addProjects, - boolean createDict, - boolean createHistoryCache, - List subFiles, - List repositories) throws IndexerException, IOException { + Set searchPaths, + boolean addProjects, + boolean createHistoryCache, + List subFiles, + List repositories) throws IndexerException, IOException { if (!env.validateUniversalCtags()) { throw new IndexerException("Didn't find Universal Ctags"); @@ -1033,10 +1028,6 @@ public void prepareIndexer(RuntimeEnvironment env, } LOGGER.info("Done generating history cache"); } - - if (createDict) { - IndexDatabase.listFrequentTokens(subFiles); - } } private void addProjects(File[] files, Map projects) { diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java index 56795831517..0c6c8d8553c 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.analysis.executables; @@ -76,7 +76,7 @@ public static void setUpClass() throws Exception { env.setHistoryEnabled(false); IndexChangedListener progress = new DefaultIndexChangedListener(); Indexer.getInstance().prepareIndexer(env, true, true, - false, null, null); + null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); Indexer.getInstance().doIndexerExecution(true, null, progress); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java index f9630c4c96c..730209be808 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java @@ -96,7 +96,7 @@ private void testIndexVersion(boolean projectsEnabled, List subFiles) th env.setProjectsEnabled(projectsEnabled); configuration.setProjectsEnabled(projectsEnabled); Indexer.getInstance().prepareIndexer(env, true, true, - false, null, null); + null, null); Indexer.getInstance().doIndexerExecution(true, null, null); IndexCheck.check(configuration, subFiles); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java index 21c37624872..d39c8151da4 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2010, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2010, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.index; @@ -206,7 +206,7 @@ public void testSymlinksWithOneAddedSymlink() throws IOException, IndexerExcepti private static void runIndexer() throws IndexerException, IOException { Indexer indexer = Indexer.getInstance(); - indexer.prepareIndexer(env, true, true, false, null, null); + indexer.prepareIndexer(env, true, true, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); indexer.doIndexerExecution(true, null, null); } diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java index 1cb247660c9..d773fe6f26c 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java @@ -142,7 +142,7 @@ public void setUpClass() throws Exception { indexer = Indexer.getInstance(); indexer.prepareIndexer( env, true, true, - false, null, null); + null, null); // Reset the state of the git project w.r.t. history based reindex. // It is the responsibility of each test that relies on the per project tunable @@ -470,7 +470,7 @@ void testGetIndexDownArgs(boolean mergeCommits, boolean renamedFiles, boolean hi HistoryGuru.getInstance().clear(); indexer.prepareIndexer( env, true, true, - false, List.of("/git"), null); + List.of("/git"), null); env.generateProjectRepositoriesMap(); // Check history cache w.r.t. the merge changeset. @@ -590,7 +590,7 @@ void testHistoryBasedReindexVsProjectWithDiverseRepos(boolean useCvs) throws Exc HistoryGuru.getInstance().clear(); indexer.prepareIndexer( env, true, true, - false, List.of("/git"), null); + List.of("/git"), null); env.setRepositories(new ArrayList<>(HistoryGuru.getInstance().getRepositories())); env.generateProjectRepositoriesMap(); @@ -644,7 +644,7 @@ void testHistoryBasedReindexWithEligibleSubRepo() throws Exception { // for the 2nd stage of indexing. indexer.prepareIndexer( env, true, true, - false, List.of("/git"), null); + List.of("/git"), null); // Verify the collected files. FileCollector fileCollector = env.getFileCollector("git"); @@ -678,7 +678,7 @@ void testHistoryBasedReindexProjectTunable(boolean historyBased) throws Exceptio HistoryGuru.getInstance().clear(); indexer.prepareIndexer( env, true, true, - false, List.of("/git"), null); + List.of("/git"), null); env.generateProjectRepositoriesMap(); verifyIndexDown(gitProject, historyBased); @@ -699,7 +699,7 @@ void testHistoryBasedReindexWithNoChange() throws Exception { HistoryGuru.getInstance().clear(); indexer.prepareIndexer( env, true, true, - false, List.of("/git"), null); + List.of("/git"), null); env.generateProjectRepositoriesMap(); verifyIndexDown(gitProject, true); @@ -734,7 +734,7 @@ void testForcedReindex(boolean historyBased) throws Exception { // Re-generate the history cache so that the git repository is ready for history based re-index. indexer.prepareIndexer( env, true, true, - false, List.of("/git"), null); + List.of("/git"), null); env.generateProjectRepositoriesMap(); // Emulate forcing reindex from scratch. diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerRepoTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerRepoTest.java index 0e6730551d9..42cfd7cf580 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerRepoTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerRepoTest.java @@ -100,7 +100,7 @@ void testPerProjectHistory(boolean globalOn) throws IndexerException, IOExceptio env, true, // search for repositories true, // scan and add projects - false, // don't create dictionary + // don't create dictionary null, // subFiles - not needed since we don't list files null); // repositories - not needed when not refreshing history env.generateProjectRepositoriesMap(); @@ -165,7 +165,7 @@ void testSymlinks() throws IndexerException, IOException { env, true, // search for repositories true, // scan and add projects - false, // don't create dictionary + // don't create dictionary null, // subFiles - not needed since we don't list files null); // repositories - not needed when not refreshing history diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java index a1c44181e75..73f2ae06fa2 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2017, 2019, Chris Fraire . * Portions Copyright (c) 2020, Ric Harris . */ @@ -104,7 +104,7 @@ void testXrefGeneration() throws Exception { env.setDataRoot(repository.getDataRoot()); env.setHistoryEnabled(false); Indexer.getInstance().prepareIndexer(env, true, true, - false, null, null); + null, null); Indexer.getInstance().doIndexerExecution(true, null, null); // There should be certain number of xref files produced. @@ -150,7 +150,7 @@ void testRescanProjects() throws Exception { env, false, // don't search for repositories true, // scan and add projects - false, // don't create dictionary + // don't create dictionary null, // subFiles - not needed since we don't list files null); // repositories - not needed when not refreshing history @@ -337,7 +337,7 @@ void testRemoveFileOnFileChange() throws Exception { // Create history cache. Indexer.getInstance().prepareIndexer(env, true, true, - false, null, List.of("mercurial")); + null, List.of("mercurial")); File historyFile = new File(env.getDataRootPath(), TandemPath.join("historycache" + path, ".gz")); assertTrue(historyFile.exists(), String.format("history cache for %s has to exist", path)); @@ -503,7 +503,7 @@ void testDefaultProjectsSingleProject() throws Exception { env.setDataRoot(repository.getDataRoot()); env.setHistoryEnabled(false); Indexer.getInstance().prepareIndexer(env, true, true, - false, null, null); + null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); assertEquals(1, env.getDefaultProjects().size()); assertEquals(new TreeSet<>(Collections.singletonList("c")), @@ -528,7 +528,7 @@ void testDefaultProjectsNonExistent() throws Exception { projectSet.add("/no-project-x32ds1"); Indexer.getInstance().prepareIndexer(env, true, true, - false, null, null); + null, null); env.setDefaultProjectsFromNames(projectSet); assertEquals(4, env.getDefaultProjects().size()); assertEquals(new TreeSet<>(Arrays.asList("lisp", "pascal", "perl", "data")), @@ -552,7 +552,7 @@ void testDefaultProjectsAll() throws Exception { defaultProjects.add("/no-project-x32ds1"); Indexer.getInstance().prepareIndexer(env, true, true, - false, null, null); + null, null); env.setDefaultProjectsFromNames(defaultProjects); Set projects = new TreeSet<>(Arrays.asList(new File(repository.getSourceRoot()).list())); assertEquals(projects.size(), env.getDefaultProjects().size()); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java index f1ba5ae5284..9d8b7095714 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.search; @@ -67,7 +67,7 @@ public static void setUpClass() throws Exception { env.setHistoryEnabled(false); Indexer.getInstance().prepareIndexer(env, true, true, - false, null, null); + null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); Indexer.getInstance().doIndexerExecution(true, null, null); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java index c25b915a85e..e5d33fbdda1 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2008, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2008, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.search.context; @@ -77,7 +77,7 @@ public static void setUpClass() throws Exception { env.setDataRoot(repository.getDataRoot()); env.setHistoryEnabled(false); Indexer.getInstance().prepareIndexer(env, true, true, - false, null, null); + null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); Indexer.getInstance().doIndexerExecution(true, null, null); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java index b503d34e890..c344191629c 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2019, Chris Fraire . */ package org.opengrok.indexer.search.context; @@ -121,7 +121,7 @@ public static void setUpClass() throws Exception { env.setHistoryEnabled(false); Indexer.getInstance().prepareIndexer(env, true, true, - false, null, null); + null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); Project proj1 = env.getProjects().get(SYMLINK1); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java index e9fdc041a88..efb28563fea 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2018, 2020, Chris Fraire . */ package org.opengrok.indexer.web; @@ -72,7 +72,7 @@ private void reindex() throws Exception { System.out.println("Generating index by using the class methods"); Indexer.getInstance().prepareIndexer(env, true, true, - false, null, null); + null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); Indexer.getInstance().doIndexerExecution(true, null, null); } diff --git a/opengrok-web/src/test/java/org/opengrok/web/DiffTest.java b/opengrok-web/src/test/java/org/opengrok/web/DiffTest.java index d5e3f91a2a7..833d94bd682 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/DiffTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/DiffTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. */ package org.opengrok.web; @@ -62,7 +62,7 @@ static void setUp() throws Exception { env, true, // search for repositories true, // scan and add projects - false, // don't create dictionary + // don't create dictionary null, // subFiles - needed when refreshing history partially null); // repositories - needed when refreshing history partially } diff --git a/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java b/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java index 81995ba34ca..e50f4f1ca5b 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java @@ -333,7 +333,7 @@ void testGetLatestRevisionViaIndex() throws Exception { env, true, // search for repositories true, // scan and add projects - false, // don't create dictionary + // don't create dictionary null, // subFiles - needed when refreshing history partially null); // repositories - needed when refreshing history partially indexer.doIndexerExecution(true, null, null); diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/AnnotationControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/AnnotationControllerTest.java index e4bc64dbf52..d84c8864869 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/AnnotationControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/AnnotationControllerTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2020, Chris Fraire . */ package org.opengrok.web.api.v1.controller; @@ -78,7 +78,7 @@ public void setUp() throws Exception { env, true, // search for repositories true, // scan and add projects - false, // don't create dictionary + // don't create dictionary null, // subFiles - needed when refreshing history partially null); // repositories - needed when refreshing history partially } diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java index c41399237e7..d4229ff0f5a 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java @@ -18,6 +18,7 @@ */ /* + * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2020, Chris Fraire . */ package org.opengrok.web.api.v1.controller; @@ -70,7 +71,7 @@ public void setUp() throws Exception { env, true, // search for repositories true, // scan and add projects - false, // don't create dictionary + // don't create dictionary null, // subFiles - needed when refreshing history partially null); // repositories - needed when refreshing history partially Indexer.getInstance().doIndexerExecution(true, Collections.singletonList("/git"), null); diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/HistoryControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/HistoryControllerTest.java index 62ab3449e1e..1a39578f0c5 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/HistoryControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/HistoryControllerTest.java @@ -79,7 +79,7 @@ public void setUp() throws Exception { env, true, // search for repositories true, // scan and add projects - false, // don't create dictionary + // don't create dictionary null, // subFiles - needed when refreshing history partially null); // repositories - needed when refreshing history partially } diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java index 862437306c2..471613e18c3 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java @@ -314,7 +314,7 @@ void testDelete() throws Exception { env, false, // don't search for repositories false, // don't scan and add projects - false, // don't create dictionary + // don't create dictionary subFiles, // subFiles - needed when refreshing history partially repos); // repositories - needed when refreshing history partially Indexer.getInstance().doIndexerExecution(true, null, null); @@ -585,7 +585,7 @@ void testListFiles() throws IOException, IndexerException { env, false, // don't search for repositories true, // add projects - false, // don't create dictionary + // don't create dictionary new ArrayList<>(), // subFiles - needed when refreshing history partially new ArrayList<>()); // repositories - needed when refreshing history partially Indexer.getInstance().doIndexerExecution(true, null, null); diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/RepositoriesControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/RepositoriesControllerTest.java index 1db94930b83..f90a7987d2d 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/RepositoriesControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/RepositoriesControllerTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2019, 2020, Chris Fraire . */ package org.opengrok.web.api.v1.controller; @@ -102,7 +102,7 @@ public void testGetRepositoryType() throws Exception { env, true, // search for repositories true, // scan and add projects - false, // don't create dictionary + // don't create dictionary null, // subFiles - needed when refreshing history partially null); // repositories - needed when refreshing history partially diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java index b1aba0e0550..f4f96d78c83 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2019, 2020, Chris Fraire . */ package org.opengrok.web.api.v1.controller; @@ -68,7 +68,7 @@ public static void setUpClass() throws Exception { env.setProjectsEnabled(false); env.setSourceRoot(repository.getSourceRoot() + File.separator + "java"); Indexer.getInstance().prepareIndexer(env, true, true, - false, null, null); + null, null); env.setDefaultProjectsFromNames(Collections.singleton("__all__")); Indexer.getInstance().doIndexerExecution(true, null, null); diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java index 6da19340841..e97148c0c84 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java @@ -18,7 +18,7 @@ */ /* - * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved. * Portions Copyright (c) 2019, 2020, Chris Fraire . */ package org.opengrok.web.api.v1.controller; @@ -107,7 +107,7 @@ public static void setUpClass() throws Exception { env.setHistoryEnabled(false); env.setProjectsEnabled(true); Indexer.getInstance().prepareIndexer(env, true, true, - false, null, null); + null, null); env.setDefaultProjectsFromNames(Collections.singleton("__all__")); Indexer.getInstance().doIndexerExecution(true, null, null); From f97c058e714c23b393245b36bcb6a3a509628056 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 8 Jul 2022 00:28:57 +0200 Subject: [PATCH 13/17] do not use the dirty flag in reduce segment count code Also, actually use the dirty flag to emit a warning message. --- .../opengrok/indexer/index/IndexDatabase.java | 23 ++++++------------- .../org/opengrok/indexer/index/Indexer.java | 10 +++----- .../analysis/executables/JarAnalyzerTest.java | 2 +- .../indexer/index/IndexCheckTest.java | 2 +- .../index/IndexDatabaseSymlinksTest.java | 2 +- .../indexer/index/IndexDatabaseTest.java | 2 +- .../opengrok/indexer/index/IndexerTest.java | 2 +- .../indexer/search/SearchEngineTest.java | 2 +- .../SearchAndContextFormatterTest.java | 2 +- .../SearchAndContextFormatterTest2.java | 2 +- .../indexer/web/SearchHelperTest.java | 2 +- .../java/org/opengrok/web/PageConfigTest.java | 2 +- .../api/v1/controller/FileControllerTest.java | 2 +- .../v1/controller/ProjectsControllerTest.java | 4 ++-- ...ggesterControllerProjectsDisabledTest.java | 2 +- .../controller/SuggesterControllerTest.java | 2 +- 16 files changed, 25 insertions(+), 38 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java index 197e8d3c177..f47d5e2f8b4 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java @@ -161,7 +161,6 @@ public class IndexDatabase { private File dirtyFile; private final Object lock = new Object(); private boolean dirty; - private boolean clearDirtyOnUpdate; private boolean running; private boolean isCountingDeltas; private boolean isWithDirectoryCounts; @@ -219,11 +218,10 @@ public IndexDatabase(Project project, IndexDownArgsFactory factory) throws IOExc /** * Update the index database for all the projects. * - * @param clearDirty clear dirty flag of index database(s) in the end * @param listener where to signal the changes to the database * @throws IOException if an error occurs */ - static CountDownLatch updateAll(boolean clearDirty, IndexChangedListener listener) throws IOException { + static CountDownLatch updateAll(IndexChangedListener listener) throws IOException { RuntimeEnvironment env = RuntimeEnvironment.getInstance(); List dbs = new ArrayList<>(); @@ -239,9 +237,6 @@ static CountDownLatch updateAll(boolean clearDirty, IndexChangedListener listene IndexerParallelizer parallelizer = RuntimeEnvironment.getInstance().getIndexerParallelizer(); CountDownLatch latch = new CountDownLatch(dbs.size()); for (IndexDatabase d : dbs) { - if (clearDirty) { - d.setClearDirtyOnUpdate(); - } final IndexDatabase db = d; if (listener != null) { db.addIndexChangedListener(listener); @@ -343,6 +338,11 @@ private void initialize() throws IOException { dirtyFile = new File(indexDir, "dirty"); dirty = dirtyFile.exists(); directories = new ArrayList<>(); + + if (dirty) { + LOGGER.log(Level.WARNING, "Index in ''{0}'' is dirty, the last indexing was likely interrupted." + + " It might be worthwhile to reindex from scratch.", indexDir); + } } } @@ -725,9 +725,7 @@ public void update() throws IOException { } if (!isInterrupted() && isDirty()) { - if (clearDirtyOnUpdate) { - unsetDirty(); - } + unsetDirty(); env.setIndexTimestamp(); } } @@ -880,9 +878,6 @@ public void reduceSegmentCount() throws IOException { wrt.forceMerge(1); elapsed.report(LOGGER, String.format("Done reducing number of segments in index%s", projectDetail), "indexer.db.reduceSegments"); - if (isDirty()) { - unsetDirty(); - } } catch (IOException e) { writerException = e; LOGGER.log(Level.SEVERE, "ERROR: reducing number of segments index", e); @@ -940,10 +935,6 @@ private void unsetDirty() { } } - void setClearDirtyOnUpdate() { - clearDirtyOnUpdate = true; - } - private File whatXrefFile(String path, boolean compress) { String xrefPath = compress ? TandemPath.join(path, ".gz") : path; return new File(xrefDir, xrefPath); diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java index 35e11803e4b..22f48fabfac 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java @@ -385,7 +385,7 @@ public static void main(String[] argv) { // And now index it all. if (runIndex) { IndexChangedListener progress = new DefaultIndexChangedListener(); - getInstance().doIndexerExecution(!reduceSegmentCount, subFiles, progress); + getInstance().doIndexerExecution(subFiles, progress); } if (reduceSegmentCount) { @@ -1066,12 +1066,11 @@ private void addProjects(File[] files, Map projects) { * by passing source code files through ctags, generating xrefs * and storing data from the source files in the index (along with history, if any). * - * @param clearDirty clear dirty flag of index database(s) in the end * @param subFiles index just some subdirectories * @param progress object to receive notifications as indexer progress is made * @throws IOException if I/O exception occurred */ - public void doIndexerExecution(boolean clearDirty, List subFiles, IndexChangedListener progress) throws IOException { + public void doIndexerExecution(List subFiles, IndexChangedListener progress) throws IOException { Statistics elapsed = new Statistics(); LOGGER.info("Starting indexing"); @@ -1079,7 +1078,7 @@ public void doIndexerExecution(boolean clearDirty, List subFiles, IndexC IndexerParallelizer parallelizer = env.getIndexerParallelizer(); final CountDownLatch latch; if (subFiles == null || subFiles.isEmpty()) { - latch = IndexDatabase.updateAll(clearDirty, progress); + latch = IndexDatabase.updateAll(progress); } else { List dbs = new ArrayList<>(); @@ -1094,9 +1093,6 @@ public void doIndexerExecution(boolean clearDirty, List subFiles, IndexC } else { db = new IndexDatabase(project); } - if (clearDirty) { - db.setClearDirtyOnUpdate(); - } int idx = dbs.indexOf(db); if (idx != -1) { db = dbs.get(idx); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java index 0c6c8d8553c..eb26113cbc6 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerTest.java @@ -79,7 +79,7 @@ public static void setUpClass() throws Exception { null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - Indexer.getInstance().doIndexerExecution(true, null, progress); + Indexer.getInstance().doIndexerExecution(null, progress); configFile = File.createTempFile("configuration", ".xml"); env.writeConfiguration(configFile); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java index 730209be808..75c5ed2bc6f 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexCheckTest.java @@ -97,7 +97,7 @@ private void testIndexVersion(boolean projectsEnabled, List subFiles) th configuration.setProjectsEnabled(projectsEnabled); Indexer.getInstance().prepareIndexer(env, true, true, null, null); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); IndexCheck.check(configuration, subFiles); } diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java index d39c8151da4..f4d27eaef6c 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseSymlinksTest.java @@ -208,7 +208,7 @@ private static void runIndexer() throws IndexerException, IOException { Indexer indexer = Indexer.getInstance(); indexer.prepareIndexer(env, true, true, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - indexer.doIndexerExecution(true, null, null); + indexer.doIndexerExecution(null, null); } private void assertSymlinkAsExpected(String message, File expectedCanonical, Path symlink) diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java index d773fe6f26c..bee4df7dda4 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexDatabaseTest.java @@ -152,7 +152,7 @@ public void setUpClass() throws Exception { env.setDefaultProjectsFromNames(new TreeSet<>(Arrays.asList("/c"))); - indexer.doIndexerExecution(true, null, null); + indexer.doIndexerExecution(null, null); env.clearFileCollector(); } diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java index 73f2ae06fa2..8d5b2330c9d 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java @@ -105,7 +105,7 @@ void testXrefGeneration() throws Exception { env.setHistoryEnabled(false); Indexer.getInstance().prepareIndexer(env, true, true, null, null); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); // There should be certain number of xref files produced. List result = null; diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java index 9d8b7095714..e3d91359d59 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/SearchEngineTest.java @@ -69,7 +69,7 @@ public static void setUpClass() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); configFile = File.createTempFile("configuration", ".xml"); env.writeConfiguration(configFile); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java index e5d33fbdda1..8d254f97cb7 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest.java @@ -79,7 +79,7 @@ public static void setUpClass() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); configFile = File.createTempFile("configuration", ".xml"); env.writeConfiguration(configFile); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java index c344191629c..f2ad8c183ee 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/search/context/SearchAndContextFormatterTest2.java @@ -128,7 +128,7 @@ public static void setUpClass() throws Exception { assertNotNull(proj1, "symlink1 project"); proj1.setTabSize(TABSIZE); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); configFile = File.createTempFile("configuration", ".xml"); env.writeConfiguration(configFile); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java index efb28563fea..eea2f2af48b 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/SearchHelperTest.java @@ -74,7 +74,7 @@ private void reindex() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, null, null); env.setDefaultProjectsFromNames(new TreeSet<>(Collections.singletonList("/c"))); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); } private SearchHelper getSearchHelper(String searchTerm) { diff --git a/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java b/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java index e50f4f1ca5b..b07f7a21c92 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/PageConfigTest.java @@ -336,7 +336,7 @@ void testGetLatestRevisionViaIndex() throws Exception { // don't create dictionary null, // subFiles - needed when refreshing history partially null); // repositories - needed when refreshing history partially - indexer.doIndexerExecution(true, null, null); + indexer.doIndexerExecution(null, null); DummyHttpServletRequest req1 = new DummyHttpServletRequest() { @Override diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java index d4229ff0f5a..b33a8d6ee84 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/FileControllerTest.java @@ -74,7 +74,7 @@ public void setUp() throws Exception { // don't create dictionary null, // subFiles - needed when refreshing history partially null); // repositories - needed when refreshing history partially - Indexer.getInstance().doIndexerExecution(true, Collections.singletonList("/git"), null); + Indexer.getInstance().doIndexerExecution(Collections.singletonList("/git"), null); } @AfterEach diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java index 471613e18c3..b4c49104bb7 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/ProjectsControllerTest.java @@ -317,7 +317,7 @@ void testDelete() throws Exception { // don't create dictionary subFiles, // subFiles - needed when refreshing history partially repos); // repositories - needed when refreshing history partially - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); for (String proj : projectsToDelete) { deleteProject(proj); @@ -588,7 +588,7 @@ void testListFiles() throws IOException, IndexerException { // don't create dictionary new ArrayList<>(), // subFiles - needed when refreshing history partially new ArrayList<>()); // repositories - needed when refreshing history partially - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); List filesFromRequest = target("projects") .path(projectName) diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java index f4f96d78c83..4705a540432 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerProjectsDisabledTest.java @@ -70,7 +70,7 @@ public static void setUpClass() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, null, null); env.setDefaultProjectsFromNames(Collections.singleton("__all__")); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); env.getSuggesterConfig().setRebuildCronConfig(null); } diff --git a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java index e97148c0c84..9636cba799f 100644 --- a/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java +++ b/opengrok-web/src/test/java/org/opengrok/web/api/v1/controller/SuggesterControllerTest.java @@ -109,7 +109,7 @@ public static void setUpClass() throws Exception { Indexer.getInstance().prepareIndexer(env, true, true, null, null); env.setDefaultProjectsFromNames(Collections.singleton("__all__")); - Indexer.getInstance().doIndexerExecution(true, null, null); + Indexer.getInstance().doIndexerExecution(null, null); env.getSuggesterConfig().setRebuildCronConfig(null); } From 393fb24cf2d22fc2b2dc4a1a21a8287185259efe Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 8 Jul 2022 00:38:17 +0200 Subject: [PATCH 14/17] actualy set interrupted to true --- .../main/java/org/opengrok/indexer/index/IndexDatabase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java index f47d5e2f8b4..0ee07d6ccc3 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java @@ -160,7 +160,7 @@ public class IndexDatabase { private CopyOnWriteArrayList listeners; private File dirtyFile; private final Object lock = new Object(); - private boolean dirty; + private boolean dirty; // Whether the index was modified either by adding or removing a document. private boolean running; private boolean isCountingDeltas; private boolean isWithDirectoryCounts; @@ -1698,6 +1698,7 @@ private void indexParallel(String dir, IndexDownArgs args) { } }))).get(); } catch (InterruptedException | ExecutionException e) { + interrupted = true; int successCount = successCounter.intValue(); double successPct = 100.0 * successCount / worksCount; String exmsg = String.format("%d successes (%.1f%%) after aborting parallel-indexing", From 33140d06c01db93eca4464860ba2e4846f6bfac6 Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 8 Jul 2022 00:38:53 +0200 Subject: [PATCH 15/17] rename --- .../java/org/opengrok/indexer/index/IndexDatabase.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java index 0ee07d6ccc3..04b373f970b 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java @@ -165,7 +165,7 @@ public class IndexDatabase { private boolean isCountingDeltas; private boolean isWithDirectoryCounts; private List directories; - private LockFactory lockfact; + private LockFactory lockFactory; private final BytesRef emptyBR = new BytesRef(""); // Directory where we store indexes @@ -195,7 +195,7 @@ public IndexDatabase() throws IOException { public IndexDatabase(Project project, IndexDownArgsFactory factory) throws IOException { indexDownArgsFactory = factory; this.project = project; - lockfact = NoLockFactory.INSTANCE; + lockFactory = NoLockFactory.INSTANCE; initialize(); } @@ -329,8 +329,8 @@ private void initialize() throws IOException { } } - lockfact = pickLockFactory(env); - indexDirectory = FSDirectory.open(indexDir.toPath(), lockfact); + lockFactory = pickLockFactory(env); + indexDirectory = FSDirectory.open(indexDir.toPath(), lockFactory); pathAccepter = env.getPathAccepter(); analyzerGuru = new AnalyzerGuru(); xrefDir = new File(env.getDataRootFile(), XREF_DIR); From 483202ed00fa179b55d842c0389c105457021c2c Mon Sep 17 00:00:00 2001 From: Vladimir Kotal Date: Fri, 8 Jul 2022 00:44:19 +0200 Subject: [PATCH 16/17] add comment --- .../main/java/org/opengrok/indexer/index/IndexDatabase.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java index 04b373f970b..3292c590042 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java @@ -637,7 +637,10 @@ public void update() throws IOException { try { if (terms != null) { uidIter = terms.iterator(); - TermsEnum.SeekStatus stat = uidIter.seekCeil(new BytesRef(startUid)); //init uid + // The seekCeil() is pretty important because it makes uidIter.term() to become non-null. + // Various indexer methods rely on this when working with the uidIter iterator - rather + // than calling uidIter.next() first thing, they check uidIter.term(). + TermsEnum.SeekStatus stat = uidIter.seekCeil(new BytesRef(startUid)); if (stat == TermsEnum.SeekStatus.END) { uidIter = null; LOGGER.log(Level.WARNING, From e9d54842b4ea8a6d93231d785f56ce05c33c7e57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vladim=C3=ADr=20Kotal?= Date: Mon, 25 Jul 2022 14:26:45 +0200 Subject: [PATCH 17/17] fix rebase --- .../src/test/java/org/opengrok/indexer/web/UtilTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/UtilTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/UtilTest.java index f6232bc41a5..c1f0384fe1b 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/UtilTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/UtilTest.java @@ -671,7 +671,6 @@ void testWriteHAD() throws Exception { env, true, // search for repositories true, // scan and add projects - false, // don't create dictionary null, // subFiles - not needed since we don't list files null); // repositories - not needed when not refreshing history env.generateProjectRepositoriesMap();