Patch config fetch (#1647)

* remove extended logging

* refactor AgentConfigurationReloadTask

* add docsObjectsTask test

* optimize task and fix test

* optimize cached agent mappings

* remove redundant logs

* update naming convention

* add requested changes

* refactor supplier

* refactor AgentConfiguration

* remove static supplier

* rename method

* move AgentDocumentation to configdocsgenerator

* add tests

* refactor tests

* remove deprecated test

* make documentation volatile

components/inspectit-ocelot-configdocsgenerator/src/main/java/inspectit/ocelot/configdocsgenerator/ConfigDocsGenerator.java

@@ -55,10 +55,10 @@ public class ConfigDocsGenerator {
     Map<String, String> specialInputDescriptionsRegex = Collections.singletonMap("_arg\\d", "The _argN-th argument with which the instrumented method was called within which this action is getting executed.");
 
     /**
-     * Map of files, which contain a set of defined objects like actions, scopes, rules & metrics
+     * Set of documentations, which contain a set of documentable objects like actions, scopes, rules & metrics for every file
      */
     @Setter
-    private Map<String, Set<String>> docsObjectsByFile = new HashMap<>();
+    private Set<AgentDocumentation> agentDocumentations = new HashSet<>();
 
     /**
      * Generates a ConfigDocumentation from a YAML String describing an {@link InspectitConfig}.
@@ -233,11 +233,11 @@ private List<ActionDocs> generateActionDocs(Map<String, GenericActionSettings> a
      * @return a set of files, which contain the provided object
      */
     private Set<String> findFiles(String name) {
-        Set<String> files = docsObjectsByFile.entrySet().stream()
-                .filter(entry -> entry.getValue().contains(name))
-                .map(Map.Entry::getKey)
+        Set<String> files = agentDocumentations.stream()
+                .filter(documentation -> documentation.getObjects().contains(name))
+                .map(AgentDocumentation::getFilePath)
                 .collect(Collectors.toSet());
-        if(files.isEmpty()) log.warn("No file found with definition of " + name);
+        if(files.isEmpty()) log.debug("No file found with definition of " + name);
 
         return files;
     }

components/inspectit-ocelot-configdocsgenerator/src/main/java/inspectit/ocelot/configdocsgenerator/model/AgentDocumentation.java

@@ -0,0 +1,24 @@
+package inspectit.ocelot.configdocsgenerator.model;
+
+import lombok.Value;
+
+import java.util.Set;
+
+/**
+ * Model, to store documentable objects of a specific yaml file.
+ * Documentable objects can be actions, scopes, rules & metrics.
+ *
+ */
+@Value
+public class AgentDocumentation {
+
+    /**
+     * file, which contains the documentable objects
+     */
+    private String filePath;
+
+    /**
+     * documentable objects of the file
+     */
+    private Set<String> objects;
+}

components/inspectit-ocelot-configdocsgenerator/src/test/java/inspectit/ocelot/configdocsgenerator/ConfigDocsGeneratorTest.java

@@ -371,8 +371,10 @@ void verifyFindFiles() throws IOException {
             docsObjects.add(ruleDocParent.getName());
             docsObjects.add(metricDoc.getName());
 
-            Map<String, Set<String>> docsObjectsByFile = Collections.singletonMap(file, docsObjects);
-            configDocsGenerator.setDocsObjectsByFile(docsObjectsByFile);
+            AgentDocumentation documentation = new AgentDocumentation(file, docsObjects);
+            Set<AgentDocumentation> agentDocumentations = Collections.singleton(documentation);
+
+            configDocsGenerator.setAgentDocumentations(agentDocumentations);
             when(actionWithDocInYaml.getFiles()).thenReturn(Collections.singleton(file));
             when(actionWithoutDocInYaml.getFiles()).thenReturn(Collections.singleton(file));
             when(scopeDoc.getFiles()).thenReturn(Collections.singleton(file));

components/inspectit-ocelot-configurationserver-ui/src/components/views/status/StatusTable.js

@@ -293,19 +293,10 @@ class StatusTable extends React.Component {
 
   agentHealthTemplate = (rowData) => {
     const { onShowHealthStateDialog } = this.props;
-    const { onShowDownloadDialog } = this.props;
-    const { metaInformation } = rowData;
+    const { healthState, metaInformation } = rowData;
+    const { health } = healthState;
     const { agentId } = metaInformation;
-    // TODO Remove console.error() after agent health issues are solved
-    let health;
-    let healthState;
-    try {
-      healthState = rowData['healthState'];
-      health = healthState['health'];
-    } catch (error) {
-      console.error(`Could not read agent health from ${agentId}`, error);
-      console.error(rowData);
-    }
+    const { onShowDownloadDialog } = this.props;
 
     let { agentCommandsEnabled, supportArchiveAvailable } = this.resolveServiceAvailability(metaInformation);
 

components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/agentconfiguration/AgentConfiguration.java

@@ -1,33 +1,50 @@
 package rocks.inspectit.ocelot.agentconfiguration;
 
-import lombok.Builder;
+import inspectit.ocelot.configdocsgenerator.model.AgentDocumentation;
 import lombok.Value;
+import lombok.extern.slf4j.Slf4j;
 import org.springframework.util.DigestUtils;
+import org.yaml.snakeyaml.Yaml;
+import rocks.inspectit.ocelot.file.FileInfo;
+import rocks.inspectit.ocelot.file.accessor.AbstractFileAccessor;
 import rocks.inspectit.ocelot.mappings.model.AgentMapping;
 
 import java.nio.charset.Charset;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 /**
  * An {@link AgentMapping} which has its configuration loaded in-memory.
  * In addition, a cryptographic hash is computed to detect changes of configurations.
  */
 @Value
+@Slf4j
 public class AgentConfiguration {
 
+    /**
+     * Used as maker in {@link AgentConfigurationManager#attributesToConfigurationCache} to mark attribute-maps
+     * for which no mapping matches.
+     */
+    public static AgentConfiguration NO_MATCHING_MAPPING = createDefault();
+
+    /**
+     * Predicate to check if a given file path ends with .yml or .yaml
+     */
+    private static final Predicate<String> HAS_YAML_ENDING = filePath -> filePath.toLowerCase()
+            .endsWith(".yml") || filePath.toLowerCase().endsWith(".yaml");
+
     /**
      * The agent mapping for which this instance represents the loaded configuration.
      */
     private AgentMapping mapping;
 
     /**
-     * The set of defined documentable objects in this configuration for each file. <br>
-     * - Key: the file path <br>
-     * - Value: the set of objects, like actions, scopes, rules & metrics
+     * The set of suppliers for documentable objects.
+     * There is a set of defined documentable objects in this configuration for each file. <br>
+     * The documentable objects will be loaded lazy.
      */
-    private Map<String, Set<String>> docsObjectsByFile;
+    private Set<AgentDocumentationSupplier> documentationSuppliers;
 
     /**
      * The merged YAML configuration for the given mapping.
@@ -39,11 +56,127 @@ public class AgentConfiguration {
      */
     private String hash;
 
-    @Builder
-    private AgentConfiguration(AgentMapping mapping, Map<String, Set<String>> docsObjectsByFile, String configYaml) {
+    private AgentConfiguration(AgentMapping mapping, Set<AgentDocumentationSupplier> documentationSuppliers, String configYaml, String hash) {
         this.mapping = mapping;
-        this.docsObjectsByFile = docsObjectsByFile;
+        this.documentationSuppliers = documentationSuppliers;
         this.configYaml = configYaml;
-        hash = DigestUtils.md5DigestAsHex(configYaml.getBytes(Charset.defaultCharset()));
+        this.hash = hash;
+    }
+
+    /**
+     * Factory method to create AgentConfigurations. Also creates a cryptographic hash.
+     *
+     * @param mapping The agent mapping for which this instance represents the loaded configuration
+     * @param fileAccessor The accessor to use for reading the files
+     *
+     * @return Created AgentConfiguration
+     */
+    public static AgentConfiguration create(AgentMapping mapping, AbstractFileAccessor fileAccessor) {
+        Set<AgentDocumentationSupplier> documentationSuppliers = new HashSet<>();
+        Object yamlResult = null;
+
+        LinkedHashSet<String> allYamlFiles = getAllYamlFilesForMapping(fileAccessor, mapping);
+        for (String path : allYamlFiles) {
+            String src = fileAccessor.readConfigurationFile(path).orElse("");
+            yamlResult = ObjectStructureMerger.loadAndMergeYaml(src, yamlResult, path);
+
+            AgentDocumentationSupplier supplier = new AgentDocumentationSupplier(() -> loadDocumentation(path, src));
+            documentationSuppliers.add(supplier);
+        }
+
+        String configYaml = yamlResult == null ? "" : new Yaml().dump(yamlResult);
+        String hash = DigestUtils.md5DigestAsHex(configYaml.getBytes(Charset.defaultCharset()));
+
+        return new AgentConfiguration(mapping, documentationSuppliers, configYaml, hash);
+    }
+
+    /**
+     * Factory method to create a default AgentConfiguration.
+     *
+     * @return Created default AgentConfiguration
+     */
+    private static AgentConfiguration createDefault() {
+        String configYaml = "";
+        String hash = DigestUtils.md5DigestAsHex(configYaml.getBytes(Charset.defaultCharset()));
+        return new AgentConfiguration(null, new HashSet<>(), configYaml, hash);
+    }
+
+    /**
+     * Get the current agent documentations. The documentations will be loaded lazy.
+     *
+     * @return The loaded agent documentations
+     */
+    public synchronized Set<AgentDocumentation> getDocumentations() {
+        if (documentationSuppliers == null) return Collections.emptySet();
+        return documentationSuppliers.stream().map(AgentDocumentationSupplier::get).collect(Collectors.toSet());
+    }
+
+    /**
+     * Returns the set of yaml files, which is defined in the agent mapping sources.
+     *
+     * @param fileAccessor the accessor to use for reading the file
+     * @param mapping the mapping, which contains a list of source file paths
+     *
+     * @return the set of yaml file paths for the provided mapping
+     * If this task has been canceled, null is returned.
+     */
+    private static LinkedHashSet<String> getAllYamlFilesForMapping(AbstractFileAccessor fileAccessor, AgentMapping mapping) {
+        LinkedHashSet<String> allYamlFiles = new LinkedHashSet<>();
+        for (String path : mapping.sources()) {
+            List<String> yamlFiles = getAllYamlFiles(fileAccessor, path);
+            allYamlFiles.addAll(yamlFiles);
+        }
+        return allYamlFiles;
+    }
+
+    /**
+     * If the given path is a yaml file, a list containing only it is returned.
+     * If the path is a directory, the absolute path of all contained yaml files is returned in alphabetical order.
+     * If it is neither, an empty list is returned.
+     *
+     * @param path the path to check for yaml files, can start with a slash which will be ignored
+     *
+     * @return a list of absolute paths of contained YAML files
+     */
+    private static List<String> getAllYamlFiles(AbstractFileAccessor fileAccessor, String path) {
+        String cleanedPath;
+        if (path.startsWith("/")) {
+            cleanedPath = path.substring(1);
+        } else {
+            cleanedPath = path;
+        }
+
+        if (fileAccessor.configurationFileExists(cleanedPath)) {
+            if (fileAccessor.configurationFileIsDirectory(cleanedPath)) {
+                List<FileInfo> fileInfos = fileAccessor.listConfigurationFiles(cleanedPath);
+
+                return fileInfos.stream()
+                        .flatMap(file -> file.getAbsoluteFilePaths(cleanedPath))
+                        .filter(HAS_YAML_ENDING)
+                        .sorted()
+                        .collect(Collectors.toList());
+            } else if (HAS_YAML_ENDING.test(cleanedPath)) {
+                return Collections.singletonList(cleanedPath);
+            }
+        }
+        return Collections.emptyList();
+    }
+
+    /**
+     * Loads all documentable objects of the yaml source string for the provided file.
+     *
+     * @param filePath the path to the yaml file
+     * @param src the yaml string
+     *
+     * @return the set of documentable objects for the provided file
+     */
+    private static AgentDocumentation loadDocumentation(String filePath, String src) {
+        Set<String> objects = Collections.emptySet();
+        try {
+            objects = DocsObjectsLoader.loadObjects(src);
+        } catch (Exception e) {
+            log.warn("Could not parse configuration: {}", filePath, e);
+        }
+        return new AgentDocumentation(filePath, objects);
     }
 }

components/inspectit-ocelot-configurationserver/src/main/java/rocks/inspectit/ocelot/agentconfiguration/AgentConfigurationManager.java

@@ -24,18 +24,15 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 
+import static rocks.inspectit.ocelot.agentconfiguration.AgentConfiguration.NO_MATCHING_MAPPING;
+
 /**
  * Manager responsible for serving the agent configuration based on the set of {@link AgentMapping}s.
  */
 @Component
 @Slf4j
 public class AgentConfigurationManager {
 
-    /**
-     * Used as maker in {@link #attributesToConfigurationCache} to mark attribute-maps for which no mapping matches.
-     */
-    private static final AgentConfiguration NO_MATCHING_MAPPING = AgentConfiguration.builder().configYaml("").build();
-
     @Autowired
     @VisibleForTesting
     InspectitServerSettings config;
@@ -120,7 +117,7 @@ private synchronized void replaceConfigurations(List<AgentConfiguration> newConf
         attributesToConfigurationCache = CacheBuilder.newBuilder()
                 .maximumSize(config.getMaxAgents())
                 .expireAfterAccess(config.getAgentEvictionDelay().toMillis(), TimeUnit.MILLISECONDS)
-                .build(new CacheLoader<Map<String, String>, AgentConfiguration>() {
+                .build(new CacheLoader<>() {
                     @Override
                     public AgentConfiguration load(Map<String, String> agentAttributes) {
                         return newConfigurations.stream()