Merge pull request opensearch-project#1126 from AndreKurait/ClusterTo…

…olsLogging

Update logging for cluster tools

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/setup.py

@@ -7,7 +7,7 @@
     package_dir={"": "src"},
     entry_points={
         "console_scripts": [
-            "cluster_tools = cluster_tools.main:main",  # Links to `main` function in `cluster_tools/main.py`
+            "cluster_tools = cluster_tools.base.main:main",  # Links to `main` function in `cluster_tools/main.py`
         ],
     },
     classifiers=[

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/src/cluster_tools/main.py → TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/src/cluster_tools/base/main.py

@@ -4,41 +4,37 @@
 import argcomplete
 from console_link.environment import Environment
 import logging
-import sys
+from datetime import datetime
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-
-
-class ClusterToolsFilter(logging.Filter):
-    def filter(self, record):
-        return record.name.startswith('cluster_tools') or record.name.startswith('tools')
+root_logger = logging.getLogger()
+root_logger.setLevel(logging.INFO)
 
+logger = logging.getLogger(__name__)
 
-def setup_logging():
-    """Sets up logging with a file handler for all logs and a stdout handler for 'cluster_tools' logs only."""
 
-    from datetime import datetime
+def setup_file_logging(tool_name: str):
+    """Sets up logging with a file handler for all logs."""
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
     logs_dir = os.getenv("SHARED_LOGS_DIR_PATH", "./logs")
-    log_file_path = os.path.join(logs_dir, f'cluster_tools/log_{timestamp}_{os.getpid()}.log')
+    host_name = os.getenv("HOSTNAME", "localhost")
+    log_file_path = os.path.join(logs_dir, f'{host_name}/cluster_tools/{timestamp}_{tool_name}_log_{os.getpid()}.log')
     os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
     file_handler = logging.FileHandler(log_file_path)
+    file_handler.setLevel(logging.INFO)
     file_handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
+    root_logger.addHandler(file_handler)
+    return file_handler
 
-    stdout_handler = logging.StreamHandler(sys.stdout)
-    stdout_handler.setFormatter(logging.Formatter('%(message)s'))
-
-    # Apply the filter to the stdout handler
-    stdout_handler.addFilter(ClusterToolsFilter())
-
-    # Add both handlers to the main logger
-    logger.addHandler(file_handler)
-    logger.addHandler(stdout_handler)
 
-    # Setup logger for this module with name
-    module_logger = logging.getLogger(__name__)
-    return module_logger
+def setup_stdout_logging():
+    """Sets up a stdout handler for 'cluster_tools' logs only."""
+    stdout_handler = logging.StreamHandler()
+    stdout_handler.setLevel(logging.INFO)
+    stdout_handler.setFormatter(logging.Formatter('%(message)s'))
+    root_module_name = __name__.split('.')[0]
+    stdout_handler.addFilter(logging.Filter(root_module_name))
+    root_logger.addHandler(stdout_handler)
+    return stdout_handler
 
 
 def list_tools():
@@ -52,9 +48,6 @@ def list_tools():
     return tools
 
 
-log_file_path = None
-
-
 def setup_parser(parser):
     parser.add_argument(
         '--config_file',
@@ -66,7 +59,7 @@ def setup_parser(parser):
     # Dynamically add subparsers for each tool
     for tool_name in list_tools():
         try:
-            tool_module = importlib.import_module(f"tools.{tool_name}")
+            tool_module = importlib.import_module(f"cluster_tools.tools.{tool_name}")
             tool_parser = subparsers.add_parser(tool_name, help=f"{tool_name} utility")
 
             # Check if the tool module has a 'define_arguments' function to define its arguments
@@ -84,7 +77,7 @@ def setup_parser(parser):
 
 
 def main(args=None):
-    global logger
+    setup_stdout_logging()
     # Create the main parser
     parser = argparse.ArgumentParser(
         description="CLI tool for managing and running different utilities."
@@ -103,17 +96,19 @@ def main(args=None):
             logger.info(f"  - {tool}")
         logger.info("\nRun `cluster_tools <tool>` to use a tool.")
     else:
-        # Setup logging for tool execution
-        logger = setup_logging()
+        # Setup file logging for tool execution
+        file_handler = setup_file_logging(args.tool)
         env = Environment(args.config_file)
         try:
             args.func(env, args)
         except Exception as e:
             logger.error(f"An error occurred while executing the tool: {e}")
             raise e
         finally:
-            if log_file_path is not None:
-                logger.info(f"Logs saved to {log_file_path}")
+            if file_handler is not None:
+                logger.info(f"\nLogs saved to {file_handler.baseFilename}")
+                file_handler.close()
+                logger.removeHandler(file_handler)
 
 
 if __name__ == "__main__":

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/src/cluster_tools/utils.py → TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/src/cluster_tools/base/utils.py

@@ -2,9 +2,6 @@
 from typing import Dict, Optional, Union, Any
 from console_link.models.cluster import HttpMethod
 from console_link.environment import Environment
-import logging
-
-logger = logging.getLogger(__name__)
 
 
 def console_curl(
@@ -30,7 +27,6 @@ def console_curl(
         http_method = HttpMethod[method.upper()]
     except KeyError:
         message = f"Invalid HTTP method: {method}"
-        logger.error(message)
         raise ValueError(message)
 
     if cluster == 'source_cluster':
@@ -39,12 +35,10 @@ def console_curl(
         cluster_obj = env.target_cluster
     else:
         message = "`cluster` must be either 'source_cluster' or 'target_cluster'."
-        logger.error(message)
         raise ValueError(message)
 
     if cluster_obj is None:
         message = f"{cluster} is not defined in the environment."
-        logger.error(message)
         raise ValueError(message)
 
     if json_data is not None:

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/src/tools/change_shards.py → TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/src/cluster_tools/tools/change_shards.py

@@ -1,6 +1,6 @@
 import argparse
 from typing import Any, Dict
-from cluster_tools.utils import console_curl
+from cluster_tools.base.utils import console_curl
 from console_link.environment import Environment
 import logging
 import uuid

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/src/tools/create_index.py → TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/src/cluster_tools/tools/create_index.py

@@ -1,6 +1,6 @@
 import argparse
 from console_link.environment import Environment
-from cluster_tools.utils import console_curl
+from cluster_tools.base.utils import console_curl
 import logging
 
 logger = logging.getLogger(__name__)

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/src/tools/disable_compatibility_mode.py → TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/src/cluster_tools/tools/disable_compatibility_mode.py

@@ -1,6 +1,6 @@
 import argparse
 from console_link.environment import Environment
-from cluster_tools.utils import console_curl
+from cluster_tools.base.utils import console_curl
 import logging
 
 logger = logging.getLogger(__name__)

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/tests/change_shards_test.py

@@ -1,7 +1,7 @@
-from src.tools.change_shards import main as change_shards
+from cluster_tools.tools.change_shards import main as change_shards
 from tests.utils import target_cluster_refresh, get_target_index_info
-from src.tools.create_index import main as create_index
-from src.cluster_tools.utils import console_curl
+from cluster_tools.tools.create_index import main as create_index
+from src.cluster_tools.base.utils import console_curl
 import argparse
 import logging
 import pytest

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/tests/create_index_test.py

@@ -1,4 +1,4 @@
-from src.tools.create_index import main as create_index
+from cluster_tools.tools.create_index import main as create_index
 from tests.utils import get_target_index_info
 import argparse
 import logging

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/tests/disable_compatibility_mode_test.py

@@ -1,5 +1,5 @@
-from src.tools.disable_compatibility_mode import main as disable_compatibility_mode
-from src.cluster_tools.utils import console_curl
+from cluster_tools.tools.disable_compatibility_mode import main as disable_compatibility_mode
+from src.cluster_tools.base.utils import console_curl
 import logging
 
 logger = logging.getLogger(__name__)

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/tests/enable_compatibility_mode_test.py

@@ -1,5 +1,5 @@
-from src.tools.enable_compatibility_mode import main as enable_compatibility_mode
-from src.cluster_tools.utils import console_curl
+from cluster_tools.tools.enable_compatibility_mode import main as enable_compatibility_mode
+from src.cluster_tools.base.utils import console_curl
 import logging
 
 logger = logging.getLogger(__name__)

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/tests/main_test.py

@@ -1,7 +1,7 @@
-from src.cluster_tools.main import main
+from src.cluster_tools.base.main import main
 import argparse
 from tests.utils import get_target_index_info
-import src.tools.create_index as create_index
+import cluster_tools.tools.create_index as create_index
 import logging
 
 logger = logging.getLogger(__name__)

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools/tests/utils.py

@@ -1,5 +1,5 @@
 from console_link.environment import Environment
-from src.cluster_tools.utils import console_curl
+from src.cluster_tools.base.utils import console_curl
 import logging
 
 logger = logging.getLogger(__name__)