1.2.0

CHANGELOG.md

@@ -4,6 +4,22 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [1.2.0] 2022-06-17
+
+### Features
+- watcher service:
+  - allowing processes to be ignored through the mapping file: **watch.ignore** (must be located in `~/.config/guapow` or `/etc/guapow`)
+    - it follows the same patterns as the `watch.map` file, but the profile name is not required (as it makes no sense). e.g:
+    ```
+    my_app_name
+    my_app_name*  
+    /bin/my_proc
+    r:/bin/.+/xpto
+    ```
+    - this feature is useful if you have general mappings that cover a lot of processes in `watch.map` (e.g: `/usr/bin/*`), but want to ignore specific ones
+    - new config property `ignored.cache` to cache all mapped patterns to memory after the first read and skip next I/O calls (default: `false`) 
+
+
 ## [1.1.1] 2022-06-08
 
 ### Fixes

README.md

@@ -48,6 +48,7 @@
     - [Watcher service](#watcher)
         - [Mapping patterns](#watch_patterns)
         - [Built-in patterns (steam)](#watch_builtin)
+        - [Ignoring processes](#watch_ignore)
         - [Settings](#watch_settings)
     - [CLI](#cli)
 7. [Improving optimizations timing](#improve_opt)
@@ -517,13 +518,24 @@ makepkg -si
     - <a name="watch_builtin">**Built-in patterns:**<a/>
         - Pre-defined words following the pattern `__word__` that are evaulated as a regex.
             - `__steam__`: defines a regex for any game launched through **Steam** (native and Proton). Example [here](#tutorial_steam).
-    
+
+- <a name="watch_ignore">Ignoring processes:</a> it is possible to define patterns to ignore specific processes through the file `~/.config/guapow/watch.ignore` (user) (or `/etc/guapow/watch.ignore` (system)). Preference: user > system (if running as **root**, system is the only option).
+    - this file follows the same mapping rules as `watch.map`, but you don't need to provide the profile names (as it makes no sense). e.g:
+        ```
+        my_app_name
+        my_app_name*  
+        /bin/my_proc
+        r:/bin/.+/xpto
+       ```
+    - this feature is useful if you have general mappings that cover a lot of processes in `watch.map` (e.g: `/usr/bin/*`), but want to ignore specific ones
+  
 - <a name="watch_settings">Settings</a>
     - Defined at the file `~/.config/guapow/watch.conf` (user) or `/etc/guapow/watch.conf` (system). Preference: user > system (if running as **root**, system is the only option).
         ```
            interval = 1 (in seconds to check for new-born applications and request optimizations)
            regex.cache = true (caches pre-compiled regex mapping patterns in memory)
-           mapping.cache = false (if 'true', caches the all mapping in memory to skip I/O calls. Changes to the mapping file won't take effect.
+           mapping.cache = false (if 'true', caches the all mapping in memory to skip I/O calls. Changes to watch.map won't have effect until the service is restarted.
+           ignored.cache = false (if 'true', caches the all ignored patterns in memory to skip I/O calls. Changes to watch.ignore won't have effect until the service is restarted.
         ```
 
 - Logs:

guapow/__init__.py

@@ -2,4 +2,4 @@
 
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
 __app_name__ = 'guapow'
-__version__ = '1.1.1'
+__version__ = '1.2.0'

guapow/service/watcher/config.py

@@ -13,12 +13,15 @@ class ProcessWatcherConfig(FileModel):
     FILE_NAME = 'watch.conf'
     FILE_MAPPING = {'interval': ('check_interval', float, None),
                     'regex.cache': ('regex_cache', bool, True),
-                    'mapping.cache': ('mapping_cache', bool, True)}
+                    'mapping.cache': ('mapping_cache', bool, True),
+                    'ignored.cache': ('ignored_cache', bool, True)}
 
-    def __init__(self, regex_cache: Optional[bool], check_interval: Optional[float], mapping_cache: Optional[bool]):
+    def __init__(self, regex_cache: Optional[bool], check_interval: Optional[float], mapping_cache: Optional[bool],
+                 ignored_cache: Optional[bool]):
         self.check_interval = check_interval
         self.regex_cache = regex_cache
         self.mapping_cache = mapping_cache
+        self.ignored_cache = ignored_cache
 
     def get_output_name(self) -> str:
         pass
@@ -29,7 +32,8 @@ def get_file_mapping(self) -> Dict[str, Tuple[str, type, Optional[object]]]:
     def is_valid(self) -> bool:
         return all((self.is_check_interval_valid(),
                     self.regex_cache is not None,
-                    self.mapping_cache is not None))
+                    self.mapping_cache is not None,
+                    self.ignored_cache is not None))
 
     def is_check_interval_valid(self):
         return self.check_interval is not None and self.check_interval > 0
@@ -44,12 +48,15 @@ def setup_valid_properties(self):
         if self.mapping_cache is None:
             self.mapping_cache = False
 
+        if self.ignored_cache is None:
+            self.ignored_cache = False
+
     def get_file_root_node_name(self) -> Optional[str]:
         pass
 
     @classmethod
     def empty(cls) -> "ProcessWatcherConfig":
-        return cls(None, None, None)
+        return cls(None, None, None, None)
 
     @classmethod
     def default(cls) -> "ProcessWatcherConfig":

guapow/service/watcher/core.py

@@ -1,23 +1,25 @@
 import asyncio
 import re
 import time
+from asyncio import create_task
 from logging import Logger
-from typing import Dict, Optional, Tuple
+from typing import Dict, Optional, Tuple, Set, Any, Union
 
 from guapow.common import network
 from guapow.common.config import OptimizerConfig
 from guapow.common.dto import OptimizationRequest
-from guapow.service.watcher import mapping
+from guapow.service.watcher import mapping, ignored
 from guapow.service.watcher.config import ProcessWatcherConfig
-from guapow.service.watcher.mapping import RegexMapper
+from guapow.service.watcher.patterns import RegexMapper, RegexType
 from guapow.service.watcher.util import map_processes
 
 
 class ProcessWatcherContext:
 
     def __init__(self, user_id: int, user_name: str, user_env: Dict[str, str], logger: Logger,
                  optimized: Dict[int, str], opt_config: OptimizerConfig, watch_config: ProcessWatcherConfig,
-                 mapping_file_path: str, machine_id: Optional[str]):
+                 mapping_file_path: str, machine_id: Optional[str], ignored_file_path: str,
+                 ignored_procs: Dict[Union[str, re.Pattern], Set[str]]):
         self.user_id = user_id
         self.user_name = user_name
         self.user_env = user_env
@@ -27,6 +29,8 @@ def __init__(self, user_id: int, user_name: str, user_env: Dict[str, str], logge
         self.watch_config = watch_config
         self.mapping_file_path = mapping_file_path
         self.machine_id = machine_id
+        self.ignored_file_path = ignored_file_path
+        self.ignored_procs = ignored_procs
 
 
 class ProcessWatcher:
@@ -41,17 +45,57 @@ def __init__(self, regex_mapper: RegexMapper, context: ProcessWatcherContext):
         self._mappings: Optional[Dict[str, str]] = None
         self._cmd_patterns: Optional[Dict[re.Pattern, str]] = None
         self._comm_patterns: Optional[Dict[re.Pattern, str]] = None
-        self._last_file_found_log: Optional[bool] = None  # controls repetitive file found logs
+
+        self._last_mapping_file_found: Optional[bool] = None  # controls repetitive file found logs
+        self._last_ignored_file_found: Optional[bool] = None  # controls repetitive file found logs
+
+        self._ignored_cached = False
+        self._ignored_exact_strs: Optional[Set[str]] = None
+        self._ignored_cmd_patterns: Optional[Set[re.Pattern]] = None
+        self._ignored_comm_patterns: Optional[Set[re.Pattern]] = None
+
+    async def _read_ignored(self) -> Optional[Tuple[Set[str], Optional[Set[re.Pattern]], Optional[Set[re.Pattern]]]]:
+        """
+        return a tuple with command patterns (cmd) and name patterns (comm)
+        """
+        if self._ignored_cached:
+            if self._ignored_exact_strs:
+                return self._ignored_exact_strs, self._ignored_cmd_patterns, self._ignored_comm_patterns
+        else:
+            file_found, ignored_strs = await ignored.read(file_path=self._context.ignored_file_path, logger=self._log,
+                                                          last_file_found_log=self._last_ignored_file_found)
+            self._last_ignored_file_found = file_found
+
+            if not self._ignored_cached and self._context.watch_config.ignored_cache:
+                self._log.debug("Caching ignored patterns to memory")
+                self._ignored_cached = True  # pre-saving the caching state (if enabled)
+
+            if ignored_strs:
+                if self._context.watch_config.ignored_cache:  # caching to memory (if enabled)
+                    self._ignored_exact_strs = ignored_strs
+
+                patterns = self._regex_mapper.map_collection(ignored_strs)
+
+                cmd_patterns, comm_patterns = None, None
+
+                if patterns:
+                    cmd_patterns, comm_patterns = patterns.get(RegexType.CMD), patterns.get(RegexType.COMM)
+
+                    if self._context.watch_config.ignored_cache:  # caching to memory (if enabled)
+                        self._ignored_cmd_patterns = cmd_patterns
+                        self._ignored_comm_patterns = comm_patterns
+
+                return ignored_strs, cmd_patterns, comm_patterns
 
     async def _read_mappings(self) -> Optional[Tuple[Dict[str, str], Optional[Dict[re.Pattern, str]], Optional[Dict[re.Pattern, str]]]]:
         if self._mapping_cached:
             if self._mappings:
                 return self._mappings, self._cmd_patterns, self._comm_patterns
         else:
-            file_found, mappings = await mapping.read(file_path=self._context.mapping_file_path, logger=self._log, last_file_found_log=self._last_file_found_log)
-            self._last_file_found_log = file_found
+            file_found, mappings = await mapping.read(file_path=self._context.mapping_file_path, logger=self._log, last_file_found_log=self._last_mapping_file_found)
+            self._last_mapping_file_found = file_found
 
-            pattern_mappings = self._regex_mapper.map(mappings)
+            pattern_mappings = self._regex_mapper.map_for_profiles(mappings)
             cmd_patterns, comm_patterns = (pattern_mappings[0], pattern_mappings[1]) if pattern_mappings else (None, None)
 
             if self._context.watch_config.mapping_cache:
@@ -60,28 +104,108 @@ async def _read_mappings(self) -> Optional[Tuple[Dict[str, str], Optional[Dict[r
 
             return mappings, cmd_patterns, comm_patterns
 
+    def _map_ignored_id(self, pid: int, comm: str) -> str:
+        return f'{pid}:{comm}'
+
+    def _is_ignored(self, ignored_id: str) -> bool:
+        if self._context.ignored_procs:
+            for ignored_ids in self._context.ignored_procs.values():
+                if ignored_id in ignored_ids:
+                    return True
+
+        return False
+
+    def _clean_old_ignore_patterns(self, ignored_exact: Set[str], current_cmd_patterns: Optional[Set[re.Pattern]],
+                                   current_comm_patterns: Optional[Set[re.Pattern]]):
+        if self._context.ignored_procs:
+            all_patterns = (ignored_exact, current_cmd_patterns, current_comm_patterns)
+            to_remove = set()
+            for pattern in self._context.ignored_procs:
+                found = False
+                for patterns in all_patterns:
+                    if patterns and pattern in patterns:
+                        found = True
+                        break
+
+                if not found:
+                    to_remove.add(pattern)
+
+            if to_remove:
+                self._log.debug(f"Cleaning old ignored patterns from context: {', '.join(str(p) for p in to_remove)}")
+                for pattern in to_remove:
+                    del self._context.ignored_procs[pattern]
+
+    def _matches_ignored(self, cmd_com: Tuple[str, str], ignored_exact: Set[str],
+                         cmd_patterns: Optional[Set[re.Pattern]], comm_patterns: Optional[Set[re.Pattern]],
+                         ignored_id: str) -> bool:
+
+        if ignored_exact or cmd_patterns or comm_patterns:
+            for idx, cmd in enumerate(cmd_com):  # 0: cmd, 1: comm
+                matched_pattern = None
+                if ignored_exact and cmd in ignored_exact:  # exact matches have higher priority than patterns
+                    matched_pattern = cmd
+                else:
+                    regex_ignored = cmd_patterns if idx == 0 else comm_patterns
+
+                    if regex_ignored:
+                        for pattern in regex_ignored:
+                            if pattern.match(cmd):
+                                matched_pattern = pattern
+                                break
+
+                if matched_pattern:
+                    ignored_ids = self._context.ignored_procs.get(matched_pattern)
+
+                    if ignored_ids is None:
+                        ignored_ids = set()
+                        self._context.ignored_procs[matched_pattern] = ignored_ids
+
+                    ignored_ids.add(ignored_id)
+                    return True
+
+        return False
+
     async def check_mappings(self):
         mapping_tuple = await self._read_mappings()
 
         if not mapping_tuple or not mapping_tuple[0]:
             return
 
         mappings, cmd_patterns, comm_patterns = mapping_tuple[0], mapping_tuple[1], mapping_tuple[2]
-
-        procs = await map_processes()
+
+        task_map_procs, task_ignored = create_task(map_processes()), create_task(self._read_ignored())
+
+        procs = await task_map_procs
 
         if not procs:
             self._log.warning('No processes alive')
             self._context.optimized.clear()
             return
 
+        ignored_exact, ignored_cmd, ignored_comm = None, None, None
+        ignored_patterns = await task_ignored
+
+        if ignored_patterns:
+            ignored_exact, ignored_cmd, ignored_comm = ignored_patterns
+
+        self._clean_old_ignore_patterns(ignored_exact, ignored_cmd, ignored_comm)
+
         tasks = []
         for pid, cmd_comm in procs.items():
+            ignored_id = self._map_ignored_id(pid, cmd_comm[1])
+
+            if self._is_ignored(ignored_id):
+                continue
+
             previously_optimized_cmd = self._context.optimized.get(pid)
 
             if previously_optimized_cmd and (previously_optimized_cmd == cmd_comm[0] or previously_optimized_cmd == cmd_comm[1]):
                 continue
 
+            if self._matches_ignored(cmd_comm, ignored_exact, ignored_cmd, ignored_comm, ignored_id):
+                self._log.info(f"Ignoring process (pid: {pid}, name: {cmd_comm[1]})")
+                continue
+
             for idx, cmd in enumerate(cmd_comm):  # 0: cmd, 1: comm
                 profile = mappings.get(cmd)  # exact matches have higher priority than patterns
                 if profile:
@@ -104,12 +228,35 @@ async def check_mappings(self):
         if tasks:
             await asyncio.gather(*tasks)
 
-        if self._context.optimized:  # removing dead processes from the context
-            pids_alive = procs.keys()
+        self._clean_dead_processes_from_context(procs)
+
+    def _clean_dead_processes_from_context(self, current_processes: Dict[int, Any]):
+        if self._context.optimized:
+            pids_alive = current_processes.keys()
             for pid in {*self._context.optimized.keys()}:
                 if pid not in pids_alive:
                     del self._context.optimized[pid]
 
+        if self._context.ignored_procs:
+            pids_alive = current_processes.keys()
+            patterns_to_remove = set()
+            for pattern, ignored_ids in self._context.ignored_procs.items():
+                to_remove = set()
+                for id_ in ignored_ids:
+                    if int(id_.split(':')[0]) not in pids_alive:
+                        to_remove.add(id_)
+
+                if to_remove:
+                    self._log.debug(f"Removing dead pids from ignored context: {', '.join(f'{p}' for p in to_remove)}")
+                    ignored_ids.difference_update(to_remove)
+
+                    if not ignored_ids:
+                        patterns_to_remove.add(pattern)
+
+            if patterns_to_remove:
+                for pattern in patterns_to_remove:
+                    del self._context.ignored_procs[pattern]
+
     async def send_request(self, pid: int, command: str, match: str, profile: str):
         request = OptimizationRequest(pid=pid, command=command, created_at=time.time(),
                                       user_name=self._context.user_name, user_env=self._context.user_env,