Merge branch 'dev' of github.com:visualDust/neetbox into dev

neetbox/config/_config.py

@@ -4,14 +4,14 @@
 # URL:    https://gong.host
 # Date:   20230413
 
-import collections
-from multiprocessing import current_process
 
 DEFAULT_GLOBAL_CONFIG = {
     "daemon": {
         "enable": True,
         "allowIpython": False,
-        "servers": [{"host": "localhost", "port": "20202"},],
+        "servers": [
+            {"host": "localhost", "port": "20202"},
+        ],
         "mode": "detached",
         "displayName": None,
         "uploadInterval": 10,

neetbox/core/registry.py

@@ -48,7 +48,7 @@ def __new__(cls, name: str) -> "Registry":
 
     # not compatible with python below 3.8
     def __init__(self, name, *args, **kwargs) -> None:
-        if not "initialized" in self:
+        if "initialized" not in self:
             self["initialized"] = True
             self.name = name
 

neetbox/daemon/__init__.py

@@ -5,8 +5,6 @@
 # Date:   20230414
 
 import json
-import os
-import platform
 import subprocess
 import time
 

neetbox/daemon/_client_apis.py

@@ -5,21 +5,16 @@
 # Date:   20230414
 
 
+from neetbox.config import get_module_level_config
 from neetbox.daemon._local_http_client import _local_http_client
+from neetbox.logging import logger
 from neetbox.utils import pkg
 from neetbox.utils.framing import get_frame_module_traceback
 
-module_name = get_frame_module_traceback().__name__
+module_name = get_frame_module_traceback().__name__  # type: ignore
 assert pkg.is_installed(
     "httpx", try_install_if_not=True
 ), f"{module_name} requires httpx which is not installed"
-import json
-import time
-
-import httpx
-
-from neetbox.config import get_module_level_config
-from neetbox.logging import logger
 
 logger = logger("NEETBOX DAEMON API")
 

neetbox/daemon/_daemon_client.py

@@ -7,6 +7,7 @@
 import json
 import time
 from threading import Thread
+from typing import Union
 
 from neetbox.config import get_module_level_config
 from neetbox.daemon._local_http_client import _local_http_client
@@ -15,14 +16,13 @@
 
 __TIME_UNIT_SEC = 0.1
 
-__upload_thread: Thread = None
+__upload_thread: Union[Thread, None] = None
 
 
 def _upload_thread(daemon_config, base_addr, display_name):
     _ctr = 0
     _api_name = "sync"
     _api_addr = f"{base_addr}/{_api_name}/{display_name}"
-    global _update_value_dict
     _disconnect_flag = False
     _disconnect_retries = 10
     while True:
@@ -36,18 +36,18 @@ def _upload_thread(daemon_config, base_addr, display_name):
         _headers = {"Content-Type": "application/json"}
         try:
             # upload data
-            resp = _local_http_client.post(_api_addr, data=_data, headers=_headers)
-            if resp.is_error: # upload failed
+            resp = _local_http_client.post(_api_addr, json=_data, headers=_headers)
+            if resp.is_error:  # upload failed
                 raise IOError(f"Failed to upload data to daemon. ({resp.status_code})")
         except Exception as e:
-            if _disconnect_flag: # already in retries
+            if _disconnect_flag:  # already in retries
                 _disconnect_retries -= 1
-                if not _disconnect_retries: # retry count down exceeded
+                if not _disconnect_retries:  # retry count down exceeded
                     logger.err(
                         "Failed to reconnect to daemon after {10} retries, Trying to launch new daemon..."
                     )
                     from neetbox.daemon import _try_attach_daemon
-                    
+
                     _try_attach_daemon()
                     time.sleep(__TIME_UNIT_SEC)
                 continue
@@ -58,7 +58,7 @@ def _upload_thread(daemon_config, base_addr, display_name):
         else:
             if not _disconnect_flag:
                 continue
-            logger.ok(f"Successfully reconnected to daemon.")
+            logger.ok("Successfully reconnected to daemon.")
             _disconnect_flag = False
             _disconnect_retries = 10
 

neetbox/daemon/_daemon_launcher.py

@@ -1,15 +1,14 @@
-print("========= Daemon Launcher =========")
-
 import argparse
 import json
-import os
 import sys
 
 from neetbox.daemon._flask_server import daemon_process
+
 # sys.stdout=open(r'D:\Projects\ML\neetbox\logdir\daemon.log', 'a+')
 
 
 # from neetbox.daemon._local_http_client import _local_http_client
+print("========= Daemon Launcher =========")
 
 
 def run():
@@ -26,6 +25,5 @@ def run():
 
 
 print("_daemon_launcher is starting with __name__ =", __name__, " and args =", sys.argv)
-
 run()
 print("_daemon_launcher: exiting")

neetbox/daemon/_flask_server.py

@@ -12,6 +12,13 @@
 from flask import Flask, abort, json, request
 
 from neetbox.config import get_module_level_config
+from neetbox.utils import pkg
+from neetbox.utils.framing import get_frame_module_traceback
+
+module_name = get_frame_module_traceback().__name__  # type: ignore
+assert pkg.is_installed(
+    "flask", try_install_if_not=True
+), f"{module_name} requires flask which is not installed"
 
 _STAT_POOL = {}
 __DAEMON_SHUTDOWN_IF_NO_UPLOAD_TIMEOUT_SEC = 60 * 60 * 12  # 12 Hours

neetbox/daemon/_local_http_client.py

@@ -1,17 +1,18 @@
-import httpx
 import logging
 
+import httpx
+
 httpx_logger = logging.getLogger("httpx")
 httpx_logger.setLevel(logging.ERROR)
-    
+
 __no_proxy = {
     "http://": None,
     "https://": None,
 }
 
 
 def __load_http_client():
-    __local_http_client = httpx.Client(proxies=__no_proxy)
+    __local_http_client = httpx.Client(proxies=__no_proxy)  # type: ignore
     return __local_http_client
 
 

neetbox/daemon/daemonable_process.py

@@ -1,9 +1,8 @@
-import multiprocessing
 import os
 import subprocess
 import sys
 import time
-from typing import *
+from typing import List, Literal
 
 is_ms_windows = "win32" in sys.platform or "cygwin" in sys.platform
 
@@ -59,7 +58,7 @@ def start(self):
                 creationflags = {
                     "attached": 0,
                     "shared": 0,
-                    "detached": subprocess.CREATE_NO_WINDOW,
+                    "detached": subprocess.CREATE_NO_WINDOW,  # type: ignore (only for windows)
                 }[self.__mode]
 
                 popen = subprocess.Popen(

neetbox/integrations/resource.py

@@ -12,7 +12,7 @@
 from functools import partial
 from random import random
 from threading import Event
-from typing import Any, Dict, Iterable, List, Union
+from typing import Dict, List, Union
 from urllib.request import urlopen
 
 import numpy as np
@@ -25,7 +25,6 @@
     TextColumn,
     TimeRemainingColumn,
     TransferSpeedColumn,
-    track,
 )
 
 from neetbox.integrations import engine

neetbox/logging/__init__.py

@@ -3,10 +3,6 @@
 
 _cfg = get_module_level_config()
 logger.set_log_dir(_cfg["logdir"])
-from neetbox.logging.logger import (
-    LogMetadata,
-    LogSplitStrategies,
-    SplitStrategyCallable,
-)
+from neetbox.logging.logger import LogSplitStrategies
 
 __all__ = ["logger", "LogSplitStrategies"]

neetbox/logging/formatting.py

@@ -5,11 +5,10 @@
 # Date:   20230318
 
 from dataclasses import dataclass
-import os
-import warnings
 from random import random
 from typing import Optional
 
+
 @dataclass
 class LogStyle:
     color: Optional[str] = None
@@ -22,7 +21,7 @@ class LogStyle:
     split_char_cmd = " > "
     split_char_identity = "/"
     split_char_txt = " | "
-    
+
     @classmethod
     def get_supported_colors(cls):
         return ["red", "green", "blue", "cyan", "yellow", "magenta"]

neetbox/logging/logger.py

@@ -401,7 +401,6 @@ def catch(
             not isclass(exception_type) or not issubclass(exception_type, BaseException)
         ):
             return self.catch()(exception_type)
-        logger = self
 
         class Catcher:
             def __init__(self, from_decorator):
@@ -415,8 +414,6 @@ def __exit__(self, type_, value, traceback_):
                     return
                 if not issubclass(type_, exception_type):
                     return False
-                from_decorator = self._from_decorator
-                catch_options = [(type_, value, traceback_)]
                 if handler:
                     handler(traceback_)
                 # logger.log(
@@ -456,19 +453,18 @@ def catch_wrapper(*args, **kwargs):
                 return catch_wrapper
 
         return Catcher(False)
-    
+
     def mention(self, func):
         @functools.wraps(func)
         def with_logging(*args, **kwargs):
-            self.log(f"Currently running: {func.__name__}",traceback=3)
+            self.log(f"Currently running: {func.__name__}", traceback=3)
             return func(*args, **kwargs)
+
         return with_logging
 
     def banner(self, text, font: Optional[str] = None):
         from pyfiglet import Figlet, FigletFont
 
-        from neetbox.utils import pkg
-
         builtin_font_list = [
             "ansiregular",
             "ansishadow",
@@ -575,7 +571,7 @@ def _bind_file(self, path):
         return self
 
     def file_bend(self) -> bool:
-        return self.file_writer != None
+        return self.file_writer is not None
 
 
 DEFAULT_LOGGER = Logger(None)

neetbox/pipeline/__init__.py

@@ -4,7 +4,6 @@
 # URL:    https://gong.host
 # Date:   20230417
 
-from neetbox.pipeline._pipe import Pipe
 from neetbox.pipeline._signal_and_slot import listen, watch
 
 __all__ = ["watch", "listen"]

neetbox/pipeline/_pipe.py

@@ -5,9 +5,6 @@
 # Date:   20230413
 
 from neetbox.core import Registry
-from neetbox.logging import logger
-from neetbox.utils import pkg
-from neetbox.utils.mvc import Singleton
 
 pipes = Registry("__pipe")
 

neetbox/torch/arch/cnn.py

@@ -4,11 +4,7 @@
 # URL:    https://gong.host
 # Date:   20230315
 
-import torch
 import torch.nn as nn
-import torch.nn.functional as F
-
-from neetbox.utils import pkg
 
 
 def ConvNxN(

neetbox/torch/arch/mask_boundary_finder.py

@@ -1,8 +1,6 @@
 import torch
 import torch.nn.functional as F
 
-from neetbox.utils import pkg
-
 
 class MaskEdgeDetecter(torch.nn.Module):
     """generate the 'edge bar' for a 0-1 mask Groundtruth of a image

neetbox/torch/profile.py

@@ -91,7 +91,7 @@ def profile(
                         f"Seems your model has an imbalanced performance peek between CUDA side synchronous test and none-sync one. Consider raising speedtest loop times (currently {speedtest} +2) to have a stable result."
                     )
                 logger.debug(
-                    f"Note that the CUDA side synchronous speedtest is more reliable since you are using a GPU."
+                    "Note that the CUDA side synchronous speedtest is more reliable since you are using a GPU."
                 )
     if profiling:
         if speedtest:

neetbox/utils/__init__.py

@@ -1,3 +1,3 @@
-from neetbox.utils._package import PipPackageHealper as pkg
+from neetbox.utils._package import pipPackageHealper as pkg
 
 __all__ = ["pkg"]

neetbox/utils/_package.py

@@ -18,7 +18,7 @@ def install(self, package, terminate=False):
         _installed = False
         while retry:
             if not retry:
-                error_str = f"Bad Input"
+                error_str = "Bad Input"
                 raise ValueError(error_str)
             print(f"{caller_name} want to install {package} via pip.")
             choice = input("Make your choice: [y]/n")
@@ -55,7 +55,7 @@ def is_installed(self, package: str, try_install_if_not: Union[str, bool] = True
             importlib.import_module(package)
             self.installed_packages.append(package)
             return True
-        except:
+        except:  # noqa
             package_name_install = (
                 package if type(try_install_if_not) is bool else try_install_if_not
             )
@@ -68,4 +68,4 @@ def is_installed(self, package: str, try_install_if_not: Union[str, bool] = True
 
 
 # singleton
-PipPackageHealper = PipPackageHealper()
+pipPackageHealper = PipPackageHealper()