Add python linter (flake8) to workflow and fix exceptions in python files [ part #25193 ]

.flake8

@@ -1,2 +1,7 @@
 [flake8]
 max-line-length = 132
+exclude = third_party
+          .*
+          out/*
+          scripts/idl/*
+          ./examples/common/QRCode/*

.github/workflows/lint.yml

@@ -213,3 +213,10 @@ jobs:
               if: always()
               run: |
                   git grep -n 'emberAfWriteAttribute' -- './*' ':(exclude).github/workflows/lint.yml' ':(exclude)src/app/util/af.h' ':(exclude)zzz_generated/app-common/app-common/zap-generated/attributes/Accessors.cpp' ':(exclude)src/app/zap-templates/templates/app/attributes/Accessors-src.zapt' ':(exclude)src/app/util/attribute-table.cpp' ':(exclude)examples/common/pigweed/rpc_services/Attributes.h' ':(exclude)src/app/util/attribute-table.h' ':(exclude)src/app/util/ember-compatibility-functions.cpp' && exit 1 || exit 0
+
+            # Run python Linter (flake8) and verify python files
+            - name: Check for errors using flake8 Python linter
+              if: always()
+              ## This large number of arguments is temporary until the patch changes reported by the linter are fully merged (all fixes in PR#25193)
+              run: |
+                  flake8 --extend-ignore=E501,W391,W291,F401,F405,F403,E711,E712,F841,E265,F541,E251,E303,F821,F402,E721,E122,W191,E122,W605,E101,E202,E201,E741,F811,E302,F523,E713

examples/common/pigweed/rpc_console/py/chip_rpc/console.py

@@ -304,11 +304,11 @@ def write_to_output(data: bytes,
 
 def _read_raw_serial(read: Callable[[], bytes], output):
     """Continuously read and pass to output."""
-    with ThreadPoolExecutor() as executor:
+    with ThreadPoolExecutor():
         while True:
             try:
                 data = read()
-            except Exception as exc:  # pylint: disable=broad-except
+            except Exception:  # pylint: disable=broad-except
                 continue
             if data:
                 output(data)

examples/platform/nxp/k32w/k32w0/scripts/detokenizer.py

@@ -53,7 +53,7 @@ def decode_string(tstr, detok):
         if s.find('$') == 0:
             return None
         return s
-    except:
+    except ValueError:
         return None
 
 
@@ -88,7 +88,7 @@ def decode_serial(serialport, outfile, database):
                     print(line, file=sys.stdout)
                     if output:
                         print(line, file=output)
-        except:
+        except Exception:
             print("Serial error or program closed", file=sys.stderr)
 
         if output:
@@ -120,7 +120,7 @@ def decode_file(infile, outfile, database):
                     # ascii decode line
                     # serial terminals may include non ascii characters
                     line = line.decode('ascii').strip()
-                except:
+                except Exception:
                     continue
                 # find token start and detokenize
                 idx = line.rfind(']')

scripts/codepregen.py

@@ -25,7 +25,7 @@
 
 try:
     from pregenerate import FindPregenerationTargets, TargetFilter
-except:
+except ImportError:
     import os
     sys.path.append(os.path.abspath(os.path.dirname(__file__)))
     from pregenerate import FindPregenerationTargets, TargetFilter
@@ -36,7 +36,7 @@
 try:
     import coloredlogs
     _has_coloredlogs = True
-except:
+except ImportError:
     _has_coloredlogs = False
 
 # Supported log levels, mapping string values required for argument

scripts/helpers/bloat_check.py

@@ -357,7 +357,7 @@ def main():
             # Output processed.
             a.delete()
 
-        except Exception as e:
+        except Exception:
             tb = traceback.format_exc()
             logging.warning('Failed to process bloat report: %s', tb)
 

scripts/py_matter_idl/matter_idl/generators/java/__init__.py

@@ -240,7 +240,7 @@ def boxed_java_type(self):
             elif t == FundamentalType.DOUBLE:
                 return "Double"
             else:
-                raise Error("Unknown fundamental type")
+                raise Exception("Unknown fundamental type")
         elif type(t) == BasicInteger:
             if t.byte_count >= 4:
                 return "Long"
@@ -277,7 +277,7 @@ def boxed_java_signature(self):
             elif t == FundamentalType.DOUBLE:
                 return "Ljava/lang/Double;"
             else:
-                raise Error("Unknown fundamental type")
+                raise Exception("Unknown fundamental type")
         elif type(t) == BasicInteger:
             if t.byte_count >= 4:
                 return "Ljava/lang/Long;"

scripts/py_matter_idl/matter_idl/generators/types.py

@@ -80,7 +80,7 @@ def idl_name(self):
         elif self == FundamentalType.DOUBLE:
             return "double"
         else:
-            raise Error("Type not handled: %r" % self)
+            raise Exception("Type not handled: %r" % self)
 
     @property
     def byte_count(self):
@@ -91,7 +91,7 @@ def byte_count(self):
         elif self == FundamentalType.DOUBLE:
             return 8
         else:
-            raise Error("Type not handled: %r" % self)
+            raise Exception("Type not handled: %r" % self)
 
     @property
     def bits(self):

scripts/py_matter_idl/matter_idl/lint/lint_rules_parser.py

@@ -14,7 +14,7 @@
 try:
     from .types import (AttributeRequirement, ClusterCommandRequirement, ClusterRequirement, RequiredAttributesRule,
                         RequiredCommandsRule)
-except:
+except ImportError:
     import sys
 
     sys.path.append(os.path.join(os.path.abspath(
@@ -102,7 +102,7 @@ def DecodeClusterFromXml(element: xml.etree.ElementTree.Element):
             required_attributes=required_attributes,
             required_commands=required_commands
         )
-    except Exception as e:
+    except Exception:
         logging.exception("Failed to decode cluster %r" % element)
         return None
 
@@ -204,7 +204,7 @@ def positive_integer(self, tokens):
         """Numbers in the grammar are integers or hex numbers.
         """
         if len(tokens) != 1:
-            raise Error("Unexpected argument counts")
+            raise Exception("Unexpected argument counts")
 
         return parseNumberString(tokens[0].value)
 
@@ -220,7 +220,7 @@ def id(self, tokens):
         """An id is a string containing an identifier
         """
         if len(tokens) != 1:
-            raise Error("Unexpected argument counts")
+            raise Exception("Unexpected argument counts")
         return tokens[0].value
 
     def ESCAPED_STRING(self, s):

scripts/py_matter_idl/matter_idl/matter_idl_parser.py

@@ -9,7 +9,7 @@
 
 try:
     from .matter_idl_types import *
-except:
+except ImportError:
     import os
     import sys
     sys.path.append(os.path.abspath(os.path.dirname(__file__)))
@@ -91,7 +91,7 @@ def positive_integer(self, tokens):
         """Numbers in the grammar are integers or hex numbers.
         """
         if len(tokens) != 1:
-            raise Error("Unexpected argument counts")
+            raise Exception("Unexpected argument counts")
 
         n = tokens[0].value
         if n.startswith('0x'):
@@ -117,14 +117,14 @@ def id(self, tokens):
         """An id is a string containing an identifier
         """
         if len(tokens) != 1:
-            raise Error("Unexpected argument counts")
+            raise Exception("Unexpected argument counts")
         return tokens[0].value
 
     def type(self, tokens):
         """A type is just a string for the type
         """
         if len(tokens) != 1:
-            raise Error("Unexpected argument counts")
+            raise Exception("Unexpected argument counts")
         return tokens[0].value
 
     def data_type(self, tokens):
@@ -134,7 +134,7 @@ def data_type(self, tokens):
         elif len(tokens) == 2:
             return DataType(name=tokens[0], max_length=tokens[1])
         else:
-            raise Error("Unexpected size for data type")
+            raise Exception("Unexpected size for data type")
 
     @v_args(inline=True)
     def constant_entry(self, id, number):

scripts/py_matter_idl/matter_idl/test_matter_idl_parser.py

@@ -17,7 +17,7 @@
 try:
     from .matter_idl_parser import CreateParser
     from .matter_idl_types import *
-except:
+except ImportError:
     import os
     import sys
     sys.path.append(os.path.abspath(os.path.dirname(__file__)))

scripts/py_matter_idl/matter_idl/test_xml_parser.py

@@ -20,7 +20,7 @@
 try:
     from matter_idl.matter_idl_types import *
     from matter_idl.zapxml import ParseSource, ParseXmls
-except:
+except ImportError:
     import os
     import sys
 

scripts/py_matter_yamltests/matter_yamltests/fixes.py

@@ -60,7 +60,7 @@ def try_apply_yaml_unrepresentable_integer_for_javascript_fixes(value):
     if type(value) is str:
         try:
             value = int(value)
-        except:
+        except ValueError:
             pass
     return value
 

scripts/py_matter_yamltests/matter_yamltests/yaml_loader.py

@@ -22,7 +22,7 @@
 
 try:
     from yaml import CSafeLoader as SafeLoader
-except:
+except ImportError:
     from yaml import SafeLoader
 
 import yaml

scripts/run-clang-tidy-on-compile-commands.py

@@ -168,7 +168,7 @@ def Check(self):
                         "Tidy %s ended with code %d", self.file, proc.returncode
                     )
                 return TidyResult(self.full_path, False)
-        except:
+        except subprocess.SubprocessError:
             traceback.print_exc()
             return TidyResult(self.full_path, False)
 

scripts/setup/nrfconnect/update_ncs.py

@@ -52,7 +52,7 @@ def get_ncs_recommended_revision():
     try:
         with open(os.path.join(chip_root, 'config/nrfconnect/.nrfconnect-recommended-revision'), 'r') as f:
             return f.readline().strip()
-    except:
+    except OSError:
         raise RuntimeError(
             "Encountered problem when trying to read .nrfconnect-recommended-revision file.")
 

scripts/tests/chiptest/yamltest_with_chip_repl_tester.py

@@ -30,7 +30,7 @@
 # ensure matter IDL is availale for import, otherwise set relative paths
 try:
     from matter_idl import matter_idl_types
-except:
+except ImportError:
     SCRIPT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '../..'))
     import sys
 

scripts/tests/java/base.py

@@ -39,7 +39,7 @@ def EnqueueLogOutput(fp, tag, q):
             try:
                 timestamp = float(line[1:18].decode())
                 line = line[19:]
-            except Exception as ex:
+            except Exception:
                 pass
         sys.stdout.buffer.write(
             (f"[{datetime.datetime.fromtimestamp(timestamp).isoformat(sep=' ')}]").encode() + tag + line)

scripts/tests/run_python_test.py

@@ -44,7 +44,7 @@ def EnqueueLogOutput(fp, tag, q):
             try:
                 timestamp = float(line[1:18].decode())
                 line = line[19:]
-            except Exception as ex:
+            except Exception:
                 pass
         sys.stdout.buffer.write(
             (f"[{datetime.datetime.fromtimestamp(timestamp).isoformat(sep=' ')}]").encode() + tag + line)

scripts/tools/nrfconnect/generate_nrfconnect_chip_factory_data.py

@@ -338,7 +338,7 @@ def generate_json(self):
             try:
                 if is_json_valid:
                     json_file.write(json_object)
-            except IOError as e:
+            except IOError:
                 log.error("Cannot save output file into directory: {}".format(self._args.output))
 
     def _add_entry(self, name: str, value: any):
@@ -372,7 +372,7 @@ def _validate_output_json(self, output_json: str):
                 schema = json.loads(schema_file.read())
                 validator = jsonschema.Draft202012Validator(schema=schema)
                 validator.validate(instance=json.loads(output_json))
-        except IOError as e:
+        except IOError:
             log.error("Provided JSON schema file is wrong: {}".format(self._args.schema))
             return False
         else:

scripts/tools/silabs/FactoryDataProvider.py

@@ -212,7 +212,7 @@ def create_nvm3injected_image(self):
                     inputImage = self.BASE_MG24_FILE
                 else:
                     raise Exception('Invalid MCU')
-            except:
+            except Exception:
                 isDeviceConnected = False
                 print("Device not connected")
                 # When no device is connected user needs to provide the mcu family for which those credentials are to be created

scripts/tools/zap/generate.py

@@ -187,7 +187,7 @@ def extractGeneratedIdl(output_dir, zap_config_path):
     if not target_path.endswith(".matter"):
         # We expect "something.zap" and don't handle corner cases of
         # multiple extensions. This is to work with existing codebase only
-        raise Error("Unexpected input zap file  %s" % self.zap_config)
+        raise Exception("Unexpected input zap file  %s" % zap_config_path)
 
     shutil.move(idl_path, target_path)
 

scripts/tools/zap/test_generate.py

@@ -116,7 +116,7 @@ def run_test_cases(self, checker: unittest.TestCase):
 
                     try:
                         subprocess.check_call(["diff", actual, expected])
-                    except:
+                    except subprocess.CalledProcessError:
                         if self.context.regenerate_golden:
                             print(
                                 f"Copying updated golden image from {actual} to {expected}")

scripts/tools/zap/zap_download.py

@@ -32,7 +32,7 @@
 try:
     import coloredlogs
     _has_coloredlogs = True
-except:
+except Exception:
     _has_coloredlogs = False
 
 # Supported log levels, mapping string values required for argument

src/controller/python/chip-device-ctrl.py

@@ -661,7 +661,7 @@ def do_closesession(self, line):
             self.devCtrl.CloseSession(args.nodeid)
         except exceptions.ChipStackException as ex:
             print(str(ex))
-        except:
+        except Exception:
             self.do_help("close-session")
 
     def do_resolve(self, line):
@@ -782,7 +782,7 @@ def do_discover(self, line):
             print('exception')
             print(str(ex))
             return
-        except:
+        except Exception:
             self.do_help("discover")
             return
 
@@ -1021,7 +1021,7 @@ def do_opencommissioningwindow(self, line):
         except exceptions.ChipStackException as ex:
             print(str(ex))
             return
-        except:
+        except Exception:
             self.do_help("open-commissioning-window")
             return
 
@@ -1146,7 +1146,7 @@ def main():
         else:
             try:
                 adapterId = int(options.bluetoothAdapter[3:])
-            except:
+            except ValueError:
                 print(
                     "Invalid bluetooth adapter: {}, adapter name looks like hci0, hci1 etc.")
                 sys.exit(-1)

src/controller/python/chip/CertificateAuthority.py

@@ -124,7 +124,7 @@ def NewFabricAdmin(self, vendorId: int, fabricId: int):
                 f"CertificateAuthority object was previously shutdown and is no longer valid!")
 
         if (vendorId is None or fabricId is None):
-            raise ValueError(f"Invalid values for fabricId and vendorId")
+            raise ValueError("Invalid values for fabricId and vendorId")
 
         for existingAdmin in self._activeAdmins:
             if (existingAdmin.fabricId == fabricId):

src/controller/python/chip/ChipBleUtility.py

@@ -62,7 +62,7 @@ def VoidPtrToUUIDString(ptr, len):
             + ptr[20:]
         )
         ptr = str(ptr)
-    except Exception as ex:
+    except Exception:
         print("ERROR: failed to convert void * to UUID")
         ptr = None