From 0ffad300413089d5993f31db96beff30a03665d6 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sun, 13 Oct 2024 09:48:31 -0700 Subject: [PATCH 1/4] Updated the apio project specification. Once released, will be reflected in the Pypi page of apio. This change bumps the developement status of apio from ALPHA to PRODUCTION/STABLE.. --- pyproject.toml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3c8d17a8..fd4de2b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,11 +8,14 @@ author = "Jesus Arroyo" author-email = "jesus.jkhlg@gmail.com " home-page = "https://github.com/FPGAwars/apio" classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Intended Audience :: Developers', 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', - 'Programming Language :: Python'] + 'Programming Language :: Python', + 'Natural Language :: English', + 'Topic :: Scientific/Engineering :: Electronic Design Automation (EDA)', +] description-file = "README.md" requires-python = ">=3.9" requires = [ From dd102acfecd6f8be5d4f69b5a9339f33b5eac5e0 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sun, 13 Oct 2024 20:51:18 -0700 Subject: [PATCH 2/4] Adding a missing change from commit 4b7ad09316d0d1778d6a20001863417e878ff568. Accessing result values as class fields rather than as dict values. --- apio/managers/system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apio/managers/system.py b/apio/managers/system.py index 8fa10328..29220d45 100644 --- a/apio/managers/system.py +++ b/apio/managers/system.py @@ -96,7 +96,7 @@ def get_usb_devices(self) -> list: # -- Get the list of the usb devices. It is read # -- from the command stdout # -- Ex: [{'hwid':'1d6b:0003'}, {'hwid':'04f2:b68b'}...] - usb_devices = self._parse_usb_devices(result["out"]) + usb_devices = self._parse_usb_devices(result.out_text) # -- Return the devices return usb_devices @@ -130,7 +130,7 @@ def get_ftdi_devices(self) -> list: # -- from the command stdout # -- Ex: [{'index': '0', 'manufacturer': 'AlhambraBits', # -- 'description': 'Alhambra II v1.0A - B07-095'}] - ftdi_devices = self._parse_ftdi_devices(result["out"]) + ftdi_devices = self._parse_ftdi_devices(result.out_text) # -- Return the devices return ftdi_devices From 00b149d870be83ca811336f0fe7d7179ff2a6e15 Mon Sep 17 00:00:00 2001 From: Zapta Date: Sun, 13 Oct 2024 21:22:00 -0700 Subject: [PATCH 3/4] Fixing a regression from commit 93fc9bc4f3bfd21568e2d66f11976831467e3b97. Now that we preserve empty stdout/err lines from scons pipes, we need to erase for the iceprog progress percentage two lines instead of one. I don't have Fumo and Tinyprog boards to test if this affect them as well so added TODOs to test and apply the same change if needed. Alhambra II upload (Iceprog) was tested successfuly. --- apio/managers/scons.py | 4 ++++ apio/managers/scons_filter.py | 41 +++++++++++++++++++++++++++-------- apio/util.py | 4 ++-- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/apio/managers/scons.py b/apio/managers/scons.py index 8c665e31..13c23ef5 100644 --- a/apio/managers/scons.py +++ b/apio/managers/scons.py @@ -60,6 +60,10 @@ def wrapper(*args, **kwargs): try: return function(*args, **kwargs) except Exception as exc: + # For debugging. Uncomment to print the exception's stack. + # import traceback + # traceback.print_tb(exc.__traceback__) + if str(exc): click.secho("Error: " + str(exc), fg="red") return exit_code diff --git a/apio/managers/scons_filter.py b/apio/managers/scons_filter.py index 8b484879..d33a3989 100644 --- a/apio/managers/scons_filter.py +++ b/apio/managers/scons_filter.py @@ -164,10 +164,10 @@ def on_line(self, pipe_id: PipeId, line: str) -> None: # -- Assign line color. line_color = self._assign_line_color( line.lower(), - { + [ (r"^warning:", "yellow"), (r"^error:", "red"), - }, + ], ) click.secho(f"{line}", fg=line_color) return @@ -178,8 +178,16 @@ def on_line(self, pipe_id: PipeId, line: str) -> None: match = re.search(pattern_fomu, line) if match: # -- Delete the previous line + # + # -- TODO: Since the commit listed below we preserve blank + # -- stdour/err lines. Iceprog emits an empty line after each + # -- percentage line so we changed it below to erase two lines. + # -- If this is also the case with tinyprog, apply the same + # -- change here. Delete this TODO when resolved. + # - Comit 93fc9bc4f3bfd21568e2d66f11976831467e3b97. + # print(CURSOR_UP + ERASE_LINE, end="", flush=True) - click.secho(f"{line}", fg="green") + click.secho(line, fg="green") return # -- Special handling for tinyprog lines. @@ -198,9 +206,17 @@ def on_line(self, pipe_id: PipeId, line: str) -> None: # -- Match all the progress bar lines except the # -- initial one (when it is 0%) if match_tinyprog and " 0%|" not in line: - # -- Delete the previous line + # -- Delete the previous line. + # + # -- TODO: Since the commit listed below we preserve blank + # -- stdour/err lines. Iceprog emits an empty line after each + # -- percentage line so we changed it below to erase two lines. + # -- If this is also the case with tinyprog, apply the same + # -- change here. Delete this TODO when resolved. + # - Comit 93fc9bc4f3bfd21568e2d66f11976831467e3b97. + # print(CURSOR_UP + ERASE_LINE, end="", flush=True) - click.secho(f"{line}") + click.secho(line) return # -- Special handling for iceprog lines. @@ -219,10 +235,17 @@ def on_line(self, pipe_id: PipeId, line: str) -> None: # -- It is a match! (iceprog is running!) # -- (or if it is the end of the writing!) # -- (or if it is the end of verifying!) - if match or "done." in line or "VERIFY OK" in line: - # -- Delete the previous line - print(CURSOR_UP + ERASE_LINE, end="", flush=True) - click.secho(line) + completed_ok = "done." in line or "VERIFY OK" in line + if match or completed_ok: + # -- Delete the previous two lines. We erase two lines because + # -- iceprog emits an empty line after each percentage line. + print( + CURSOR_UP + ERASE_LINE + CURSOR_UP + ERASE_LINE, + end="", + flush=True, + ) + line_color = "green" if completed_ok else None + click.secho(line, fg=line_color) return # Handling the rest of the stdout lines. diff --git a/apio/util.py b/apio/util.py index 2009f69d..055c7162 100644 --- a/apio/util.py +++ b/apio/util.py @@ -686,14 +686,14 @@ def exec_command(*args, **kwargs) -> CommandResult: if isinstance(pipe, AsyncPipe): lines = pipe.get_buffer() text = "\n".join(lines) - out_text = text.strip() + out_text = text # -- If stderr pipe is an AsyncPipe, extract its text. pipe = flags["stderr"] if isinstance(pipe, AsyncPipe): lines = pipe.get_buffer() text = "\n".join(lines) - err_text = text.strip() + err_text = text # -- All done. result = CommandResult(out_text, err_text, exit_code) From 3938ccfc6e03bbc24e52aa40a1fb30a506729766 Mon Sep 17 00:00:00 2001 From: Zapta Date: Mon, 14 Oct 2024 00:24:02 -0700 Subject: [PATCH 4/4] Updated the scons pipe filter of the icrprog program. 1. Added a range detector. The filter is now active only within the range of the iceprog output lines. 2. Changed the line erasure model. We track with a flag if the previous line should be erased before printing the current line (for the percents progress indicator). This change was tested with Alhambra II. The Fumo and Tinyprog filters should be changed in the same way but I currently don't have boards to test to leaving as is. --- apio/managers/scons_filter.py | 139 ++++++++++++++++++++++------------ 1 file changed, 90 insertions(+), 49 deletions(-) diff --git a/apio/managers/scons_filter.py b/apio/managers/scons_filter.py index d33a3989..eab3a7ab 100644 --- a/apio/managers/scons_filter.py +++ b/apio/managers/scons_filter.py @@ -42,15 +42,15 @@ class RangeEvents(Enum): END_AFTER = 4 # Range ends, after the current line. -class SectionDetector: - """Base classifier of a range of lines within the sequence of stdout/err +class RangeDetector: + """Base detector of a range of lines within the sequence of stdout/err lines recieves from the scons subprocess.""" def __init__(self): self._in_range = False def update(self, pipe_id: PipeId, line: str) -> bool: - """Updates the section classifier with the next stdout/err line. + """Updates the range detector with the next stdout/err line. return True iff detector classified this line to be within a range.""" prev_state = self._in_range @@ -85,11 +85,12 @@ def classify_line( raise NotImplementedError("Should be implemented by a subclass") -class PnrSectionDetector(SectionDetector): - """Implements a RangeDetector for the nextpnr command verbose log lines.""" +class PnrRangeDetector(RangeDetector): + """Implements a RangeDetector for the nextpnr command verbose + log lines.""" def classify_line(self, pipe_id: PipeId, line: str) -> RangeEvents: - # -- Brek line into words. + # -- Break line into words. tokens = line.split() # -- Range start: A nextpnr command on stdout without @@ -108,6 +109,29 @@ def classify_line(self, pipe_id: PipeId, line: str) -> RangeEvents: return None +class IceProgRangeDetector(RangeDetector): + """Implements a RangeDetector for the iceprog command output.""" + + def __init__(self): + super().__init__() + # -- Indicates if the last line should be erased before printing the + # -- next one. This happens with interactive progress meters. + self.pending_erasure = False + + def classify_line(self, pipe_id: PipeId, line: str) -> RangeEvents: + # -- Range start: A nextpnr command on stdout without + # -- the -q (quiet) flag. + if pipe_id == PipeId.STDOUT and line.startswith("iceprog"): + self.pending_erasure = False + return RangeEvents.START_AFTER + + # Range end: The end message of nextnpr. + if pipe_id == PipeId.STDERR and line.startswith("Bye."): + return RangeEvents.END_AFTER + + return None + + class SconsFilter: """Implements the filtering and printing of the stdout/err streams of the scons subprocess. Accepts a line one at a time, detects lines ranges of @@ -115,7 +139,8 @@ class SconsFilter: stdout.""" def __init__(self): - self._pnr_detector = PnrSectionDetector() + self._pnr_detector = PnrRangeDetector() + self._iceprog_detector = IceProgRangeDetector() def on_stdout_line(self, line: str) -> None: """Stdout pipe calls this on each line.""" @@ -150,8 +175,9 @@ def on_line(self, pipe_id: PipeId, line: str) -> None: from other programs. See the PNR detector for an example. """ - # -- Update the classifiers + # -- Update the range detectors. in_pnr_verbose_range = self._pnr_detector.update(pipe_id, line) + in_iceprog_range = self._iceprog_detector.update(pipe_id, line) # -- Handle the line while in the nextpnr verbose log range. if pipe_id == PipeId.STDERR and in_pnr_verbose_range: @@ -172,6 +198,52 @@ def on_line(self, pipe_id: PipeId, line: str) -> None: click.secho(f"{line}", fg=line_color) return + # -- Special handling for iceprog line range. + if pipe_id == PipeId.STDERR and in_iceprog_range: + # -- Iceprog prints blank likes that are used as line erasers. + # -- We don't need them here. + if len(line) == 0: + return + + # -- If the last iceprog line was a to-be-erased line, erase it + # -- now and clear the flag. + if self._iceprog_detector.pending_erasure: + print( + CURSOR_UP + ERASE_LINE, + end="", + flush=True, + ) + self._iceprog_detector.pending_erasure = False + + # -- Determine if the current line should be erased before we will + # -- print the next line. + # -- + # -- Match outputs like these "addr 0x001400 3%" + # -- Regular expression remainder: + # -- ^ --> Match the begining of the line + # -- \s --> Match one blank space + # -- [0-9A-F]+ one or more hexadecimal digit + # -- \d{1,2} one or two decimal digits + pattern = r"^addr\s0x[0-9A-F]+\s+\d{1,2}%" + + # -- Calculate if there is a match! + match = re.search(pattern, line) + + # -- If the line is to be erased set the flag. + if match: + self._iceprog_detector.pending_erasure = True + + # -- Determine line color by its content and print it. + line_color = self._assign_line_color( + line, + [ + (r"^done.", "green"), + (r"^VERIFY OK", "green"), + ], + ) + click.secho(line, fg=line_color) + return + # -- Special handling for Fumo lines. if pipe_id == PipeId.STDOUT: pattern_fomu = r"^Download\s*\[=*" @@ -179,12 +251,11 @@ def on_line(self, pipe_id: PipeId, line: str) -> None: if match: # -- Delete the previous line # - # -- TODO: Since the commit listed below we preserve blank - # -- stdour/err lines. Iceprog emits an empty line after each - # -- percentage line so we changed it below to erase two lines. - # -- If this is also the case with tinyprog, apply the same - # -- change here. Delete this TODO when resolved. - # - Comit 93fc9bc4f3bfd21568e2d66f11976831467e3b97. + # -- NOTE: If the progress line will scroll instead of + # -- overwriting each other, try to add erasure of a second + # -- line. This is due to the commit below which restored + # -- empty lines. + # - Commit 93fc9bc4f3bfd21568e2d66f11976831467e3b97. # print(CURSOR_UP + ERASE_LINE, end="", flush=True) click.secho(line, fg="green") @@ -208,46 +279,16 @@ def on_line(self, pipe_id: PipeId, line: str) -> None: if match_tinyprog and " 0%|" not in line: # -- Delete the previous line. # - # -- TODO: Since the commit listed below we preserve blank - # -- stdour/err lines. Iceprog emits an empty line after each - # -- percentage line so we changed it below to erase two lines. - # -- If this is also the case with tinyprog, apply the same - # -- change here. Delete this TODO when resolved. - # - Comit 93fc9bc4f3bfd21568e2d66f11976831467e3b97. + # -- NOTE: If the progress line will scroll instead of + # -- overwriting each other, try to add erasure of a second + # -- line. This is due to the commit below which restored + # -- empty lines. + # - Commit 93fc9bc4f3bfd21568e2d66f11976831467e3b97. # print(CURSOR_UP + ERASE_LINE, end="", flush=True) click.secho(line) return - # -- Special handling for iceprog lines. - if pipe_id == PipeId.STDERR: - # -- Match outputs like these "addr 0x001400 3%" - # -- Regular expression remainder: - # -- ^ --> Match the begining of the line - # -- \s --> Match one blank space - # -- [0-9A-F]+ one or more hexadecimal digit - # -- \d{1,2} one or two decimal digits - pattern = r"^addr\s0x[0-9A-F]+\s+\d{1,2}%" - - # -- Calculate if there is a match! - match = re.search(pattern, line) - - # -- It is a match! (iceprog is running!) - # -- (or if it is the end of the writing!) - # -- (or if it is the end of verifying!) - completed_ok = "done." in line or "VERIFY OK" in line - if match or completed_ok: - # -- Delete the previous two lines. We erase two lines because - # -- iceprog emits an empty line after each percentage line. - print( - CURSOR_UP + ERASE_LINE + CURSOR_UP + ERASE_LINE, - end="", - flush=True, - ) - line_color = "green" if completed_ok else None - click.secho(line, fg=line_color) - return - # Handling the rest of the stdout lines. if pipe_id == PipeId.STDOUT: # Default stdout line coloring.