From 89d6f99291ef246a5a6b1c780f52ba7245e96702 Mon Sep 17 00:00:00 2001 From: Muhammad Bilal Sakhawat Date: Wed, 21 Jul 2021 12:26:11 +0500 Subject: [PATCH 01/10] Adding support for intermediate run start points --- riscof/Templates/setup/model/riscof_model.py | 2 ++ .../Templates/setup/reference/riscof_model.py | 2 ++ .../setup/sail_cSim/riscof_sail_cSim.py | 2 ++ riscof/framework/main.py | 6 ++-- riscof/framework/test.py | 34 +++++++++++++++---- riscof/main.py | 27 +++++++++------ riscof/utils.py | 6 ++++ 7 files changed, 59 insertions(+), 20 deletions(-) diff --git a/riscof/Templates/setup/model/riscof_model.py b/riscof/Templates/setup/model/riscof_model.py index 122a8cc..8967352 100644 --- a/riscof/Templates/setup/model/riscof_model.py +++ b/riscof/Templates/setup/model/riscof_model.py @@ -75,6 +75,8 @@ def build(self, isa_yaml, platform_yaml): # build your RTL here def runTests(self, testList, cgf_file=None): + if os.path.exists(self.work_dir+ "/Makefile." + self.name[:-1]): + os.remove(self.work_dir+ "/Makefile." + self.name[:-1]) make = utils.makeUtil(makefilePath=os.path.join(self.work_dir, "Makefile." + self.name[:-1])) make.makeCommand = 'make -j' + self.num_jobs for file in testList: diff --git a/riscof/Templates/setup/reference/riscof_model.py b/riscof/Templates/setup/reference/riscof_model.py index b34fdbe..f7a51ad 100644 --- a/riscof/Templates/setup/reference/riscof_model.py +++ b/riscof/Templates/setup/reference/riscof_model.py @@ -79,6 +79,8 @@ def build(self, isa_yaml, platform_yaml): # build your RTL here def runTests(self, testList, cgf_file=None): + if os.path.exists(self.work_dir+ "/Makefile." + self.name[:-1]): + os.remove(self.work_dir+ "/Makefile." + self.name[:-1]) make = utils.makeUtil(makefilePath=os.path.join(self.work_dir, "Makefile." + self.name[:-1])) make.makeCommand = self.make + ' -j' + self.num_jobs for file in testList: diff --git a/riscof/Templates/setup/sail_cSim/riscof_sail_cSim.py b/riscof/Templates/setup/sail_cSim/riscof_sail_cSim.py index e20a028..258ee1f 100644 --- a/riscof/Templates/setup/sail_cSim/riscof_sail_cSim.py +++ b/riscof/Templates/setup/sail_cSim/riscof_sail_cSim.py @@ -80,6 +80,8 @@ def build(self, isa_yaml, platform_yaml): def runTests(self, testList, cgf_file=None): + if os.path.exists(self.work_dir+ "/Makefile." + self.name[:-1]): + os.remove(self.work_dir+ "/Makefile." + self.name[:-1]) make = utils.makeUtil(makefilePath=os.path.join(self.work_dir, "Makefile." + self.name[:-1])) make.makeCommand = self.make + ' -j' + self.num_jobs for file in testList: diff --git a/riscof/framework/main.py b/riscof/framework/main.py index 54e453d..e943132 100644 --- a/riscof/framework/main.py +++ b/riscof/framework/main.py @@ -151,7 +151,7 @@ def run_coverage(base, dut_isa_spec, dut_platform_spec, work_dir, cgf_file=None) return results, for_html, test_stats, coverpoints -def run(dut, base, dut_isa_spec, dut_platform_spec, work_dir): +def run(dut, base, dut_isa_spec, dut_platform_spec, work_dir, cntr_args): ''' Entry point for the framework module. This function initializes and sets up the required variables for the tests to run. @@ -165,6 +165,8 @@ def run(dut, base, dut_isa_spec, dut_platform_spec, work_dir): :param dut_platform_spec: The absolute path to the checked yaml containing the DUT platform specification. + + :param cntr_args: testfile, no_ref_run, no_dut_run :type dut_platform_spec: str @@ -188,7 +190,7 @@ def run(dut, base, dut_isa_spec, dut_platform_spec, work_dir): logger.info("Running Build for Reference") base.build(dut_isa_spec, dut_platform_spec) - results = test.run_tests(dut, base, ispec['hart0'], pspec, work_dir) + results = test.run_tests(dut, base, ispec['hart0'], pspec, work_dir, cntr_args) return results diff --git a/riscof/framework/test.py b/riscof/framework/test.py index 14ad514..c893568 100644 --- a/riscof/framework/test.py +++ b/riscof/framework/test.py @@ -300,11 +300,13 @@ def generate_test_pool(ispec, pspec, workdir): with open(os.path.join(workdir,"test_list.yaml"),"w") as tfile: yaml.dump(test_list,tfile) + with open(os.path.join(workdir,"test_pool.yaml"),"w") as tpfile: + yaml.dump(test_pool,tpfile) return (test_list, test_pool) -def run_tests(dut, base, ispec, pspec, work_dir): +def run_tests(dut, base, ispec, pspec, work_dir, cntr_args): ''' Function to run the tests for the DUT. @@ -315,6 +317,8 @@ def run_tests(dut, base, ispec, pspec, work_dir): :param ispec: The isa specifications of the DUT. :param pspec: The platform specifications of the DUT. + + :param cntr_args: testfile, no_ref_run, no_dut_run :type ispec: dict @@ -323,13 +327,29 @@ def run_tests(dut, base, ispec, pspec, work_dir): :return: A list of dictionary objects containing the necessary information required to generate the report. ''' - test_list, test_pool = generate_test_pool(ispec, pspec, work_dir) + if cntr_args[0]: + with open(os.path.join(work_dir,"test_pool.yaml"),"r") as tpfile: + test_pool = yaml.load(tpfile) + with open(os.path.join(work_dir,"test_list.yaml"),"r") as tfile: + test_list = yaml.load(tfile) + else: + test_list, test_pool = generate_test_pool(ispec, pspec, work_dir) results = [] - logger.info("Running Tests on DUT.") - dut.runTests(test_list) - logger.info("Running Tests on Reference Model.") - base.runTests(test_list) - + if cntr_args[1]: + logger.info("Running Tests on DUT.") + dut.runTests(test_list) + logger.info("Tests run on DUT done.") + raise SystemExit + elif cntr_args[2]: + logger.info("Running Tests on Reference Model.") + base.runTests(test_list) + logger.info("Tests run on Reference done.") + raise SystemExit + else: + logger.info("Running Tests on DUT.") + dut.runTests(test_list) + logger.info("Running Tests on Reference Model.") + base.runTests(test_list) logger.info("Initiating signature checking.") for entry in test_pool: diff --git a/riscof/main.py b/riscof/main.py index e5eb30e..a931651 100644 --- a/riscof/main.py +++ b/riscof/main.py @@ -140,7 +140,7 @@ def execute(): if not os.path.exists(work_dir): logger.debug('Creating new work directory: ' + work_dir) os.mkdir(work_dir) - else: + elif not args.testfile: logger.debug('Removing old work directory: ' + work_dir) shutil.rmtree(work_dir) logger.debug('Creating new work directory: ' + work_dir) @@ -196,14 +196,17 @@ def execute(): #Run riscv_config on inputs isa_file = dut.isa_spec platform_file = dut.platform_spec - - - try: - isa_file = checker.check_isa_specs( isa_file, work_dir, True) - platform_file = checker.check_platform_specs( platform_file, work_dir, True) - except ValidationError as msg: - logger.error(msg) - return 1 + + if args.testfile: + isa_file = work_dir+ '/' + (isa_file.rsplit('/', 1)[1]).rsplit('.')[0] + "_checked.yaml" + platform_file = work_dir+ '/' + (platform_file.rsplit('/', 1)[1]).rsplit('.')[0] + "_checked.yaml" + else: + try: + isa_file = checker.check_isa_specs( isa_file, work_dir, True) + platform_file = checker.check_platform_specs( platform_file, work_dir, True) + except ValidationError as msg: + logger.error(msg) + return 1 isa_specs = utils.load_yaml(isa_file)['hart0'] platform_specs = utils.load_yaml(platform_file) @@ -291,7 +294,9 @@ def execute(): with open(platform_file, "r") as platfile: pspecs = platfile.read() - + + cntr_args = [args.testfile,args.no_ref_run,args.no_dut_run] + report_objects = {} report_objects['date'] = (datetime.now( pytz.timezone('GMT'))).strftime("%Y-%m-%d %H:%M GMT") @@ -312,7 +317,7 @@ def execute(): report_objects['platform_specs'] = pspecs report_objects['results'] = framework.run(dut, base, isa_file, - platform_file, work_dir) + platform_file, work_dir, cntr_args) report_objects['num_passed'] = 0 report_objects['num_failed'] = 0 diff --git a/riscof/utils.py b/riscof/utils.py index bf6fc68..2c083f8 100644 --- a/riscof/utils.py +++ b/riscof/utils.py @@ -598,6 +598,12 @@ def riscof_cmdline_args(): metavar= 'PATH', default=str(pathlib.Path('./riscof_work').absolute()) ) + run.add_argument('--testfile',action='store_true', + help="Run tests from already generated testlist file") + run.add_argument('--no-ref-run',action='store_true', + help="Do not run tests on Reference") + run.add_argument('--no-dut-run',action='store_true', + help="Do not run tests on DUT") testlist = subparsers.add_parser('testlist', help='Generate the test list for the given DUT and suite.',formatter_class=SortingHelpFormatter) testlist.add_argument('--work-dir', From 7d62bc284df08deb637f22a9af828738601862ae Mon Sep 17 00:00:00 2001 From: Muhammad Bilal Sakhawat Date: Wed, 21 Jul 2021 12:28:14 +0500 Subject: [PATCH 02/10] =?UTF-8?q?Bump=20version:=201.21.0=20=E2=86=92=201.?= =?UTF-8?q?21.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- riscof/__init__.py | 2 +- setup.cfg | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/riscof/__init__.py b/riscof/__init__.py index 48d477c..6874d34 100644 --- a/riscof/__init__.py +++ b/riscof/__init__.py @@ -4,4 +4,4 @@ __author__ = """InCore Semiconductors Pvt Ltd""" __email__ = 'info@incoresemi.com' -__version__ = '1.21.0' +__version__ = '1.21.1' diff --git a/setup.cfg b/setup.cfg index f8c2874..67d8b18 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.21.0 +current_version = 1.21.1 commit = True tag = True diff --git a/setup.py b/setup.py index 3f1856f..1d81553 100644 --- a/setup.py +++ b/setup.py @@ -23,7 +23,7 @@ def read_requires(): test_requirements = [ ] setup(name="riscof", - version='1.21.0', + version='1.21.1', description="RISC-V Architectural Test Framework", long_description=readme + '\n\n', classifiers=[ From 789adcb69e1f98aa4513ddedb08e27de972ed79f Mon Sep 17 00:00:00 2001 From: Muhammad Bilal Sakhawat Date: Wed, 21 Jul 2021 12:42:02 +0500 Subject: [PATCH 03/10] Update the CHANGELOG.md --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa19779..01b0854 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.21.1] - 2021-07-21 +- Added support for intermediate run start points #1 +- --testfile is added to directly run tests on the target and reference (assumes the testlist has been correctly generated) +- --no-ref-run is added so that tests do not run on reference and quit before signature comparison +- --no-dut-run is added so that tests do not run on DUT and quit before signature comparison + ## [1.21.0] - 2021-07-21 - Changed CI script from gitlab to github actions - Removing hosted cgf files From 5bf6efd7105074fe4d000bb7ab72acb4054eff54 Mon Sep 17 00:00:00 2001 From: Muhammad Bilal Sakhawat Date: Sat, 24 Jul 2021 13:41:37 +0500 Subject: [PATCH 04/10] Add --dbfile argument, Made changes suggested by Pawan and Neel, Update the CHANGELOG.md --- CHANGELOG.md | 2 ++ riscof/dbgen.py | 3 +++ riscof/framework/main.py | 2 +- riscof/framework/test.py | 44 ++++++++++++++++++++++------------------ riscof/main.py | 32 ++++++++++++++++++++++++----- riscof/utils.py | 12 +++++++++-- 6 files changed, 67 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01b0854..aa0ca14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,9 +4,11 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [1.21.1] - 2021-07-21 - Added support for intermediate run start points #1 +- --dbfile is added to generate the test-list and then run tests on the target and reference - --testfile is added to directly run tests on the target and reference (assumes the testlist has been correctly generated) - --no-ref-run is added so that tests do not run on reference and quit before signature comparison - --no-dut-run is added so that tests do not run on DUT and quit before signature comparison +- Added a timestamp comment on the generated database.yaml and testlist.yaml files ## [1.21.0] - 2021-07-21 - Changed CI script from gitlab to github actions diff --git a/riscof/dbgen.py b/riscof/dbgen.py index 41b2ef4..3738ee2 100644 --- a/riscof/dbgen.py +++ b/riscof/dbgen.py @@ -3,6 +3,8 @@ import git import sys import re +import pytz +from datetime import datetime from riscof.utils import yaml import riscof.utils as utils import collections @@ -189,5 +191,6 @@ def generate(): continue db[fpath] = {'commit_id': '-', **temp} with open(dbfile, "w") as wrfile: + wrfile.write('# database generated on ' + (datetime.now(pytz.timezone('GMT'))).strftime("%Y-%m-%d %H:%M GMT")+'\n') yaml.dump(orderdict(db), wrfile) diff --git a/riscof/framework/main.py b/riscof/framework/main.py index e943132..f67aa8a 100644 --- a/riscof/framework/main.py +++ b/riscof/framework/main.py @@ -166,7 +166,7 @@ def run(dut, base, dut_isa_spec, dut_platform_spec, work_dir, cntr_args): :param dut_platform_spec: The absolute path to the checked yaml containing the DUT platform specification. - :param cntr_args: testfile, no_ref_run, no_dut_run + :param cntr_args: dbfile, testfile, no_ref_run, no_dut_run :type dut_platform_spec: str diff --git a/riscof/framework/test.py b/riscof/framework/test.py index c893568..a38686e 100644 --- a/riscof/framework/test.py +++ b/riscof/framework/test.py @@ -5,6 +5,7 @@ from pathlib import Path import difflib import ast +import pytz import random from datetime import datetime @@ -221,7 +222,7 @@ def eval_macro(macro, spec): return (False,[]) -def generate_test_pool(ispec, pspec, workdir): +def generate_test_pool(ispec, pspec, workdir, dbfile = None): ''' Funtion to select the tests which are applicable for the DUT and generate the macros necessary for each test. @@ -244,7 +245,11 @@ def generate_test_pool(ispec, pspec, workdir): spec = {**ispec, **pspec} test_pool = [] test_list = {} - db = utils.load_yaml(constants.framework_db) + if dbfile is not None: + with open(dbfile,"r") as db_file: + db = yaml.load(db_file) + else: + db = utils.load_yaml(constants.framework_db) for file in db: macros = [] cov_labels = [] @@ -285,6 +290,7 @@ def generate_test_pool(ispec, pspec, workdir): else: work_dir = os.path.join(workdir,entry[0].replace("suite/", '')) Path(work_dir).mkdir(parents=True, exist_ok=True) + temp['commit_id']=entry[1] temp['work_dir']=work_dir temp['macros']=entry[2] temp['isa']=entry[3] @@ -299,9 +305,8 @@ def generate_test_pool(ispec, pspec, workdir): sys.exit(1) with open(os.path.join(workdir,"test_list.yaml"),"w") as tfile: + tfile.write('# testlist generated on ' + (datetime.now(pytz.timezone('GMT'))).strftime("%Y-%m-%d %H:%M GMT")+'\n') yaml.dump(test_list,tfile) - with open(os.path.join(workdir,"test_pool.yaml"),"w") as tpfile: - yaml.dump(test_pool,tpfile) return (test_list, test_pool) @@ -318,7 +323,7 @@ def run_tests(dut, base, ispec, pspec, work_dir, cntr_args): :param pspec: The platform specifications of the DUT. - :param cntr_args: testfile, no_ref_run, no_dut_run + :param cntr_args: dbfile, testfile, no_ref_run, no_dut_run :type ispec: dict @@ -327,20 +332,18 @@ def run_tests(dut, base, ispec, pspec, work_dir, cntr_args): :return: A list of dictionary objects containing the necessary information required to generate the report. ''' - if cntr_args[0]: - with open(os.path.join(work_dir,"test_pool.yaml"),"r") as tpfile: - test_pool = yaml.load(tpfile) - with open(os.path.join(work_dir,"test_list.yaml"),"r") as tfile: + if cntr_args[1] is not None: + with open(cntr_args[1],"r") as tfile: test_list = yaml.load(tfile) else: - test_list, test_pool = generate_test_pool(ispec, pspec, work_dir) + test_list, test_pool = generate_test_pool(ispec, pspec, work_dir, cntr_args[0]) results = [] - if cntr_args[1]: + if cntr_args[2]: logger.info("Running Tests on DUT.") dut.runTests(test_list) logger.info("Tests run on DUT done.") raise SystemExit - elif cntr_args[2]: + elif cntr_args[3]: logger.info("Running Tests on Reference Model.") base.runTests(test_list) logger.info("Tests run on Reference done.") @@ -352,20 +355,21 @@ def run_tests(dut, base, ispec, pspec, work_dir, cntr_args): base.runTests(test_list) logger.info("Initiating signature checking.") - for entry in test_pool: - work_dir = test_list[entry[0]]['work_dir'] - res = os.path.join(test_list[entry[0]]['work_dir'],dut.name[:-1]+".signature") - ref = os.path.join(test_list[entry[0]]['work_dir'],base.name[:-1]+".signature") + for file in test_list: + testentry = test_list[file] + work_dir = testentry['work_dir'] + res = os.path.join(testentry['work_dir'],dut.name[:-1]+".signature") + ref = os.path.join(testentry['work_dir'],base.name[:-1]+".signature") result, diff = compare_signature(res, ref) res = { 'name': - entry[0], + file, 'res': result, 'commit_id': - entry[1], + testentry['commit_id'], 'log': - 'commit_id:' + entry[1] + "\nMACROS:\n" + "\n".join(entry[2]) + + 'commit_id:' + testentry['commit_id'] + "\nMACROS:\n" + "\n".join(testentry['macros']) + "" if result == "Passed" else diff, 'path': work_dir, @@ -374,7 +378,7 @@ def run_tests(dut, base, ispec, pspec, work_dir, cntr_args): } results.append(res) - logger.info('Following ' + str(len(test_pool)) + ' tests have been run :\n') + logger.info('Following ' + str(len(test_list)) + ' tests have been run :\n') logger.info('{0:<50s} : {1:<40s} : {2}'.format('TEST NAME', 'COMMIT ID', 'STATUS')) for res in results: diff --git a/riscof/main.py b/riscof/main.py index a931651..8e85271 100644 --- a/riscof/main.py +++ b/riscof/main.py @@ -140,7 +140,13 @@ def execute(): if not os.path.exists(work_dir): logger.debug('Creating new work directory: ' + work_dir) os.mkdir(work_dir) - elif not args.testfile: + elif args.command=='run': + if args.testfile is None: + logger.debug('Removing old work directory: ' + work_dir) + shutil.rmtree(work_dir) + logger.debug('Creating new work directory: ' + work_dir) + os.mkdir(work_dir) + else: logger.debug('Removing old work directory: ' + work_dir) shutil.rmtree(work_dir) logger.debug('Creating new work directory: ' + work_dir) @@ -197,7 +203,7 @@ def execute(): isa_file = dut.isa_spec platform_file = dut.platform_spec - if args.testfile: + if args.command=='run' and args.testfile is not None: isa_file = work_dir+ '/' + (isa_file.rsplit('/', 1)[1]).rsplit('.')[0] + "_checked.yaml" platform_file = work_dir+ '/' + (platform_file.rsplit('/', 1)[1]).rsplit('.')[0] + "_checked.yaml" else: @@ -211,8 +217,8 @@ def execute(): isa_specs = utils.load_yaml(isa_file)['hart0'] platform_specs = utils.load_yaml(platform_file) - if args.command=='gendb' or args.command=='run' or \ - args.command=='testlist' or args.command == 'coverage' : + if args.command=='gendb' or args.command=='testlist' or \ + args.command=='coverage' : logger.info("Generating database for suite: "+args.suite) work_dir = args.work_dir constants.suite = args.suite @@ -223,6 +229,18 @@ def execute(): logger.info('Database File Generated: '+constants.framework_db) constants.env = args.env logger.info('Env path set to'+constants.env) + elif args.command=='run' : + if args.dbfile is None and args.testfile is None: + logger.info("Generating database for suite: "+args.suite) + work_dir = args.work_dir + constants.suite = args.suite + constants.framework_db = os.path.join(work_dir,"database.yaml") + logger.debug('Suite used: '+constants.suite) + logger.debug('ENV used: '+ args.env) + dbgen.generate() + logger.info('Database File Generated: '+constants.framework_db) + constants.env = args.env + logger.info('Env path set to'+constants.env) if args.command == 'testlist': test_routines.generate_test_pool(isa_specs, platform_specs, work_dir) @@ -294,8 +312,12 @@ def execute(): with open(platform_file, "r") as platfile: pspecs = platfile.read() + + if args.dbfile is not None and args.testfile is not None: + logger.error("dbfile and testfile are mutually exclusive arguments") + raise SystemExit - cntr_args = [args.testfile,args.no_ref_run,args.no_dut_run] + cntr_args = [args.dbfile,args.testfile,args.no_ref_run,args.no_dut_run] report_objects = {} report_objects['date'] = (datetime.now( diff --git a/riscof/utils.py b/riscof/utils.py index 2c083f8..42fbb74 100644 --- a/riscof/utils.py +++ b/riscof/utils.py @@ -598,8 +598,16 @@ def riscof_cmdline_args(): metavar= 'PATH', default=str(pathlib.Path('./riscof_work').absolute()) ) - run.add_argument('--testfile',action='store_true', - help="Run tests from already generated testlist file") + run.add_argument('--dbfile', + type= lambda p: str(pathlib.Path(p).absolute()), + action='store', + help='The Path to the custom suite directory.', + metavar= 'PATH') + run.add_argument('--testfile', + type= lambda p: str(pathlib.Path(p).absolute()), + action='store', + help='The Path to the custom suite directory.', + metavar= 'PATH') run.add_argument('--no-ref-run',action='store_true', help="Do not run tests on Reference") run.add_argument('--no-dut-run',action='store_true', From 54857acf43862e2c9288b5bc946bd1f7671b91a8 Mon Sep 17 00:00:00 2001 From: Muhammad Bilal Sakhawat Date: Sat, 24 Jul 2021 14:05:04 +0500 Subject: [PATCH 05/10] Change FXLEN macro to FLEN --- CHANGELOG.md | 3 ++- riscof/framework/test.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa0ca14..1de01d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,13 +2,14 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [1.21.1] - 2021-07-21 +## [1.21.1] - 2021-07-24 - Added support for intermediate run start points #1 - --dbfile is added to generate the test-list and then run tests on the target and reference - --testfile is added to directly run tests on the target and reference (assumes the testlist has been correctly generated) - --no-ref-run is added so that tests do not run on reference and quit before signature comparison - --no-dut-run is added so that tests do not run on DUT and quit before signature comparison - Added a timestamp comment on the generated database.yaml and testlist.yaml files +- Change FXLEN macro to FLEN #8 ## [1.21.0] - 2021-07-21 - Changed CI script from gitlab to github actions diff --git a/riscof/framework/test.py b/riscof/framework/test.py index a38686e..a8ee4e8 100644 --- a/riscof/framework/test.py +++ b/riscof/framework/test.py @@ -276,9 +276,9 @@ def generate_test_pool(ispec, pspec, workdir, dbfile = None): xlen = '128' macros.append("XLEN=" + xlen) if re.match(r"^[^(Z,z)]+D.*$",isa): - macros.append("FXLEN=64") + macros.append("FLEN=64") elif re.match(r"^[^(Z,z)]+F.*$",isa): - macros.append("FXLEN=32") + macros.append("FLEN=32") test_pool.append( (file, db[file]['commit_id'], macros, db[file]['isa'],cov_labels)) logger.info("Selecting Tests.") From 1c98ee67cf277aaeb20dd41351faa583b28815c0 Mon Sep 17 00:00:00 2001 From: Muhammad Bilal Sakhawat Date: Mon, 26 Jul 2021 11:10:14 +0500 Subject: [PATCH 06/10] Update the plusargs help description and add dbfile, testfile arguments in mutually exclusive plusarg group --- riscof/framework/test.py | 6 ++---- riscof/main.py | 4 ---- riscof/utils.py | 9 +++++---- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/riscof/framework/test.py b/riscof/framework/test.py index a8ee4e8..c4dde3b 100644 --- a/riscof/framework/test.py +++ b/riscof/framework/test.py @@ -246,8 +246,7 @@ def generate_test_pool(ispec, pspec, workdir, dbfile = None): test_pool = [] test_list = {} if dbfile is not None: - with open(dbfile,"r") as db_file: - db = yaml.load(db_file) + db = utils.load_yaml(dbfile) else: db = utils.load_yaml(constants.framework_db) for file in db: @@ -333,8 +332,7 @@ def run_tests(dut, base, ispec, pspec, work_dir, cntr_args): required to generate the report. ''' if cntr_args[1] is not None: - with open(cntr_args[1],"r") as tfile: - test_list = yaml.load(tfile) + test_list = utils.load_yaml(cntr_args[1]) else: test_list, test_pool = generate_test_pool(ispec, pspec, work_dir, cntr_args[0]) results = [] diff --git a/riscof/main.py b/riscof/main.py index 8e85271..f275dcb 100644 --- a/riscof/main.py +++ b/riscof/main.py @@ -312,10 +312,6 @@ def execute(): with open(platform_file, "r") as platfile: pspecs = platfile.read() - - if args.dbfile is not None and args.testfile is not None: - logger.error("dbfile and testfile are mutually exclusive arguments") - raise SystemExit cntr_args = [args.dbfile,args.testfile,args.no_ref_run,args.no_dut_run] diff --git a/riscof/utils.py b/riscof/utils.py index 42fbb74..a70efdc 100644 --- a/riscof/utils.py +++ b/riscof/utils.py @@ -598,15 +598,16 @@ def riscof_cmdline_args(): metavar= 'PATH', default=str(pathlib.Path('./riscof_work').absolute()) ) - run.add_argument('--dbfile', + run_group = run.add_mutually_exclusive_group() + run_group.add_argument('--dbfile', type= lambda p: str(pathlib.Path(p).absolute()), action='store', - help='The Path to the custom suite directory.', + help='The Path to the database file.', metavar= 'PATH') - run.add_argument('--testfile', + run_group.add_argument('--testfile', type= lambda p: str(pathlib.Path(p).absolute()), action='store', - help='The Path to the custom suite directory.', + help='The Path to the testlist file.', metavar= 'PATH') run.add_argument('--no-ref-run',action='store_true', help="Do not run tests on Reference") From db7520ae77c318ab3f812fb76d5bc6241e4b8cfa Mon Sep 17 00:00:00 2001 From: Muhammad Bilal Sakhawat Date: Mon, 26 Jul 2021 11:44:08 +0500 Subject: [PATCH 07/10] Update the documentation for RISCOF run command --- docs/source/commands.rst | 4 ++++ docs/source/installation.rst | 27 ++++++++++++++++----------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/docs/source/commands.rst b/docs/source/commands.rst index c681d1e..a6ac1ea 100644 --- a/docs/source/commands.rst +++ b/docs/source/commands.rst @@ -160,6 +160,10 @@ Optional arguments from the cli: ``./riscof_work`` - no-browser: when used, RISCOF skips automatically opening the html report in the default web browser. +- dbfile: The path to the database file, from which testlist will be generated +- testfile: The path to the testlist file on which tests will be run +- no-ref-run: when used, RISCOF will not run tests on Reference and will quit before signatures comparison +- no-dut-run: when used, RISCOF will not run tests on DUT and will quit before signatures comparison All artifacts of this command are generated in the ``work_dir`` directory. Typicall artifacts will include: diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 97ba989..7e7b7f6 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -267,17 +267,22 @@ Once you have installed RISCOF you can execute ``riscof --help`` to print the he -h, --help show this help message and exit Action 'run' - - usage: riscof run [-h] [--config PATH] --suite PATH --env PATH [--no-browser] - [--work-dir PATH] - - optional arguments: - --config PATH The Path to the config file. [Default=./config.ini] - --env PATH The Path to the custom env directory. - --no-browser Do not open the browser for showing the test report. - --suite PATH The Path to the custom suite directory. - --work-dir PATH The Path to the work-dir. - -h, --help show this help message and exit + + usage: riscof run [-h] [--config PATH] --suite PATH --env PATH [--no-browser] [--work-dir PATH] [--dbfile PATH | --testfile PATH] + [--no-ref-run] [--no-dut-run] + + optional arguments: + --config PATH The Path to the config file. [Default=./config.ini] + --dbfile PATH The Path to the database file. + --env PATH The Path to the custom env directory. + --no-browser Do not open the browser for showing the test report. + --no-dut-run Do not run tests on DUT + --no-ref-run Do not run tests on Reference + --suite PATH The Path to the custom suite directory. + --testfile PATH The Path to the testlist file. + --work-dir PATH The Path to the work-dir. + -h, --help show this help message and exit + Action 'testlist' From 805791a17c996d9fb64dee22df160ed6fdc82ccd Mon Sep 17 00:00:00 2001 From: Muhammad Bilal Sakhawat Date: Fri, 30 Jul 2021 11:04:44 +0500 Subject: [PATCH 08/10] minor fixes for dbfile and testfile arguments --- riscof/main.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/riscof/main.py b/riscof/main.py index f275dcb..e4a4e23 100644 --- a/riscof/main.py +++ b/riscof/main.py @@ -141,7 +141,7 @@ def execute(): logger.debug('Creating new work directory: ' + work_dir) os.mkdir(work_dir) elif args.command=='run': - if args.testfile is None: + if args.dbfile is None and args.testfile is None: logger.debug('Removing old work directory: ' + work_dir) shutil.rmtree(work_dir) logger.debug('Creating new work directory: ' + work_dir) @@ -203,7 +203,7 @@ def execute(): isa_file = dut.isa_spec platform_file = dut.platform_spec - if args.command=='run' and args.testfile is not None: + if args.command=='run' and (args.testfile is not None or args.dbfile is not None): isa_file = work_dir+ '/' + (isa_file.rsplit('/', 1)[1]).rsplit('.')[0] + "_checked.yaml" platform_file = work_dir+ '/' + (platform_file.rsplit('/', 1)[1]).rsplit('.')[0] + "_checked.yaml" else: @@ -241,6 +241,13 @@ def execute(): logger.info('Database File Generated: '+constants.framework_db) constants.env = args.env logger.info('Env path set to'+constants.env) + elif args.dbfile is None: + constants.suite = args.suite + constants.framework_db = os.path.join(work_dir,"database.yaml") + constants.env = args.env + else: + constants.suite = args.suite + constants.env = args.env if args.command == 'testlist': test_routines.generate_test_pool(isa_specs, platform_specs, work_dir) From dae00a5306ba79488351507e591533f201dd0005 Mon Sep 17 00:00:00 2001 From: Muhammad Bilal Sakhawat Date: Fri, 30 Jul 2021 11:58:58 +0500 Subject: [PATCH 09/10] made build stage of the plugins dependent on the --no--run arguments --- riscof/framework/main.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/riscof/framework/main.py b/riscof/framework/main.py index f67aa8a..c74238c 100644 --- a/riscof/framework/main.py +++ b/riscof/framework/main.py @@ -184,11 +184,18 @@ def run(dut, base, dut_isa_spec, dut_platform_spec, work_dir, cntr_args): #Loading Specs ispec = utils.load_yaml(dut_isa_spec) pspec = utils.load_yaml(dut_platform_spec) - - logger.info("Running Build for DUT") - dut.build(dut_isa_spec, dut_platform_spec) - logger.info("Running Build for Reference") - base.build(dut_isa_spec, dut_platform_spec) + + if cntr_args[2]: + logger.info("Running Build for DUT") + dut.build(dut_isa_spec, dut_platform_spec) + elif cntr_args[3]: + logger.info("Running Build for Reference") + base.build(dut_isa_spec, dut_platform_spec) + else: + logger.info("Running Build for DUT") + dut.build(dut_isa_spec, dut_platform_spec) + logger.info("Running Build for Reference") + base.build(dut_isa_spec, dut_platform_spec) results = test.run_tests(dut, base, ispec['hart0'], pspec, work_dir, cntr_args) From 548d8158a52302e49efe7d013b966606b5c7a2bb Mon Sep 17 00:00:00 2001 From: Muhammad Bilal Sakhawat Date: Tue, 3 Aug 2021 12:42:41 +0500 Subject: [PATCH 10/10] fix a typo arch_test.py --- riscof/arch_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/riscof/arch_test.py b/riscof/arch_test.py index 83f97f8..20017be 100644 --- a/riscof/arch_test.py +++ b/riscof/arch_test.py @@ -21,7 +21,7 @@ def get_version(path): tags = repo.tags remote = repo.remote() url = remote.url - if (tags) and (url==constants.https_url or url == ssh_url): + if (tags) and (url==constants.https_url or url == constants.ssh_url): ver_dict['commit'] = str(commit) for tag in tags: if tag.commit == commit: