Merge pull request #12 from bilalsakhawat-10xe/migration-fix

Adding support for intermediate run start points

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.21.1] - 2021-07-24
+- Added support for intermediate run start points #1
+- --dbfile is added to generate the test-list and then run tests on the target and reference
+- --testfile is added to directly run tests on the target and reference (assumes the testlist has been correctly generated)
+- --no-ref-run is added so that tests do not run on reference and quit before signature comparison
+- --no-dut-run is added so that tests do not run on DUT and quit before signature comparison 
+- Added a timestamp comment on the generated database.yaml and testlist.yaml files
+- Change FXLEN macro to FLEN #8
+
 ## [1.21.0] - 2021-07-21
 - Changed CI script from gitlab to github actions
 - Removing hosted cgf files

docs/source/commands.rst

@@ -160,6 +160,10 @@ Optional arguments from the cli:
   ``./riscof_work``
 - no-browser: when used, RISCOF skips automatically opening the html report in the default web
   browser.
+- dbfile: The path to the database file, from which testlist will be generated 
+- testfile: The path to the testlist file on which tests will be run
+- no-ref-run: when used, RISCOF will not run tests on Reference and will quit before signatures comparison
+- no-dut-run: when used, RISCOF will not run tests on DUT and will quit before signatures comparison 
 
 All artifacts of this command are generated in the ``work_dir`` directory. Typicall artifacts will
 include:

docs/source/installation.rst

@@ -267,17 +267,22 @@ Once you have installed RISCOF you can execute ``riscof --help`` to print the he
     	  -h, --help       show this help message and exit
     	
     Action 'run'
-    
-    	usage: riscof run [-h] [--config PATH] --suite PATH --env PATH [--no-browser]
-    	                  [--work-dir PATH]
-    	
-    	optional arguments:
-    	  --config PATH    The Path to the config file. [Default=./config.ini]
-    	  --env PATH       The Path to the custom env directory.
-    	  --no-browser     Do not open the browser for showing the test report.
-    	  --suite PATH     The Path to the custom suite directory.
-    	  --work-dir PATH  The Path to the work-dir.
-    	  -h, --help       show this help message and exit
+
+	usage: riscof run [-h] [--config PATH] --suite PATH --env PATH [--no-browser] [--work-dir PATH] [--dbfile PATH | --testfile PATH]
+	                  [--no-ref-run] [--no-dut-run]
+	
+	optional arguments:
+	  --config PATH    The Path to the config file. [Default=./config.ini]
+	  --dbfile PATH    The Path to the database file.
+	  --env PATH       The Path to the custom env directory.
+	  --no-browser     Do not open the browser for showing the test report.
+	  --no-dut-run     Do not run tests on DUT
+	  --no-ref-run     Do not run tests on Reference
+	  --suite PATH     The Path to the custom suite directory.
+	  --testfile PATH  The Path to the testlist file.
+	  --work-dir PATH  The Path to the work-dir.
+	  -h, --help       show this help message and exit
+
     	
     Action 'testlist'
     

riscof/Templates/setup/model/riscof_model.py

@@ -75,6 +75,8 @@ def build(self, isa_yaml, platform_yaml):
         # build your RTL here
 
     def runTests(self, testList, cgf_file=None):
+        if os.path.exists(self.work_dir+ "/Makefile." + self.name[:-1]):
+            os.remove(self.work_dir+ "/Makefile." + self.name[:-1])
         make = utils.makeUtil(makefilePath=os.path.join(self.work_dir, "Makefile." + self.name[:-1]))
         make.makeCommand = 'make -j' + self.num_jobs
         for file in testList:

riscof/Templates/setup/reference/riscof_model.py

@@ -79,6 +79,8 @@ def build(self, isa_yaml, platform_yaml):
         # build your RTL here
 
     def runTests(self, testList, cgf_file=None):
+        if os.path.exists(self.work_dir+ "/Makefile." + self.name[:-1]):
+            os.remove(self.work_dir+ "/Makefile." + self.name[:-1])
         make = utils.makeUtil(makefilePath=os.path.join(self.work_dir, "Makefile." + self.name[:-1]))
         make.makeCommand = self.make + ' -j' + self.num_jobs
         for file in testList:

riscof/Templates/setup/sail_cSim/riscof_sail_cSim.py

@@ -80,6 +80,8 @@ def build(self, isa_yaml, platform_yaml):
 
 
     def runTests(self, testList, cgf_file=None):
+        if os.path.exists(self.work_dir+ "/Makefile." + self.name[:-1]):
+            os.remove(self.work_dir+ "/Makefile." + self.name[:-1])
         make = utils.makeUtil(makefilePath=os.path.join(self.work_dir, "Makefile." + self.name[:-1]))
         make.makeCommand = self.make + ' -j' + self.num_jobs
         for file in testList:

riscof/__init__.py

@@ -4,4 +4,4 @@
 
 __author__ = """InCore Semiconductors Pvt Ltd"""
 __email__ = 'info@incoresemi.com'
-__version__ = '1.21.0'
+__version__ = '1.21.1'

riscof/arch_test.py

@@ -21,7 +21,7 @@ def get_version(path):
         tags = repo.tags
         remote = repo.remote()
         url = remote.url
-        if (tags) and (url==constants.https_url or url == ssh_url):
+        if (tags) and (url==constants.https_url or url == constants.ssh_url):
             ver_dict['commit'] = str(commit)
             for tag in tags:
                 if tag.commit == commit:

riscof/dbgen.py

@@ -3,6 +3,8 @@
 import git
 import sys
 import re
+import pytz
+from datetime import datetime
 from riscof.utils import yaml
 import riscof.utils as utils
 import collections
@@ -189,5 +191,6 @@ def generate():
                 continue
             db[fpath] = {'commit_id': '-', **temp}
     with open(dbfile, "w") as wrfile:
+        wrfile.write('# database generated on ' + (datetime.now(pytz.timezone('GMT'))).strftime("%Y-%m-%d %H:%M GMT")+'\n')
         yaml.dump(orderdict(db),
                   wrfile)

riscof/framework/main.py

@@ -151,7 +151,7 @@ def run_coverage(base, dut_isa_spec, dut_platform_spec, work_dir, cgf_file=None)
 
     return results, for_html, test_stats, coverpoints
 
-def run(dut, base, dut_isa_spec, dut_platform_spec, work_dir):
+def run(dut, base, dut_isa_spec, dut_platform_spec, work_dir, cntr_args):
     '''
         Entry point for the framework module. This function initializes and sets up the required
         variables for the tests to run.
@@ -165,6 +165,8 @@ def run(dut, base, dut_isa_spec, dut_platform_spec, work_dir):
 
         :param dut_platform_spec: The absolute path to the checked yaml containing
             the DUT platform specification.
+            
+        :param cntr_args: dbfile, testfile, no_ref_run, no_dut_run
 
         :type dut_platform_spec: str
 
@@ -182,13 +184,20 @@ def run(dut, base, dut_isa_spec, dut_platform_spec, work_dir):
     #Loading Specs
     ispec = utils.load_yaml(dut_isa_spec)
     pspec = utils.load_yaml(dut_platform_spec)
+
+    if cntr_args[2]:
+        logger.info("Running Build for DUT")
+        dut.build(dut_isa_spec, dut_platform_spec)
+    elif cntr_args[3]:
+        logger.info("Running Build for Reference")
+        base.build(dut_isa_spec, dut_platform_spec)
+    else:
+        logger.info("Running Build for DUT")
+        dut.build(dut_isa_spec, dut_platform_spec)
+        logger.info("Running Build for Reference")
+        base.build(dut_isa_spec, dut_platform_spec)
 
-    logger.info("Running Build for DUT")
-    dut.build(dut_isa_spec, dut_platform_spec)
-    logger.info("Running Build for Reference")
-    base.build(dut_isa_spec, dut_platform_spec)
-
-    results = test.run_tests(dut, base, ispec['hart0'], pspec, work_dir)
+    results = test.run_tests(dut, base, ispec['hart0'], pspec, work_dir, cntr_args)
 
     return results
 

riscof/framework/test.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 import difflib
 import ast
+import pytz
 import random
 from datetime import datetime
 
@@ -221,7 +222,7 @@ def eval_macro(macro, spec):
         return (False,[])
 
 
-def generate_test_pool(ispec, pspec, workdir):
+def generate_test_pool(ispec, pspec, workdir, dbfile = None):
     '''
         Funtion to select the tests which are applicable for the DUT and generate the macros
         necessary for each test.
@@ -244,7 +245,10 @@ def generate_test_pool(ispec, pspec, workdir):
     spec = {**ispec, **pspec}
     test_pool = []
     test_list = {}
-    db = utils.load_yaml(constants.framework_db)
+    if dbfile is not None:
+        db = utils.load_yaml(dbfile)
+    else:
+        db = utils.load_yaml(constants.framework_db)
     for file in db:
         macros = []
         cov_labels = []
@@ -271,9 +275,9 @@ def generate_test_pool(ispec, pspec, workdir):
                 xlen = '128'
             macros.append("XLEN=" + xlen)
             if re.match(r"^[^(Z,z)]+D.*$",isa):
-                macros.append("FXLEN=64")
+                macros.append("FLEN=64")
             elif re.match(r"^[^(Z,z)]+F.*$",isa):
-                macros.append("FXLEN=32")
+                macros.append("FLEN=32")
             test_pool.append(
                 (file, db[file]['commit_id'], macros, db[file]['isa'],cov_labels))
     logger.info("Selecting Tests.")
@@ -285,6 +289,7 @@ def generate_test_pool(ispec, pspec, workdir):
         else:
             work_dir = os.path.join(workdir,entry[0].replace("suite/", ''))
         Path(work_dir).mkdir(parents=True, exist_ok=True)
+        temp['commit_id']=entry[1]
         temp['work_dir']=work_dir
         temp['macros']=entry[2]
         temp['isa']=entry[3]
@@ -299,12 +304,13 @@ def generate_test_pool(ispec, pspec, workdir):
         sys.exit(1)
 
     with open(os.path.join(workdir,"test_list.yaml"),"w") as tfile:
+        tfile.write('# testlist generated on ' + (datetime.now(pytz.timezone('GMT'))).strftime("%Y-%m-%d %H:%M GMT")+'\n')
         yaml.dump(test_list,tfile)
 
     return (test_list, test_pool)
 
 
-def run_tests(dut, base, ispec, pspec, work_dir):
+def run_tests(dut, base, ispec, pspec, work_dir, cntr_args):
     '''
         Function to run the tests for the DUT.
 
@@ -315,6 +321,8 @@ def run_tests(dut, base, ispec, pspec, work_dir):
         :param ispec: The isa specifications of the DUT.
 
         :param pspec: The platform specifications of the DUT.
+        
+        :param cntr_args: dbfile, testfile, no_ref_run, no_dut_run
 
         :type ispec: dict
 
@@ -323,29 +331,43 @@ def run_tests(dut, base, ispec, pspec, work_dir):
         :return: A list of dictionary objects containing the necessary information
             required to generate the report.
     '''
-    test_list, test_pool = generate_test_pool(ispec, pspec, work_dir)
+    if cntr_args[1] is not None:
+        test_list = utils.load_yaml(cntr_args[1])
+    else:
+        test_list, test_pool = generate_test_pool(ispec, pspec, work_dir, cntr_args[0])
     results = []
-    logger.info("Running Tests on DUT.")
-    dut.runTests(test_list)
-    logger.info("Running Tests on Reference Model.")
-    base.runTests(test_list)
-
+    if cntr_args[2]:
+        logger.info("Running Tests on DUT.")
+        dut.runTests(test_list)
+        logger.info("Tests run on DUT done.")
+        raise SystemExit
+    elif cntr_args[3]:
+        logger.info("Running Tests on Reference Model.")
+        base.runTests(test_list)
+        logger.info("Tests run on Reference done.")
+        raise SystemExit
+    else:
+        logger.info("Running Tests on DUT.")
+        dut.runTests(test_list)
+        logger.info("Running Tests on Reference Model.")
+        base.runTests(test_list)
 
     logger.info("Initiating signature checking.")
-    for entry in test_pool:
-        work_dir = test_list[entry[0]]['work_dir']
-        res = os.path.join(test_list[entry[0]]['work_dir'],dut.name[:-1]+".signature")
-        ref = os.path.join(test_list[entry[0]]['work_dir'],base.name[:-1]+".signature")
+    for file in test_list:
+        testentry = test_list[file]
+        work_dir = testentry['work_dir']
+        res = os.path.join(testentry['work_dir'],dut.name[:-1]+".signature")
+        ref = os.path.join(testentry['work_dir'],base.name[:-1]+".signature")
         result, diff = compare_signature(res, ref)
         res = {
             'name':
-            entry[0],
+            file,
             'res':
             result,
             'commit_id':
-            entry[1],
+            testentry['commit_id'],
             'log':
-            'commit_id:' + entry[1] + "\nMACROS:\n" + "\n".join(entry[2]) +
+            'commit_id:' + testentry['commit_id'] + "\nMACROS:\n" + "\n".join(testentry['macros']) +
             "" if result == "Passed" else diff,
             'path':
             work_dir,
@@ -354,7 +376,7 @@ def run_tests(dut, base, ispec, pspec, work_dir):
         }
         results.append(res)
 
-    logger.info('Following ' + str(len(test_pool)) + ' tests have been run :\n')
+    logger.info('Following ' + str(len(test_list)) + ' tests have been run :\n')
     logger.info('{0:<50s} : {1:<40s} : {2}'.format('TEST NAME', 'COMMIT ID',
                                                    'STATUS'))
     for res in results:

riscof/main.py

@@ -140,6 +140,12 @@ def execute():
         if not os.path.exists(work_dir):
             logger.debug('Creating new work directory: ' + work_dir)
             os.mkdir(work_dir)
+        elif args.command=='run':
+            if args.dbfile is None and args.testfile is None:
+                logger.debug('Removing old work directory: ' + work_dir)
+                shutil.rmtree(work_dir)
+                logger.debug('Creating new work directory: ' + work_dir)
+                os.mkdir(work_dir)
         else:
             logger.debug('Removing old work directory: ' + work_dir)
             shutil.rmtree(work_dir)
@@ -196,20 +202,23 @@ def execute():
         #Run riscv_config on inputs
         isa_file = dut.isa_spec
         platform_file = dut.platform_spec
-
-
-        try:
-            isa_file = checker.check_isa_specs( isa_file, work_dir, True)
-            platform_file = checker.check_platform_specs( platform_file, work_dir, True)
-        except ValidationError as msg:
-            logger.error(msg)
-            return 1
+
+        if args.command=='run' and (args.testfile is not None or args.dbfile is not None):
+            isa_file = work_dir+ '/' + (isa_file.rsplit('/', 1)[1]).rsplit('.')[0] + "_checked.yaml"
+            platform_file = work_dir+ '/' + (platform_file.rsplit('/', 1)[1]).rsplit('.')[0] + "_checked.yaml"
+        else:
+            try:
+                isa_file = checker.check_isa_specs( isa_file, work_dir, True)
+                platform_file = checker.check_platform_specs( platform_file, work_dir, True)
+            except ValidationError as msg:
+                logger.error(msg)
+                return 1
 
         isa_specs = utils.load_yaml(isa_file)['hart0']
         platform_specs = utils.load_yaml(platform_file)
 
-    if args.command=='gendb' or args.command=='run' or \
-            args.command=='testlist' or args.command == 'coverage' :
+    if args.command=='gendb' or args.command=='testlist' or \
+            args.command=='coverage' :
         logger.info("Generating database for suite: "+args.suite)
         work_dir = args.work_dir
         constants.suite = args.suite
@@ -220,6 +229,25 @@ def execute():
         logger.info('Database File Generated: '+constants.framework_db)
         constants.env = args.env
         logger.info('Env path set to'+constants.env)
+    elif args.command=='run' :
+        if args.dbfile is None and args.testfile is None:
+            logger.info("Generating database for suite: "+args.suite)
+            work_dir = args.work_dir
+            constants.suite = args.suite
+            constants.framework_db = os.path.join(work_dir,"database.yaml")
+            logger.debug('Suite used: '+constants.suite)
+            logger.debug('ENV used: '+ args.env)
+            dbgen.generate()
+            logger.info('Database File Generated: '+constants.framework_db)
+            constants.env = args.env
+            logger.info('Env path set to'+constants.env)
+        elif args.dbfile is None:
+            constants.suite = args.suite
+            constants.framework_db = os.path.join(work_dir,"database.yaml")
+            constants.env = args.env
+        else:
+            constants.suite = args.suite
+            constants.env = args.env
 
     if args.command == 'testlist':
         test_routines.generate_test_pool(isa_specs, platform_specs, work_dir)
@@ -291,7 +319,9 @@ def execute():
 
         with open(platform_file, "r") as platfile:
             pspecs = platfile.read()
-
+
+        cntr_args = [args.dbfile,args.testfile,args.no_ref_run,args.no_dut_run]
+
         report_objects = {}
         report_objects['date'] = (datetime.now(
             pytz.timezone('GMT'))).strftime("%Y-%m-%d %H:%M GMT")
@@ -312,7 +342,7 @@ def execute():
         report_objects['platform_specs'] = pspecs
 
         report_objects['results'] = framework.run(dut, base, isa_file,
-                                                  platform_file, work_dir)
+                                                  platform_file, work_dir, cntr_args)
 
         report_objects['num_passed'] = 0
         report_objects['num_failed'] = 0