Generate (also historical) (O)ID tables

.gitignore

@@ -13,3 +13,5 @@ test/oqs_test_groups
 # test artifacts
 tmp
 interop.log
+# pycache
+oqs-template/__pycache__

oqs-template/generate.py

@@ -215,4 +215,5 @@ def load_config(include_disabled_sigs=False):
 populate('ALGORITHMS.md', config2, '<!---')
 populate('README.md', config2, '<!---')
 print("All files generated")
-
+os.environ["LIBOQS_DOCS_DIR"]=os.path.join(os.environ["LIBOQS_SRC_DIR"], "docs")
+import generate_oid_nid_table

oqs-template/generate_oid_nid_table.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+
+import argparse
+import sys
+from tabulate import tabulate
+import yaml
+import os
+
+import generatehelpers
+
+config = {}
+
+def gen_sig_table(oqslibdocdir):
+  liboqs_sig_docs_dir = os.path.join(oqslibdocdir, 'algorithms', 'sig')
+  liboqs_sigs = {}
+  for root, _, files in os.walk(liboqs_sig_docs_dir):
+    for fil in files:
+      if fil.endswith(".yml"):
+        with open(os.path.join(root, fil), mode='r', encoding='utf-8') as f:
+           algyml = yaml.safe_load(f.read())
+        liboqs_sigs[algyml['name']]=algyml
+
+  table = [['Algorithm', 'Implementation Version',
+          'NIST round', 'Claimed NIST Level', 'Code Point', 'OID']]
+  claimed_nist_level = 0
+  for sig in sorted(config['sigs'], key=lambda s: s['family']):
+    for variant in sig['variants']:
+        if variant['security'] == 128:
+            claimed_nist_level = 1
+        elif variant['security'] == 192:
+            claimed_nist_level = 3
+        elif variant['security'] == 256:
+            claimed_nist_level = 5
+        else:
+            sys.exit("variant['security'] value malformed.")
+
+        if sig['family'].startswith('SPHINCS'):
+            sig['family'] = 'SPHINCS+'
+
+        if variant['name'].startswith('dilithium2'):
+            claimed_nist_level = 2
+
+        try: 
+            table.append([variant['name'], liboqs_sigs[sig['family']]['spec-version'],
+                          liboqs_sigs[sig['family']]['nist-round'], claimed_nist_level, variant['code_point'],
+                          variant['oid']])
+            for hybrid in variant['mix_with']:
+                table.append([variant['name'] + ' **hybrid with** ' + hybrid['name'],
+                              liboqs_sigs[sig['family']]['spec-version'],
+                              liboqs_sigs[sig['family']]['nist-round'],
+                              claimed_nist_level,
+                              hybrid['code_point'],
+                              hybrid['oid']])
+        except KeyError as ke:
+            # Non-existant NIDs mean this alg is not supported any more
+            pass
+
+        if 'extra_nids' in variant:
+          for i in range(len(variant['extra_nids']['old'])):
+            table.append([variant['name'], variant['extra_nids']['old'][i]['implementation_version'],
+                          variant['extra_nids']['old'][i]['nist-round'], claimed_nist_level, variant['extra_nids']['old'][i]['code_point'],
+                          variant['extra_nids']['old'][i]['oid']])
+            for hybrid in variant['extra_nids']['old'][i]['mix_with']:
+                table.append([variant['name'] + ' **hybrid with** ' + hybrid['name'],
+                              variant['extra_nids']['old'][i]['implementation_version'],
+                              variant['extra_nids']['old'][i]['nist-round'],
+                              claimed_nist_level,
+                              hybrid['code_point'],
+                              hybrid['oid']])
+
+  with open(os.path.join('oqs-template', 'oqs-sig-info.md'), mode='w', encoding='utf-8') as f:
+    f.write(tabulate(table, tablefmt="pipe", headers="firstrow"))
+  print("Written oqs-sig-info.md")
+
+def gen_kem_table(oqslibdocdir):
+  liboqs_kem_docs_dir = os.path.join(oqslibdocdir, 'algorithms', 'kem')
+  liboqs_kems = {}
+  for root, _, files in os.walk(liboqs_kem_docs_dir):
+    for fil in files:
+      if fil.endswith(".yml"):
+        with open(os.path.join(root, fil), mode='r', encoding='utf-8') as f:
+           algyml = yaml.safe_load(f.read())
+        liboqs_kems[algyml['name']]=algyml
+  if 'SIKE' in liboqs_kems:
+      liboqs_kems['SIDH']=liboqs_kems['SIKE']
+  # TODO: Workaround for wrong upstream name for Kyber:
+  liboqs_kems['CRYSTALS-Kyber']=liboqs_kems['Kyber']
+
+  table_header = ['Family', 'Implementation Version', 'Variant', 'NIST round', 'Claimed NIST Level',
+           'Code Point', 'Hybrid Elliptic Curve (if any)']
+  table = []
+  hybrid_elliptic_curve = ''
+  for kem in sorted(config['kems'], key=lambda k: k['family']):
+    if kem['bit_security'] == 128:
+        claimed_nist_level = 1
+        hybrid_elliptic_curve = 'secp256_r1'
+    elif kem['bit_security'] == 192:
+        claimed_nist_level = 3
+        hybrid_elliptic_curve = 'secp384_r1'
+    elif kem['bit_security'] == 256:
+        claimed_nist_level = 5
+        hybrid_elliptic_curve = 'secp521_r1'
+    else:
+        sys.exit("kem['bit_security'] value malformed.")
+
+    if 'implementation_version' in kem:
+        implementation_version = kem['implementation_version']
+    else:
+        if kem['family'] in liboqs_kems:
+            implementation_version = liboqs_kems[kem['family']]['spec-version']
+
+    if kem['name_group'].startswith('sidhp503') or kem['name_group'].startswith('sikep503'):
+        claimed_nist_level = 2
+
+    try: 
+       table.append([kem['family'], implementation_version,
+                     kem['name_group'], liboqs_kems[kem['family']]['nist-round'], claimed_nist_level,
+                     kem['nid'], ""])
+       table.append([kem['family'], implementation_version,
+                     kem['name_group'], liboqs_kems[kem['family']]['nist-round'], claimed_nist_level,
+                     kem['nid_hybrid'], hybrid_elliptic_curve])
+    except KeyError as ke:
+       # Non-existant NIDs mean this alg is not supported any more
+       pass
+
+    if 'extra_nids' in kem:
+        if 'current' in kem['extra_nids']: # assume "current" NIDs to mean liboqs-driven NIST round information:
+            for entry in kem['extra_nids']['current']:
+                table.append([kem['family'], implementation_version,
+                              kem['name_group'], liboqs_kems[kem['family']]['nist-round'], claimed_nist_level,
+                              entry['nid'], 
+                              entry['hybrid_group'] if 'hybrid_group' in entry else ""])
+        if 'old' in kem['extra_nids']:
+            for entry in kem['extra_nids']['old']:
+                table.append([kem['family'], entry['implementation_version'],
+                              kem['name_group'], entry['nist-round'], claimed_nist_level,
+                              entry['nid'],
+                              entry['hybrid_group'] if 'hybrid_group' in entry else ""])
+
+  # sort by:  family, version, security level, variant, hybrid
+  table.sort(key = lambda row: "{:s}|{:s}|{:d}|{:s}|{:s}".format(row[0], row[1], row[3], row[2], row[5]))
+
+  table = [table_header] + table
+
+  with open(os.path.join('oqs-template', 'oqs-kem-info.md'), mode='w', encoding='utf-8') as f:
+    f.write(tabulate(table, tablefmt="pipe", headers="firstrow"))
+    f.write("\n")
+  print("Written oqs-kem-info.md")
+
+# main:
+with open(os.path.join('oqs-template', 'generate.yml'), mode='r', encoding='utf-8') as f:
+    config = yaml.safe_load(f.read())
+
+if 'LIBOQS_DOCS_DIR' not in os.environ:
+   parser = argparse.ArgumentParser()
+   parser.add_argument('--liboqs-docs-dir', dest="liboqs_docs_dir", required=True)
+   args = parser.parse_args()
+   oqsdocsdir = args.liboqs_docs_dir
+else:
+   oqsdocsdir = os.environ["LIBOQS_DOCS_DIR"]
+
+config = generatehelpers.complete_config(config, oqsdocsdir)
+
+gen_kem_table(oqsdocsdir)
+gen_sig_table(oqsdocsdir)

oqs-template/generatehelpers.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+
+import copy
+import glob
+import jinja2
+import jinja2.ext
+import os
+import shutil
+import subprocess
+import yaml
+import json
+import sys
+
+def file_get_contents(filename, encoding=None):
+    with open(filename, mode='r', encoding=encoding) as fh:
+        return fh.read()
+
+def file_put_contents(filename, s, encoding=None):
+    with open(filename, mode='w', encoding=encoding) as fh:
+        fh.write(s)
+
+def get_kem_nistlevel(alg, docsdir):
+    # translate family names in generate.yml to directory names for liboqs algorithm datasheets
+    if alg['family'] == 'CRYSTALS-Kyber': datasheetname = 'kyber'
+    elif alg['family'] == 'SIDH': datasheetname = 'sike'
+    elif alg['family'] == 'NTRU-Prime': datasheetname = 'ntruprime'
+    else: datasheetname = alg['family'].lower()
+    # load datasheet
+    try:
+        algymlfilename = os.path.join(docsdir, 'algorithms', 'kem', '{:s}.yml'.format(datasheetname))
+        algyml = yaml.safe_load(file_get_contents(algymlfilename, encoding='utf-8'))
+    except: # check alternate location in "oldalgs" folder
+        algymlfilename = os.path.join("oqs-template", 'oldalgdocs', 'kem', '{:s}.yml'.format(datasheetname))
+        algyml = yaml.safe_load(file_get_contents(algymlfilename, encoding='utf-8'))
+
+    # hacks to match names
+    def matches(name, alg):
+        def simplify(s):
+            return s.lower().replace('_', '').replace('-', '')
+        if 'FrodoKEM' in name: name = name.replace('FrodoKEM', 'Frodo')
+        if 'Saber-KEM' in name: name = name.replace('-KEM', '')
+        if '-90s' in name: name = name.replace('-90s', '').replace('Kyber', 'Kyber90s')
+        if simplify(name) == simplify(alg['name_group']): return True
+        return False
+    # find the variant that matches
+    for variant in algyml['parameter-sets']:
+        if matches(variant['name'], alg):
+            return variant['claimed-nist-level']
+    # Information file for algorithms no longer supported by liboqs:
+    oldalgs = yaml.safe_load(file_get_contents(os.path.join("oqs-template", "oldalgs.yml"), encoding='utf-8'))
+    name = alg['name_group']
+    if name in oldalgs:
+        return oldalgs[name]['claimed-nist-level']
+    return None
+
+def get_sig_nistlevel(family, alg, docsdir):
+    # translate family names in generate.yml to directory names for liboqs algorithm datasheets
+    if family['family'] == 'CRYSTALS-Dilithium': datasheetname = 'dilithium'
+    elif family['family'] == 'SPHINCS-Haraka': datasheetname = 'sphincs'
+    elif family['family'] == 'SPHINCS-SHA256': datasheetname = 'sphincs'
+    elif family['family'] == 'SPHINCS-SHAKE256': datasheetname = 'sphincs'
+    elif family['family'] == 'SPHINCS-SHA2': datasheetname = 'sphincs'
+    elif family['family'] == 'SPHINCS-SHAKE': datasheetname = 'sphincs'
+    else: datasheetname = family['family'].lower()
+    # load datasheet
+    algymlfilename = os.path.join(docsdir, 'algorithms', 'sig', '{:s}.yml'.format(datasheetname))
+    algyml = yaml.safe_load(file_get_contents(algymlfilename, encoding='utf-8'))
+    # hacks to match names
+    def matches(name, alg):
+        def simplify(s):
+            return s.lower().replace('_', '').replace('-', '').replace('+', '')
+        if simplify(name) == simplify(alg['name']): return True
+        return False
+    # find the variant that matches
+    for variant in algyml['parameter-sets']:
+        if matches(variant['name'], alg):
+            return variant['claimed-nist-level']
+    # Information file for algorithms no longer supported by liboqs:
+    oldalgs = yaml.safe_load(file_get_contents(os.path.join("oqs-template", "oldalgs.yml"), encoding='utf-8'))
+    name = alg['name']
+    if name in oldalgs:
+        return oldalgs[name]['claimed-nist-level']
+    return None
+
+def nist_to_bits(nistlevel):
+   if nistlevel==1 or nistlevel==2:
+      return 128
+   elif nistlevel==3 or nistlevel==4:
+      return 192
+   elif nistlevel==5:
+      return 256
+   else: 
+      return None
+
+def complete_config(config, oqsdocsdir = None):
+   if oqsdocsdir == None:
+      if 'LIBOQS_DOCS_DIR' not in os.environ:
+        print("Must include LIBOQS_DOCS_DIR in environment")
+        exit(1)
+      oqsdocsdir = os.environ["LIBOQS_DOCS_DIR"]
+   for kem in config['kems']:
+      if not "bit_security" in kem.keys():
+         bits_level = nist_to_bits(get_kem_nistlevel(kem, oqsdocsdir))
+         if bits_level == None: 
+             print("Cannot find security level for {:s} {:s}".format(kem['family'], kem['name_group']))
+             exit(1)
+         kem['bit_security'] = bits_level
+   for famsig in config['sigs']:
+      for sig in famsig['variants']:
+         if not "security" in sig.keys():
+            bits_level = nist_to_bits(get_sig_nistlevel(famsig, sig, oqsdocsdir))
+            if bits_level == None: 
+                if sig['name'].startswith("rainbowI"):
+                    bits_level=128
+                else:
+                    print("Cannot find security level for {:s} {:s}".format(famsig['family'], sig['name']))
+                    exit(1)
+            sig['security'] = bits_level
+   return config
+