From 4682948c1e3a9ea8a4222c8015cc616c3214a725 Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Wed, 18 Jan 2023 13:59:30 +0100 Subject: [PATCH 1/5] test: driver-only: fix disparities in ECP Signed-off-by: Valerio Setti --- tests/scripts/analyze_outcomes.py | 1 - tests/suites/test_suite_ecp.function | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index eeded5f62801..11607e1ed2c7 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -166,7 +166,6 @@ def do_analyze_driver_vs_reference(outcome_file, args): # they will be removed by upcoming work 'psa_crypto_se_driver_hal', # #6856 'random', # #6856 - 'ecp', # #6856 'pk', # #6857 'x509parse', # #6858 'x509write', # #6858 diff --git a/tests/suites/test_suite_ecp.function b/tests/suites/test_suite_ecp.function index 394253de53f2..c8a0a8284474 100644 --- a/tests/suites/test_suite_ecp.function +++ b/tests/suites/test_suite_ecp.function @@ -811,7 +811,7 @@ exit: } /* END_CASE */ -/* BEGIN_CASE depends_on:MBEDTLS_ECDH_C:MBEDTLS_ECDSA_C */ +/* BEGIN_CASE */ void mbedtls_ecp_group_metadata(int id, int bit_size, int crv_type, char *P, char *A, char *B, char *G_x, char *G_y, char *N, @@ -903,9 +903,13 @@ void mbedtls_ecp_group_metadata(int id, int bit_size, int crv_type, // Check curve type, and if it can be used for ECDH/ECDSA TEST_EQUAL(mbedtls_ecp_get_type(&grp), crv_type); +#if defined(MBEDTLS_ECDH_C) TEST_EQUAL(mbedtls_ecdh_can_do(id), 1); +#endif +#if defined(MBEDTLS_ECDSA_C) TEST_EQUAL(mbedtls_ecdsa_can_do(id), crv_type == MBEDTLS_ECP_TYPE_SHORT_WEIERSTRASS); +#endif // Copy group and compare with original TEST_EQUAL(mbedtls_ecp_group_copy(&grp_cpy, &grp), 0); From 7e57920511e9e64b1f599487a41262324292cd66 Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Wed, 18 Jan 2023 14:13:59 +0100 Subject: [PATCH 2/5] test: driver-only: fix disparities in psa_crypto_se_driver_hal Signed-off-by: Valerio Setti --- tests/scripts/analyze_outcomes.py | 1 - .../suites/test_suite_psa_crypto_se_driver_hal.data | 12 ++++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 11607e1ed2c7..746521703fd5 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -164,7 +164,6 @@ def do_analyze_driver_vs_reference(outcome_file, args): 'ecdsa', # the software implementation that's excluded # the following lines should not be needed, # they will be removed by upcoming work - 'psa_crypto_se_driver_hal', # #6856 'random', # #6856 'pk', # #6857 'x509parse', # #6858 diff --git a/tests/suites/test_suite_psa_crypto_se_driver_hal.data b/tests/suites/test_suite_psa_crypto_se_driver_hal.data index 2bcf4e4b7bc1..22b0570bf499 100644 --- a/tests/suites/test_suite_psa_crypto_se_driver_hal.data +++ b/tests/suites/test_suite_psa_crypto_se_driver_hal.data @@ -178,25 +178,25 @@ Key registration: key id max volatile register_key_smoke_test:TEST_SE_PERSISTENT_LIFETIME:7:PSA_KEY_ID_VOLATILE_MAX:1:PSA_ERROR_INVALID_ARGUMENT Import-sign-verify: sign in driver, ECDSA -depends_on:MBEDTLS_ECDSA_C:MBEDTLS_ECP_C:PSA_WANT_ECC_SECP_R1_256 +depends_on:PSA_WANT_ALG_ECDSA:PSA_WANT_ECC_SECP_R1_256 sign_verify:SIGN_IN_DRIVER_AND_PARALLEL_CREATION:PSA_KEY_TYPE_ECC_KEY_PAIR( PSA_ECC_FAMILY_SECP_R1 ):PSA_ALG_ECDSA_ANY:0:"49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee":"54686973206973206e6f74206120686173682e" Import-sign-verify: sign in driver then export_public, ECDSA -depends_on:MBEDTLS_ECDSA_C:MBEDTLS_ECP_C:PSA_WANT_ECC_SECP_R1_256 +depends_on:PSA_WANT_ALG_ECDSA:PSA_WANT_ECC_SECP_R1_256 sign_verify:SIGN_IN_DRIVER_THEN_EXPORT_PUBLIC:PSA_KEY_TYPE_ECC_KEY_PAIR( PSA_ECC_FAMILY_SECP_R1 ):PSA_ALG_ECDSA_ANY:0:"49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee":"54686973206973206e6f74206120686173682e" Import-sign-verify: sign in software, ECDSA -depends_on:MBEDTLS_ECDSA_C:MBEDTLS_ECP_C:PSA_WANT_ECC_SECP_R1_256 +depends_on:PSA_WANT_ALG_ECDSA:PSA_WANT_ECC_SECP_R1_256 sign_verify:SIGN_IN_SOFTWARE_AND_PARALLEL_CREATION:PSA_KEY_TYPE_ECC_KEY_PAIR( PSA_ECC_FAMILY_SECP_R1 ):PSA_ALG_ECDSA_ANY:0:"49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee":"54686973206973206e6f74206120686173682e" Generate-sign-verify: sign in driver, ECDSA -depends_on:MBEDTLS_ECDSA_C:MBEDTLS_ECP_C:PSA_WANT_ECC_SECP_R1_256 +depends_on:PSA_WANT_ALG_ECDSA:PSA_WANT_ECC_SECP_R1_256 sign_verify:SIGN_IN_DRIVER_AND_PARALLEL_CREATION:PSA_KEY_TYPE_ECC_KEY_PAIR( PSA_ECC_FAMILY_SECP_R1 ):PSA_ALG_ECDSA_ANY:256:"49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee":"54686973206973206e6f74206120686173682e" Generate-sign-verify: sign in driver then export_public, ECDSA -depends_on:MBEDTLS_ECDSA_C:MBEDTLS_ECP_C:PSA_WANT_ECC_SECP_R1_256 +depends_on:PSA_WANT_ALG_ECDSA:PSA_WANT_ECC_SECP_R1_256 sign_verify:SIGN_IN_DRIVER_THEN_EXPORT_PUBLIC:PSA_KEY_TYPE_ECC_KEY_PAIR( PSA_ECC_FAMILY_SECP_R1 ):PSA_ALG_ECDSA_ANY:256:"49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee":"54686973206973206e6f74206120686173682e" Generate-sign-verify: sign in software, ECDSA -depends_on:MBEDTLS_ECDSA_C:MBEDTLS_ECP_C:PSA_WANT_ECC_SECP_R1_256 +depends_on:PSA_WANT_ALG_ECDSA:PSA_WANT_ECC_SECP_R1_256 sign_verify:SIGN_IN_SOFTWARE_AND_PARALLEL_CREATION:PSA_KEY_TYPE_ECC_KEY_PAIR( PSA_ECC_FAMILY_SECP_R1 ):PSA_ALG_ECDSA_ANY:256:"49c9a8c18c4b885638c431cf1df1c994131609b580d4fd43a0cab17db2f13eee":"54686973206973206e6f74206120686173682e" From 3002c992470481cc6505e215b5e6e45d44bc28cc Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Wed, 18 Jan 2023 17:28:36 +0100 Subject: [PATCH 3/5] test: extend analyze_outcomes.py in order to skip only some test in a suite Signed-off-by: Valerio Setti --- tests/scripts/analyze_outcomes.py | 35 +++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 746521703fd5..6101a1c28478 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -61,24 +61,32 @@ def analyze_coverage(results, outcomes): # fixed this branch to have full coverage of test cases. results.warning('Test case not executed: {}', key) -def analyze_driver_vs_reference(outcomes, component_ref, component_driver, ignored_tests): +def analyze_driver_vs_reference(outcomes, component_ref, component_driver, + ignored_suites, ignored_test=None): """Check that all tests executed in the reference component are also executed in the corresponding driver component. - Skip test suites provided in ignored_tests list. + Skip: + - full test suites provided in ignored_suites list + - only some specific test inside a test suite, for which the corresponding + output string is provided """ available = check_test_cases.collect_available_test_cases() result = True for key in available: # Skip ignored test suites - test_suite = key.split(';')[0] # retrieve test suit name - test_suite = test_suite.split('.')[0] # retrieve main part of test suit name - if test_suite in ignored_tests: + full_test_suite = key.split(';')[0] # retrieve full test suit name + test_string = key.split(';')[1] # retrieve the text string of this test + test_suite = full_test_suite.split('.')[0] # retrieve main part of test suit name + if test_suite in ignored_suites: continue # Continue if test was not executed by any component hits = outcomes[key].hits() if key in outcomes else 0 if hits == 0: continue + if ((full_test_suite in ignored_test) and + (test_string in ignored_test[full_test_suite])): + continue # Search for tests that run in reference component and not in driver component driver_test_passed = False reference_test_passed = False @@ -129,13 +137,14 @@ def do_analyze_coverage(outcome_file, args): def do_analyze_driver_vs_reference(outcome_file, args): """Perform driver vs reference analyze.""" - ignored_tests = ['test_suite_' + x for x in args['ignored_suites']] + ignored_suites = ['test_suite_' + x for x in args['ignored_suites']] outcomes = read_outcome_file(outcome_file) print("\n*** Analyze driver {} vs reference {} ***\n".format( args['component_driver'], args['component_ref'])) return analyze_driver_vs_reference(outcomes, args['component_ref'], - args['component_driver'], ignored_tests) + args['component_driver'], ignored_suites, + args['ignored_tests']) # List of tasks with a function that can handle this task and additional arguments if required TASKS = { @@ -154,7 +163,11 @@ def do_analyze_driver_vs_reference(outcome_file, args): 'ignored_suites': [ 'shax', 'mdx', # the software implementations that are being excluded 'md', # the legacy abstraction layer that's being excluded - ]}}, + ], + 'ignored_tests': { + } + } + }, 'analyze_driver_vs_reference_ecdsa': { 'test_function': do_analyze_driver_vs_reference, 'args': { @@ -170,7 +183,11 @@ def do_analyze_driver_vs_reference(outcome_file, args): 'x509write', # #6858 'debug', # #6860 'ssl', # #6860 - ]}}, + ], + 'ignored_tests': { + } + } + }, } def main(): From 9cb0f7a4237d538dcccb6d7fcf2b3a7048b3d4d5 Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Wed, 18 Jan 2023 17:29:29 +0100 Subject: [PATCH 4/5] test: driver-only: fix disparities in random Signed-off-by: Valerio Setti --- tests/scripts/analyze_outcomes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 6101a1c28478..4938b3b87a42 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -177,7 +177,6 @@ def do_analyze_driver_vs_reference(outcome_file, args): 'ecdsa', # the software implementation that's excluded # the following lines should not be needed, # they will be removed by upcoming work - 'random', # #6856 'pk', # #6857 'x509parse', # #6858 'x509write', # #6858 @@ -185,6 +184,9 @@ def do_analyze_driver_vs_reference(outcome_file, args): 'ssl', # #6860 ], 'ignored_tests': { + 'test_suite_random': [ + 'PSA classic wrapper: ECDSA signature (SECP256R1)', + ], } } }, From 00c1ccb08c9421fd2ddb3c9b99b557180e12a699 Mon Sep 17 00:00:00 2001 From: Valerio Setti Date: Thu, 2 Feb 2023 11:33:31 +0100 Subject: [PATCH 5/5] depends.py: fix typo and slightly reorganized code Signed-off-by: Valerio Setti --- tests/scripts/analyze_outcomes.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py index 4938b3b87a42..2cdcbf1f3e5d 100755 --- a/tests/scripts/analyze_outcomes.py +++ b/tests/scripts/analyze_outcomes.py @@ -74,16 +74,16 @@ def analyze_driver_vs_reference(outcomes, component_ref, component_driver, result = True for key in available: - # Skip ignored test suites - full_test_suite = key.split(';')[0] # retrieve full test suit name - test_string = key.split(';')[1] # retrieve the text string of this test - test_suite = full_test_suite.split('.')[0] # retrieve main part of test suit name - if test_suite in ignored_suites: - continue # Continue if test was not executed by any component hits = outcomes[key].hits() if key in outcomes else 0 if hits == 0: continue + # Skip ignored test suites + full_test_suite = key.split(';')[0] # retrieve full test suite name + test_string = key.split(';')[1] # retrieve the text string of this test + test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name + if test_suite in ignored_suites: + continue if ((full_test_suite in ignored_test) and (test_string in ignored_test[full_test_suite])): continue