Merge pull request #378 from atesgoral/atesgoral/success-checking

Success checking
atesgoral · Jul 5, 2024 · a0dae4f · a0dae4f
2 parents 6c87680 + ebba8fb
commit a0dae4f
Show file tree

Hide file tree

Showing 44 changed files with 16 additions and 8 deletions.
diff --git a/gulpfile.js b/gulpfile.js
@@ -113,9 +113,9 @@ function inspect() {
 
       return {
         path: parsedPath,
-        level: level,
-        source: source,
-        program: program,
+        level,
+        source,
+        program,
       };
     } catch (e) {
       if (parsedPath) {
@@ -134,15 +134,15 @@ function benchmark() {
 
       const runs = data.level.examples.slice(0);
 
-      inboxGenerator.seed(123);
+      inboxGenerator.seed(Date.now());
 
-      while (runs.length < 10) {
+      while (runs.length < 100) {
         const inbox = inboxGenerator.generate(data.level.number);
         const outbox = outboxGenerator.generate(data.level.number, inbox);
 
         runs.push({
-          inbox: inbox,
-          outbox: outbox,
+          inbox,
+          outbox,
         });
       }
 
@@ -190,6 +190,14 @@ function benchmark() {
         throw 'Program always failing';
       }
 
+      if (data.successRatio !== 1) {
+        if (runs[0].success && data.path.type !== 'specific') {
+          throw `Non-specific program failing on novel inputs (${Math.round(
+            100 * data.successRatio,
+          )}% pass)`;
+        }
+      }
+
       data.averageSteps = successfulRuns.length
         ? Math.round(
             successfulRuns.reduce((totalSteps, run) => {
@@ -198,7 +206,7 @@ function benchmark() {
           )
         : 0;
     } catch (e) {
-      console.error(chalk.red(data.path.full));
+      console.error(chalk.red(file.data.path.full));
       console.error(' ', chalk.red(e));
       throw e;
     }

diff --git a/...bler-Handler-7.21/14.19.unroll-ad-pro.asm → ...Handler-7.21/14.19.unroll-ad-pro.asm.skip b/...bler-Handler-7.21/14.19.unroll-ad-pro.asm → ...Handler-7.21/14.19.unroll-ad-pro.asm.skip
diff --git a/...bler-Handler-7.21/19.19.unroll-mrflip.asm → ...Handler-7.21/19.19.unroll-mrflip.asm.skip b/...bler-Handler-7.21/19.19.unroll-mrflip.asm → ...Handler-7.21/19.19.unroll-mrflip.asm.skip
diff --git a/...r-Handler-7.21/20.18.unroll-viamodulo.asm → ...dler-7.21/20.18.unroll-viamodulo.asm.skip b/...r-Handler-7.21/20.18.unroll-viamodulo.asm → ...dler-7.21/20.18.unroll-viamodulo.asm.skip
diff --git a/...Scrambler-Handler-7.21/7.21-atesgoral.asm → ...bler-Handler-7.21/7.21-atesgoral.asm.skip b/...Scrambler-Handler-7.21/7.21-atesgoral.asm → ...bler-Handler-7.21/7.21-atesgoral.asm.skip
diff --git a/...ee-Sort-34.78/158.68.unroll-viamodulo.asm → ...rt-34.78/158.68.unroll-viamodulo.asm.skip b/...ee-Sort-34.78/158.68.unroll-viamodulo.asm → ...rt-34.78/158.68.unroll-viamodulo.asm.skip
diff --git a/...ons/28-Three-Sort-34.78/28.115-eiTTio.asm → ...8-Three-Sort-34.78/28.115-eiTTio.asm.skip b/...ons/28-Three-Sort-34.78/28.115-eiTTio.asm → ...8-Three-Sort-34.78/28.115-eiTTio.asm.skip
diff --git a/...ee-Sort-34.78/28.117.obsolete-esseger.asm → ...rt-34.78/28.117.obsolete-esseger.asm.skip b/...ee-Sort-34.78/28.117.obsolete-esseger.asm → ...rt-34.78/28.117.obsolete-esseger.asm.skip
diff --git a/...ree-Sort-34.78/29.117.obsolete-Eirik0.asm → ...ort-34.78/29.117.obsolete-Eirik0.asm.skip b/...ree-Sort-34.78/29.117.obsolete-Eirik0.asm → ...ort-34.78/29.117.obsolete-Eirik0.asm.skip
diff --git a/...hree-Sort-34.78/31.123.obsolete-Mygod.asm → ...Sort-34.78/31.123.obsolete-Mygod.asm.skip b/...hree-Sort-34.78/31.123.obsolete-Mygod.asm → ...Sort-34.78/31.123.obsolete-Mygod.asm.skip
diff --git a/...rt-34.78/32.128.obsolete-albertferras.asm → ....78/32.128.obsolete-albertferras.asm.skip b/...rt-34.78/32.128.obsolete-albertferras.asm → ....78/32.128.obsolete-albertferras.asm.skip
diff --git a/...ns/28-Three-Sort-34.78/33.85-steambap.asm → ...-Three-Sort-34.78/33.85-steambap.asm.skip b/...ns/28-Three-Sort-34.78/33.85-steambap.asm → ...-Three-Sort-34.78/33.85-steambap.asm.skip
diff --git a/...ort-34.78/34.139.obsolete-clarfonthey.asm → ...4.78/34.139.obsolete-clarfonthey.asm.skip b/...ort-34.78/34.139.obsolete-clarfonthey.asm → ...4.78/34.139.obsolete-clarfonthey.asm.skip
diff --git a/...ns/28-Three-Sort-34.78/47.75-jdashton.asm → ...-Three-Sort-34.78/47.75-jdashton.asm.skip b/...ns/28-Three-Sort-34.78/47.75-jdashton.asm → ...-Three-Sort-34.78/47.75-jdashton.asm.skip
diff --git a/...ions/28-Three-Sort-34.78/49.77-eiTTio.asm → ...28-Three-Sort-34.78/49.77-eiTTio.asm.skip b/...ions/28-Three-Sort-34.78/49.77-eiTTio.asm → ...28-Three-Sort-34.78/49.77-eiTTio.asm.skip
diff --git a/solutions/28-Three-Sort-34.78/62.72-popq.asm → ...s/28-Three-Sort-34.78/62.72-popq.asm.skip b/solutions/28-Three-Sort-34.78/62.72-popq.asm → ...s/28-Three-Sort-34.78/62.72-popq.asm.skip
diff --git a/...-Sort-34.78/62.73.obsolete-AlanDeSmet.asm → ...-34.78/62.73.obsolete-AlanDeSmet.asm.skip b/...-Sort-34.78/62.73.obsolete-AlanDeSmet.asm → ...-34.78/62.73.obsolete-AlanDeSmet.asm.skip
diff --git a/...Sort-34.78/62.75.obsolete-clarfonthey.asm → ...34.78/62.75.obsolete-clarfonthey.asm.skip b/...Sort-34.78/62.75.obsolete-clarfonthey.asm → ...34.78/62.75.obsolete-clarfonthey.asm.skip
diff --git a/...ring-Reverse-11.122/14.103-FireGoblin.asm → ...Reverse-11.122/14.103-FireGoblin.asm.skip b/...ring-Reverse-11.122/14.103-FireGoblin.asm → ...Reverse-11.122/14.103-FireGoblin.asm.skip
diff --git a/...al-17.167/27.96.hashtable-ironwallaby.asm → ....167/27.96.hashtable-ironwallaby.asm.skip b/...al-17.167/27.96.hashtable-ironwallaby.asm → ....167/27.96.hashtable-ironwallaby.asm.skip
diff --git a/...-Duplicate-Removal-17.167/42.63-ocoss.asm → ...icate-Removal-17.167/42.63-ocoss.asm.skip b/...-Duplicate-Removal-17.167/42.63-ocoss.asm → ...icate-Removal-17.167/42.63-ocoss.asm.skip
diff --git a/...licate-Removal-17.167/48.70-WolfWings.asm → ...e-Removal-17.167/48.70-WolfWings.asm.skip b/...licate-Removal-17.167/48.70-WolfWings.asm → ...e-Removal-17.167/48.70-WolfWings.asm.skip
diff --git a/...6-Alphabetizer-39.109/25.88-halchihal.asm → ...habetizer-39.109/25.88-halchihal.asm.skip b/...6-Alphabetizer-39.109/25.88-halchihal.asm → ...habetizer-39.109/25.88-halchihal.asm.skip
diff --git a/...nator-14.76/142.40.obsolete-viamodulo.asm → ...-14.76/142.40.obsolete-viamodulo.asm.skip b/...nator-14.76/142.40.obsolete-viamodulo.asm → ...-14.76/142.40.obsolete-viamodulo.asm.skip
diff --git a/...-Prime-Factory-28.399/28.247-jdashton.asm → ...e-Factory-28.399/28.247-jdashton.asm.skip b/...-Prime-Factory-28.399/28.247-jdashton.asm → ...e-Factory-28.399/28.247-jdashton.asm.skip
diff --git a/...Floor-34.714/188.447.insertion-mrflip.asm → ...-34.714/188.447.insertion-mrflip.asm.skip b/...Floor-34.714/188.447.insertion-mrflip.asm → ...-34.714/188.447.insertion-mrflip.asm.skip
diff --git a/...Sorting-Floor-34.714/19.734-marcinsmu.asm → ...ng-Floor-34.714/19.734-marcinsmu.asm.skip b/...Sorting-Floor-34.714/19.734-marcinsmu.asm → ...ng-Floor-34.714/19.734-marcinsmu.asm.skip
diff --git a/...or-34.714/196.446.insertion-viamodulo.asm → ....714/196.446.insertion-viamodulo.asm.skip b/...or-34.714/196.446.insertion-viamodulo.asm → ....714/196.446.insertion-viamodulo.asm.skip
diff --git a/...-Floor-34.714/20.648.selection-mrflip.asm → ...r-34.714/20.648.selection-mrflip.asm.skip b/...-Floor-34.714/20.648.selection-mrflip.asm → ...r-34.714/20.648.selection-mrflip.asm.skip
diff --git a/....714/20.651.selection-sniperrifle2004.asm → ...20.651.selection-sniperrifle2004.asm.skip b/....714/20.651.selection-sniperrifle2004.asm → ...20.651.selection-sniperrifle2004.asm.skip
diff --git a/...rting-Floor-34.714/20.689-polarathene.asm → ...-Floor-34.714/20.689-polarathene.asm.skip b/...rting-Floor-34.714/20.689-polarathene.asm → ...-Floor-34.714/20.689-polarathene.asm.skip
diff --git a/...g-Floor-34.714/208.446.exploit-mrflip.asm → ...or-34.714/208.446.exploit-mrflip.asm.skip b/...g-Floor-34.714/208.446.exploit-mrflip.asm → ...or-34.714/208.446.exploit-mrflip.asm.skip
diff --git a/....714/27.639.insertion-sniperrifle2004.asm → ...27.639.insertion-sniperrifle2004.asm.skip b/....714/27.639.insertion-sniperrifle2004.asm → ...27.639.insertion-sniperrifle2004.asm.skip
diff --git a/...-Floor-34.714/28.537.insertion-mrflip.asm → ...r-34.714/28.537.insertion-mrflip.asm.skip b/...-Floor-34.714/28.537.insertion-mrflip.asm → ...r-34.714/28.537.insertion-mrflip.asm.skip
diff --git a/...r-34.714/28.705.selection-clarfonthey.asm → ...714/28.705.selection-clarfonthey.asm.skip b/...r-34.714/28.705.selection-clarfonthey.asm → ...714/28.705.selection-clarfonthey.asm.skip
diff --git a/...Floor-34.714/30.1061.gnomesort-eiTTio.asm → ...-34.714/30.1061.gnomesort-eiTTio.asm.skip b/...Floor-34.714/30.1061.gnomesort-eiTTio.asm → ...-34.714/30.1061.gnomesort-eiTTio.asm.skip
diff --git a/...oor-34.714/30.536.insertion-viamodulo.asm → ...4.714/30.536.insertion-viamodulo.asm.skip b/...oor-34.714/30.536.insertion-viamodulo.asm → ...4.714/30.536.insertion-viamodulo.asm.skip
diff --git a/...41-Sorting-Floor-34.714/31.546-eiTTio.asm → ...rting-Floor-34.714/31.546-eiTTio.asm.skip b/...41-Sorting-Floor-34.714/31.546-eiTTio.asm → ...rting-Floor-34.714/31.546-eiTTio.asm.skip
diff --git a/...r-34.714/31.938.insertion-clarfonthey.asm → ...714/31.938.insertion-clarfonthey.asm.skip b/...r-34.714/31.938.insertion-clarfonthey.asm → ...714/31.938.insertion-clarfonthey.asm.skip
diff --git a/...r-34.714/32.1404.bubblesort-hastebrot.asm → ...714/32.1404.bubblesort-hastebrot.asm.skip b/...r-34.714/32.1404.bubblesort-hastebrot.asm → ...714/32.1404.bubblesort-hastebrot.asm.skip
diff --git a/...rting-Floor-34.714/33.570-polarathene.asm → ...-Floor-34.714/33.570-polarathene.asm.skip b/...rting-Floor-34.714/33.570-polarathene.asm → ...-Floor-34.714/33.570-polarathene.asm.skip
diff --git a/...loor-34.714/43.621.insertion-dubaaron.asm → ...34.714/43.621.insertion-dubaaron.asm.skip b/...loor-34.714/43.621.insertion-dubaaron.asm → ...34.714/43.621.insertion-dubaaron.asm.skip
diff --git a/...loor-34.714/69.1534.mergesort-IAmWave.asm → ...34.714/69.1534.mergesort-IAmWave.asm.skip b/...loor-34.714/69.1534.mergesort-IAmWave.asm → ...34.714/69.1534.mergesort-IAmWave.asm.skip
diff --git a/...-Floor-34.714/79.488.insertion-mrflip.asm → ...r-34.714/79.488.insertion-mrflip.asm.skip b/...-Floor-34.714/79.488.insertion-mrflip.asm → ...r-34.714/79.488.insertion-mrflip.asm.skip