From d36faa65c2bae7fcf323243326cf6144e7cf2925 Mon Sep 17 00:00:00 2001
From: wpbonelli <wbonelli@ucar.edu>
Date: Wed, 22 May 2024 15:20:04 -0400
Subject: [PATCH] test(release): fix distribution example script check (#1841)

Remove break statement likely leftover from development/debugging, which caused only the first example model to get a test run. This is why the examples script issue addressed by MODFLOW-USGS/modflow-devtools#156, where models were written to runall.[sh/bat] in an order which did not respect flow model dependencies, was not caught before.

After this PR, the runall.[sh/bat] script will be tested, as well as 3 of the separate example scripts picked at random.
---
 distribution/check_dist.py | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/distribution/check_dist.py b/distribution/check_dist.py
index 17c0198e4e2..bfd237e7e81 100644
--- a/distribution/check_dist.py
+++ b/distribution/check_dist.py
@@ -3,10 +3,12 @@
 from os import environ
 from pathlib import Path
 from pprint import pprint
+from random import shuffle
 
 import pytest
 
 from modflow_devtools.markers import no_parallel
+from modflow_devtools.misc import run_cmd
 
 
 # OS-specific extensions
@@ -175,20 +177,34 @@ def test_examples(dist_dir_path, full):
     if not full:
         pytest.skip(reason="examples not included in minimal distribution")
 
+    # check examples directory
     examples_path = dist_dir_path / "examples"
     assert examples_path.is_dir()
-    assert (examples_path / f"runall{_scext}").is_file()
+
+    # print examples found
     example_paths = [
         p for p in examples_path.glob("*") if p.is_dir() and p.stem.startswith("ex")
     ]
+    assert any(example_paths)
     print(f"{len(example_paths)} example models found:")
     pprint(example_paths)
-    for p in example_paths:
-        script_path = p / f"run{_scext}"
-        if not script_path.is_file():
-            continue
-        pprint(subprocess.check_output([str(script_path)], cwd=p).decode().split())
-        break
+
+    # pick some examples at random to test run individually
+    n = 3
+    shuffle(example_paths)
+    script_paths = [next(iter(p.rglob(f"*run{_scext}"))) for p in example_paths[:n]]
+    print(f"Testing {n} randomly selected example model scripts:")
+    pprint(script_paths)
+    for script_path in script_paths:
+        out, err, ret = run_cmd(str(script_path), cwd=script_path.parent)
+        assert not ret, out + err
+
+    # check comprehensive examples script and give it a test run
+    script_path = examples_path / f"runall{_scext}"
+    print(f"Testing comprehensive examples script: {script_path}")
+    assert script_path.is_file()
+    out, err, ret = run_cmd(str(script_path), cwd=script_path.parent)
+    assert not ret, out + err
 
 
 @no_parallel