Property Testing: test generator coverage

[doug-q]

We do not have confidence in the behaviour of the Arbitrary strategies that we use. proptest has no functionality analagous to QuickChecks Analysing test case distribution which is used to classify examples and verify that all categories are adequately covered. I do think this is possible to achieve in rust(see below), we could publish this independently of hugr or perhaps upstream to proptest.

Prototype proptest coverage

This borrow-checks but is otherwise untested:

#[derive(Debug,Clone)]
struct CoverageStrategy<Inner: Strategy>(
    Inner,
    std::sync::mpsc::Sender<Option<Inner::Value>>
);

impl<Inner: Strategy> CoverageStrategy<Inner> {
    fn report(&self, v: Inner::Value) -> Result<(), Reason>{
        self.1.send(Some(v)).map_err(|e| e.to_string().into())
    }
}

impl<Inner: Strategy> Strategy for CoverageStrategy<Inner> {
    type Tree = Inner::Tree;
    type Value = Inner::Value;

    fn new_tree(&self, runner: &mut TestRunner) -> NewTree<Self> {
        let tree = self.0.new_tree(runner)?;
        self.report(ValueTree::current(&tree))?;
        Ok(tree)
    }
}

pub fn coverage<V: std::fmt::Debug + Sync + Send + 'static>(expected: HashMap<String,f64>, classify: impl Fn(&V) -> String + Sync + Send, subject: impl Strategy<Value=V>) {
    let (tx_v, rx_v) = std::sync::mpsc::channel();

    let results = std::thread::scope(|scope| {
        let thread = scope.spawn(move || {
            let mut results: HashMap<String,usize> = Default::default();
            while let Some(v) = rx_v.recv().unwrap() {
                let label = classify(&v);
                if let Some(old) = results.insert(label.clone(), 1) {
                    results.insert(label, old+1);
                }
            }
            results
        });
        let mut tr = TestRunner::new(Default::default());
        tr.run(&CoverageStrategy(subject, tx_v), |_| Ok(())).unwrap();
        thread.join().unwrap();
    });

    println!("coverage: expected:\n {expected:?}\nresults:\n{results:?}")
}