Improve and Document Test Structure (#507)

* refactor: clean up setting minimum log level in`main.spec.ts` * refactor: starting to clean up `util` tests * refactor: restructure `util` folder * refactor: enforce non-recursive nature of `requireAllTestsInFolder` * refactor: statistics tests now use the new `-tests` suffix scheme * refactor: `slicing` tests folder * refactor: `flowr` test folder * refactor: `dataflow` folder + sanity check for requiring all testfiles * refactor, test-fix: deal with new sanity check findings * refactor: `benchmark-slicer` test folder * lint-fix: deal with unused import lintr issues * doc: document tests and how their basic structure works
flowr-analysis · Nov 22, 2023 · 596b70f · 596b70f · github-actions · Nov 22, 2023
1 parent aefbf04
commit 596b70f
Show file tree

Hide file tree

Showing 70 changed files with 457 additions and 259 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/src/util/log.ts b/src/util/log.ts
@@ -3,7 +3,6 @@ import { createStream, type Options } from 'rotating-file-stream'
 
 export class FlowrLogger extends Logger<ILogObj> {
 	/** by keeping track of all children we can propagate updates of the settings (e.g., in tests) */
-
 	private readonly childLoggers: Logger<ILogObj>[] = []
 
 	public getSubLogger(

diff --git a/test/functionality/helper/ast-builder.ts → test/functionality/_helper/ast-builder.ts b/test/functionality/helper/ast-builder.ts → test/functionality/_helper/ast-builder.ts
diff --git a/test/functionality/_helper/collect-tests.ts b/test/functionality/_helper/collect-tests.ts
@@ -0,0 +1,18 @@
+import fs from 'node:fs'
+import path from 'path'
+
+/**
+ * Require all files in a folder that end with a suffix, will never recurse into subfolders
+ * @param folder - the folder to require all files from
+ * @param suffix - the suffix which the files of interest must have
+ */
+export function requireAllTestsInFolder(folder: string, suffix = '-tests.ts'): void {
+	for(const fileBuff of fs.readdirSync(folder, { recursive: false })) {
+		const file = fileBuff.toString()
+		if(file.endsWith(suffix)) {
+			require(path.join(folder,file))
+		} else if(!file.endsWith('.spec.ts')) {
+			throw new Error(`Unexpected file ${file} in ${folder}, neither matches the import suffix '${suffix}', nor the test suffix '.spec.ts'. This is a sanity check so no files are lost. Please restructure your folders if this is intended.`)
+		}
+	}
+}
diff --git a/test/functionality/_helper/environment-builder.ts b/test/functionality/_helper/environment-builder.ts
@@ -0,0 +1,7 @@
+import { NodeId } from '../../../src/r-bridge'
+import { IdentifierDefinition } from '../../../src/dataflow'
+import { LocalScope } from '../../../src/dataflow/environments/scopes'
+
+export function variable(name: string, definedAt: NodeId): IdentifierDefinition {
+	return { name, kind: 'variable', scope: LocalScope, used: 'always', nodeId: '_0', definedAt }
+}
diff --git a/test/functionality/helper/net.ts → test/functionality/_helper/net.ts b/test/functionality/helper/net.ts → test/functionality/_helper/net.ts
diff --git a/test/functionality/helper/network.ts → test/functionality/_helper/network.ts b/test/functionality/helper/network.ts → test/functionality/_helper/network.ts
diff --git a/test/functionality/helper/provider.ts → test/functionality/_helper/provider.ts b/test/functionality/helper/provider.ts → test/functionality/_helper/provider.ts
diff --git a/test/functionality/helper/shell.ts → test/functionality/_helper/shell.ts b/test/functionality/helper/shell.ts → test/functionality/_helper/shell.ts
diff --git a/test/functionality/helper/version.ts → test/functionality/_helper/version.ts b/test/functionality/helper/version.ts → test/functionality/_helper/version.ts
diff --git a/test/functionality/benchmark/slicer.spec.ts → ...tionality/benchmark-slicer/slicer.spec.ts b/test/functionality/benchmark/slicer.spec.ts → ...tionality/benchmark-slicer/slicer.spec.ts
@@ -18,7 +18,7 @@ async function retrieveStatsSafe(slicer: BenchmarkSlicer, request: { request: st
 	return { stats, statInfo }
 }
 
-describe('The Benchmark Slicer', () => {
+describe('Benchmark Slicer', () => {
 	describe('Stats by parsing text-based inputs', function() {
 		this.timeout('15min')
 		it('Simple slice for simple line', async() => {

diff --git a/test/functionality/dataflow/dataflow.spec.ts b/test/functionality/dataflow/dataflow.spec.ts
@@ -1,16 +1,15 @@
+import { requireAllTestsInFolder } from '../_helper/collect-tests'
+import path from 'path'
+
 describe('Dataflow', () => {
-	require('./environments/environments')
+	describe('Environments', () =>
+		requireAllTestsInFolder(path.join(__dirname, 'environments'))
+	)
+
+	describe('Graph', () =>
+		requireAllTestsInFolder(path.join(__dirname, 'graph'))
+	)
 
-	describe('Graph', () => {
-		require('./graph/equal')
-	})
 
-	describe('Extraction', () => {
-		require('./elements/atomic')
-		require('./elements/expression-lists')
-		describe('Functions', () => {
-			require('./elements/functions/function-definition')
-			require('./elements/functions/function-call')
-		})
-	})
+	require('./processing-of-elements/processing-of-elements')
 })
diff --git a/test/functionality/dataflow/elements/expression-lists.ts b/test/functionality/dataflow/elements/expression-lists.ts
diff --git a/test/functionality/dataflow/elements/loops/loops.ts b/test/functionality/dataflow/elements/loops/loops.ts
diff --git a/test/functionality/dataflow/environments/environments.ts b/test/functionality/dataflow/environments/environments.ts
diff --git a/...y/dataflow/environments/initialization.ts → ...flow/environments/initialization-tests.ts b/...y/dataflow/environments/initialization.ts → ...flow/environments/initialization-tests.ts
diff --git a/...ity/dataflow/environments/modification.ts → ...taflow/environments/modification-tests.ts b/...ity/dataflow/environments/modification.ts → ...taflow/environments/modification-tests.ts
@@ -3,8 +3,8 @@ import { DefaultEnvironmentMemory, IEnvironment, initializeCleanEnvironments } f
 import { guard } from '../../../../src/util/assert'
 import { expect } from 'chai'
 import { appendEnvironments, define, overwriteEnvironments } from '../../../../src/dataflow/environments'
-import { variable } from './environments'
 import { GlobalScope, LocalScope } from '../../../../src/dataflow/environments/scopes'
+import { variable } from '../../_helper/environment-builder'
 
 /** if you pass multiple `definedAt`, this will expect the node to have multiple definitions */
 function existsDefinedAt(name: string, definedAt: NodeId[], result: IEnvironment | undefined, message?: string) {
@@ -17,7 +17,7 @@ function existsDefinedAt(name: string, definedAt: NodeId[], result: IEnvironment
 	expect(got.map(d => d.definedAt), `${name} should be defined at ${JSON.stringify(definedAt)}. ${message ?? ''}`).to.deep.equal(definedAt)
 }
 
-describe('Overwrite', () => {
+describe('Modification', () => {
 	describe('Global', () => {
 		it('Different variables', () => {
 			let clean = initializeCleanEnvironments()

diff --git a/...ionality/dataflow/environments/resolve.ts → ...ty/dataflow/environments/resolve-tests.ts b/...ionality/dataflow/environments/resolve.ts → ...ty/dataflow/environments/resolve-tests.ts
@@ -1,8 +1,8 @@
 import { define, initializeCleanEnvironments, resolveByName } from '../../../../src/dataflow/environments'
-import { variable } from './environments'
 import { expect } from 'chai'
 import { guard } from '../../../../src/util/assert'
 import { GlobalScope, LocalScope } from '../../../../src/dataflow/environments/scopes'
+import { variable } from '../../_helper/environment-builder'
 
 describe('Resolve', () => {
 	describe('ByName', () => {

diff --git a/test/functionality/dataflow/graph/equal.ts → ...nctionality/dataflow/graph/equal-tests.ts b/test/functionality/dataflow/graph/equal.ts → ...nctionality/dataflow/graph/equal-tests.ts
@@ -12,7 +12,7 @@ function test(cmp: (x: boolean) => void, a: DataflowGraph, b: DataflowGraph, tex
 	}
 }
 
-describe('Graph Equality', () => {
+describe('Equal', () => {
 	const raw = (name: string, a: DataflowGraph, b: DataflowGraph, text: string, cmp: (x: boolean) => void) => {
 		return it(name, () => {
 			// as the comparison is relatively quick, we allow explicit checks for commutativity

diff --git a/...functionality/dataflow/elements/atomic.ts → ...essing-of-elements/atomic/atomic-tests.ts b/...functionality/dataflow/elements/atomic.ts → ...essing-of-elements/atomic/atomic-tests.ts
@@ -3,15 +3,15 @@
  * Yet, some constructs (like for-loops) require the combination of statements, they are included as well.
  * This will not include functions!
  */
-import { assertDataflow, withShell } from '../../helper/shell'
-import { DataflowGraph, EdgeType, initializeCleanEnvironments } from '../../../../src/dataflow'
-import { RAssignmentOpPool, RNonAssignmentBinaryOpPool, RUnaryOpPool } from '../../helper/provider'
-import { appendEnvironments, define } from '../../../../src/dataflow/environments'
-import { UnnamedArgumentPrefix } from '../../../../src/dataflow/internal/process/functions/argument'
-import { GlobalScope, LocalScope } from '../../../../src/dataflow/environments/scopes'
-import { MIN_VERSION_PIPE } from '../../../../src/r-bridge/lang-4.x/ast/model/versions'
+import { assertDataflow, withShell } from '../../../_helper/shell'
+import { DataflowGraph, EdgeType, initializeCleanEnvironments } from '../../../../../src/dataflow'
+import { RAssignmentOpPool, RNonAssignmentBinaryOpPool, RUnaryOpPool } from '../../../_helper/provider'
+import { appendEnvironments, define } from '../../../../../src/dataflow/environments'
+import { UnnamedArgumentPrefix } from '../../../../../src/dataflow/internal/process/functions/argument'
+import { GlobalScope, LocalScope } from '../../../../../src/dataflow/environments/scopes'
+import { MIN_VERSION_PIPE } from '../../../../../src/r-bridge/lang-4.x/ast/model/versions'
 
-describe('Atomic dataflow information', withShell((shell) => {
+describe('Atomic (dataflow information)', withShell((shell) => {
 	describe('uninteresting leafs', () => {
 		for(const input of ['42', '"test"', 'TRUE', 'NA', 'NULL']) {
 			assertDataflow(input, shell, input, new DataflowGraph())

diff --git a/...nality/dataflow/elements/lists/if-then.ts → ...lements/expression-lists/if-then-tests.ts b/...nality/dataflow/elements/lists/if-then.ts → ...lements/expression-lists/if-then-tests.ts
@@ -2,7 +2,7 @@ import {
 	DataflowGraph, EdgeType,
 	initializeCleanEnvironments
 } from '../../../../../src/dataflow'
-import { assertDataflow, withShell } from '../../../helper/shell'
+import { assertDataflow, withShell } from '../../../_helper/shell'
 import { appendEnvironments, define } from '../../../../../src/dataflow/environments'
 import { GlobalScope, LocalScope } from '../../../../../src/dataflow/environments/scopes'
 

diff --git a/...tionality/dataflow/elements/lists/read.ts → ...f-elements/expression-lists/read-tests.ts b/...tionality/dataflow/elements/lists/read.ts → ...f-elements/expression-lists/read-tests.ts
@@ -1,6 +1,6 @@
 import { NodeId } from '../../../../../src/r-bridge'
 import { DataflowGraph, EdgeType, initializeCleanEnvironments } from '../../../../../src/dataflow'
-import { assertDataflow, withShell } from '../../../helper/shell'
+import { assertDataflow, withShell } from '../../../_helper/shell'
 import { define } from '../../../../../src/dataflow/environments'
 import { LocalScope } from '../../../../../src/dataflow/environments/scopes'
 

diff --git a/.../dataflow/elements/lists/uninteresting.ts → ...s/expression-lists/uninteresting-tests.ts b/.../dataflow/elements/lists/uninteresting.ts → ...s/expression-lists/uninteresting-tests.ts
@@ -1,4 +1,4 @@
-import { assertDataflow, withShell } from '../../../helper/shell'
+import { assertDataflow, withShell } from '../../../_helper/shell'
 import { DataflowGraph } from '../../../../../src/dataflow'
 
 describe('Lists without variable references ', withShell(shell => {
Benchmark suite	Current: `596b70f`	Previous: `243959c`	Ratio
`Total per-file`	`3804.5559240909092` ms (`5170.359319412824`)	`3190.570812409091` ms (`3747.790963743062`)	`1.19`
`Retrieve AST from R code`	`74.56589218181819` ms (`146.8099284297801`)	`68.74045054545455` ms (`132.55959690808635`)	`1.08`
`Normalize R AST`	`96.26604622727274` ms (`155.31980088528658`)	`96.19992009090909` ms (`156.15063334620123`)	`1.00`
`Produce dataflow information`	`67.10777218181818` ms (`171.15683868512687`)	`66.09041786363636` ms (`169.92667998599904`)	`1.02`
`Total per-slice`	`2.0174989008245396` ms (`1.4109740347352995`)	`1.8411702253695927` ms (`1.267065280512554`)	`1.10`
`Static slicing`	`1.42280678296949` ms (`1.2731817766313591`)	`1.3684034386686694` ms (`1.1799623517395228`)	`1.04`
`Reconstruct code`	`0.5771566862856531` ms (`0.3880877652226172`)	`0.4553098527401606` ms (`0.2299566392692304`)	`1.27`
`failed to reconstruct/re-parse`	`0` #	`0` #	`NaN`
`times hit threshold`	`0` #	`0` #	`NaN`
`reduction (characters)`	`0.7329390759026896` #	`0.7329390759026896` #	`1`
`reduction (normalized tokens)`	`0.720988345209971` #	`0.720988345209971` #	`1`
Benchmark suite	Current: `596b70f`	Previous: `243959c`	Ratio
`Total per-file`	`5263.65605614` ms (`5814.868226927445`)	`5419.399718899999` ms (`6111.491408384184`)	`0.97`
`Retrieve AST from R code`	`76.85367336` ms (`67.01097143149245`)	`85.22318365999999` ms (`79.41151212889127`)	`0.90`
`Normalize R AST`	`112.33219194` ms (`69.67744944436191`)	`113.45254984` ms (`69.72126242848063`)	`0.99`
`Produce dataflow information`	`162.1001241` ms (`275.13110196515186`)	`164.38322226` ms (`279.00889908327076`)	`0.99`
`Total per-slice`	`8.437385266529795` ms (`14.054690917342569`)	`8.82827828085479` ms (`14.575511768060265`)	`0.96`
`Static slicing`	`7.941200117842794` ms (`13.94011342956602`)	`8.228727295296194` ms (`14.45423896853327`)	`0.97`
`Reconstruct code`	`0.48773673947577545` ms (`0.24228534846179905`)	`0.5898725311439794` ms (`0.30780610835907124`)	`0.83`
`failed to reconstruct/re-parse`	`9` #	`9` #	`1`
`times hit threshold`	`967` #	`967` #	`1`
`reduction (characters)`	`0.898713819973478` #	`0.8987761232201357` #	`1.00`
`reduction (normalized tokens)`	`0.8579790415512589` #	`0.8582032343145828` #	`1.00`