Support Library Loads and Installations as Potential Side Effects (#944)

src/cli/repl/server/connection.ts

@@ -33,12 +33,12 @@ import { DataflowGraph } from '../../../dataflow/graph/graph'
 import * as tmp from 'tmp'
 import fs from 'fs'
 import type { RParseRequests } from '../../../r-bridge/retriever'
-import { autoSelectLibrary } from '../../../reconstruct/auto-select/auto-select-defaults'
 import { makeMagicCommentHandler } from '../../../reconstruct/auto-select/magic-comments'
 import type { LineageRequestMessage, LineageResponseMessage } from './messages/lineage'
 import { requestLineageMessage } from './messages/lineage'
 import { getLineage } from '../commands/lineage'
 import { guard } from '../../../util/assert'
+import { doNotAutoSelect } from '../../../reconstruct/auto-select/auto-select-defaults'
 
 /**
  * Each connection handles a single client, answering to its requests.
@@ -227,7 +227,7 @@ export class FlowRServerConnection {
 
 		fileInformation.pipeline.updateRequest({
 			criterion:    request.criterion,
-			autoSelectIf: request.noMagicComments ? autoSelectLibrary : makeMagicCommentHandler(autoSelectLibrary)
+			autoSelectIf: request.noMagicComments ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect)
 		})
 
 		void fileInformation.pipeline.allRemainingSteps(true).then(results => {

src/cli/slicer-app.ts

@@ -11,8 +11,8 @@ import type { SingleSlicingCriterion, SlicingCriteria } from '../slicing/criteri
 import type { ReconstructionResult } from '../reconstruct/reconstruct'
 import type { NodeId } from '../r-bridge/lang-4.x/ast/model/processing/node-id'
 import { stats2string } from '../benchmark/stats/print'
-import { autoSelectLibrary } from '../reconstruct/auto-select/auto-select-defaults'
 import { makeMagicCommentHandler } from '../reconstruct/auto-select/magic-comments'
+import { doNotAutoSelect } from '../reconstruct/auto-select/auto-select-defaults'
 
 export interface SlicerCliOptions {
 	verbose:             boolean
@@ -48,7 +48,7 @@ async function getSlice() {
 		options['input-is-text']
 			? { request: 'text', content: options.input }
 			: { request: 'file', content: options.input },
-		options['no-magic-comments'] ? autoSelectLibrary : makeMagicCommentHandler(autoSelectLibrary)
+		options['no-magic-comments'] ? doNotAutoSelect : makeMagicCommentHandler(doNotAutoSelect)
 	)
 
 	let mappedSlices: { criterion: SingleSlicingCriterion, id: NodeId }[] = []

src/dataflow/environments/built-in.ts

@@ -186,7 +186,7 @@ registerBuiltInFunctions(true,  ['[', '[['],
 registerBuiltInFunctions(true,  ['$', '@'],                                        processAccess,             { treatIndicesAsString: true }                                               )
 registerBuiltInFunctions(true,  ['if', 'ifelse'],                                  processIfThenElse,         {}                                                                           )
 registerBuiltInFunctions(true,  ['get'],                                           processGet,                {}                                                                           )
-registerBuiltInFunctions(false, ['library'],                                       processLibrary,            {}                                                                           )
+registerBuiltInFunctions(false, ['library', 'require'],                            processLibrary,            {}                                                                           )
 registerBuiltInFunctions(true,  ['<-', '='],                                       processAssignment,         { canBeReplacement: true }                                                   )
 registerBuiltInFunctions(true,  [':=', 'assign'],                                  processAssignment,         {}                                                                           )
 registerBuiltInFunctions(true,  ['delayedAssign'],                                 processAssignment,         { quoteSource: true }                                                        )
@@ -203,6 +203,12 @@ registerBuiltInFunctions(true,  ['repeat'],
 registerBuiltInFunctions(true,  ['while'],                                         processWhileLoop,          {}                                                                           )
 registerBuiltInFunctions(true,  ['options'],                                       defaultBuiltInProcessor,   { hasUnknownSideEffects: true, forceArgs: 'all' as const }                   )
 registerBuiltInFunctions(true,  ['on.exit', 'sys.on.exit'],                        defaultBuiltInProcessor,   { hasUnknownSideEffects: true }                                              )
+/* library and require is handled above */
+registerBuiltInFunctions(true,  ['requireNamespace', 'loadNamespace', 'attachNamespace', 'asNamespace'], defaultBuiltInProcessor, { hasUnknownSideEffects: true }                                         )
+/* downloader and installer functions (R, devtools, BiocManager) */
+registerBuiltInFunctions(true,  ['library.dynam', 'install.packages','install', 'install_github', 'install_gitlab', 'install_bitbucket', 'install_url', 'install_git', 'install_svn', 'install_local', 'install_version', 'update_packages'], defaultBuiltInProcessor, { hasUnknownSideEffects: true }                               )
+/* weird env attachments */
+registerBuiltInFunctions(true,  ['attach'],                                        defaultBuiltInProcessor,   { hasUnknownSideEffects: true }                                              )
 
 /* they are all mapped to `<-` but we separate super assignments */
 registerReplacementFunctions({ makeMaybe: true },  ['<-', '<<-'], '[', '[[', '$', '@', 'names', 'dimnames', 'attributes', 'attr', 'class', 'levels', 'rownames', 'colnames')

src/dataflow/internal/process/functions/call/built-in/built-in-library.ts

@@ -12,21 +12,21 @@ import { RType } from '../../../../../../r-bridge/lang-4.x/ast/model/type'
 import { wrapArgumentsUnnamed } from '../argument/make-argument'
 
 
-/* we currently do not mark this as an unknown side effect, as we can enable/disable this with a toggle */
 export function processLibrary<OtherInfo>(
 	name: RSymbol<OtherInfo & ParentInformation>,
 	args: readonly RFunctionArgument<OtherInfo & ParentInformation>[],
 	rootId: NodeId,
 	data: DataflowProcessorInformation<OtherInfo & ParentInformation>
 ): DataflowInformation {
+	/* we do not really know what loading the library does and what side effects it causes, hence we mark it as an unknown side effect */
 	if(args.length !== 1) {
 		dataflowLogger.warn(`Currently only one-arg library-likes are allows (for ${name.content}), skipping`)
-		return processKnownFunctionCall({ name, args, rootId, data }).information
+		return processKnownFunctionCall({ name, args, rootId, data, hasUnknownSideEffect: true }).information
 	}
 	const nameToLoad = unpackArgument(args[0])
 	if(nameToLoad === undefined || nameToLoad.type !== RType.Symbol) {
 		dataflowLogger.warn('No library name provided, skipping')
-		return processKnownFunctionCall({ name, args, rootId, data }).information
+		return processKnownFunctionCall({ name, args, rootId, data, hasUnknownSideEffect: true }).information
 	}
 
 	// treat as a function call but convert the first argument to a string
@@ -40,6 +40,8 @@ export function processLibrary<OtherInfo>(
 			str:    nameToLoad.content
 		}
 	}
-
-	return processKnownFunctionCall({ name, args: wrapArgumentsUnnamed([newArg], data.completeAst.idMap), rootId, data }).information
+	return processKnownFunctionCall({
+		name, args:                 wrapArgumentsUnnamed([newArg], data.completeAst.idMap), rootId, data,
+		hasUnknownSideEffect: true
+	}).information
 }

src/dataflow/internal/process/functions/call/known-call-handling.ts

@@ -16,16 +16,18 @@ import { dataflowLogger } from '../../../../logger'
 import { VertexType } from '../../../../graph/vertex'
 
 export interface ProcessKnownFunctionCallInput<OtherInfo> extends ForceArguments {
-	readonly name:          RSymbol<OtherInfo & ParentInformation>
-	readonly args:          readonly (RNode<OtherInfo & ParentInformation> | RFunctionArgument<OtherInfo & ParentInformation>)[]
-	readonly rootId:        NodeId
-	readonly data:          DataflowProcessorInformation<OtherInfo & ParentInformation>
-	/* should arguments be processed from right to left? This does not affect the order recorded in the call but of the environments */
-	readonly reverseOrder?: boolean
+	readonly name:                  RSymbol<OtherInfo & ParentInformation>
+	readonly args:                  readonly (RNode<OtherInfo & ParentInformation> | RFunctionArgument<OtherInfo & ParentInformation>)[]
+	readonly rootId:                NodeId
+	readonly data:                  DataflowProcessorInformation<OtherInfo & ParentInformation>
+	/** should arguments be processed from right to left? This does not affect the order recorded in the call but of the environments */
+	readonly reverseOrder?:         boolean
 	/** which arguments are to be marked as {@link EdgeType#NonStandardEvaluation|non-standard-evaluation}? */
-	readonly markAsNSE?:    readonly number[]
-	/* allows passing a data processor in-between each argument */
-	readonly patchData?:    (data: DataflowProcessorInformation<OtherInfo & ParentInformation>, arg: number) => DataflowProcessorInformation<OtherInfo & ParentInformation>
+	readonly markAsNSE?:            readonly number[]
+	/** allows passing a data processor in-between each argument */
+	readonly patchData?:            (data: DataflowProcessorInformation<OtherInfo & ParentInformation>, arg: number) => DataflowProcessorInformation<OtherInfo & ParentInformation>
+	/** Does the call have a side effect that we do not know a lot about which may have further consequences? */
+	readonly hasUnknownSideEffect?: boolean
 }
 
 export interface ProcessKnownFunctionCallResult {
@@ -56,7 +58,7 @@ export function markNonStandardEvaluationEdges(
 }
 
 export function processKnownFunctionCall<OtherInfo>(
-	{ name,args, rootId,data, reverseOrder = false, markAsNSE = undefined, forceArgs, patchData = d => d }: ProcessKnownFunctionCallInput<OtherInfo>
+	{ name,args, rootId,data, reverseOrder = false, markAsNSE = undefined, forceArgs, patchData = d => d, hasUnknownSideEffect }: ProcessKnownFunctionCallInput<OtherInfo>
 ): ProcessKnownFunctionCallResult {
 	const functionName = processDataflowFor(name, data)
 
@@ -85,6 +87,10 @@ export function processKnownFunctionCall<OtherInfo>(
 		args:                reverseOrder ? [...callArgs].reverse() : callArgs
 	})
 
+	if(hasUnknownSideEffect) {
+		finalGraph.markIdForUnknownSideEffects(rootId)
+	}
+
 	const inIds = remainingReadInArgs
 	const fnRef = { nodeId: rootId, name: functionCallName, controlDependencies: data.controlDependencies, call: true as const }
 	inIds.push(fnRef)

src/reconstruct/auto-select/auto-select-defaults.ts

@@ -1,6 +1,5 @@
-import type { NoInfo, RNode } from '../../r-bridge/lang-4.x/ast/model/model'
+import type { RNode } from '../../r-bridge/lang-4.x/ast/model/model'
 import type { ParentInformation, NormalizedAst } from '../../r-bridge/lang-4.x/ast/model/processing/decorate'
-import { RType } from '../../r-bridge/lang-4.x/ast/model/type'
 
 /**
  * The structure of the predicate that should be used to determine
@@ -19,16 +18,3 @@ export type AutoSelectPredicate = (node: RNode<ParentInformation>, fullAst: Norm
 export function doNotAutoSelect(_node: RNode): boolean {
 	return false
 }
-
-const libraryFunctionCall = /^(library|require|((require|load|attach)Namespace))$/
-
-/**
- * A variant of the {@link AutoSelectPredicate} which does its best
- * to select any kind of library import automatically.
- */
-export function autoSelectLibrary<Info = NoInfo>(node: RNode<Info>): boolean {
-	if(node.type !== RType.FunctionCall || !node.named) {
-		return false
-	}
-	return libraryFunctionCall.test(node.functionName.content)
-}

src/reconstruct/reconstruct.ts

@@ -31,7 +31,7 @@ import type { StatefulFoldFunctions } from '../r-bridge/lang-4.x/ast/model/proce
 import { foldAstStateful } from '../r-bridge/lang-4.x/ast/model/processing/stateful-fold'
 import type { NodeId } from '../r-bridge/lang-4.x/ast/model/processing/node-id'
 import type { AutoSelectPredicate } from './auto-select/auto-select-defaults'
-import { autoSelectLibrary } from './auto-select/auto-select-defaults'
+import { doNotAutoSelect } from './auto-select/auto-select-defaults'
 
 type Selection = ReadonlySet<NodeId>
 interface PrettyPrintLine {
@@ -491,7 +491,7 @@ function removeOuterExpressionListIfApplicable(result: PrettyPrintLine[], linesW
  *
  * @returns The number of lines for which `autoSelectIf` triggered, as well as the reconstructed code itself.
  */
-export function reconstructToCode(ast: NormalizedAst, selection: Selection, autoSelectIf: AutoSelectPredicate = autoSelectLibrary): ReconstructionResult {
+export function reconstructToCode(ast: NormalizedAst, selection: Selection, autoSelectIf: AutoSelectPredicate = doNotAutoSelect): ReconstructionResult {
 	if(reconstructLogger.settings.minLevel <= LogLevel.Trace) {
 		reconstructLogger.trace(`reconstruct ast with ids: ${JSON.stringify([...selection])}`)
 	}

test/functionality/benchmark/slicer.spec.ts

@@ -135,8 +135,8 @@ cat(d)`
 				tokensNoComments:                  { min: 13, max: 35, median: 19, mean: 22.333333333333332, std: 9.285592184789413,  total: 67 },
 				normalizedTokens:                  { min: 8,  max: 19, median: 11, mean: (8+11+19)/3,        std: 4.642796092394707,  total: 38 },
 				normalizedTokensNoComments:        { min: 8,  max: 19, median: 11, mean: (8+11+19)/3,        std: 4.642796092394707,  total: 38 },
-				dataflowNodes:                     { min: 3,  max: 14, median: 6,  mean: (3+6+14)/3,         std: 4.642796092394707,  total: 23 },
-				linesWithAutoSelected:             { min: 1,  max: 1,  median: 1,  mean: 1,                  std: 0,                  total: 3  } // always select one library statement
+				dataflowNodes:                     { min: 5,  max: 16, median: 8,  mean: (5+8+16)/3,         std: 4.642796092394707,  total: 29 },
+				linesWithAutoSelected:             { min: 0,  max: 0,  median: 0,  mean: 0,                  std: 0,                  total: 0  }
 			}, statInfo)
 
 			assert.deepStrictEqual(stats.perSliceMeasurements.sliceCriteriaSizes, {

test/functionality/slicing/static-program-slices/calls-tests.ts

@@ -682,5 +682,12 @@ x`)
 		'x\non.exit(function() 3)', ['1@x'],
 		'x\non.exit(function() 3)'
 		)
+		assertSliced(label('Library Loads and Installations', [
+			'functions-with-global-side-effects', 'name-normal', 'strings', 'call-normal', 'unnamed-arguments', 'newlines', 'library-loading'
+		]), shell,
+		/* w should be included as it defined the package to be loaded by the library call */
+		'v\nlibrary(x)\nrequire(y)\nw <- "x"\nattachNamespace(w)\nloadNamespace("x")', ['1@v'],
+		'v\nlibrary(x)\nrequire(y)\nw <- "x"\nattachNamespace(w)\nloadNamespace("x")'
+		)
 	})
 }))