From eb02c11d3514ba3399433f4b87001381e137b57d Mon Sep 17 00:00:00 2001 From: saem Date: Thu, 19 Oct 2023 13:27:40 -0700 Subject: [PATCH 01/12] fix `discardCheck` bug --- compiler/front/cli_reporter.nim | 2 -- compiler/sem/semstmts.nim | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/compiler/front/cli_reporter.nim b/compiler/front/cli_reporter.nim index 71e729bda9c..f504bef7dfd 100644 --- a/compiler/front/cli_reporter.nim +++ b/compiler/front/cli_reporter.nim @@ -1206,7 +1206,6 @@ proc reportBody*(conf: ConfigRef, r: SemReport): string = of rsemUseOrDiscardExpr: let n = r.wrongNode - result.add( "expression '", n.render, @@ -1217,7 +1216,6 @@ proc reportBody*(conf: ConfigRef, r: SemReport): string = if r.ast.info.line != n.info.line or r.ast.info.fileIndex != n.info.fileIndex: - result.add "; start of expression here: " & conf$r.ast.info if r.ast.typ.kind == tyProc: diff --git a/compiler/sem/semstmts.nim b/compiler/sem/semstmts.nim index 680f3424f13..8614ba98807 100644 --- a/compiler/sem/semstmts.nim +++ b/compiler/sem/semstmts.nim @@ -307,15 +307,15 @@ proc semTry(c: PContext, n: PNode; flags: TExprFlags): PNode = if n[0].isError: return wrapError(c.config, n) for i in 1 ..< n.len: - n[i][^1] = discardCheck(c, n[i][^1], flags) - if n[i][^1].isError: + n[i] = discardCheck(c, n[i], flags) + if n[i].isError: return wrapError(c.config, n) if typ == c.enforceVoidContext: result.typ = c.enforceVoidContext else: if n.lastSon.kind == nkFinally: - n[^1][^1] = discardCheck(c, n[^1][^1], flags) - if n[^1][^1].isError: + n[^1] = discardCheck(c, n[^1], flags) + if n[^1].isError: return wrapError(c.config, n) n[0] = fitNode(c, typ, n[0], n[0].info) for i in 1..last: From 9d0523b66d87ce78a725b3586d2f43b3b4a500ca Mon Sep 17 00:00:00 2001 From: saem Date: Thu, 19 Oct 2023 13:28:12 -0700 Subject: [PATCH 02/12] docs typo in `astrepr` also there is a bug when it comes to debugging types... sigh --- compiler/utils/astrepr.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/utils/astrepr.nim b/compiler/utils/astrepr.nim index a10a4fb5fe2..a03a317e6e1 100644 --- a/compiler/utils/astrepr.nim +++ b/compiler/utils/astrepr.nim @@ -617,7 +617,7 @@ proc treeRepr*( else: addi indent, rconf.formatKind(typ.kind) + style.kind if trfShowTypeSym notin rconf and typ.sym != nil: - # If the name is not show in verbose manner as a field, print it + # If the name is not shown in verbose manner as a field, print it # beforehand as a regular string. add " \"" add typ.sym.name.s + style.identLit From 8cd70d7f452af368311785dce90bf5f94b0f923d Mon Sep 17 00:00:00 2001 From: saem Date: Thu, 19 Oct 2023 14:28:13 -0700 Subject: [PATCH 03/12] formatting --- compiler/front/options.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/compiler/front/options.nim b/compiler/front/options.nim index 8f83a430300..ce0c76e9c08 100644 --- a/compiler/front/options.nim +++ b/compiler/front/options.nim @@ -462,7 +462,6 @@ template changedOpts(conf: ConfigRef, body: untyped) = body let after = conf.localOptions let removed = (optHints in before) and (optHints notin after) - else: body From 10ac525c29d3e4e5b0bcb1c815c5181aa305b415 Mon Sep 17 00:00:00 2001 From: saem Date: Thu, 19 Oct 2023 14:28:51 -0700 Subject: [PATCH 04/12] initial port of `dust` added to to `tools` build in `koch` --- tools/dust/boring.nim | 163 +++++++++++++++++++++++++++++++++++++++++ tools/dust/dust.nim | 142 +++++++++++++++++++++++++++++++++++ tools/dust/hashing.nim | 10 +++ tools/dust/mutate.nim | 71 ++++++++++++++++++ tools/dust/pass.nim | 39 ++++++++++ tools/dust/spec.nim | 84 +++++++++++++++++++++ tools/koch/koch.nim | 2 + 7 files changed, 511 insertions(+) create mode 100644 tools/dust/boring.nim create mode 100644 tools/dust/dust.nim create mode 100644 tools/dust/hashing.nim create mode 100644 tools/dust/mutate.nim create mode 100644 tools/dust/pass.nim create mode 100644 tools/dust/spec.nim diff --git a/tools/dust/boring.nim b/tools/dust/boring.nim new file mode 100644 index 00000000000..d709c65ec33 --- /dev/null +++ b/tools/dust/boring.nim @@ -0,0 +1,163 @@ +##[ + +the boring bits that really aren't very relevant to dust. + +]## + +import std/times +import std/os +import std/parseopt + +import + compiler / ast / [ + idents, + lineinfos, + ], + compiler / front / [ + cmdlinehelper, + commands, + condsyms, + main, + msgs, + nimconf, + options, + optionsprocessor, + ], + compiler / modules / [ + modules, + modulegraphs, + ], + compiler / utils / pathutils + +const + NimCfg* {.strdefine.} = "nim".addFileExt "cfg" + +template excludeAllNotes(config: ConfigRef; n: typed) = + config.notes.excl n + when compiles(config.mainPackageNotes): + config.mainPackageNotes.excl n + when compiles(config.foreignPackageNotes): + config.foreignPackageNotes.excl n + +proc cmdLine(pass: TCmdLinePass, cmd: openArray[string]; config: ConfigRef) = + ## parse the command-line into the config + var p = initOptParser(cmd) + var argsCount = 0 + + config.commandLine.setLen 0 # some bug + while true: + next(p) + case p.kind + of cmdEnd: + break + of cmdLongOption, cmdShortOption: + config.commandLine.add " " + config.commandLine.add: + if p.kind == cmdLongOption: "--" else: "-" + config.commandLine.add p.key.quoteShell + if p.val.len > 0: + config.commandLine.add ':' + config.commandLine.add p.val.quoteShell + if p.key == " ": + p.key = "-" + if processArgument(pass, p, argsCount, config): + break + else: + # Main part of the configuration processing - + # `optionsprocessor.processSwitch` processes input switches a second + # time and puts them in necessary configuration fields. + let res = processSwitch(pass, p, config) + for e in procSwitchResultToEvents(config, pass, p.key, p.val, res): + config.cliEventLogger(e) + of cmdArgument: + config.commandLine.add " " + config.commandLine.add p.key.quoteShell + if processArgument(pass, p, argsCount, config): + break + + when false: + if pass == passCmd2: + if {optRun, optWasNimscript} * config.globalOptions == {} and + config.arguments.len > 0 and + config.command.normalize notin ["run", "e"]: + rawMessage(config, errGenerated, errArgsNeedRunOption) + +proc helpOnError(config: ConfigRef) = + const + Usage = """ + dust [options] [projectfile] + + Options: Same options that the Nim compiler supports. + """ + showMsg(config, Usage) + msgQuit 0 + +proc reset*(graph: ModuleGraph) = + ## reset the module graph so it is ready for recompilation + # we're not dirty if we don't have a fileindex + if graph.config.projectMainIdx != InvalidFileIdx: + # mark the program as dirty + graph.markDirty graph.config.projectMainIdx + # mark dependencies as dirty + graph.markClientsDirty graph.config.projectMainIdx + # reset the error counter + graph.config.errorCounter = 0 + +proc compile*(graph: ModuleGraph) = + ## compile a module graph + reset graph + let config = graph.config + config.lastCmdTime = epochTime() + if config.libpath notin config.searchPaths: + config.searchPaths.add config.libpath # make sure we can import + + config.setErrorMaxHighMaybe # for now, we honor errorMax + defineSymbol(config, "nimcheck") # useful for static: reasons + + graph.suggestMode = true # needed for dirty flags + compileProject graph # process the graph + +proc setup*(cache: IdentCache; config: ConfigRef; graph: ModuleGraph): bool = + proc noop(graph: ModuleGraph) {.used.} = discard + let prog = NimProg(supportsStdinFile: true, + processCmdLine: cmdLine) #, mainCommand: mainCommand) + initDefinesProg(prog, config, "dust") + if paramCount() == 0: + helpOnError(config) + else: + let argv = getExecArgs() + processCmdLineAndProjectPath(prog, config, argv) + result = loadConfigsAndProcessCmdLine(prog, cache, config, graph, argv) + +proc parentDir(file: AbsoluteFile): AbsoluteDir = + result = AbsoluteDir(file) / RelativeDir".." + +proc loadConfig*(graph: ModuleGraph; fn: AbsoluteFile) = + ## use the ident cache to load the project config for the given filename + var result = graph.config + + #excludeAllNotes(result, hintConf) + #excludeAllNotes(result, hintLineTooLong) + + initDefines(result.symbols) + + let compilerPath = AbsoluteFile findExe"nim" + result.prefixDir = parentDir(compilerPath) / RelativeDir".." + result.projectPath = parentDir(fn) + + when false: + let cfg = fn.string & ExtSep & "cfg" + if fileExists(cfg): + if not readConfigFile(cfg.AbsoluteFile, graph.cache, result): + raise newException(ValueError, "couldn't parse " & cfg) + else: + let cwd = getCurrentDir() + setCurrentDir $result.projectPath + try: + discard loadConfigs(NimCfg.RelativeFile, graph.cache, result) + finally: + setCurrentDir cwd + + incl result, optStaticBoundsCheck + excl result, optWarns + excl result, optHints \ No newline at end of file diff --git a/tools/dust/dust.nim b/tools/dust/dust.nim new file mode 100644 index 00000000000..7ed55ab43f2 --- /dev/null +++ b/tools/dust/dust.nim @@ -0,0 +1,142 @@ +import std/os + +{.define(nimcore).} + +import + compiler / ast / [ + ast, + astalgo, + idents, + lineinfos, + parser, + report_enums, # legacy reports stupidity + ], + compiler / front / options, + compiler / modules / modulegraphs, + compiler / sem / [ + passes, + sem, + ], + compiler / utils / [ astrepr, pathutils, ] + +# legacy reports stupidity +from compiler/ast/reports import Report, location, kind + +import std/options as std_options # due to legacy reports stupidity + +import spec +import boring +import mutate + +template semcheck(body: untyped) {.dirty.} = + ## perform the complete setup and compilation process + cache = newIdentCache() + config = newConfigRef(uhoh) + graph = newModuleGraph(cache, config) + graph.loadConfig(filename) + + # perform boring setup of the config using the cache + if not setup(cache, config, graph): + echo "crashing due to error during setup" + quit 1 + + config.verbosity = compVerbosityMin # reduce spam + + # create a new module graph + #graph = newModuleGraph(cache, config) + + body + registerPass graph, semPass # perform semcheck + compile graph # run the compile + inc counter + +proc calculateScore(config: ConfigRef; n: PNode): int = + when defined(dustFewerLines): + result = config.linesCompiled + else: + result = size(n) + +proc dust*(filename: AbsoluteFile) = + var + graph: ModuleGraph + cache: IdentCache + config: ConfigRef + errorKind: ReportKind + best: PNode + counter = 0 + score: int + remains: Remains + rendered: string + + proc uhoh(config: ConfigRef, rep: Report): TErrorHandling = + ## capture the first error + if config.severity(rep) == rsevError: + if std_options.unsafeGet(rep.location).fileIndex == config.projectMainIdx: + if errorKind == repNone: + errorKind = rep.kind + elif errorKind == rep.kind: + config.structuredReportHook = nil + + # in the first pass, we add the program to our cache + semcheck: + # basically, just taking advantage of cache and config values... + best = toPNode(parseString(readFile(filename.string), + cache = cache, config = config, line = 0, + filename = filename.string)) + score = size(best) + remains.add best + assert len(remains) > 0 + + # if the semcheck passes, we have nothing to do + if config.errorCounter == 0: + echo "error: " & filename.string & " passes the semcheck" + quit 1 + + # otherwise, we have an interesting error message to pursue + echo "interesting: ", errorKind, + " first of ", config.errorCounter, " errors" + + # make note of the expected number of errors + let expected = config.errorCounter + + while len(remains) > 0: + echo rendered + echo "remaining: ", len(remains), " best: ", score + let node = pop(remains) + + semcheck: + try: + writeFile(filename.string, $node) + except IndexError: + echo "cheating to get around rendering bug" + continue + + # extra errors are a problem + if config.errorCounter > expected: + echo "(unexpected errors)" + # if we didn't unhook the errors, + # it means we didn't find the error we were looking for + elif config.structuredReportHook != nil: + echo "(uninteresting errors)" + # i guess this node is a viable reproduction + else: + let z = calculateScore(config, node) + if z < score: + echo "(new high score)" + best = node + score = z + rendered = $best + for mutant in mutations(node): + remains.add mutant + + if not best.isNil: + debug best + echo "=== minimal after ", counter, "/", remains.count, " semchecks; scored ", score + echo best + writeFile(filename.string, $best) + +when isMainModule: + if paramCount() > 0: + dust paramStr(paramCount()).AbsoluteFile + else: + echo "supply a source file to inspect" \ No newline at end of file diff --git a/tools/dust/hashing.nim b/tools/dust/hashing.nim new file mode 100644 index 00000000000..ed0877449e8 --- /dev/null +++ b/tools/dust/hashing.nim @@ -0,0 +1,10 @@ +include compiler/sem/sighashes # because too few things are exported + +const + considerAll = {ConsiderFlag.low .. ConsiderFlag.high} + +proc hashNode*(n: PNode; flags: set[ConsiderFlag] = considerAll): SigHash = + var c: MD5Context + md5Init c + hashTree(c, n, flags) + md5Final(c, result.MD5Digest) \ No newline at end of file diff --git a/tools/dust/mutate.nim b/tools/dust/mutate.nim new file mode 100644 index 00000000000..68a4e0634ec --- /dev/null +++ b/tools/dust/mutate.nim @@ -0,0 +1,71 @@ +import + compiler / modules / modulegraphs, + compiler / ast / [ lineinfos, renderer, ast, astalgo, ] + +import spec +import hashing + +proc node(k: TNodeKind): PNode = PNode(kind: k) + +type + Transformer = proc (n: PNode): PNode + +proc transform*(n: PNode; fn: Transformer): PNode = + assert not n.isNil + result = fn(n) + if result.isNil: + result = shallowCopy n + for i, child in pairs(n): + result.sons[i] = transform(child, fn) + +proc rebuild*(n: PNode; fn: Transformer): PNode = + assert not n.isNil + result = fn(n) + if cast[uint](result) == cast[uint](n): + result = shallowCopy n + if n.kind notin unsunny: + result.sons.setLen 0 + for child in items(n): + let rebuilt = rebuild(child, fn) + if not rebuilt.isNil: + result.add rebuilt + +proc removeIndex(n: PNode; index: int): PNode = + var index = index + let z = size(n) + proc remover(n: PNode): PNode = + if index == 0: + result = nil + else: + result = n + dec index + result = rebuild(n, remover) + +proc discardIndex(n: PNode; index: int): PNode = + var index = index + proc discarder(n: PNode): PNode = + if index == 0: + result = nkDiscardStmt.node + else: + dec index + result = transform(n, discarder) + +proc emptyIndex(n: PNode; index: int): PNode = + var index = index + proc emptier(n: PNode): PNode = + if index == 0: + result = nkEmpty.node + else: + dec index + result = transform(n, emptier) + +iterator mutations*(n: PNode): PNode = + for i in 1 ..< size(n): + yield removeIndex(n, i) + yield emptyIndex(n, i) + yield discardIndex(n, i) + +proc init*(remains: var Remains; n: PNode) = + let h = hashNode(n) + if h notin remains: + remains.add(n, h) \ No newline at end of file diff --git a/tools/dust/pass.nim b/tools/dust/pass.nim new file mode 100644 index 00000000000..fa8673b2ce0 --- /dev/null +++ b/tools/dust/pass.nim @@ -0,0 +1,39 @@ +import + compiler / ast / [ lineinfos, ast, astalgo ] + compiler / front / options, + compiler / modules / modulegraphs, + compiler / sem / passes + +import dust/spec +import dust/mutate +import dust/hashing + +var remains*: Remains + +proc opener(graph: ModuleGraph; module: PSym): PPassContext {.nosinks.} = + ## the opener learns when we're compiling the test file + result = DustContext(mainIndex: graph.config.projectMainIdx) + +when false: + proc closer(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = + ## the closer don't do shit + discard + +proc rewriter(context: PPassContext, n: PNode): PNode {.nosinks.} = + template c: DustContext = DustContext(context) + # if this isn't the main project, + if n.info.fileIndex != c.mainIndex: + # yield the original node + result = n + else: + if c.ignore: + # we already yielded the program, so just return nil + result = nil + else: + # yield the next available permutation (once) + result = pop(remains) + assert result.kind == nkStmtList + c.ignore = true + +const + dustPass* = makePass(opener, rewriter, nil) \ No newline at end of file diff --git a/tools/dust/spec.nim b/tools/dust/spec.nim new file mode 100644 index 00000000000..195cc6e7c53 --- /dev/null +++ b/tools/dust/spec.nim @@ -0,0 +1,84 @@ +import std/strutils +import std/sets + +import + compiler / modules / modulegraphs, + compiler / ast / [ lineinfos, renderer, ast ] + +import hashing + +const + unsunny* = {nkCharLit..nkUInt64Lit} + + {nkFloatLit..nkFloat128Lit} + + {nkStrLit..nkTripleStrLit} + + {nkSym, nkIdent} + +type + Attempt = object + node: PNode + sig: SigHash + size: int + + Remains* = object + signatures: HashSet[SigHash] + attempts: seq[Attempt] + + DustContext* = ref object of PPassContext + mainIndex*: FileIndex + ignore*: bool + +proc len*(r: Remains): int = len(r.attempts) + +proc count*(r: Remains): int = len(r.signatures) + +proc pop*(remains: var Remains): PNode = + assert len(remains) > 0, "pop from empty remains" + result = pop(remains.attempts).node + +proc size*(n: PNode): int = + assert not n.isNil + result = 1 + if n.kind notin unsunny: + for child in items(n.sons): + inc result, size(child) + +proc contains*(remains: Remains; h: SigHash): bool = + h in remains.signatures + +proc contains*(remains: Remains; n: PNode): bool = + hashNode(n) in remains + +proc newAttempt(n: PNode): Attempt = + var n = copyTree(n) + result = Attempt(node: n, sig: hashNode(n), size: size(n)) + +proc newAttempt(n: PNode; sig: SigHash): Attempt = + var n = copyTree(n) + assert hashNode(n) == sig + result = Attempt(node: n, sig: sig, size: size(n)) + +proc add(remains: var Remains; a: Attempt) = + if a.size > 1: + if not containsOrIncl(remains.signatures, a.sig): + remains.attempts.add a + +proc add*(remains: var Remains; n: PNode) = + if not n.isNil: + remains.add newAttempt(n) + +proc add*(remains: var Remains; n: PNode; sig: SigHash) = + if not n.isNil: + remains.add newAttempt(n, sig) + +proc next*(remains: Remains): PNode = + if len(remains) > 0: + result = remains.attempts[^1].node + +proc massageMessage*(s: string): string = + result = s.splitLines()[0] + for c in {';', ':'}: + let i = find(result, c) + if i != -1: + result = result[0 .. i - 1] + +export `$` \ No newline at end of file diff --git a/tools/koch/koch.nim b/tools/koch/koch.nim index 2a8bf161cdd..a5755179a46 100644 --- a/tools/koch/koch.nim +++ b/tools/koch/koch.nim @@ -199,6 +199,8 @@ proc buildTool(toolname, args: string) = copyFile(dest="bin" / splitFile(toolname).name.exe, source=toolname.exe) proc buildTools(args: string = "") = + nimCompileFold("Compile dust", "tools/dust/dust.nim", + options = "-d:release $# $#" % [defineSourceMetadata(), args]) bundleNimsuggest(args) nimCompileFold("Compile nimgrep", "tools/nimgrep.nim", options = "-d:release " & defineSourceMetadata() & " " & args) From 82d3bb05b7aa9656230ddb9d1c5350adbcce96d1 Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Fri, 20 Oct 2023 12:44:43 -0700 Subject: [PATCH 05/12] replace `unsunny` with `nkWithSons` usage Co-authored-by: Clyybber --- tools/dust/mutate.nim | 2 +- tools/dust/spec.nim | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/tools/dust/mutate.nim b/tools/dust/mutate.nim index 68a4e0634ec..b8273b31357 100644 --- a/tools/dust/mutate.nim +++ b/tools/dust/mutate.nim @@ -23,7 +23,7 @@ proc rebuild*(n: PNode; fn: Transformer): PNode = result = fn(n) if cast[uint](result) == cast[uint](n): result = shallowCopy n - if n.kind notin unsunny: + if n.kind in nkWithSons: result.sons.setLen 0 for child in items(n): let rebuilt = rebuild(child, fn) diff --git a/tools/dust/spec.nim b/tools/dust/spec.nim index 195cc6e7c53..64c6313788b 100644 --- a/tools/dust/spec.nim +++ b/tools/dust/spec.nim @@ -7,12 +7,6 @@ import import hashing -const - unsunny* = {nkCharLit..nkUInt64Lit} + - {nkFloatLit..nkFloat128Lit} + - {nkStrLit..nkTripleStrLit} + - {nkSym, nkIdent} - type Attempt = object node: PNode @@ -38,7 +32,7 @@ proc pop*(remains: var Remains): PNode = proc size*(n: PNode): int = assert not n.isNil result = 1 - if n.kind notin unsunny: + if n.kind in nkWithSons: for child in items(n.sons): inc result, size(child) From b7d16c6776fc8ef6b3da1044da90712e4768b045 Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Fri, 20 Oct 2023 12:46:20 -0700 Subject: [PATCH 06/12] remove unused imports Co-authored-by: zerbina <100542850+zerbina@users.noreply.github.com> --- tools/dust/boring.nim | 2 -- tools/dust/mutate.nim | 1 - 2 files changed, 3 deletions(-) diff --git a/tools/dust/boring.nim b/tools/dust/boring.nim index d709c65ec33..0f467a8e0a5 100644 --- a/tools/dust/boring.nim +++ b/tools/dust/boring.nim @@ -17,9 +17,7 @@ import cmdlinehelper, commands, condsyms, - main, msgs, - nimconf, options, optionsprocessor, ], diff --git a/tools/dust/mutate.nim b/tools/dust/mutate.nim index b8273b31357..ab3f3e3902b 100644 --- a/tools/dust/mutate.nim +++ b/tools/dust/mutate.nim @@ -1,5 +1,4 @@ import - compiler / modules / modulegraphs, compiler / ast / [ lineinfos, renderer, ast, astalgo, ] import spec From c4d8399dc9ef30377a8a4226facb7a9efbed305f Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Fri, 20 Oct 2023 12:48:29 -0700 Subject: [PATCH 07/12] `PNode` usage-style changes. Co-authored-by: zerbina <100542850+zerbina@users.noreply.github.com> --- tools/dust/mutate.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/dust/mutate.nim b/tools/dust/mutate.nim index ab3f3e3902b..66f1d23a16f 100644 --- a/tools/dust/mutate.nim +++ b/tools/dust/mutate.nim @@ -4,7 +4,7 @@ import import spec import hashing -proc node(k: TNodeKind): PNode = PNode(kind: k) +proc node(k: TNodeKind): PNode = newNode(k) type Transformer = proc (n: PNode): PNode @@ -15,15 +15,15 @@ proc transform*(n: PNode; fn: Transformer): PNode = if result.isNil: result = shallowCopy n for i, child in pairs(n): - result.sons[i] = transform(child, fn) + result[i] = transform(child, fn) proc rebuild*(n: PNode; fn: Transformer): PNode = assert not n.isNil result = fn(n) - if cast[uint](result) == cast[uint](n): + if result == n: result = shallowCopy n if n.kind in nkWithSons: - result.sons.setLen 0 + result.setLen 0 for child in items(n): let rebuilt = rebuild(child, fn) if not rebuilt.isNil: From 43827db0b681da300512310bbda0dbc05c96144f Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Fri, 20 Oct 2023 12:49:44 -0700 Subject: [PATCH 08/12] fix: single statement modules no longer assert Co-authored-by: zerbina <100542850+zerbina@users.noreply.github.com> --- tools/dust/pass.nim | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/dust/pass.nim b/tools/dust/pass.nim index fa8673b2ce0..5d87963f283 100644 --- a/tools/dust/pass.nim +++ b/tools/dust/pass.nim @@ -32,7 +32,6 @@ proc rewriter(context: PPassContext, n: PNode): PNode {.nosinks.} = else: # yield the next available permutation (once) result = pop(remains) - assert result.kind == nkStmtList c.ignore = true const From 4f306399d1e8f6d6f7aa5dd18a3daf67012a40db Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Fri, 20 Oct 2023 12:50:49 -0700 Subject: [PATCH 09/12] small efficiency improvement Co-authored-by: zerbina <100542850+zerbina@users.noreply.github.com> --- tools/dust/spec.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/dust/spec.nim b/tools/dust/spec.nim index 64c6313788b..f860d3e7997 100644 --- a/tools/dust/spec.nim +++ b/tools/dust/spec.nim @@ -70,7 +70,7 @@ proc next*(remains: Remains): PNode = proc massageMessage*(s: string): string = result = s.splitLines()[0] - for c in {';', ':'}: + for c in [';', ':']: let i = find(result, c) if i != -1: result = result[0 .. i - 1] From 0698a03a76edd9e5a867fa5f9ac267d4b28e0efc Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Tue, 31 Oct 2023 12:20:42 -0700 Subject: [PATCH 10/12] get dust compiling and sorta working --- tools/dust/boring.nim | 29 +++++++++++++++++++++++++++-- tools/dust/dust.nim | 6 ++++-- tools/dust/mutate.nim | 2 +- tools/dust/pass.nim | 38 -------------------------------------- tools/dust/spec.nim | 7 ------- 5 files changed, 32 insertions(+), 50 deletions(-) delete mode 100644 tools/dust/pass.nim diff --git a/tools/dust/boring.nim b/tools/dust/boring.nim index 0f467a8e0a5..011840dac86 100644 --- a/tools/dust/boring.nim +++ b/tools/dust/boring.nim @@ -8,6 +8,8 @@ import std/times import std/os import std/parseopt +from std/strutils import endsWith + import compiler / ast / [ idents, @@ -15,7 +17,7 @@ import ], compiler / front / [ cmdlinehelper, - commands, + # commands, condsyms, msgs, options, @@ -27,6 +29,10 @@ import ], compiler / utils / pathutils +from compiler / front / commands import procSwitchResultToEvents, + cliEventLogger, + showMsg + const NimCfg* {.strdefine.} = "nim".addFileExt "cfg" @@ -37,6 +43,25 @@ template excludeAllNotes(config: ConfigRef; n: typed) = when compiles(config.foreignPackageNotes): config.foreignPackageNotes.excl n +proc processArgument(pass: TCmdLinePass; p: OptParser; + argsCount: var int; config: ConfigRef): bool = + if argsCount == 0: + if p.key.endsWith(".nim"): + config.setCmd cmdCompileToC + config.projectName = unixToNativePath(p.key) + config.arguments = cmdLineRest(p) + result = true + elif pass != passCmd2: setCommandEarly(config, p.key) + else: + if pass == passCmd1: config.commandArgs.add p.key + if argsCount == 1: + # support UNIX style filenames everywhere for portable build scripts: + if config.projectName.len == 0 and config.inputMode == pimFile: + config.projectName = unixToNativePath(p.key) + config.arguments = cmdLineRest(p) + result = true + inc argsCount + proc cmdLine(pass: TCmdLinePass, cmd: openArray[string]; config: ConfigRef) = ## parse the command-line into the config var p = initOptParser(cmd) @@ -85,7 +110,7 @@ proc helpOnError(config: ConfigRef) = Usage = """ dust [options] [projectfile] - Options: Same options that the Nim compiler supports. + Options: Same options that the Nimskull compiler supports. """ showMsg(config, Usage) msgQuit 0 diff --git a/tools/dust/dust.nim b/tools/dust/dust.nim index 7ed55ab43f2..68764c47d79 100644 --- a/tools/dust/dust.nim +++ b/tools/dust/dust.nim @@ -20,7 +20,8 @@ import compiler / utils / [ astrepr, pathutils, ] # legacy reports stupidity -from compiler/ast/reports import Report, location, kind +from compiler / ast / reports import Report, location, kind +from compiler / front / cli_reporter import reportFull import std/options as std_options # due to legacy reports stupidity @@ -71,7 +72,8 @@ proc dust*(filename: AbsoluteFile) = proc uhoh(config: ConfigRef, rep: Report): TErrorHandling = ## capture the first error if config.severity(rep) == rsevError: - if std_options.unsafeGet(rep.location).fileIndex == config.projectMainIdx: + if std_options.isSome(rep.location) and + std_options.unsafeGet(rep.location).fileIndex == config.projectMainIdx: if errorKind == repNone: errorKind = rep.kind elif errorKind == rep.kind: diff --git a/tools/dust/mutate.nim b/tools/dust/mutate.nim index 66f1d23a16f..07de3b2fd72 100644 --- a/tools/dust/mutate.nim +++ b/tools/dust/mutate.nim @@ -23,7 +23,7 @@ proc rebuild*(n: PNode; fn: Transformer): PNode = if result == n: result = shallowCopy n if n.kind in nkWithSons: - result.setLen 0 + result.sons.setLen 0 for child in items(n): let rebuilt = rebuild(child, fn) if not rebuilt.isNil: diff --git a/tools/dust/pass.nim b/tools/dust/pass.nim deleted file mode 100644 index 5d87963f283..00000000000 --- a/tools/dust/pass.nim +++ /dev/null @@ -1,38 +0,0 @@ -import - compiler / ast / [ lineinfos, ast, astalgo ] - compiler / front / options, - compiler / modules / modulegraphs, - compiler / sem / passes - -import dust/spec -import dust/mutate -import dust/hashing - -var remains*: Remains - -proc opener(graph: ModuleGraph; module: PSym): PPassContext {.nosinks.} = - ## the opener learns when we're compiling the test file - result = DustContext(mainIndex: graph.config.projectMainIdx) - -when false: - proc closer(graph: ModuleGraph; context: PPassContext, n: PNode): PNode = - ## the closer don't do shit - discard - -proc rewriter(context: PPassContext, n: PNode): PNode {.nosinks.} = - template c: DustContext = DustContext(context) - # if this isn't the main project, - if n.info.fileIndex != c.mainIndex: - # yield the original node - result = n - else: - if c.ignore: - # we already yielded the program, so just return nil - result = nil - else: - # yield the next available permutation (once) - result = pop(remains) - c.ignore = true - -const - dustPass* = makePass(opener, rewriter, nil) \ No newline at end of file diff --git a/tools/dust/spec.nim b/tools/dust/spec.nim index f860d3e7997..cdd4db2468e 100644 --- a/tools/dust/spec.nim +++ b/tools/dust/spec.nim @@ -68,11 +68,4 @@ proc next*(remains: Remains): PNode = if len(remains) > 0: result = remains.attempts[^1].node -proc massageMessage*(s: string): string = - result = s.splitLines()[0] - for c in [';', ':']: - let i = find(result, c) - if i != -1: - result = result[0 .. i - 1] - export `$` \ No newline at end of file From 79b09959385875171c5d10c4fa01eab6527fc311 Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Tue, 31 Oct 2023 12:21:03 -0700 Subject: [PATCH 11/12] slap together an initial test --- tools/dust/tests/mminimizeme.nim | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 tools/dust/tests/mminimizeme.nim diff --git a/tools/dust/tests/mminimizeme.nim b/tools/dust/tests/mminimizeme.nim new file mode 100644 index 00000000000..7c59ee78f17 --- /dev/null +++ b/tools/dust/tests/mminimizeme.nim @@ -0,0 +1,6 @@ +let thisIsFine = 1 + +proc foo() = + result = 10 # this is not fine + +echo foo(), " ", thisIsFine \ No newline at end of file From 51c744378e8d81fd5ab26d198dfb90ca97dc8255 Mon Sep 17 00:00:00 2001 From: Saem Ghani Date: Thu, 18 Jan 2024 15:38:54 -0800 Subject: [PATCH 12/12] dust mostly works the problem is that right now we generate more and more errors over time, so the minimization process diverges, rather than converging. --- compiler/front/options.nim | 8 +++++--- compiler/sem/semstmts.nim | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/compiler/front/options.nim b/compiler/front/options.nim index 9755253bdab..43d43bc6902 100644 --- a/compiler/front/options.nim +++ b/compiler/front/options.nim @@ -555,9 +555,11 @@ proc getReportHook*(conf: ConfigRef): ReportHook = proc report*(conf: ConfigRef, inReport: Report): TErrorHandling = ## Write `inReport` assert inReport.kind != repNone, "Cannot write out empty report" - assert(conf.structuredReportHook != nil, - "Cannot write report with empty report hook") - return conf.structuredReportHook(conf, inReport) + # TODO: use a no-op structuredHook in dust instead of nil + # assert(conf.structuredReportHook != nil, + # "Cannot write report with empty report hook") + if conf.structuredReportHook != nil: + return conf.structuredReportHook(conf, inReport) proc canReport*(conf: ConfigRef, id: NodeId): bool = ## Check whether report with given ID can actually be written out, or it diff --git a/compiler/sem/semstmts.nim b/compiler/sem/semstmts.nim index d17ec6bbf63..5548725930b 100644 --- a/compiler/sem/semstmts.nim +++ b/compiler/sem/semstmts.nim @@ -1424,6 +1424,7 @@ proc symForVar(c: PContext, n: PNode): PSym = let hasPragma = n.kind == nkPragmaExpr resultNode = newSymGNode(skForVar, (if hasPragma: n[0] else: n), c) + semmedNode = if hasPragma: copyNodeWithKids(n) else: resultNode result = getDefNameSymOrRecover(resultNode) styleCheckDef(c.config, result) @@ -1431,13 +1432,14 @@ proc symForVar(c: PContext, n: PNode): PSym = if hasPragma: let pragma = pragmaDecl(c, result, n[1], forVarPragmas) if pragma.kind == nkError: - n[1] = pragma + semmedNode[0] = resultNode + semmedNode[1] = pragma if resultNode.kind == nkError or hasPragma and n[1].kind == nkError: result = newSym(skError, result.name, nextSymId(c.idgen), result.owner, n.info) result.typ = c.errorType - result.ast = c.config.wrapError(n) + result.ast = c.config.wrapError(semmedNode) proc semSingleForVar(c: PContext, formal: PType, view: ViewTypeKind, n: PNode): PNode = ## Semantically analyses a single definition of a variable in the context of @@ -1446,7 +1448,7 @@ proc semSingleForVar(c: PContext, formal: PType, view: ViewTypeKind, n: PNode): let v = symForVar(c, n) if v.kind == skError: - return c.config.wrapError(n) + return v.ast if getCurrOwner(c).kind == skModule: incl(v.flags, sfGlobal)