From 570f141438888db2bb770062646c0d0d315b5003 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 21 Jan 2025 06:16:35 +0100 Subject: [PATCH] Infrastructure for `enso --docs` option & signature generator (#10291) --- .../main/scala/org/enso/polyglot/Module.scala | 8 - .../scala/org/enso/polyglot/TopScope.scala | 23 +- .../src/main/java/org/enso/runner/Main.java | 143 ++++--- .../java/org/enso/runner/EngineMainTest.java | 7 +- .../org/enso/compiler/dump/DocsDispatch.java | 53 +++ .../enso/compiler/dump/DocsEmitMarkdown.java | 61 +++ .../compiler/dump/DocsEmitSignatures.java | 63 +++ .../org/enso/compiler/dump/DocsGenerate.java | 118 ++++++ .../org/enso/compiler/dump/DocsUtils.java | 155 +++++++ .../org/enso/compiler/dump/DocsVisit.java | 85 ++++ .../compiler/phase/ImportResolverForIR.java | 2 +- .../scala/org/enso/compiler/Compiler.scala | 325 +++++++++------ .../enso/compiler/phase/ImportResolver.scala | 2 +- .../compiler/dump/test/DocsGenerateTest.java | 392 ++++++++++++++++++ .../enso/compiler/test/SerdeCompilerTest.java | 2 +- .../test/SerializationManagerTest.java | 2 +- .../org/enso/interpreter/runtime/Module.java | 5 +- .../runtime/scope/TopLevelScope.java | 49 ++- .../org/enso/test/utils/ProjectUtils.java | 32 ++ 19 files changed, 1287 insertions(+), 240 deletions(-) create mode 100644 engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsDispatch.java create mode 100644 engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsEmitMarkdown.java create mode 100644 engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsEmitSignatures.java create mode 100644 engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsGenerate.java create mode 100644 engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsUtils.java create mode 100644 engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsVisit.java create mode 100644 engine/runtime-integration-tests/src/test/java/org/enso/compiler/dump/test/DocsGenerateTest.java diff --git a/engine/polyglot-api/src/main/scala/org/enso/polyglot/Module.scala b/engine/polyglot-api/src/main/scala/org/enso/polyglot/Module.scala index 8ac6aadf4c27..05ad055ad287 100644 --- a/engine/polyglot-api/src/main/scala/org/enso/polyglot/Module.scala +++ b/engine/polyglot-api/src/main/scala/org/enso/polyglot/Module.scala @@ -44,14 +44,6 @@ class Module(private val value: Value) { def evalExpression(code: String): Value = value.invokeMember(EVAL_EXPRESSION, code) - /** Triggers generation of documentation from module sources. - * - * @return value with `GENERATE_DOCS` invoked on it. - */ - def generateDocs(): Value = { - value.invokeMember(GENERATE_DOCS) - } - /** Triggers gathering of import statements from module sources. * * @return value with `GATHER_IMPORT_STATEMENTS` invoked on it. diff --git a/engine/polyglot-api/src/main/scala/org/enso/polyglot/TopScope.scala b/engine/polyglot-api/src/main/scala/org/enso/polyglot/TopScope.scala index 3b3150f7376d..afedd4d1e856 100644 --- a/engine/polyglot-api/src/main/scala/org/enso/polyglot/TopScope.scala +++ b/engine/polyglot-api/src/main/scala/org/enso/polyglot/TopScope.scala @@ -34,7 +34,26 @@ class TopScope(private val value: Value) { value.invokeMember(UNREGISTER_MODULE, qualifiedName): Unit } - def compile(shouldCompileDependencies: Boolean): Unit = { - value.invokeMember(COMPILE, shouldCompileDependencies) + def compile( + shouldCompileDependencies: Boolean + ): Unit = { + compile(shouldCompileDependencies, None) } + def compile( + shouldCompileDependencies: Boolean, + generateDocs: Option[String] + ): Unit = { + val docsArg = generateDocs.map { + case "api" => "api" + case "md" => "md" + case other => + throw new IllegalStateException("Invalid docs format: " + other) + } + value.invokeMember( + COMPILE, + shouldCompileDependencies, + docsArg.getOrElse(false) + ) + } + } diff --git a/engine/runner/src/main/java/org/enso/runner/Main.java b/engine/runner/src/main/java/org/enso/runner/Main.java index d7054f698bf2..1f20840d329a 100644 --- a/engine/runner/src/main/java/org/enso/runner/Main.java +++ b/engine/runner/src/main/java/org/enso/runner/Main.java @@ -146,8 +146,13 @@ private static Options buildOptions() { .build(); var docs = cliOptionBuilder() + .hasArg(true) + .numberOfArgs(1) + .optionalArg(true) .longOpt(DOCS_OPTION) - .desc("Runs the Enso documentation generator.") + .desc( + "Runs the Enso documentation generator. Additional argument may specify format -" + + " either the default `md` or `api`.") .build(); var preinstall = cliOptionBuilder() @@ -531,8 +536,16 @@ void printHelp() { new HelpFormatter().printHelp(LanguageInfo.ID, CLI_OPTIONS); } - /** Terminates the process with a failure exit code. */ - private RuntimeException exitFail() { + /** + * Terminates the process with a failure exit code. + * + * @param error the error message to send to stderr before terminating the process, can be {@code + * null} + */ + private RuntimeException exitFail(String error) { + if (error != null) { + stderr(error); + } return doExit(1); } @@ -588,8 +601,7 @@ private void createNew( Template.fromString(n) .getOrElse( () -> { - logger.error("Unknown project template name: '" + n + "'."); - throw exitFail(); + throw exitFail("Unknown project template name: '" + n + "'."); })); PackageManager$.MODULE$ @@ -632,8 +644,7 @@ private void compile( boolean logMasking) { var file = new File(packagePath); if (!file.exists() || !file.isDirectory()) { - println("No package exists at " + file + "."); - throw exitFail(); + throw exitFail("No package exists at " + file + "."); } var context = @@ -652,11 +663,11 @@ private void compile( var topScope = context.getTopScope(); try { - topScope.compile(shouldCompileDependencies); + topScope.compile(shouldCompileDependencies, scala.Option.empty()); throw exitSuccess(); } catch (Throwable t) { logger.error("Unexpected internal error", t); - throw exitFail(); + throw exitFail("Unexpected internal error"); } finally { context.context().close(); } @@ -697,7 +708,7 @@ private void handleRun( throws IOException { var fileAndProject = Utils.findFileAndProject(path, projectPath); if (fileAndProject == null) { - throw exitFail(); + throw exitFail("Cannot find " + path + " and " + projectPath); } var projectMode = fileAndProject._1(); var file = fileAndProject._2(); @@ -709,12 +720,10 @@ private void handleRun( mainFile = pkg.mainFile(); if (!mainFile.exists()) { - println("Main file does not exist."); - throw exitFail(); + throw exitFail("Main file does not exist."); } } else { - println(result.failed().get().getMessage()); - throw exitFail(); + throw exitFail(result.failed().get().getMessage()); } } @@ -737,8 +746,7 @@ private void handleRun( if (inspect) { if (enableDebugServer) { - println("Cannot use --inspect and --repl and --run at once"); - throw exitFail(); + throw exitFail("Cannot use --inspect and --repl and --run at once"); } options.put("inspect", ""); } @@ -757,8 +765,7 @@ private void handleRun( var mainModuleName = pkg.moduleNameForFile(pkg.mainFile()).toString(); runPackage(context, mainModuleName, file, additionalArgs); } else { - System.err.println(result.failed().get().getMessage()); - throw exitFail(); + throw exitFail(result.failed().get().getMessage()); } } else { runSingleFile(context, file, additionalArgs); @@ -785,12 +792,15 @@ private void handleRun( * @param enableIrCaches are the IR caches enabled */ private void genDocs( - String projectPath, Level logLevel, boolean logMasking, boolean enableIrCaches) { - if (projectPath.isEmpty()) { - println("Path hasn't been provided."); - throw exitFail(); + String docsFormat, + String projectPath, + Level logLevel, + boolean logMasking, + boolean enableIrCaches) { + if (projectPath == null || projectPath.isEmpty()) { + throw exitFail("Specify path to a project with --in-project option"); } - generateDocsFrom(projectPath, logLevel, logMasking, enableIrCaches); + generateDocsFrom(docsFormat, projectPath, logLevel, logMasking, enableIrCaches); throw exitSuccess(); } @@ -799,7 +809,7 @@ private void genDocs( * path. */ private void generateDocsFrom( - String path, Level logLevel, boolean logMasking, boolean enableIrCaches) { + String docsFormat, String path, Level logLevel, boolean logMasking, boolean enableIrCaches) { var executionContext = new PolyglotContext( ContextFactory.create() @@ -816,17 +826,8 @@ private void generateDocsFrom( var main = pkg.map(x -> x.mainFile()); if (main.exists(x -> x.exists())) { - var mainFile = main.get(); - var mainModuleName = pkg.get().moduleNameForFile(mainFile).toString(); var topScope = executionContext.getTopScope(); - var mainModule = topScope.getModule(mainModuleName); - var generated = mainModule.generateDocs(); - println(generated.toString()); - - // TODO: - // - go through executed code and get all HTML docs - // with their corresponding atoms/methods etc. - // - Save those to files + topScope.compile(false, scala.Option.apply(docsFormat == null ? "md" : docsFormat)); } } @@ -840,15 +841,14 @@ private void generateDocsFrom( */ private void preinstallDependencies(String projectPath, Level logLevel) { if (projectPath == null) { - println("Dependency installation is only available for projects."); - throw exitFail(); + throw exitFail("Dependency installation is only available for projects."); } try { DependencyPreinstaller.preinstallDependencies(new File(projectPath), logLevel); throw exitSuccess(); } catch (RuntimeException error) { logger.error("Dependency installation failed: " + error.getMessage(), error); - throw exitFail(); + throw exitFail("Dependency installation failed: " + error.getMessage()); } } @@ -878,12 +878,11 @@ private void runMain( var mainType = mainModule.getAssociatedType(); var mainFun = mainModule.getMethod(mainType, mainMethodName); if (mainFun.isEmpty()) { - System.err.println( + throw exitFail( "The module " + mainModule.getName() + " does not contain a `main` " + "function. It could not be run."); - throw exitFail(); } var main = mainFun.get(); if (!DEFAULT_MAIN_METHOD_NAME.equals(mainMethodName)) { @@ -909,7 +908,7 @@ private void runMain( var res = main.execute(listOfArgs.reverse()); if (!res.isNull()) { var textRes = res.isString() ? res.asString() : res.toString(); - println(textRes); + stdout(textRes); if (res.isException()) { try { throw res.throwException(); @@ -926,7 +925,7 @@ private void runMain( throw doExit(e.getExitStatus()); } else { printPolyglotException(e, rootPkgPath); - throw exitFail(); + throw exitFail(e.getMessage()); } } } @@ -1004,7 +1003,7 @@ private void displayVersion(boolean useJson) { var customVersion = CurrentVersion.getVersion().toString(); var versionDescription = VersionDescription.make("Enso Compiler and Runtime", true, false, List.of(), customVersion); - println(versionDescription.asString(useJson)); + stdout(versionDescription.asString(useJson)); } /** Parses the log level option. */ @@ -1017,8 +1016,7 @@ private Level parseLogLevel(String levelOption) { Stream.of(Level.values()) .map(x -> x.toString().toLowerCase()) .collect(Collectors.joining(", ")); - System.err.println("Invalid log level. Possible values are " + possible + "."); - throw exitFail(); + throw exitFail("Invalid log level. Possible values are " + possible + "."); } else { return found.get(); } @@ -1029,8 +1027,7 @@ private URI parseUri(String string) { try { return new URI(string); } catch (URISyntaxException ex) { - System.err.println("`" + string + "` is not a valid URI."); - throw exitFail(); + throw exitFail("`" + string + "` is not a valid URI."); } } @@ -1079,12 +1076,11 @@ final void mainEntry(CommandLine line, Level logLevel, boolean logMasking) throw .map(x -> Path.of(x)) .getOrElse( () -> { - logger.error( + throw exitFail( "When uploading, the " + IN_PROJECT_OPTION + " is mandatory " + "to specify which project to upload."); - throw exitFail(); }); try { @@ -1098,7 +1094,7 @@ final void mainEntry(CommandLine line, Level logLevel, boolean logMasking) throw } catch (UploadFailedError ex) { // We catch this error to avoid printing an unnecessary stack trace. // The error itself is already logged. - throw exitFail(); + throw exitFail(ex.getMessage()); } } @@ -1108,14 +1104,13 @@ final void mainEntry(CommandLine line, Level logLevel, boolean logMasking) throw .map(x -> Path.of(x)) .getOrElse( () -> { - logger.error("The " + IN_PROJECT_OPTION + " is mandatory."); - throw exitFail(); + throw exitFail("The " + IN_PROJECT_OPTION + " is mandatory."); }); try { ProjectUploader.updateManifest(projectRoot, logLevel); } catch (Throwable err) { err.printStackTrace(); - throw exitFail(); + throw exitFail(err.getMessage()); } throw exitSuccess(); } @@ -1164,14 +1159,18 @@ final void mainEntry(CommandLine line, Level logLevel, boolean logMasking) throw } if (line.hasOption(DOCS_OPTION)) { genDocs( - line.getOptionValue(IN_PROJECT_OPTION), logLevel, logMasking, shouldEnableIrCaches(line)); + line.getOptionValue(DOCS_OPTION), + line.getOptionValue(IN_PROJECT_OPTION), + logLevel, + logMasking, + shouldEnableIrCaches(line)); } if (line.hasOption(PREINSTALL_OPTION)) { preinstallDependencies(line.getOptionValue(IN_PROJECT_OPTION), logLevel); } if (line.getOptions().length == 0) { printHelp(); - throw exitFail(); + throw exitFail(null); } } @@ -1253,8 +1252,8 @@ protected Map parseSystemProperties(CommandLine cmdLine) { } else if (items.length == 1) { props.put(items[0], "true"); } else { - println("Argument to " + SYSTEM_PROPERTY + " must be in the form ="); - throw exitFail(); + throw exitFail( + "Argument to " + SYSTEM_PROPERTY + " must be in the form ="); } } return props; @@ -1298,7 +1297,7 @@ private void printPolyglotException(PolyglotException exception, File relativeTo exception.isSyntaxError(), msg, relativeTo, - this::println, + this::stderr, fnLangId, fnRootName, fnSourceSection); @@ -1314,10 +1313,14 @@ private static final scala.collection.immutable.List join( return scala.collection.immutable.$colon$colon$.MODULE$.apply(head, tail); } - void println(String msg) { + void stdout(String msg) { System.out.println(msg); } + void stderr(String msg) { + System.err.println(msg); + } + private void launch(String[] args) throws IOException, InterruptedException, URISyntaxException { var line = preprocessArguments(args); @@ -1339,14 +1342,13 @@ private void launch(String[] args) throws IOException, InterruptedException, URI } var shouldLaunchJvm = current == null || !current.equals(jvm); if (!shouldLaunchJvm) { - println(JVM_OPTION + " option has no effect - already running in JVM " + current); + stderr(JVM_OPTION + " option has no effect - already running in JVM " + current); } else { var commandAndArgs = new ArrayList(); if (jvm == null) { var javaExe = JavaFinder.findJavaExecutable(); if (javaExe == null) { - println("Cannot find java executable"); - throw exitFail(); + throw exitFail("Cannot find java executable"); } commandAndArgs.add(javaExe); } else { @@ -1432,9 +1434,8 @@ final CommandLine preprocessArguments(String... args) { System.currentTimeMillis() - startParsing); return line; } catch (Exception e) { - println(e.getMessage()); printHelp(); - throw exitFail(); + throw exitFail(e.getMessage()); } } @@ -1461,12 +1462,11 @@ private void launch(CommandLine line, Level logLevel, boolean logMasking) { LanguageServerApi.launchLanguageServer(line, conf, logLevel); throw exitSuccess(); } catch (WrongOption e) { - System.err.println(e.getMessage()); - throw exitFail(); + throw exitFail(e.getMessage()); } } else { if (line.hasOption(LANGUAGE_SERVER_NATIVE_OPTION)) { - System.out.println( + stderr( "\"--" + LANGUAGE_SERVER_NATIVE_OPTION + "\" has no effect without --\"" @@ -1487,12 +1487,10 @@ private void launch(CommandLine line, Level logLevel, boolean logMasking) { if (logger.isDebugEnabled()) { logger.error("Error during execution", ex); } - System.out.println("Command failed with an error: " + ex); - throw exitFail(); + throw exitFail("Command failed with an error: " + ex.getMessage()); } } catch (WrongOption e) { - System.err.println(e.getMessage()); - throw exitFail(); + throw exitFail(e.getMessage()); } } } @@ -1508,7 +1506,7 @@ protected String getLanguageId() { * @param dir directory with other files that should be older than base * @return */ - private static boolean checkOutdatedLauncher(File base, File dir) { + private boolean checkOutdatedLauncher(File base, File dir) { var needsCheck = base.canExecute(); if (needsCheck) { var files = dir.listFiles(); @@ -1516,8 +1514,7 @@ private static boolean checkOutdatedLauncher(File base, File dir) { var baseTime = base.lastModified(); for (var f : files) { if (baseTime < f.lastModified()) { - System.err.println( - "File " + base + " is older than " + f + " consider running in --jvm mode"); + stderr("File " + base + " is older than " + f + " consider running in --jvm mode"); return false; } } diff --git a/engine/runner/src/test/java/org/enso/runner/EngineMainTest.java b/engine/runner/src/test/java/org/enso/runner/EngineMainTest.java index 91d42c609f40..226594f05c6b 100644 --- a/engine/runner/src/test/java/org/enso/runner/EngineMainTest.java +++ b/engine/runner/src/test/java/org/enso/runner/EngineMainTest.java @@ -100,7 +100,12 @@ RuntimeException doExit(int exitCode) { } @Override - void println(String msg) { + void stdout(String msg) { + linesOut.add(msg); + } + + @Override + void stderr(String msg) { linesOut.add(msg); } diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsDispatch.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsDispatch.java new file mode 100644 index 000000000000..c253917837d0 --- /dev/null +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsDispatch.java @@ -0,0 +1,53 @@ +package org.enso.compiler.dump; + +import java.io.IOException; +import org.enso.compiler.core.ir.Module; +import org.enso.compiler.core.ir.module.scope.Definition; +import org.enso.compiler.core.ir.module.scope.definition.Method; +import org.enso.pkg.QualifiedName; + +/** + * Class to use from {@link DocsGenerate} to dispatch individual IR elements to provided visitor. + */ +abstract class DocsDispatch { + static DocsDispatch create(DocsVisit visitor, java.io.PrintWriter writer) { + return new DocsDispatch() { + @Override + boolean dispatchModule(QualifiedName name, Module ir) throws IOException { + return visitor.visitModule(name, ir, writer); + } + + @Override + void dispatchMethod(Definition.Type t, Method.Explicit m) throws IOException { + visitor.visitMethod(t, m, writer); + } + + @Override + void dispatchConversion(Method.Conversion c) throws IOException { + visitor.visitConversion(c, writer); + } + + @Override + boolean dispatchType(Definition.Type t) throws IOException { + return visitor.visitType(t, writer); + } + + @Override + void dispatchConstructor(Definition.Type t, Definition.Data d) throws IOException { + visitor.visitConstructor(t, d, writer); + } + }; + } + + private DocsDispatch() {} + + abstract boolean dispatchModule(QualifiedName name, Module ir) throws IOException; + + abstract void dispatchMethod(Definition.Type t, Method.Explicit m) throws IOException; + + abstract void dispatchConversion(Method.Conversion c) throws IOException; + + abstract boolean dispatchType(Definition.Type t) throws IOException; + + abstract void dispatchConstructor(Definition.Type t, Definition.Data d) throws IOException; +} diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsEmitMarkdown.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsEmitMarkdown.java new file mode 100644 index 000000000000..fa16256b150c --- /dev/null +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsEmitMarkdown.java @@ -0,0 +1,61 @@ +package org.enso.compiler.dump; + +import java.io.IOException; +import java.io.PrintWriter; +import org.enso.compiler.core.IR; +import org.enso.compiler.core.ir.Module; +import org.enso.compiler.core.ir.module.scope.Definition; +import org.enso.compiler.core.ir.module.scope.definition.Method; +import org.enso.compiler.pass.resolve.DocumentationComments; +import org.enso.compiler.pass.resolve.DocumentationComments$; +import org.enso.pkg.QualifiedName; + +/** Visitor that emits documentation in markdown format. */ +final class DocsEmitMarkdown implements DocsVisit { + + @Override + public boolean visitUnknown(IR ir, PrintWriter w) { + return true; + } + + @Override + public boolean visitModule(QualifiedName name, Module module, PrintWriter w) throws IOException { + w.println("## Documentation for " + name); + writeDocs(module, w); + return true; + } + + @Override + public void visitMethod(Definition.Type t, Method.Explicit m, PrintWriter w) throws IOException { + w.println("#### method " + m.methodName().name()); + writeDocs(m, w); + } + + @Override + public void visitConversion(Method.Conversion c, PrintWriter w) throws IOException { + w.println("#### conversion " + c.methodName().name()); + writeDocs(c, w); + } + + private void writeDocs(IR b, PrintWriter w) throws IOException { + var option = b.passData().get(DocumentationComments$.MODULE$); + if (option.isDefined()) { + var doc = (DocumentationComments.Doc) option.get(); + w.println(doc.documentation()); + w.println(); + w.println(); + } + } + + @Override + public boolean visitType(Definition.Type t, PrintWriter w) throws IOException { + w.println("#### **type** " + t.name().name()); + return true; + } + + @Override + public void visitConstructor(Definition.Type t, Definition.Data d, PrintWriter w) + throws IOException { + w.println("#### data " + d.name().name()); + } +} diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsEmitSignatures.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsEmitSignatures.java new file mode 100644 index 000000000000..1ac5e2124a94 --- /dev/null +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsEmitSignatures.java @@ -0,0 +1,63 @@ +package org.enso.compiler.dump; + +import static org.enso.scala.wrapper.ScalaConversions.asJava; + +import java.io.IOException; +import java.io.PrintWriter; +import org.enso.compiler.core.IR; +import org.enso.compiler.core.ir.Module; +import org.enso.compiler.core.ir.module.scope.Definition; +import org.enso.compiler.core.ir.module.scope.definition.Method; +import org.enso.pkg.QualifiedName; + +/** Visitor that emits documentation in markdown format. */ +final class DocsEmitSignatures implements DocsVisit { + + @Override + public boolean visitUnknown(IR ir, PrintWriter w) throws IOException { + w.println("- Unknown IR " + ir.getClass().getName()); + return true; + } + + @Override + public boolean visitModule(QualifiedName name, Module module, PrintWriter w) throws IOException { + w.println("## Enso Signatures 1.0"); + w.println("## module " + name); + return true; + } + + @Override + public void visitMethod(Definition.Type t, Method.Explicit m, PrintWriter w) throws IOException { + if (t != null) { + w.append(" - "); + } else { + if (m.typeName().isDefined()) { + var fqn = DocsUtils.toFqnOrSimpleName(m.typeName().get()); + w.append(fqn + "."); + } + } + w.println(DocsVisit.toSignature(m)); + } + + @Override + public void visitConversion(Method.Conversion c, PrintWriter w) throws IOException { + w.println("#### conversion " + c.methodName().name()); + } + + @Override + public boolean visitType(Definition.Type t, PrintWriter w) throws IOException { + var sb = new StringBuilder(); + sb.append("- type ").append(t.name().name()); + for (var a : asJava(t.params())) { + sb.append(" ").append(DocsVisit.toSignature(a)); + } + w.println(sb.toString()); + return true; + } + + @Override + public void visitConstructor(Definition.Type t, Definition.Data d, PrintWriter w) + throws IOException { + w.println(" - " + DocsVisit.toSignature(d)); + } +} diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsGenerate.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsGenerate.java new file mode 100644 index 000000000000..bf27672d0a28 --- /dev/null +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsGenerate.java @@ -0,0 +1,118 @@ +package org.enso.compiler.dump; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.IdentityHashMap; +import org.enso.compiler.context.CompilerContext; +import org.enso.compiler.core.IR; +import org.enso.compiler.core.ir.Module; +import org.enso.compiler.core.ir.module.scope.Definition; +import org.enso.compiler.core.ir.module.scope.definition.Method; +import org.enso.filesystem.FileSystem; +import org.enso.pkg.QualifiedName; +import scala.collection.immutable.Seq; +import scala.jdk.CollectionConverters; + +/** Generator of documentation for an Enso project. */ +public final class DocsGenerate { + private DocsGenerate() {} + + /** + * Iterate over all provide modules and generate documentation using {@code pkg}'s {@link + * FileSystem}. + * + * @param abstract file to operate with + * @param visitor visitor to use to generate the output + * @param pkg library to generate the documentation for + * @param modules parsed modules found in the library + * @return directory where the output was generated + * @throws IOException when I/O problem occurs + */ + public static File write( + DocsVisit visitor, org.enso.pkg.Package pkg, Iterable modules) + throws IOException { + var fs = pkg.fileSystem(); + var docs = fs.getChild(pkg.root(), "docs"); + var api = fs.getChild(docs, "api"); + fs.createDirectories(api); + + for (var module : modules) { + var ir = module.getIr(); + assert ir != null : "need IR for " + module; + if (ir.isPrivate()) { + continue; + } + var moduleName = module.getName(); + var dir = createPkg(fs, api, moduleName); + var md = fs.getChild(dir, moduleName.item() + ".md"); + try (var mdWriter = fs.newBufferedWriter(md); + var pw = new PrintWriter(mdWriter)) { + visitModule(visitor, moduleName, ir, pw); + } + } + return api; + } + + private static File createPkg(FileSystem fs, File root, QualifiedName pkg) + throws IOException { + var dir = root; + for (var item : pkg.pathAsJava()) { + dir = fs.getChild(dir, item); + } + fs.createDirectories(dir); + return dir; + } + + public static void visitModule( + DocsVisit visitor, QualifiedName moduleName, Module ir, PrintWriter w) throws IOException { + var dispatch = DocsDispatch.create(visitor, w); + + if (dispatch.dispatchModule(moduleName, ir)) { + var moduleBindings = asJava(ir.bindings()); + var alreadyDispatched = new IdentityHashMap(); + for (var b : moduleBindings) { + if (alreadyDispatched.containsKey(b)) { + continue; + } + switch (b) { + case Definition.Type t -> { + if (dispatch.dispatchType(t)) { + for (var d : asJava(t.members())) { + if (!d.isPrivate()) { + dispatch.dispatchConstructor(t, d); + } + } + for (var mb : moduleBindings) { + if (mb instanceof Method.Explicit m) { + if (m.isStaticWrapperForInstanceMethod() || m.isPrivate()) { + alreadyDispatched.put(m, m); + continue; + } + var p = m.methodReference().typePointer(); + if (p.isDefined()) { + var methodTypeName = p.get().name(); + if (methodTypeName.equals(t.name().name())) { + dispatch.dispatchMethod(t, m); + alreadyDispatched.put(m, m); + } + } + } + } + } + } + case Method.Explicit m -> { + if (!m.isPrivate()) { + dispatch.dispatchMethod(null, m); + } + } + case Method.Conversion c -> dispatch.dispatchConversion(c); + default -> throw new AssertionError("unknown type " + b.getClass()); + } + } + } + } + + private static Iterable asJava(Seq seq) { + return CollectionConverters.IterableHasAsJava(seq).asJava(); + } +} diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsUtils.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsUtils.java new file mode 100644 index 000000000000..931f1c2ac823 --- /dev/null +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsUtils.java @@ -0,0 +1,155 @@ +package org.enso.compiler.dump; + +import static org.enso.scala.wrapper.ScalaConversions.asJava; + +import java.util.ArrayList; +import java.util.List; +import org.enso.compiler.core.ConstantsNames; +import org.enso.compiler.core.IR; +import org.enso.compiler.core.ir.DefinitionArgument; +import org.enso.compiler.core.ir.Expression; +import org.enso.compiler.core.ir.Function.Lambda; +import org.enso.compiler.core.ir.Name; +import org.enso.compiler.core.ir.expression.Application; +import org.enso.compiler.core.ir.module.scope.Definition; +import org.enso.compiler.core.ir.module.scope.definition.Method; +import org.enso.compiler.core.ir.type.Set; +import org.enso.compiler.data.BindingsMap; +import org.enso.compiler.pass.resolve.MethodDefinitions; +import org.enso.compiler.pass.resolve.TypeNames$; +import org.enso.compiler.pass.resolve.TypeSignatures; +import org.enso.compiler.pass.resolve.TypeSignatures$; +import org.enso.pkg.QualifiedName; + +final class DocsUtils { + private static final String ANY = "Standard.Base.Any.Any"; + + private DocsUtils() {} + + static String toSignature(Method.Explicit m) { + var sb = new StringBuilder(); + sb.append(m.methodName().name()); + if (m.body() instanceof Lambda fn) { + var first = m.isStatic(); + for (var a : asJava(fn.arguments())) { + if (first) { + first = false; + continue; + } + sb.append(" ").append(toSignature(a)); + } + var ret = extractTypeOrAny(fn.body()); + sb.append(" -> ").append(ret); + } + return sb.toString(); + } + + static String toSignature(Definition.Data d) { + var sb = new StringBuilder(); + sb.append(d.name().name()); + for (var a : asJava(d.arguments())) { + sb.append(" ").append(toSignature(a)); + } + return sb.toString(); + } + + static String toFqnOrSimpleName(Name ir) { + var typeNameOpt = ir.passData().get(MethodDefinitions.INSTANCE); + if (typeNameOpt.isDefined()) { + var typeName = (BindingsMap.Resolution) typeNameOpt.get(); + return typeName.target().qualifiedName().toString(); + } + return ir.name(); + } + + static String toSignature(DefinitionArgument a) { + var sb = new StringBuilder(); + if (a.suspended()) { + sb.append("~"); + } + var name = a.name().name(); + sb.append(name); + if (!ConstantsNames.SELF_ARGUMENT.equals(name)) { + var type = extractTypeOrAny(a); + sb.append(":").append(type); + } + if (a.defaultValue().isDefined()) { + sb.append("="); + } + return sb.toString(); + } + + private static String extractTypeOrAny(IR ir) { + var meta = ir.passData().get(TypeSignatures$.MODULE$); + if (meta.isDefined()) { + var sigMeta = (TypeSignatures.Signature) meta.get(); + var sigFqn = extractFqnOrNull(sigMeta.signature()); + if (sigFqn != null) { + return sigFqn.toString(); + } + var type = + switch (sigMeta.signature()) { + case Application.Prefix app -> { + var typeConstructor = extractFqnOrNull(app.function()); + if (typeConstructor == null) { + yield null; + } + var sb = new StringBuilder(); + sb.append("("); + sb.append(typeConstructor); + for (var a : asJava(app.arguments())) { + var fqn = extractTypeOrAny(a.value()); + sb.append(" "); + sb.append(fqn); + } + sb.append(")"); + yield sb.toString(); + } + case Set.Union union -> extractSet(asJava(union.operands()), "|"); + case Set.Intersection inter -> extractSet(collectInter(inter, new ArrayList<>()), "&"); + default -> ANY; + }; + return type; + } else { + if (ir instanceof Application.Force force) { + return extractTypeOrAny(force.target()); + } + var fqn = extractFqnOrNull(ir); + return fqn == null ? ANY : fqn.toString(); + } + } + + private static List collectInter(Expression ir, List append) { + if (ir instanceof Set.Intersection inter) { + var left = collectInter(inter.left(), append); + return collectInter(inter.right(), left); + } else { + append.add(ir); + return append; + } + } + + private static String extractSet(List operands, String sep) { + var sb = new StringBuilder(); + for (var op : operands) { + if (sb.isEmpty()) { + sb.append("("); + } else { + sb.append(sep); + } + var opType = extractTypeOrAny(op); + sb.append(opType); + } + sb.append(")"); + return sb.toString(); + } + + private static QualifiedName extractFqnOrNull(IR ir) { + var typeNameOpt = ir.passData().get(TypeNames$.MODULE$); + if (typeNameOpt.isDefined()) { + var typeName = (BindingsMap.Resolution) typeNameOpt.get(); + return typeName.target().qualifiedName(); + } + return null; + } +} diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsVisit.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsVisit.java new file mode 100644 index 000000000000..39d1463b3fdb --- /dev/null +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsVisit.java @@ -0,0 +1,85 @@ +package org.enso.compiler.dump; + +import java.io.IOException; +import java.io.PrintWriter; +import org.enso.compiler.core.IR; +import org.enso.compiler.core.ir.DefinitionArgument; +import org.enso.compiler.core.ir.Module; +import org.enso.compiler.core.ir.module.scope.Definition; +import org.enso.compiler.core.ir.module.scope.definition.Method; +import org.enso.pkg.QualifiedName; + +/** + * Visitor for processing documentation elements in a module. Pass into {@link + * DocsGenerate#visitModule}. This interface also includes various static methods to help working + * with the {@link IR}. + */ +public interface DocsVisit { + boolean visitModule(QualifiedName name, Module ir, PrintWriter writer) throws IOException; + + boolean visitUnknown(IR ir, PrintWriter w) throws IOException; + + void visitMethod(Definition.Type t, Method.Explicit m, PrintWriter writer) throws IOException; + + void visitConversion(Method.Conversion c, PrintWriter w) throws IOException; + + boolean visitType(Definition.Type t, PrintWriter w) throws IOException; + + void visitConstructor(Definition.Type t, Definition.Data d, PrintWriter w) throws IOException; + + // + // helper methods + // + + /** + * Converts a method into textual representation of its signature. + * + * @param method the method to process + * @return text representing the method name and its signature (if any) + */ + public static String toSignature(Method.Explicit method) { + return DocsUtils.toSignature(method); + } + + /** + * Converts a constructor into textual representation of its signature. + * + * @param cons the constructor to process + * @return text representing the constructor name and its signature (if any) + */ + public static String toSignature(Definition.Data cons) { + return DocsUtils.toSignature(cons); + } + + /** + * Converts an argument definition into textual representation of its signature. + * + * @param arg the argument to process + * @return text representing the argument + */ + public static String toSignature(DefinitionArgument arg) { + return DocsUtils.toSignature(arg); + } + + // + // Standard visitor implementations + // + + /** + * Generates markdown files with documentation content. + * + * @return new instance of visitor generating markdown documentation format + */ + public static DocsVisit createMarkdown() { + return new DocsEmitMarkdown(); + } + + /** + * Ignore comments, but generate fully qualified signatures of the visible elements + * + * @return new instance of visitor generating just signatures + */ + public static DocsVisit createSignatures() { + return new DocsEmitSignatures(); + } +} diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/phase/ImportResolverForIR.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/phase/ImportResolverForIR.java index 671be7c2435b..c041dfeda1c3 100644 --- a/engine/runtime-compiler/src/main/java/org/enso/compiler/phase/ImportResolverForIR.java +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/phase/ImportResolverForIR.java @@ -361,7 +361,7 @@ private BindingsMap loadBindingsMap(CompilerContext.Module mod) { u.ir(null); u.compilationStage(CompilationStage.INITIAL); }); - getCompiler().ensureParsed(mod, false); + getCompiler().ensureParsed(mod, false, false); bindingsMap = mod.getBindingsMap(); } return bindingsMap; diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala index 3484c211d670..a00d0cce24b6 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala @@ -1,11 +1,14 @@ package org.enso.compiler +import scala.jdk.CollectionConverters.IterableHasAsJava import org.enso.compiler.context.{ CompilerContext, FreshNameSupply, InlineContext, ModuleContext } +import org.enso.compiler.dump.DocsGenerate +import org.enso.compiler.dump.DocsVisit import org.enso.compiler.context.CompilerContext.Module import org.enso.compiler.core.CompilerError import org.enso.compiler.core.Implicits.AsMetadata @@ -139,12 +142,14 @@ class Compiler( * but no results will be written * @param useGlobalCacheLocations whether or not the compilation result should * be written to the global cache + * @param generateDocs should a documenation be generied * @return future to track subsequent serialization of the library */ def compile( shouldCompileDependencies: Boolean, shouldWriteCache: Boolean, - useGlobalCacheLocations: Boolean + useGlobalCacheLocations: Boolean, + generateDocs: Option[String] ): Future[java.lang.Boolean] = { getPackageRepository.getMainProjectPackage match { case None => @@ -186,13 +191,25 @@ class Compiler( shouldCompileDependencies ) + if (generateDocs.isDefined) { + val v = if (generateDocs.get == "api") { + DocsVisit.createSignatures() + } else { + DocsVisit.createMarkdown(); + } + val outDir = DocsGenerate.write(v, pkg, packageModules.asJava) + printDiagnostic(s"Documentation generated to ${outDir}") + } + if (shouldWriteCache) { context.serializeLibrary( this, pkg.libraryName, useGlobalCacheLocations ) - } else CompletableFuture.completedFuture(true) + } else { + CompletableFuture.completedFuture(true) + } } } } @@ -206,7 +223,7 @@ class Compiler( parseModule( module, irCachingEnabled && !context.isInteractive(module), - isGenDocs = true + true ) module } @@ -220,7 +237,8 @@ class Compiler( private def runInternal( modules: List[Module], generateCode: Boolean, - shouldCompileDependencies: Boolean + shouldCompileDependencies: Boolean, + generateDocs: Boolean = false ): CompilerResult = { @scala.annotation.tailrec def go( @@ -233,7 +251,8 @@ class Compiler( runCompilerPipeline( modulesToCompile, generateCode, - shouldCompileDependencies + shouldCompileDependencies, + generateDocs ) val pending = packageRepository.getPendingModules.toList @@ -246,12 +265,17 @@ class Compiler( private def runCompilerPipeline( modules: List[Module], generateCode: Boolean, - shouldCompileDependencies: Boolean + shouldCompileDependencies: Boolean, + generateDocs: Boolean ): List[Module] = { initialize() modules.foreach(m => try { - parseModule(m, irCachingEnabled && !context.isInteractive(m)) + parseModule( + m, + irCachingEnabled && !context.isInteractive(m), + generateDocs + ) } catch { case e: Throwable => context.log( @@ -293,12 +317,20 @@ class Compiler( ) ) context.updateModule(module, _.invalidateCache()) - parseModule(module, irCachingEnabled && !context.isInteractive(module)) + parseModule( + module, + irCachingEnabled && !context.isInteractive(module), + generateDocs + ) importedModules .filter(isLoadedFromSource) .foreach(m => { if (m.getBindingsMap == null) { - parseModule(m, irCachingEnabled && !context.isInteractive(module)) + parseModule( + m, + irCachingEnabled && !context.isInteractive(module), + generateDocs + ) } }) runImportsAndExportsResolution(module, generateCode) @@ -309,7 +341,7 @@ class Compiler( if (irCachingEnabled) { requiredModules.foreach { module => - ensureParsed(module, !context.isInteractive(module)) + ensureParsed(module, !context.isInteractive(module), generateDocs) } } requiredModules.foreach { module => @@ -322,9 +354,10 @@ class Compiler( ) { val moduleContext = ModuleContext( - module = module, - freshNameSupply = Some(freshNameSupply), - compilerConfig = config + module = module, + freshNameSupply = Some(freshNameSupply), + compilerConfig = config, + isGeneratingDocs = generateDocs ) val compilerOutput = runGlobalTypingPasses(context.getIr(module), moduleContext) @@ -338,142 +371,147 @@ class Compiler( ) } } - requiredModules.foreach { module => - if ( - !context - .getCompilationStage(module) - .isAtLeast( - CompilationStage.AFTER_STATIC_PASSES - ) - ) { + if (!generateDocs) { - val moduleContext = ModuleContext( - module = module, - freshNameSupply = Some(freshNameSupply), - compilerConfig = config, - pkgRepo = Some(packageRepository) - ) - val compilerOutput = - runMethodBodyPasses(context.getIr(module), moduleContext) - context.updateModule( - module, - { u => - u.ir(compilerOutput) - u.compilationStage(CompilationStage.AFTER_STATIC_PASSES) - } - ) - } - } + requiredModules.foreach { module => + if ( + !context + .getCompilationStage(module) + .isAtLeast( + CompilationStage.AFTER_STATIC_PASSES + ) + ) { - requiredModules.foreach { module => - if ( - !context - .getCompilationStage(module) - .isAtLeast( - CompilationStage.AFTER_TYPE_INFERENCE_PASSES + val moduleContext = ModuleContext( + module = module, + freshNameSupply = Some(freshNameSupply), + compilerConfig = config, + pkgRepo = Some(packageRepository), + isGeneratingDocs = generateDocs ) - ) { - - val moduleContext = ModuleContext( - module = module, - freshNameSupply = Some(freshNameSupply), - compilerConfig = config, - pkgRepo = Some(packageRepository) - ) - val compilerOutput = - runFinalTypeInferencePasses(context.getIr(module), moduleContext) - context.updateModule( - module, - { u => - u.ir(compilerOutput) - u.compilationStage(CompilationStage.AFTER_TYPE_INFERENCE_PASSES) - } - ) + val compilerOutput = + runMethodBodyPasses(context.getIr(module), moduleContext) + context.updateModule( + module, + { u => + u.ir(compilerOutput) + u.compilationStage(CompilationStage.AFTER_STATIC_PASSES) + } + ) + } } - } - runErrorHandling(requiredModules) + requiredModules.foreach { module => + if ( + !context + .getCompilationStage(module) + .isAtLeast( + CompilationStage.AFTER_TYPE_INFERENCE_PASSES + ) + ) { - val requiredModulesWithScope = requiredModules.map { module => - if ( - !context - .getCompilationStage(module) - .isAtLeast( - CompilationStage.AFTER_RUNTIME_STUBS + val moduleContext = ModuleContext( + module = module, + freshNameSupply = Some(freshNameSupply), + compilerConfig = config, + pkgRepo = Some(packageRepository), + isGeneratingDocs = generateDocs ) - ) { - val moduleScopeBuilder = module.getScopeBuilder() - context.runStubsGenerator(module, moduleScopeBuilder) - context.updateModule( - module, - { u => - u.compilationStage(CompilationStage.AFTER_RUNTIME_STUBS) - } - ) - (module, moduleScopeBuilder) - } else { - (module, module.getScopeBuilder) + val compilerOutput = + runFinalTypeInferencePasses(context.getIr(module), moduleContext) + context.updateModule( + module, + { u => + u.ir(compilerOutput) + u.compilationStage(CompilationStage.AFTER_TYPE_INFERENCE_PASSES) + } + ) + } } - } - requiredModulesWithScope.foreach { case (module, moduleScopeBuilder) => - if ( - !context - .getCompilationStage(module) - .isAtLeast( - CompilationStage.AFTER_CODEGEN - ) - ) { + runErrorHandling(requiredModules) - if (generateCode) { - context.log( - Compiler.defaultLogLevel, - "Generating code for module [{0}].", - context.getModuleName(module) + val requiredModulesWithScope = requiredModules.map { module => + if ( + !context + .getCompilationStage(module) + .isAtLeast( + CompilationStage.AFTER_RUNTIME_STUBS + ) + ) { + val moduleScopeBuilder = module.getScopeBuilder() + context.runStubsGenerator(module, moduleScopeBuilder) + context.updateModule( + module, + { u => + u.compilationStage(CompilationStage.AFTER_RUNTIME_STUBS) + } ) - - context.truffleRunCodegen(module, moduleScopeBuilder, config) + (module, moduleScopeBuilder) + } else { + (module, module.getScopeBuilder) } - context.updateModule( - module, - { u => - u.compilationStage(CompilationStage.AFTER_CODEGEN) - } - ) + } + requiredModulesWithScope.foreach { case (module, moduleScopeBuilder) => if ( - shouldCompileDependencies || (!context.isInteractive( - module - ) && context.isModuleInRootPackage(module)) + !context + .getCompilationStage(module) + .isAtLeast( + CompilationStage.AFTER_CODEGEN + ) ) { - val shouldStoreCache = - generateCode && - irCachingEnabled && !context.wasLoadedFromCache(module) + + if (generateCode) { + context.log( + Compiler.defaultLogLevel, + "Generating code for module [{0}].", + context.getModuleName(module) + ) + + context.truffleRunCodegen(module, moduleScopeBuilder, config) + } + context.updateModule( + module, + { u => + u.compilationStage(CompilationStage.AFTER_CODEGEN) + } + ) + if ( - shouldStoreCache && !hasErrors(module) && - !context.isInteractive(module) && !context.isSynthetic(module) + shouldCompileDependencies || (!context.isInteractive( + module + ) && context.isModuleInRootPackage(module)) ) { - if (isInteractiveMode) { - context.notifySerializeModule(context.getModuleName(module)) - } else { - context.serializeModule( - this, - module, - useGlobalCacheLocations, - true - ) + val shouldStoreCache = + generateCode && + irCachingEnabled && !context.wasLoadedFromCache(module) + if ( + shouldStoreCache && !hasErrors(module) && + !context.isInteractive(module) && !context.isSynthetic(module) + ) { + if (isInteractiveMode) { + context.notifySerializeModule(context.getModuleName(module)) + } else { + context.serializeModule( + this, + module, + useGlobalCacheLocations, + true + ) + } } + } else { + context.log( + Compiler.defaultLogLevel, + "Skipping serialization for [{0}].", + context.getModuleName(module) + ) } - } else { - context.log( - Compiler.defaultLogLevel, - "Skipping serialization for [{0}].", - context.getModuleName(module) - ) } } - } + } requiredModules } @@ -529,7 +567,11 @@ class Compiler( private def ensureParsedAndAnalyzed(module: Module): Unit = { if (module.getBindingsMap() == null) { - ensureParsed(module, irCachingEnabled && !context.isInteractive(module)) + ensureParsed( + module, + irCachingEnabled && !context.isInteractive(module), + false + ) } if (context.isSynthetic(module)) { // Synthetic modules need to be import-analyzed @@ -559,7 +601,11 @@ class Compiler( * @param module - the scope from which docs are generated. */ def gatherImportStatements(module: Module): Array[String] = { - ensureParsed(module, irCachingEnabled && !context.isInteractive(module)) + ensureParsed( + module, + irCachingEnabled && !context.isInteractive(module), + false + ) val importedModules = context.getIr(module).imports.flatMap { case imp: Import.Module => imp.name.parts.take(2).map(_.name) match { @@ -583,7 +629,7 @@ class Compiler( private def parseModule( module: Module, useCaches: Boolean, - isGenDocs: Boolean = false + generateDocs: Boolean ): Unit = { context.log( Compiler.defaultLogLevel, @@ -599,7 +645,7 @@ class Compiler( return } - uncachedParseModule(module, isGenDocs) + uncachedParseModule(module, generateDocs) } /** Retrieve module bindings from cache, if available. @@ -616,7 +662,10 @@ class Compiler( } else None } - private def uncachedParseModule(module: Module, isGenDocs: Boolean): Unit = { + private def uncachedParseModule( + module: Module, + generateDocs: Boolean + ): Unit = { context.log( Compiler.defaultLogLevel, "Loading module [{0}] from source.", @@ -628,7 +677,7 @@ class Compiler( module = module, freshNameSupply = Some(freshNameSupply), compilerConfig = config, - isGeneratingDocs = isGenDocs + isGeneratingDocs = generateDocs ) val src = context.getCharacters(module) @@ -689,10 +738,14 @@ class Compiler( */ def ensureParsed(module: Module): Unit = { val useCaches = irCachingEnabled && !context.isInteractive(module) - ensureParsed(module, useCaches) + ensureParsed(module, useCaches, false) } - def ensureParsed(module: Module, useCaches: Boolean): Unit = { + def ensureParsed( + module: Module, + useCaches: Boolean, + generateDocs: Boolean + ): Unit = { if ( !context .getCompilationStage(module) @@ -700,7 +753,7 @@ class Compiler( CompilationStage.AFTER_PARSING ) ) { - parseModule(module, useCaches) + parseModule(module, useCaches, generateDocs) } } diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/phase/ImportResolver.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/phase/ImportResolver.scala index 5b431b38a1dc..c5dfc4d4190e 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/phase/ImportResolver.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/phase/ImportResolver.scala @@ -61,7 +61,7 @@ final class ImportResolver(compiler: Compiler) extends ImportResolverForIR { u.invalidateCache() } ) - compiler.ensureParsed(current, false) + compiler.ensureParsed(current, false, false) return analyzeModule(current) } // put the list of resolved imports in the module metadata diff --git a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/dump/test/DocsGenerateTest.java b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/dump/test/DocsGenerateTest.java new file mode 100644 index 000000000000..80a18fa1a512 --- /dev/null +++ b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/dump/test/DocsGenerateTest.java @@ -0,0 +1,392 @@ +package org.enso.compiler.dump.test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; +import org.enso.compiler.Compiler; +import org.enso.compiler.core.IR; +import org.enso.compiler.core.ir.Module; +import org.enso.compiler.core.ir.module.scope.Definition; +import org.enso.compiler.core.ir.module.scope.definition.Method; +import org.enso.compiler.dump.DocsGenerate; +import org.enso.compiler.dump.DocsVisit; +import org.enso.interpreter.runtime.EnsoContext; +import org.enso.pkg.QualifiedName; +import org.enso.test.utils.ContextUtils; +import org.enso.test.utils.ProjectUtils; +import org.graalvm.polyglot.Context; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class DocsGenerateTest { + @ClassRule public static final TemporaryFolder TEMP = new TemporaryFolder(); + + private static Context ctx; + private static EnsoContext leak; + private static Compiler compiler; + + public DocsGenerateTest() {} + + @BeforeClass + public static void initCtx() { + ctx = ContextUtils.defaultContextBuilder().build(); + leak = ContextUtils.leakContext(ctx); + } + + @AfterClass + public static void closeCtx() { + ctx.close(); + ctx = null; + leak = null; + } + + @Test + public void simpleType() throws Exception { + var code = + """ + type Calc + Zero + One x + Two x y + + create v = Calc.One v + sum self = self.x+self.y + + main = Calc.create 42 + """; + + var v = new MockVisitor(); + + generateDocumentation("Calc", code, v); + + assertEquals("One type found", 1, v.visitType.size()); + assertEquals("Three constructors", 3, v.visitConstructor.size()); + + var typeMethods = + v.visitMethod.stream() + .filter(p -> p.t() != null) + .map( + p -> { + assertEquals("Name of the type is", "Calc", p.t().name().name()); + return p; + }) + .toList(); + var moduleMethods = new ArrayList<>(v.visitMethod); + moduleMethods.removeAll(typeMethods); + + assertEquals( + "Two type methods: " + + typeMethods.stream() + .map( + p -> { + var typePref = p.t() != null ? p.t().name().name() + "." : ""; + return typePref + p.ir().methodName().name(); + }) + .toList(), + 2, + typeMethods.size()); + + assertEquals("One module method", 1, moduleMethods.size()); + assertEquals("main", moduleMethods.get(0).ir().methodName().name()); + } + + @Test + public void functionArgumentTypes() throws Exception { + var code = + """ + from Standard.Base import Integer + + sum x:Integer y:Integer -> Integer = x+y + """; + + var v = new MockVisitor(); + generateDocumentation("Sum", code, v); + + assertEquals("One method only", 1, v.visitMethod.size()); + assertNull("No type associated", v.visitMethod.get(0).t()); + var sum = v.visitMethod.get(0).ir(); + assertEquals( + "sum x:Standard.Base.Data.Numbers.Integer y:Standard.Base.Data.Numbers.Integer ->" + + " Standard.Base.Data.Numbers.Integer", + DocsVisit.toSignature(sum)); + } + + @Test + public void suspendAndDefault() throws Exception { + var code = + """ + from Standard.Base import Integer + + sum ~x:Integer y:Integer=10 = x+y + """; + + var v = new MockVisitor(); + generateDocumentation("Suspend", code, v); + + assertEquals("One method only", 1, v.visitMethod.size()); + assertNull("No type associated", v.visitMethod.get(0).t()); + var sum = v.visitMethod.get(0).ir(); + assertEquals( + "sum ~x:Standard.Base.Data.Numbers.Integer y:Standard.Base.Data.Numbers.Integer= ->" + + " Standard.Base.Any.Any", + DocsVisit.toSignature(sum)); + } + + @Test + public void constructorSignature() throws Exception { + var code = + """ + from Standard.Base import Integer + + type Result + Sum ~x:Integer y:Integer=10 + """; + + var v = new MockVisitor(); + generateDocumentation("TypeResult", code, v); + + assertEquals("No methods", 0, v.visitMethod.size()); + assertEquals("One constructor", 1, v.visitConstructor.size()); + var sum = v.visitConstructor.get(0); + assertEquals( + "Sum ~x:Standard.Base.Data.Numbers.Integer y:Standard.Base.Data.Numbers.Integer=", + DocsVisit.toSignature(sum)); + } + + @Test + public void instanceMethodSignature() throws Exception { + var code = + """ + from Standard.Base import Integer + + type Result + sum self y = self+y + """; + + var v = new MockVisitor(); + generateDocumentation("InstanceResult", code, v); + + assertEquals("No methods", 1, v.visitMethod.size()); + var p = v.visitMethod.get(0); + assertEquals("Result", p.t().name().name()); + var sum = p.ir(); + assertEquals( + "sum self y:Standard.Base.Any.Any -> Standard.Base.Any.Any", DocsVisit.toSignature(sum)); + } + + @Test + public void staticMethodSignature() throws Exception { + var code = + """ + from Standard.Base import Integer + + type Result + sum ~x:Integer y:Integer=10 = x+y + """; + + var v = new MockVisitor(); + generateDocumentation("PrivateResult", code, v); + + assertEquals("One sum method", 1, v.visitMethod.size()); + var p = v.visitMethod.get(0); + assertEquals("Result", p.t().name().name()); + var sum = p.ir(); + assertEquals( + "sum ~x:Standard.Base.Data.Numbers.Integer y:Standard.Base.Data.Numbers.Integer= ->" + + " Standard.Base.Any.Any", + DocsVisit.toSignature(sum)); + } + + @Test + public void privateAreHidden() throws Exception { + var code = + """ + type Result + private Zero + private One x + + private create v = Result.One v + private power self = self.x*self.x + """; + + var v = new MockVisitor(); + generateDocumentation("StaticResult", code, v); + + assertEquals("No methods", 0, v.visitMethod.size()); + assertEquals("No constructors", 0, v.visitConstructor.size()); + } + + @Test + public void vectorWithElements() throws Exception { + var code = + """ + from Standard.Base import Vector, Text + + values a:Text -> Vector Text = [a] + """; + + var v = new MockVisitor(); + generateDocumentation("VectorText", code, v); + + assertEquals("One methods", 1, v.visitMethod.size()); + assertEquals("No constructors", 0, v.visitConstructor.size()); + + var p = v.visitMethod.get(0); + assertNull("It is a module method", p.t()); + + var m = p.ir(); + assertEquals("values", m.methodName().name()); + assertEquals( + "Generates vector with argument type as return type", + "values a:Standard.Base.Data.Text.Text -> (Standard.Base.Data.Vector.Vector" + + " Standard.Base.Data.Text.Text)", + DocsVisit.toSignature(m)); + } + + @Test + public void unionTypes() throws Exception { + var code = + """ + type A + type B + type C + + one a:A -> A | B | C = A + """; + + var v = new MockVisitor(); + generateDocumentation("Union", code, v); + + assertEquals("One methods", 1, v.visitMethod.size()); + assertEquals("No constructors", 0, v.visitConstructor.size()); + + var p = v.visitMethod.get(0); + assertNull("It is a module method", p.t()); + + var m = p.ir(); + assertEquals("one", m.methodName().name()); + assertEquals( + "Generates vector with argument type as return type", + "one a:local.Union.Main.A -> (local.Union.Main.A|local.Union.Main.B|local.Union.Main.C)", + DocsVisit.toSignature(m)); + } + + @Test + public void intersectionTypes() throws Exception { + var code = + """ + type A + type B + type C + + one a:A -> A & B & C = a + """; + + var v = new MockVisitor(); + generateDocumentation("Inter", code, v); + + assertEquals("One methods", 1, v.visitMethod.size()); + assertEquals("No constructors", 0, v.visitConstructor.size()); + + var p = v.visitMethod.get(0); + assertNull("It is a module method", p.t()); + + var m = p.ir(); + var sig = DocsVisit.toSignature(m); + assertEquals("one", m.methodName().name()); + assertEquals( + "Generates vector with argument type as return type", + "one a:local.Inter.Main.A -> (local.Inter.Main.A&local.Inter.Main.B&local.Inter.Main.C)", + sig); + } + + private static void generateDocumentation(String name, String code, DocsVisit v) + throws IOException { + var pathCalc = TEMP.newFolder(name); + ProjectUtils.createProject(name, code, pathCalc.toPath()); + ProjectUtils.generateProjectDocs( + "api", + ContextUtils.defaultContextBuilder(), + pathCalc.toPath(), + (context) -> { + var enso = ContextUtils.leakContext(context); + var modules = enso.getTopScope().getModules(); + var optMod = + modules.stream().filter(m -> m.getName().toString().contains(name)).findFirst(); + assertTrue( + "Found " + name + " in " + modules.stream().map(m -> m.getName()).toList(), + optMod.isPresent()); + var mod = optMod.get(); + assertEquals("local." + name + ".Main", mod.getName().toString()); + var ir = mod.getIr(); + assertNotNull("Ir for " + mod + " found", ir); + + try { + DocsGenerate.visitModule(v, mod.getName(), ir, null); + } catch (IOException e) { + throw raise(RuntimeException.class, e); + } + }); + } + + private static final class MockVisitor implements DocsVisit { + private final List visitModule = new ArrayList<>(); + private final List visitType = new ArrayList<>(); + private final List visitConstructor = new ArrayList<>(); + private final List visitUnknown = new ArrayList<>(); + private final List> visitMethod = new ArrayList<>(); + private final List visitConversion = new ArrayList<>(); + + @Override + public boolean visitModule(QualifiedName name, Module ir, PrintWriter writer) + throws IOException { + visitModule.add(ir); + return true; + } + + @Override + public boolean visitUnknown(IR ir, PrintWriter w) throws IOException { + visitUnknown.add(ir); + return true; + } + + @Override + public void visitMethod(Definition.Type t, Method.Explicit m, PrintWriter writer) + throws IOException { + visitMethod.add(new TypeAnd<>(t, m)); + } + + @Override + public void visitConversion(Method.Conversion c, PrintWriter w) throws IOException { + visitConversion.add(c); + } + + @Override + public boolean visitType(Definition.Type t, PrintWriter w) throws IOException { + visitType.add(t); + return true; + } + + @Override + public void visitConstructor(Definition.Type t, Definition.Data d, PrintWriter w) + throws IOException { + visitConstructor.add(d); + } + } + + @SuppressWarnings("unchecked") + private static E raise(Class type, Exception t) throws E { + throw (E) t; + } + + record TypeAnd(Definition.Type t, IRElement ir) {} +} diff --git a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerdeCompilerTest.java b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerdeCompilerTest.java index 437419b7a6e4..def609c384a0 100644 --- a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerdeCompilerTest.java +++ b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerdeCompilerTest.java @@ -69,7 +69,7 @@ private void parseSerializedModule(String projectName, String forbiddenMessage) futures.add(future); return null; }); - futures.add(compiler.compile(false, true, true)); + futures.add(compiler.compile(false, true, true, scala.Option.empty())); for (var f : futures) { var persisted = f.get(10, TimeUnit.SECONDS); assertEquals("Fib_Test library has been fully persisted", true, persisted); diff --git a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializationManagerTest.java b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializationManagerTest.java index e0dfa4f6b5d3..4f439eeb3081 100644 --- a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializationManagerTest.java +++ b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializationManagerTest.java @@ -89,7 +89,7 @@ public void serializeLibrarySuggestions() Object result = ensoContext .getCompiler() - .compile(false, true, false) + .compile(false, true, false, scala.Option.empty()) .get(COMPILE_TIMEOUT_SECONDS, TimeUnit.SECONDS); Assert.assertEquals(Boolean.TRUE, result); diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java index c46da21bab26..1fb62a430711 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/Module.java @@ -695,7 +695,10 @@ private static Object evalExpression( } private static Object generateDocs(Module module, EnsoContext context) { - return context.getCompiler().generateDocs(module.asCompilerModule()); + var compilerModule = module.asCompilerModule(); + var res = context.getCompiler().generateDocs(compilerModule); + assert res == compilerModule; + return module; } @CompilerDirectives.TruffleBoundary diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/scope/TopLevelScope.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/scope/TopLevelScope.java index 3c6aee846d8d..fffc805a3bc1 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/scope/TopLevelScope.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/scope/TopLevelScope.java @@ -1,21 +1,10 @@ package org.enso.interpreter.runtime.scope; -import com.oracle.truffle.api.CompilerDirectives; -import com.oracle.truffle.api.TruffleFile; -import com.oracle.truffle.api.dsl.Bind; -import com.oracle.truffle.api.dsl.Specialization; -import com.oracle.truffle.api.interop.ArityException; -import com.oracle.truffle.api.interop.InteropLibrary; -import com.oracle.truffle.api.interop.UnknownIdentifierException; -import com.oracle.truffle.api.interop.UnsupportedMessageException; -import com.oracle.truffle.api.interop.UnsupportedTypeException; -import com.oracle.truffle.api.library.ExportLibrary; -import com.oracle.truffle.api.library.ExportMessage; -import com.oracle.truffle.api.nodes.Node; import java.io.File; import java.util.Collection; import java.util.Optional; import java.util.concurrent.ExecutionException; + import org.enso.common.MethodNames; import org.enso.compiler.PackageRepository; import org.enso.editions.LibraryName; @@ -30,6 +19,21 @@ import org.enso.pkg.QualifiedName; import org.enso.scala.wrapper.ScalaConversions; +import com.oracle.truffle.api.CompilerDirectives; +import com.oracle.truffle.api.TruffleFile; +import com.oracle.truffle.api.dsl.Bind; +import com.oracle.truffle.api.dsl.Specialization; +import com.oracle.truffle.api.interop.ArityException; +import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.UnknownIdentifierException; +import com.oracle.truffle.api.interop.UnsupportedMessageException; +import com.oracle.truffle.api.interop.UnsupportedTypeException; +import com.oracle.truffle.api.library.ExportLibrary; +import com.oracle.truffle.api.library.ExportMessage; +import com.oracle.truffle.api.nodes.Node; + +import scala.Option; + /** Represents the top scope of Enso execution, containing all the importable modules. */ @ExportLibrary(InteropLibrary.class) public final class TopLevelScope extends EnsoObject { @@ -184,13 +188,28 @@ private static Object leakContext(EnsoContext context) { @CompilerDirectives.TruffleBoundary private static Object compile(Object[] arguments, EnsoContext context) throws UnsupportedTypeException, ArityException { - boolean shouldCompileDependencies = Types.extractArguments(arguments, Boolean.class); - boolean shouldWriteCache = !context.isIrCachingDisabled(); boolean useGlobalCache = context.isUseGlobalCache(); + boolean shouldCompileDependencies; + scala.Option generateDocs; + switch (arguments.length) { + case 2 -> { + shouldCompileDependencies = Boolean.TRUE.equals(arguments[0]); + generateDocs = switch (arguments[1]) { + case Boolean b when !b -> Option.empty(); + case String s -> Option.apply(s); + default -> Option.empty(); + }; + } + default -> { + shouldCompileDependencies = Types.extractArguments(arguments, Boolean.class); + generateDocs = Option.empty(); + } + } + boolean shouldWriteCache = !context.isIrCachingDisabled(); try { return context .getCompiler() - .compile(shouldCompileDependencies, shouldWriteCache, useGlobalCache) + .compile(shouldCompileDependencies, shouldWriteCache, useGlobalCache, generateDocs) .get(); } catch (InterruptedException e) { throw new RuntimeException(e); diff --git a/lib/java/test-utils/src/main/java/org/enso/test/utils/ProjectUtils.java b/lib/java/test-utils/src/main/java/org/enso/test/utils/ProjectUtils.java index 5eb9a183f974..40704d9f6b30 100644 --- a/lib/java/test-utils/src/main/java/org/enso/test/utils/ProjectUtils.java +++ b/lib/java/test-utils/src/main/java/org/enso/test/utils/ProjectUtils.java @@ -15,6 +15,7 @@ import org.graalvm.polyglot.Context.Builder; import org.graalvm.polyglot.Value; import org.slf4j.LoggerFactory; +import scala.Option; /** Utility methods for creating and running Enso projects. */ public class ProjectUtils { @@ -108,6 +109,37 @@ public static void testProjectRun( } } + /** + * Tests running the project located in the given {@code projDir}. Is equal to running {@code enso + * --run }. + * + * @param docsFormat format of the documentation to generate + * @param ctxBuilder A context builder that might be initialized with some specific options. + * @param projDir Root directory of the project. + * @param whenDone callback when generated + */ + public static void generateProjectDocs( + String docsFormat, Context.Builder ctxBuilder, Path projDir, Consumer whenDone) { + if (!(projDir.toFile().exists() && projDir.toFile().isDirectory())) { + throw new IllegalArgumentException( + "Project directory " + projDir + " must already be created"); + } + try (var ctx = + ctxBuilder + .option(RuntimeOptions.PROJECT_ROOT, projDir.toAbsolutePath().toString()) + .option(RuntimeOptions.STRICT_ERRORS, "true") + .option(RuntimeOptions.DISABLE_IR_CACHES, "true") + .build()) { + var polyCtx = new PolyglotContext(ctx); + var mainSrcPath = projDir.resolve("src").resolve("Main.enso"); + if (!mainSrcPath.toFile().exists()) { + throw new IllegalArgumentException("Main module not found in " + projDir); + } + polyCtx.getTopScope().compile(false, Option.apply(docsFormat)); + whenDone.accept(polyCtx.context()); + } + } + /** * Just a wrapper for {@link ProjectUtils#testProjectRun(Builder, Path, Consumer)}. *