Add a rough outline of a command line driver. (#548)

This is actually code that I wrote a long time ago but didn't get added
to the repository, trying to tidy that up. This is the last code.
Nothing much interesting here, just a skeleton of a CLI. But seemed
better to add it than to not.

driver/BUILD

@@ -0,0 +1,63 @@
+# Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+# Exceptions. See /LICENSE for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
+load("//bazel/fuzzing:rules.bzl", "cc_fuzz_test")
+
+package(default_visibility = ["//visibility:public"])
+
+cc_library(
+    name = "driver",
+    srcs = ["driver.cpp"],
+    hdrs = ["driver.h"],
+    textual_hdrs = ["flags.def"],
+    deps = [
+        "//diagnostics:diagnostic_emitter",
+        "//lexer:tokenized_buffer",
+        "//source:source_buffer",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
+cc_test(
+    name = "driver_test",
+    srcs = ["driver_test.cpp"],
+    deps = [
+        ":driver",
+        "//lexer:tokenized_buffer_test_helpers",
+        "@llvm-project//llvm:Support",
+        "@llvm-project//llvm:gmock",
+        "@llvm-project//llvm:gtest",
+        "@llvm-project//llvm:gtest_main",
+    ],
+)
+
+cc_fuzz_test(
+    name = "driver_fuzzer",
+    srcs = ["driver_fuzzer.cpp"],
+    corpus = glob(["fuzzer_corpus/*"]),
+    deps = [
+        ":driver",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
+cc_binary(
+    name = "carbon",
+    srcs = ["driver_main.cpp"],
+    deps = [
+        ":driver",
+        "@llvm-project//llvm:Support",
+    ],
+)
+
+# FIXME: No support for LLVM's lit-style command line & FileCheck tests.
+#
+#lit_test(
+#    name = "carbon_test.carbon",
+#    data = [
+#        ":carbon",
+#        "@llvm-project//llvm:FileCheck",
+#    ],
+#)

driver/carbon_test.carbon

@@ -0,0 +1,24 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// FIXME: We need to figure out how to integrate this style of integration
+// testing with Bazel.
+//
+// RUN: driver/carbon dump-tokens %s | FileCheck %s --check-prefix=TOKENS
+
+fn run(String program) {
+  return True;
+}
+
+// TOKENS: token: { index:  0, kind:       'FnKeyword', line: 4, column:  1, indent: 1, spelling: 'fn' }
+// TOKENS: token: { index:  1, kind:      'Identifier', line: 4, column:  4, indent: 1, spelling: 'run', identifier: 0 }
+// TOKENS: token: { index:  2, kind:       'OpenParen', line: 4, column:  7, indent: 1, spelling: '(', closing_token: 5 }
+// TOKENS: token: { index:  3, kind:      'Identifier', line: 4, column:  8, indent: 1, spelling: 'String', identifier: 1 }
+// TOKENS: token: { index:  4, kind:      'Identifier', line: 4, column: 15, indent: 1, spelling: 'program', identifier: 2 }
+// TOKENS: token: { index:  5, kind:      'CloseParen', line: 4, column: 22, indent: 1, spelling: ')', opening_token: 2 }
+// TOKENS: token: { index:  6, kind:  'OpenCurlyBrace', line: 4, column: 24, indent: 1, spelling: '{', closing_token: 10 }
+// TOKENS: token: { index:  7, kind:   'ReturnKeyword', line: 5, column:  3, indent: 3, spelling: 'return' }
+// TOKENS: token: { index:  8, kind:      'Identifier', line: 5, column: 10, indent: 3, spelling: 'True', identifier: 3 }
+// TOKENS: token: { index:  9, kind:            'Semi', line: 5, column: 14, indent: 3, spelling: ';' }
+// TOKENS: token: { index: 10, kind: 'CloseCurlyBrace', line: 6, column:  1, indent: 1, spelling: '}', opening_token: 6 }

driver/driver.cpp

@@ -0,0 +1,141 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "driver/driver.h"
+
+#include "diagnostics/diagnostic_emitter.h"
+#include "lexer/tokenized_buffer.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Format.h"
+#include "source/source_buffer.h"
+
+namespace Carbon {
+
+namespace {
+
+enum class Subcommand {
+#define CARBON_SUBCOMMAND(Name, ...) Name,
+#include "driver/flags.def"
+  Unknown,
+};
+
+auto GetSubcommand(llvm::StringRef name) -> Subcommand {
+  return llvm::StringSwitch<Subcommand>(name)
+#define CARBON_SUBCOMMAND(Name, Spelling, ...) .Case(Spelling, Subcommand::Name)
+#include "driver/flags.def"
+      .Default(Subcommand::Unknown);
+}
+
+}  // namespace
+
+auto Driver::RunFullCommand(llvm::ArrayRef<llvm::StringRef> args) -> bool {
+  if (args.empty()) {
+    error_stream << "ERROR: No subcommand specified.\n";
+    return false;
+  }
+
+  llvm::StringRef subcommand_text = args[0];
+  llvm::SmallVector<llvm::StringRef, 16> subcommand_args(
+      std::next(args.begin()), args.end());
+  switch (GetSubcommand(subcommand_text)) {
+    case Subcommand::Unknown:
+      error_stream << "ERROR: Unknown subcommand '" << subcommand_text
+                   << "'.\n";
+      return false;
+
+#define CARBON_SUBCOMMAND(Name, ...) \
+  case Subcommand::Name:             \
+    return Run##Name##Subcommand(subcommand_args);
+#include "driver/flags.def"
+  }
+  llvm_unreachable("All subcommands handled!");
+}
+
+auto Driver::RunHelpSubcommand(llvm::ArrayRef<llvm::StringRef> args) -> bool {
+  // FIXME: We should support getting detailed help on a subcommand by looking
+  // for it as a positional parameter here.
+  if (!args.empty()) {
+    ReportExtraArgs("help", args);
+    return false;
+  }
+
+  output_stream << "List of subcommands:\n\n";
+
+  constexpr llvm::StringLiteral SubcommandsAndHelp[][2] = {
+#define CARBON_SUBCOMMAND(Name, Spelling, HelpText) {Spelling, HelpText},
+#include "driver/flags.def"
+  };
+
+  int max_subcommand_width = 0;
+  for (auto subcommand_and_help : SubcommandsAndHelp) {
+    max_subcommand_width = std::max(
+        max_subcommand_width, static_cast<int>(subcommand_and_help[0].size()));
+  }
+
+  for (auto subcommand_and_help : SubcommandsAndHelp) {
+    llvm::StringRef subcommand_text = subcommand_and_help[0];
+    // FIXME: We should wrap this to the number of columns left after the
+    // subcommand on the terminal, and using a hanging indent.
+    llvm::StringRef help_text = subcommand_and_help[1];
+    output_stream << "  "
+                  << llvm::left_justify(subcommand_text, max_subcommand_width)
+                  << " - " << help_text << "\n";
+  }
+
+  output_stream << "\n";
+  return true;
+}
+
+auto Driver::RunDumpTokensSubcommand(llvm::ArrayRef<llvm::StringRef> args)
+    -> bool {
+  if (args.empty()) {
+    error_stream << "ERROR: No input file specified.\n";
+    return false;
+  }
+
+  llvm::StringRef input_file_name = args.front();
+  args = args.drop_front();
+  if (!args.empty()) {
+    ReportExtraArgs("dump-tokens", args);
+    return false;
+  }
+
+  auto source = SourceBuffer::CreateFromFile(input_file_name);
+  if (!source) {
+    error_stream << "ERROR: Unable to open input source file: ";
+    llvm::handleAllErrors(source.takeError(),
+                          [&](const llvm::ErrorInfoBase& ei) {
+                            ei.log(error_stream);
+                            error_stream << "\n";
+                          });
+    return false;
+  }
+  auto tokenized_source =
+      TokenizedBuffer::Lex(*source, ConsoleDiagnosticConsumer());
+  if (tokenized_source.HasErrors()) {
+    error_stream << "ERROR: Unable to tokenize source file '" << input_file_name
+                 << "'!\n";
+    return false;
+  }
+  tokenized_source.Print(output_stream);
+  return true;
+}
+
+auto Driver::ReportExtraArgs(llvm::StringRef subcommand_text,
+                             llvm::ArrayRef<llvm::StringRef> args) -> void {
+  error_stream << "ERROR: Unexpected additional arguments to the '"
+               << subcommand_text << "' subcommand:";
+  for (auto arg : args) {
+    error_stream << " " << arg;
+  }
+
+  error_stream << "\n";
+}
+
+}  // namespace Carbon

driver/driver.h

@@ -0,0 +1,70 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef DRIVER_DRIVER_H_
+#define DRIVER_DRIVER_H_
+
+#include <cstdint>
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace Carbon {
+
+// Command line interface driver.
+//
+// Provides simple API to parse and run command lines for Carbon.  It is
+// generally expected to be used to implement command line tools for working
+// with the language.
+class Driver {
+ public:
+  // Default constructed driver uses stderr for all error and informational
+  // output.
+  Driver() : output_stream(llvm::outs()), error_stream(llvm::errs()) {}
+
+  // Constructs a driver with any error or informational output directed to a
+  // specified stream.
+  Driver(llvm::raw_ostream& output_stream, llvm::raw_ostream& error_stream)
+      : output_stream(output_stream), error_stream(error_stream) {}
+
+  // Parses the given arguments into both a subcommand to select the operation
+  // to perform and any arguments to that subcommand.
+  //
+  // Returns true if the operation succeeds. If the operation fails, returns
+  // false and any information about the failure is printed to the registered
+  // error stream (stderr by default).
+  auto RunFullCommand(llvm::ArrayRef<llvm::StringRef> args) -> bool;
+
+  // Subcommand that prints available help text to the error stream.
+  //
+  // Optionally one positional parameter may be provided to select a particular
+  // subcommand or detailed section of help to print.
+  //
+  // Returns true if appropriate help text was found and printed. If an invalid
+  // positional parameter (or flag) is provided, returns false.
+  auto RunHelpSubcommand(llvm::ArrayRef<llvm::StringRef> args) -> bool;
+
+  // Subcommand that dumps the token information for the provided source file.
+  //
+  // Requires exactly one positional parameter to designate the source file to
+  // read. May be `-` to read from stdin.
+  //
+  // Returns true if the operation succeeds. If the operation fails, this
+  // returns false and any information about the failure is printed to the
+  // registered error stream (stderr by default).
+  auto RunDumpTokensSubcommand(llvm::ArrayRef<llvm::StringRef> args) -> bool;
+
+ private:
+  auto ReportExtraArgs(llvm::StringRef subcommand_text,
+                       llvm::ArrayRef<llvm::StringRef> args) -> void;
+
+  llvm::raw_ostream& output_stream;
+  llvm::raw_ostream& error_stream;
+};
+
+}  // namespace Carbon
+
+#endif  // DRIVER_DRIVER_H_

driver/driver_fuzzer.cpp

@@ -0,0 +1,82 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <cstdint>
+#include <cstring>
+#include <numeric>
+#include <string>
+
+#include "driver/driver.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace Carbon {
+
+static auto Read(const unsigned char*& data, size_t& size, int& output)
+    -> bool {
+  if (size < sizeof(output)) {
+    return false;
+  }
+  std::memcpy(&output, data, sizeof(output));
+  size -= sizeof(output);
+  data += sizeof(output);
+  return true;
+}
+
+extern "C" auto LLVMFuzzerTestOneInput(const unsigned char* data, size_t size)
+    -> int {
+  // First use the data to compute the number of arguments. Note that for
+  // scaling reasons we don't allow 2^31 arguments, even empty ones. Simply
+  // creating the vector of those won't work. We limit this to 2^20 arguments
+  // total.
+  int num_args;
+  if (!Read(data, size, num_args) || num_args < 0 || num_args > (1 << 20)) {
+    return 0;
+  }
+
+  // Now use the data to compute the length of each argument. We don't want to
+  // exhaust all memory, so bound the search space to using 2^17 bytes of
+  // memory for the argument text itself.
+  size_t arg_length_sum = 0;
+  llvm::SmallVector<int, 16> arg_lengths(num_args);
+  for (int& arg_length : arg_lengths) {
+    if (!Read(data, size, arg_length) || arg_length < 0) {
+      return 0;
+    }
+    arg_length_sum += arg_length;
+    if (arg_length_sum > (1 << 17)) {
+      return 0;
+    }
+  }
+
+  // Ensure we have enough data for all the arguments.
+  if (size < arg_length_sum) {
+    return 0;
+  }
+
+  // Lastly, read the contents of each argument out of the data.
+  llvm::SmallVector<llvm::StringRef, 16> args;
+  args.reserve(num_args);
+  for (int arg_length : arg_lengths) {
+    args.push_back(
+        llvm::StringRef(reinterpret_cast<const char*>(data), arg_length));
+    data += arg_length;
+    size -= arg_length;
+  }
+
+  std::string error_text;
+  llvm::raw_string_ostream error_stream(error_text);
+  llvm::raw_null_ostream output_stream;
+  Driver d(output_stream, error_stream);
+  if (!d.RunFullCommand(args)) {
+    error_stream.flush();
+    if (error_text.find("ERROR:") == std::string::npos) {
+      llvm::errs() << "No error message on a failure!\n";
+      return 1;
+    }
+  }
+  return 0;
+}
+}  // namespace Carbon