Skip to content

Commit

Permalink
[WIP] RIIR HtmlDocCk
Browse files Browse the repository at this point in the history
  • Loading branch information
fmease committed May 30, 2024
1 parent 0a59f11 commit 263abce
Show file tree
Hide file tree
Showing 19 changed files with 910 additions and 314 deletions.
550 changes: 248 additions & 302 deletions Cargo.lock

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ members = [
"src/tools/miri/cargo-miri",
"src/tools/rustdoc-themes",
"src/tools/unicode-table-generator",
"src/tools/htmldocck",
"src/tools/jsondocck",
"src/tools/jsondoclint",
"src/tools/llvm-bitcode-linker",
Expand Down
3 changes: 2 additions & 1 deletion src/bootstrap/src/core/build_steps/clippy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,8 @@ lint_any!(
CollectLicenseMetadata, "src/tools/collect-license-metadata", "collect-license-metadata";
Compiletest, "src/tools/compiletest", "compiletest";
CoverageDump, "src/tools/coverage-dump", "coverage-dump";
Jsondocck, "src/tools/jsondocck", "jsondocck";
HtmldocCk, "src/tools/htmldocck", "htmldocck";
JsondocCk, "src/tools/jsondocck", "jsondocck";
Jsondoclint, "src/tools/jsondoclint", "jsondoclint";
LintDocs, "src/tools/lint-docs", "lint-docs";
LlvmBitcodeLinker, "src/tools/llvm-bitcode-linker", "llvm-bitcode-linker";
Expand Down
15 changes: 10 additions & 5 deletions src/bootstrap/src/core/build_steps/test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1766,13 +1766,18 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler));
}

if mode == "rustdoc" {
// Use the beta compiler for htmldocck.
let compiler = compiler.with_stage(0);
cmd.arg("--htmldocck-path").arg(builder.ensure(tool::HtmlDocCk { compiler, target }));
}

if mode == "rustdoc-json" {
// Use the beta compiler for jsondocck
let json_compiler = compiler.with_stage(0);
cmd.arg("--jsondocck-path")
.arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target }));
// Use the beta compiler for jsondocck.
let compiler = compiler.with_stage(0);
cmd.arg("--jsondocck-path").arg(builder.ensure(tool::JsonDocCk { compiler, target }));
cmd.arg("--jsondoclint-path")
.arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target }));
.arg(builder.ensure(tool::JsonDocLint { compiler, target }));
}

if mode == "coverage-map" {
Expand Down
1 change: 1 addition & 0 deletions src/bootstrap/src/core/build_steps/tool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,7 @@ bootstrap_tool!(
RustInstaller, "src/tools/rust-installer", "rust-installer";
RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes";
LintDocs, "src/tools/lint-docs", "lint-docs";
HtmlDocCk, "src/tools/htmldocck", "htmldocck";
JsonDocCk, "src/tools/jsondocck", "jsondocck";
JsonDocLint, "src/tools/jsondoclint", "jsondoclint";
HtmlChecker, "src/tools/html-checker", "html-checker";
Expand Down
3 changes: 2 additions & 1 deletion src/bootstrap/src/core/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -764,7 +764,8 @@ impl<'a> Builder<'a> {
clippy::CollectLicenseMetadata,
clippy::Compiletest,
clippy::CoverageDump,
clippy::Jsondocck,
clippy::HtmldocCk,
clippy::JsondocCk,
clippy::Jsondoclint,
clippy::LintDocs,
clippy::LlvmBitcodeLinker,
Expand Down
5 changes: 4 additions & 1 deletion src/tools/compiletest/src/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -193,9 +193,12 @@ pub struct Config {
/// The coverage-dump executable.
pub coverage_dump_path: Option<PathBuf>,

/// The Python executable to use for LLDB and htmldocck.
/// The Python executable to use for LLDB.
pub python: String,

/// The htmldocck executable.
pub htmldocck_path: Option<String>,

/// The jsondocck executable.
pub jsondocck_path: Option<String>,

Expand Down
1 change: 1 addition & 0 deletions src/tools/compiletest/src/header/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ impl ConfigBuilder {
"--compile-lib-path=",
"--run-lib-path=",
"--python=",
// FIXME(fmease): Do we need to set htmldocck-path to "", too?
"--jsondocck-path=",
"--src-base=",
"--build-base=",
Expand Down
4 changes: 4 additions & 0 deletions src/tools/compiletest/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,9 @@ pub fn parse_config(args: Vec<String>) -> Config {
.optopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH")
.optopt("", "rust-demangler-path", "path to rust-demangler to use in tests", "PATH")
.optopt("", "coverage-dump-path", "path to coverage-dump to use in tests", "PATH")
// FIXME(fmease): fix docs here
.reqopt("", "python", "path to python to use for doc tests", "PATH")
.optopt("", "htmldocck-path", "path to htmldocck to use for doc tests", "PATH")
.optopt("", "jsondocck-path", "path to jsondocck to use for doc tests", "PATH")
.optopt("", "jsondoclint-path", "path to jsondoclint to use for doc tests", "PATH")
.optopt("", "valgrind-path", "path to Valgrind executable for Valgrind tests", "PROGRAM")
Expand Down Expand Up @@ -235,6 +237,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
rust_demangler_path: matches.opt_str("rust-demangler-path").map(PathBuf::from),
coverage_dump_path: matches.opt_str("coverage-dump-path").map(PathBuf::from),
python: matches.opt_str("python").unwrap(),
htmldocck_path: matches.opt_str("htmldocck-path"),
jsondocck_path: matches.opt_str("jsondocck-path"),
jsondoclint_path: matches.opt_str("jsondoclint-path"),
valgrind_path: matches.opt_str("valgrind-path"),
Expand Down Expand Up @@ -617,6 +620,7 @@ fn common_inputs_stamp(config: &Config) -> Stamp {

if let Some(ref rustdoc_path) = config.rustdoc_path {
stamp.add_path(&rustdoc_path);
// FIXME(fmease): Remove this one once the rewrite is completed.
stamp.add_path(&rust_src_dir.join("src/etc/htmldocck.py"));
}

Expand Down
13 changes: 10 additions & 3 deletions src/tools/compiletest/src/runtest.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3030,9 +3030,16 @@ impl<'test> TestCx<'test> {
if self.props.check_test_line_numbers_match {
self.check_rustdoc_test_option(proc_res);
} else {
let root = self.config.find_rust_src_root().unwrap();
let mut cmd = Command::new(&self.config.python);
cmd.arg(root.join("src/etc/htmldocck.py")).arg(&out_dir).arg(&self.testpaths.file);
// FIXME(fmease): Temporary commented out code:
// FIXME(fmease): I don't like this unwrap!
let mut cmd = Command::new(self.config.htmldocck_path.as_ref().unwrap());
cmd.arg("--doc-dir").arg(&out_dir).arg("--template").arg(&self.testpaths.file);

// let root = self.config.find_rust_src_root().unwrap();
// let mut cmd = Command::new(&self.config.python);
// cmd.arg(root.join("src/etc/htmldocck.py"));
// cmd.arg(&out_dir).arg(&self.testpaths.file);

if self.config.bless {
cmd.arg("--bless");
}
Expand Down
11 changes: 11 additions & 0 deletions src/tools/htmldocck/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
[package]
name = "htmldocck"
version = "0.1.0"
description = "A test framework for rustdoc's HTML backend"
edition = "2021"

[dependencies]
getopts = "0.2"
regex = "1.8" # 1.8 to avoid memchr 2.6.0, as 2.5.0 is pinned in the workspace
shlex = "1.3.0"
unicode-width = "0.1.4"
69 changes: 69 additions & 0 deletions src/tools/htmldocck/src/cache.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
use std::{
collections::{hash_map::Entry, HashMap},
path::Path,
};

use crate::error::DiagCtxt;

pub(crate) struct Cache<'a> {
root: &'a Path,
// FIXME: `&'a str`s
files: HashMap<String, String>,
// FIXME: `&'a str`, comment what this is for -- `-`
last_path: Option<String>,
}

impl<'a> Cache<'a> {
pub(crate) fn new(root: &'a Path) -> Self {
Self { root, files: HashMap::new(), last_path: None }
}

// FIXME: check file vs. dir (`@has <PATH>` vs. `@has-dir <PATH>`)
/// Check if the path points to an existing entity.
pub(crate) fn has(&mut self, path: String, dcx: &mut DiagCtxt) -> Result<bool, ()> {
// FIXME: should we use `try_exists` over `exists` instead? matters the most for `@!has <PATH>`.
let path = self.resolve(path, dcx)?;

Ok(self.files.contains_key(&path) || Path::new(&path).exists())
}

/// Load the contents of the given path.
pub(crate) fn load(&mut self, path: String, dcx: &mut DiagCtxt) -> Result<&str, ()> {
let path = self.resolve(path, dcx)?;

Ok(match self.files.entry(path) {
Entry::Occupied(entry) => entry.into_mut(),
Entry::Vacant(entry) => {
// FIXME: better message, location
let data =
std::fs::read_to_string(self.root.join(entry.key())).map_err(|error| {
dcx.emit(&format!("failed to read file: {error}"), None, None)
})?;
entry.insert(data)
}
})
}

// FIXME: &str -> &str if possible
fn resolve(&mut self, path: String, dcx: &mut DiagCtxt) -> Result<String, ()> {
if path == "-" {
// FIXME: no cloning
return self
.last_path
.clone()
// FIXME better diag, location
.ok_or_else(|| {
dcx.emit(
"attempt to use `-` ('previous path') in the very first command",
None,
None,
)
});
}

// While we could normalize the `path` at this point by
// using `std::path::absolute`, it's likely not worth it.
self.last_path = Some(path.clone());
Ok(path)
}
}
50 changes: 50 additions & 0 deletions src/tools/htmldocck/src/channel.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// FIXME: document that the "channel" is indeed a channel *URL*!

use std::{borrow::Cow, sync::OnceLock};

use crate::error::DiagCtxt;

const PLACEHOLDER: &str = "{{channel}}";
const ENV_VAR_KEY: &str = "DOC_RUST_LANG_ORG_CHANNEL";

pub(crate) fn instantiate<'a>(input: &'a str, dcx: &mut DiagCtxt) -> Result<Cow<'a, str>, ()> {
let Some(channel) = channel(dcx)? else { return Ok(input.into()) };
Ok(input.replace(PLACEHOLDER, channel).into())
}

#[allow(dead_code)] // FIXME
pub(crate) fn anonymize<'a>(input: &'a str, dcx: &'_ mut DiagCtxt) -> Result<Cow<'a, str>, ()> {
let Some(channel) = channel(dcx)? else { return Ok(input.into()) };
Ok(input.replace(channel, PLACEHOLDER).into())
}

fn channel(dcx: &mut DiagCtxt) -> Result<Option<&'static str>, ()> {
static CHANNEL_URL: OnceLock<Option<String>> = OnceLock::new();

// FIXME: Use `get_or_try_init` here (instead of `get`→`set`→`get`) if/once stabilized (on beta).

if let Some(channel_url) = CHANNEL_URL.get() {
return Ok(channel_url.as_deref());
}

let channel_url = match std::env::var(ENV_VAR_KEY) {
Ok(url) => Some(url),
// FIXME: should we make the channel mandatory instead?
Err(std::env::VarError::NotPresent) => None,
Err(std::env::VarError::NotUnicode(var)) => {
// FIXME: better diag
// FIXME: Use `OsStr::display` (instead of `to_string_lossy`) if/once stabilized (on beta).
dcx.emit(
&format!("env var `{ENV_VAR_KEY}` is not valid UTF-8: `{}`", var.to_string_lossy()),
None,
None,
);
return Err(());
}
};

// unwrap: The static item is locally scoped and no other thread tries to initialize it.
CHANNEL_URL.set(channel_url).unwrap();
// unwrap: Initialized above.
Ok(CHANNEL_URL.get().unwrap().as_deref())
}
79 changes: 79 additions & 0 deletions src/tools/htmldocck/src/check.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
use crate::cache::Cache;
use crate::error::{DiagCtxt, Source};
use crate::{channel, Command, CommandKind};

impl Command<'_> {
pub(crate) fn check(self, cache: &mut Cache<'_>, dcx: &mut DiagCtxt) -> Result<(), ()> {
let result = self.kind.check(cache, self.source.clone(), dcx)?;

if result == self.negated {
// FIXME: better diag
dcx.emit("check failed", self.source, None);
return Err(());
}

Ok(())
}
}

impl CommandKind {
// FIXME: implement all checks!
fn check(
self,
cache: &mut Cache<'_>,
_source: Source<'_>, // FIXME: unused
dcx: &mut DiagCtxt,
) -> Result<bool, ()> {
Ok(match self {
Self::HasFile { path } => cache.has(path, dcx)?, // FIXME: check if it's actually a file
Self::HasDir { path } => cache.has(path, dcx)?, // FIXME: check if it's actually a directory
Self::Has { path, xpath, text } => {
let _data = cache.load(path, dcx)?;
_ = xpath;
_ = text;
true // FIXME
}
Self::HasRaw { path, text } => {
let data = cache.load(path, dcx)?;

if text.is_empty() {
// fast path
return Ok(true);
}

let text = channel::instantiate(&text, dcx)?;
let text = text.replace(|c: char| c.is_ascii_whitespace(), " ");
let data = data.replace(|c: char| c.is_ascii_whitespace(), " ");

data.contains(&text)
}
Self::Matches { path, xpath, pattern } => {
let _data = cache.load(path, dcx)?;
_ = xpath;
_ = pattern;

true // FIXME
}
Self::MatchesRaw { path, pattern } => pattern.is_match(cache.load(path, dcx)?),
Self::Count { path, xpath, text, count } => {
let _data = cache.load(path, dcx)?;
_ = xpath;
_ = text;
_ = count;
true // FIXME
}
Self::Files { path, files } => {
let _data = cache.load(path, dcx)?;
_ = files;
true // FIXME
}
Self::Snapshot { name, path, xpath } => {
let _data = cache.load(path, dcx)?;
_ = name;
_ = path;
_ = xpath;
true // FIXME
}
})
}
}
47 changes: 47 additions & 0 deletions src/tools/htmldocck/src/config.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
use std::path::PathBuf;

use crate::error::DiagCtxt;

pub(crate) struct Config {
/// The path to the directory that contains the generated HTML documentation.
pub(crate) doc_dir: PathBuf,
/// The path to the test file the docs were generated for and which may contain check commands.
pub(crate) template: String,
/// Whether to automatically update snapshot files.
#[allow(dead_code)] // FIXME
pub(crate) bless: bool,
}

impl Config {
pub(crate) fn parse(args: &[String], dcx: &mut DiagCtxt) -> Result<Self, ()> {
const DOC_DIR_OPT: &str = "doc-dir";
const TEMPLATE_OPT: &str = "template";
const BLESS_FLAG: &str = "bless";

let mut opts = getopts::Options::new();
opts.reqopt("", DOC_DIR_OPT, "Path to the documentation directory", "<PATH>")
.reqopt("", TEMPLATE_OPT, "Path to the template file", "<PATH>")
.optflag("", BLESS_FLAG, "Whether to automatically update snapshot files");

// We may not assume the presence of the first argument. On some platforms,
// it's possible to pass an empty array of arguments to `execve`.
let program = args.get(0).map(|arg| arg.as_str()).unwrap_or("htmldocck");
let args = args.get(1..).unwrap_or_default();

match opts.parse(args) {
Ok(matches) => Ok(Self {
doc_dir: matches.opt_str(DOC_DIR_OPT).unwrap().into(),
template: matches.opt_str(TEMPLATE_OPT).unwrap(),
bless: matches.opt_present(BLESS_FLAG),
}),
Err(err) => {
let mut err = err.to_string();
err.push_str("\n\n");
err.push_str(&opts.short_usage(program));
err.push_str(&opts.usage(""));
dcx.emit(&err, None, None);
Err(())
}
}
}
}
Loading

0 comments on commit 263abce

Please sign in to comment.