1//! Optional checks for file types other than Rust source2//!3//! Handles python tool version management via a virtual environment in4//! `build/venv`.5//!6//! # Functional outline7//!8//! 1. Run tidy with an extra option: `--extra-checks=py,shell`,9//!    `--extra-checks=py:lint`, or similar. Optionally provide specific10//!    configuration after a double dash (`--extra-checks=py -- foo.py`)11//! 2. Build configuration based on args/environment:12//!    - Formatters by default are in check only mode13//!    - If in CI (TIDY_PRINT_DIFF=1 is set), check and print the diff14//!    - If `--bless` is provided, formatters may run15//!    - Pass any additional config after the `--`. If no files are specified,16//!      use a default.17//! 3. Print the output of the given command. If it fails and `TIDY_PRINT_DIFF`18//!    is set, rerun the tool to print a suggestion diff (for e.g. CI)1920use std::ffi::{OsStr, OsString};21use std::path::{Path, PathBuf};22use std::process::Command;23use std::str::FromStr;24use std::{env, fmt, fs, io};2526use crate::diagnostics::TidyCtx;2728mod rustdoc_js;2930#[cfg(test)]31mod tests;3233const MIN_PY_REV: (u32, u32) = (3, 9);34const MIN_PY_REV_STR: &str = "≥3.9";3536/// Path to find the python executable within a virtual environment37#[cfg(target_os = "windows")]38const REL_PY_PATH: &[&str] = &["Scripts", "python.exe"];39#[cfg(not(target_os = "windows"))]40const REL_PY_PATH: &[&str] = &["bin", "python3"];4142const RUFF_CONFIG_PATH: &[&str] = &["src", "tools", "tidy", "config", "ruff.toml"];43/// Location within build directory44const RUFF_CACHE_PATH: &[&str] = &["cache", "ruff_cache"];45const PIP_REQ_PATH: &[&str] = &["src", "tools", "tidy", "config", "requirements.txt"];4647const SPELLCHECK_DIRS: &[&str] = &["compiler", "library", "src/bootstrap", "src/librustdoc"];48const SPELLCHECK_VER: &str = "1.38.1";4950pub fn check(51    root_path: &Path,52    outdir: &Path,53    librustdoc_path: &Path,54    tools_path: &Path,55    npm: &Path,56    cargo: &Path,57    extra_checks: Option<Vec<String>>,58    pos_args: Vec<String>,59    tidy_ctx: TidyCtx,60) {61    // Split comma-separated args up62    let mut lint_args = match extra_checks {63        Some(s) => s64            .iter()65            .map(|s| {66                if s == "spellcheck:fix" {67                    eprintln!("warning: `spellcheck:fix` is no longer valid, use `--extra-checks=spellcheck --bless`");68                }69                (ExtraCheckArg::from_str(s), s)70            })71            .filter_map(|(res, src)| match res {72                Ok(arg) => {73                    Some(arg)74                }75                Err(err) => {76                    // only warn because before bad extra checks would be silently ignored.77                    eprintln!("warning: bad extra check argument {src:?}: {err:?}");78                    None79                }80            })81            .collect(),82        None => vec![],83    };84    lint_args.retain(|ck| ck.is_non_if_installed_or_matches(root_path, outdir));85    if lint_args.iter().any(|ck| ck.auto) {86        crate::files_modified_batch_filter(87            &tidy_ctx.base_commit,88            tidy_ctx.is_running_on_ci(),89            &mut lint_args,90            |ck, path| ck.is_non_auto_or_matches(path),91        );92    }9394    macro_rules! extra_check {95        ($lang:ident, $kind:ident) => {96            lint_args.iter().any(|arg| arg.matches(ExtraCheckLang::$lang, ExtraCheckKind::$kind))97        };98    }99100    let python_lint = extra_check!(Py, Lint);101    let python_fmt = extra_check!(Py, Fmt);102    let shell_lint = extra_check!(Shell, Lint);103    let cpp_fmt = extra_check!(Cpp, Fmt);104    let spellcheck = extra_check!(Spellcheck, None);105    let js_lint = extra_check!(Js, Lint);106    let js_typecheck = extra_check!(Js, Typecheck);107108    let mut py_path = None;109110    let (cfg_args, file_args): (Vec<_>, Vec<_>) = pos_args111        .iter()112        .map(OsStr::new)113        .partition(|arg| arg.to_str().is_some_and(|s| s.starts_with('-')));114115    if python_lint || python_fmt || cpp_fmt {116        // Since python lint, format and cpp format share python env, we need to ensure python env is installed before running those checks.117        let p = py_prepare(root_path, outdir, &tidy_ctx);118        if p.is_none() {119            return;120        }121        py_path = p;122    }123124    if python_lint {125        check_python_lint(126            root_path,127            outdir,128            &cfg_args,129            &file_args,130            py_path.as_ref().unwrap(),131            &tidy_ctx,132        );133    }134135    if python_fmt {136        check_python_fmt(137            root_path,138            outdir,139            &cfg_args,140            &file_args,141            py_path.as_ref().unwrap(),142            &tidy_ctx,143        );144    }145146    if cpp_fmt {147        check_cpp_fmt(root_path, &cfg_args, &file_args, py_path.as_ref().unwrap(), &tidy_ctx);148    }149150    if shell_lint {151        check_shell_lint(root_path, &cfg_args, &file_args, &tidy_ctx);152    }153154    if spellcheck {155        check_spellcheck(root_path, outdir, cargo, &tidy_ctx);156    }157158    if js_lint || js_typecheck {159        // Since js lint and format share node env, we need to ensure node env is installed before running those checks.160        if js_prepare(root_path, outdir, npm, &tidy_ctx).is_none() {161            return;162        }163    }164165    if js_lint {166        check_js_lint(outdir, librustdoc_path, tools_path, &tidy_ctx);167    }168169    if js_typecheck {170        check_js_typecheck(outdir, librustdoc_path, &tidy_ctx);171    }172}173174fn py_prepare(root_path: &Path, outdir: &Path, tidy_ctx: &TidyCtx) -> Option<PathBuf> {175    let mut check = tidy_ctx.start_check("extra_checks:py_prepare");176177    let venv_path = outdir.join("venv");178    let mut reqs_path = root_path.to_owned();179    reqs_path.extend(PIP_REQ_PATH);180181    match get_or_create_venv(&venv_path, &reqs_path) {182        Ok(p) => Some(p),183        Err(e) => {184            check.error(e);185            None186        }187    }188}189190fn js_prepare(root_path: &Path, outdir: &Path, npm: &Path, tidy_ctx: &TidyCtx) -> Option<()> {191    let mut check = tidy_ctx.start_check("extra_checks:js_prepare");192193    if let Err(e) = rustdoc_js::npm_install(root_path, outdir, npm) {194        check.error(e.to_string());195        return None;196    }197198    Some(())199}200201fn show_bless_help(mode: &str, action: &str, bless: bool) {202    if !bless {203        eprintln!("rerun tidy with `--extra-checks={mode} --bless` to {action}");204    }205}206207fn show_diff() -> bool {208    std::env::var("TIDY_PRINT_DIFF").is_ok_and(|v| v.eq_ignore_ascii_case("true") || v == "1")209}210211fn check_spellcheck(root_path: &Path, outdir: &Path, cargo: &Path, tidy_ctx: &TidyCtx) {212    let mut check = tidy_ctx.start_check("extra_checks:spellcheck");213214    let bless = tidy_ctx.is_bless_enabled();215216    let config_path = root_path.join("typos.toml");217    let mut args = vec!["-c", config_path.as_os_str().to_str().unwrap()];218    args.extend_from_slice(SPELLCHECK_DIRS);219220    if bless {221        eprintln!("spellchecking files and fixing typos");222        args.push("--write-changes");223    } else {224        eprintln!("spellchecking files");225    }226227    if let Err(e) =228        spellcheck_runner(root_path, &outdir, &cargo, &args, tidy_ctx.is_running_on_ci())229    {230        show_bless_help("spellcheck", "fix typos", bless);231        check.error(e);232    }233}234235fn check_js_lint(outdir: &Path, librustdoc_path: &Path, tools_path: &Path, tidy_ctx: &TidyCtx) {236    let mut check = tidy_ctx.start_check("extra_checks:js_lint");237238    let bless = tidy_ctx.is_bless_enabled();239240    if bless {241        eprintln!("linting javascript files and applying suggestions");242    } else {243        eprintln!("linting javascript files");244    }245246    if let Err(e) = rustdoc_js::lint(outdir, librustdoc_path, tools_path, bless) {247        show_bless_help("js:lint", "apply esplint suggestion", bless);248        check.error(e);249        return;250    }251252    if let Err(e) = rustdoc_js::es_check(outdir, librustdoc_path) {253        check.error(e);254    }255}256257fn check_js_typecheck(outdir: &Path, librustdoc_path: &Path, tidy_ctx: &TidyCtx) {258    let mut check = tidy_ctx.start_check("extra_checks:js_typecheck");259260    eprintln!("typechecking javascript files");261    if let Err(e) = rustdoc_js::typecheck(outdir, librustdoc_path) {262        check.error(e);263    }264}265266fn check_shell_lint(267    root_path: &Path,268    cfg_args: &Vec<&OsStr>,269    file_args: &Vec<&OsStr>,270    tidy_ctx: &TidyCtx,271) {272    let mut check = tidy_ctx.start_check("extra_checks:shell_lint");273274    eprintln!("linting shell files");275276    let mut file_args_shc = file_args.clone();277    let files;278    if file_args.is_empty() {279        match find_with_extension(root_path, None, &[OsStr::new("sh")]) {280            Ok(f) => files = f,281            Err(e) => {282                check.error(e);283                return;284            }285        }286287        file_args_shc.extend(files.iter().map(|p| p.as_os_str()));288    }289290    if let Err(e) = shellcheck_runner(&merge_args(&cfg_args, &file_args_shc)) {291        check.error(e);292    }293}294295fn check_python_lint(296    root_path: &Path,297    outdir: &Path,298    cfg_args: &Vec<&OsStr>,299    file_args: &Vec<&OsStr>,300    py_path: &Path,301    tidy_ctx: &TidyCtx,302) {303    let mut check = tidy_ctx.start_check("extra_checks:python_lint");304305    let bless = tidy_ctx.is_bless_enabled();306307    let args: &[&OsStr] = if bless {308        eprintln!("linting python files and applying suggestions");309        &["check".as_ref(), "--fix".as_ref()]310    } else {311        eprintln!("linting python files");312        &["check".as_ref()]313    };314315    let res = run_ruff(root_path, outdir, py_path, &cfg_args, &file_args, args);316317    if res.is_err() && !bless && show_diff() {318        eprintln!("\npython linting failed! Printing diff suggestions:");319320        let diff_res = run_ruff(321            root_path,322            outdir,323            py_path,324            &cfg_args,325            &file_args,326            &["check".as_ref(), "--diff".as_ref()],327        );328        // `ruff check --diff` will return status 0 if there are no suggestions.329        if diff_res.is_err() {330            show_bless_help("py:lint", "apply ruff suggestions", bless);331        }332    }333    if let Err(e) = res {334        check.error(e);335    }336}337338fn check_python_fmt(339    root_path: &Path,340    outdir: &Path,341    cfg_args: &Vec<&OsStr>,342    file_args: &Vec<&OsStr>,343    py_path: &Path,344    tidy_ctx: &TidyCtx,345) {346    let mut check = tidy_ctx.start_check("extra_checks:python_fmt");347348    let bless = tidy_ctx.is_bless_enabled();349350    let mut args: Vec<&OsStr> = vec!["format".as_ref()];351    if bless {352        eprintln!("formatting python files");353    } else {354        eprintln!("checking python file formatting");355        args.push("--check".as_ref());356    }357358    let res = run_ruff(root_path, outdir, py_path, &cfg_args, &file_args, &args);359360    if res.is_err() && !bless {361        if show_diff() {362            eprintln!("\npython formatting does not match! Printing diff:");363364            let _ = run_ruff(365                root_path,366                outdir,367                py_path,368                &cfg_args,369                &file_args,370                &["format".as_ref(), "--diff".as_ref()],371            );372        }373        show_bless_help("py:fmt", "reformat Python code", bless);374    }375376    if let Err(e) = res {377        check.error(e);378    }379}380381fn check_cpp_fmt(382    root_path: &Path,383    cfg_args: &Vec<&OsStr>,384    file_args: &Vec<&OsStr>,385    py_path: &Path,386    tidy_ctx: &TidyCtx,387) {388    let mut check = tidy_ctx.start_check("extra_checks:cpp_fmt");389390    let bless = tidy_ctx.is_bless_enabled();391392    let mut cfg_args_clang_format = cfg_args.clone();393    let mut file_args_clang_format = file_args.clone();394    let config_path = root_path.join(".clang-format");395    let mut config_file_arg = OsString::from("file:");396    config_file_arg.push(&config_path);397    cfg_args_clang_format.extend(&["--style".as_ref(), config_file_arg.as_ref()]);398    if bless {399        eprintln!("formatting C++ files");400        cfg_args_clang_format.push("-i".as_ref());401    } else {402        eprintln!("checking C++ file formatting");403        cfg_args_clang_format.extend(&["--dry-run".as_ref(), "--Werror".as_ref()]);404    }405    let files;406    if file_args_clang_format.is_empty() {407        let llvm_wrapper = root_path.join("compiler/rustc_llvm/llvm-wrapper");408        match find_with_extension(409            root_path,410            Some(llvm_wrapper.as_path()),411            &[OsStr::new("h"), OsStr::new("cpp")],412        ) {413            Ok(f) => files = f,414            Err(e) => {415                check.error(e);416                return;417            }418        }419        file_args_clang_format.extend(files.iter().map(|p| p.as_os_str()));420    }421    let args = merge_args(&cfg_args_clang_format, &file_args_clang_format);422    let res = py_runner(py_path, false, None, "clang-format", &args);423424    if res.is_err() && !bless && show_diff() {425        eprintln!("\nclang-format linting failed! Printing diff suggestions:");426427        let mut cfg_args_diff = cfg_args.clone();428        cfg_args_diff.extend(&["--style".as_ref(), config_file_arg.as_ref()]);429        for file in file_args {430            let mut formatted = String::new();431            let mut diff_args = cfg_args_diff.clone();432            diff_args.push(file);433            let _ = py_runner(py_path, false, Some(&mut formatted), "clang-format", &diff_args);434            if formatted.is_empty() {435                eprintln!(436                    "failed to obtain the formatted content for '{}'",437                    file.to_string_lossy()438                );439                continue;440            }441            let actual = std::fs::read_to_string(file).unwrap_or_else(|e| {442                panic!("failed to read the C++ file at '{}' due to '{e}'", file.to_string_lossy())443            });444            if formatted != actual {445                let diff = similar::TextDiff::from_lines(&actual, &formatted);446                eprintln!(447                    "{}",448                    diff.unified_diff().context_radius(4).header(449                        &format!("{} (actual)", file.to_string_lossy()),450                        &format!("{} (formatted)", file.to_string_lossy())451                    )452                );453            }454        }455        show_bless_help("cpp:fmt", "reformat C++ code", bless);456    }457458    if let Err(e) = res {459        check.error(e);460    }461}462463fn run_ruff(464    root_path: &Path,465    outdir: &Path,466    py_path: &Path,467    cfg_args: &[&OsStr],468    file_args: &[&OsStr],469    ruff_args: &[&OsStr],470) -> Result<(), Error> {471    let mut cfg_args_ruff = cfg_args.to_vec();472    let mut file_args_ruff = file_args.to_vec();473474    let mut cfg_path = root_path.to_owned();475    cfg_path.extend(RUFF_CONFIG_PATH);476    let mut cache_dir = outdir.to_owned();477    cache_dir.extend(RUFF_CACHE_PATH);478479    cfg_args_ruff.extend([480        "--config".as_ref(),481        cfg_path.as_os_str(),482        "--cache-dir".as_ref(),483        cache_dir.as_os_str(),484    ]);485486    if file_args_ruff.is_empty() {487        file_args_ruff.push(root_path.as_os_str());488    }489490    let mut args: Vec<&OsStr> = ruff_args.to_vec();491    args.extend(merge_args(&cfg_args_ruff, &file_args_ruff));492    py_runner(py_path, true, None, "ruff", &args)493}494495/// Helper to create `cfg1 cfg2 -- file1 file2` output496fn merge_args<'a>(cfg_args: &[&'a OsStr], file_args: &[&'a OsStr]) -> Vec<&'a OsStr> {497    let mut args = cfg_args.to_owned();498    args.push("--".as_ref());499    args.extend(file_args);500    args501}502503/// Run a python command with given arguments. `py_path` should be a virtualenv.504///505/// Captures `stdout` to a string if provided, otherwise prints the output.506fn py_runner(507    py_path: &Path,508    as_module: bool,509    stdout: Option<&mut String>,510    bin: &'static str,511    args: &[&OsStr],512) -> Result<(), Error> {513    let mut cmd = Command::new(py_path);514    if as_module {515        cmd.arg("-m").arg(bin).args(args);516    } else {517        let bin_path = py_path.with_file_name(bin);518        cmd.arg(bin_path).args(args);519    }520    let status = if let Some(stdout) = stdout {521        let output = cmd.output()?;522        if let Ok(s) = std::str::from_utf8(&output.stdout) {523            stdout.push_str(s);524        }525        output.status526    } else {527        cmd.status()?528    };529    if status.success() { Ok(()) } else { Err(Error::FailedCheck(bin)) }530}531532/// Create a virtuaenv at a given path if it doesn't already exist, or validate533/// the install if it does. Returns the path to that venv's python executable.534fn get_or_create_venv(venv_path: &Path, src_reqs_path: &Path) -> Result<PathBuf, Error> {535    let mut py_path = venv_path.to_owned();536    py_path.extend(REL_PY_PATH);537538    if !has_py_tools(venv_path, src_reqs_path)? {539        let dst_reqs_path = venv_path.join("requirements.txt");540        eprintln!("removing old virtual environment");541        if venv_path.is_dir() {542            fs::remove_dir_all(venv_path).unwrap_or_else(|_| {543                panic!("failed to remove directory at {}", venv_path.display())544            });545        }546        create_venv_at_path(venv_path)?;547        install_requirements(&py_path, src_reqs_path, &dst_reqs_path)?;548    }549550    verify_py_version(&py_path)?;551    Ok(py_path)552}553554fn has_py_tools(venv_path: &Path, src_reqs_path: &Path) -> Result<bool, Error> {555    let dst_reqs_path = venv_path.join("requirements.txt");556    if let Ok(req) = fs::read_to_string(&dst_reqs_path) {557        if req == fs::read_to_string(src_reqs_path)? {558            return Ok(true);559        }560        eprintln!("requirements.txt file mismatch");561    }562563    Ok(false)564}565566/// Attempt to create a virtualenv at this path. Cycles through all expected567/// valid python versions to find one that is installed.568fn create_venv_at_path(path: &Path) -> Result<(), Error> {569    /// Preferred python versions in order. Newest to oldest then current570    /// development versions571    const TRY_PY: &[&str] = &[572        "python3.13",573        "python3.12",574        "python3.11",575        "python3.10",576        "python3.9",577        "python3",578        "python",579        "python3.14",580    ];581582    let mut sys_py = None;583    let mut found = Vec::new();584585    for py in TRY_PY {586        match verify_py_version(Path::new(py)) {587            Ok(_) => {588                sys_py = Some(*py);589                break;590            }591            // Skip not found errors592            Err(Error::Io(e)) if e.kind() == io::ErrorKind::NotFound => (),593            // Skip insufficient version errors594            Err(Error::Version { installed, .. }) => found.push(installed),595            // just log and skip unrecognized errors596            Err(e) => eprintln!("note: error running '{py}': {e}"),597        }598    }599600    let Some(sys_py) = sys_py else {601        let ret = if found.is_empty() {602            Error::MissingReq("python3", "python file checks", None)603        } else {604            found.sort();605            found.dedup();606            Error::Version {607                program: "python3",608                required: MIN_PY_REV_STR,609                installed: found.join(", "),610            }611        };612        return Err(ret);613    };614615    // First try venv, which should be packaged in the Python3 standard library.616    // If it is not available, try to create the virtual environment using the617    // virtualenv package.618    if try_create_venv(sys_py, path, "venv").is_ok() {619        return Ok(());620    }621    try_create_venv(sys_py, path, "virtualenv")622}623624fn try_create_venv(python: &str, path: &Path, module: &str) -> Result<(), Error> {625    eprintln!(626        "creating virtual environment at '{}' using '{python}' and '{module}'",627        path.display()628    );629    let out = Command::new(python).args(["-m", module]).arg(path).output().unwrap();630631    if out.status.success() {632        return Ok(());633    }634635    let stderr = String::from_utf8_lossy(&out.stderr);636    let err = if stderr.contains(&format!("No module named {module}")) {637        Error::Generic(format!(638            r#"{module} not found: you may need to install it:639`{python} -m pip install {module}`640If you see an error about "externally managed environment" when running the above command,641either install `{module}` using your system package manager642(e.g. `sudo apt-get install {python}-{module}`) or create a virtual environment manually, install643`{module}` in it and then activate it before running tidy.644"#645        ))646    } else {647        Error::Generic(format!(648            "failed to create venv at '{}' using {python} -m {module}: {stderr}",649            path.display()650        ))651    };652    Err(err)653}654655/// Parse python's version output (`Python x.y.z`) and ensure we have a656/// suitable version.657fn verify_py_version(py_path: &Path) -> Result<(), Error> {658    let out = Command::new(py_path).arg("--version").output()?;659    let outstr = String::from_utf8_lossy(&out.stdout);660    let vers = outstr.trim().split_ascii_whitespace().nth(1).unwrap().trim();661    let mut vers_comps = vers.split('.');662    let major: u32 = vers_comps.next().unwrap().parse().unwrap();663    let minor: u32 = vers_comps.next().unwrap().parse().unwrap();664665    if (major, minor) < MIN_PY_REV {666        Err(Error::Version {667            program: "python",668            required: MIN_PY_REV_STR,669            installed: vers.to_owned(),670        })671    } else {672        Ok(())673    }674}675676fn install_requirements(677    py_path: &Path,678    src_reqs_path: &Path,679    dst_reqs_path: &Path,680) -> Result<(), Error> {681    let stat = Command::new(py_path)682        .args(["-m", "pip", "install", "--upgrade", "pip"])683        .status()684        .expect("failed to launch pip");685    if !stat.success() {686        return Err(Error::Generic(format!("pip install failed with status {stat}")));687    }688689    let stat = Command::new(py_path)690        .args(["-m", "pip", "install", "--quiet", "--require-hashes", "-r"])691        .arg(src_reqs_path)692        .status()?;693    if !stat.success() {694        return Err(Error::Generic(format!(695            "failed to install requirements at {}",696            src_reqs_path.display()697        )));698    }699    fs::copy(src_reqs_path, dst_reqs_path)?;700    assert_eq!(701        fs::read_to_string(src_reqs_path).unwrap(),702        fs::read_to_string(dst_reqs_path).unwrap()703    );704    Ok(())705}706707/// Returns `Ok` if shellcheck is installed, `Err` otherwise.708fn has_shellcheck() -> Result<(), Error> {709    match Command::new("shellcheck").arg("--version").status() {710        Ok(_) => Ok(()),711        Err(e) if e.kind() == io::ErrorKind::NotFound => Err(Error::MissingReq(712            "shellcheck",713            "shell file checks",714            Some(715                "see <https://github.com/koalaman/shellcheck#installing> \716                for installation instructions"717                    .to_owned(),718            ),719        )),720        Err(e) => Err(e.into()),721    }722}723724/// Check that shellcheck is installed then run it at the given path725fn shellcheck_runner(args: &[&OsStr]) -> Result<(), Error> {726    has_shellcheck()?;727728    let status = Command::new("shellcheck").args(args).status()?;729    if status.success() { Ok(()) } else { Err(Error::FailedCheck("shellcheck")) }730}731732/// Ensure that spellchecker is installed then run it at the given path733fn spellcheck_runner(734    src_root: &Path,735    outdir: &Path,736    cargo: &Path,737    args: &[&str],738    is_ci: bool,739) -> Result<(), Error> {740    let bin_path = ensure_version_or_cargo_install(741        outdir,742        cargo,743        "typos-cli",744        "typos",745        SPELLCHECK_VER,746        is_ci,747    )?;748    match Command::new(bin_path).current_dir(src_root).args(args).status() {749        Ok(status) => {750            if status.success() {751                Ok(())752            } else {753                Err(Error::FailedCheck("typos"))754            }755        }756        Err(err) => Err(Error::Generic(format!("failed to run typos tool: {err:?}"))),757    }758}759760/// Check git for tracked files matching an extension761fn find_with_extension(762    root_path: &Path,763    find_dir: Option<&Path>,764    extensions: &[&OsStr],765) -> Result<Vec<PathBuf>, Error> {766    // Untracked files show up for short status and are indicated with a leading `?`767    // -C changes git to be as if run from that directory768    let stat_output =769        Command::new("git").arg("-C").arg(root_path).args(["status", "--short"]).output()?.stdout;770771    if String::from_utf8_lossy(&stat_output).lines().filter(|ln| ln.starts_with('?')).count() > 0 {772        eprintln!("found untracked files, ignoring");773    }774775    let mut output = Vec::new();776    let binding = {777        let mut command = Command::new("git");778        command.arg("-C").arg(root_path).args(["ls-files"]);779        if let Some(find_dir) = find_dir {780            command.arg(find_dir);781        }782        command.output()?783    };784    let tracked = String::from_utf8_lossy(&binding.stdout);785786    for line in tracked.lines() {787        let line = line.trim();788        let path = Path::new(line);789790        let Some(ref extension) = path.extension() else {791            continue;792        };793        if extensions.contains(extension) {794            output.push(root_path.join(path));795        }796    }797798    Ok(output)799}800801/// Check if the given executable is installed and the version is expected.802fn ensure_version(build_dir: &Path, bin_name: &str, version: &str) -> Result<PathBuf, Error> {803    let bin_path = build_dir.join("misc-tools").join("bin").join(bin_name);804805    match Command::new(&bin_path).arg("--version").output() {806        Ok(output) => {807            let Some(v) = str::from_utf8(&output.stdout).unwrap().trim().split_whitespace().last()808            else {809                return Err(Error::Generic("version check failed".to_string()));810            };811812            if v != version {813                return Err(Error::Version { program: "", required: "", installed: v.to_string() });814            }815            Ok(bin_path)816        }817        Err(e) => Err(Error::Io(e)),818    }819}820821/// If the given executable is installed with the given version, use that,822/// otherwise install via cargo.823fn ensure_version_or_cargo_install(824    build_dir: &Path,825    cargo: &Path,826    pkg_name: &str,827    bin_name: &str,828    version: &str,829    is_ci: bool,830) -> Result<PathBuf, Error> {831    if let Ok(bin_path) = ensure_version(build_dir, bin_name, version) {832        return Ok(bin_path);833    }834835    eprintln!("building external tool {bin_name} from package {pkg_name}@{version}");836837    let tool_root_dir = build_dir.join("misc-tools");838    let tool_bin_dir = tool_root_dir.join("bin");839    let bin_path = tool_bin_dir.join(bin_name).with_extension(env::consts::EXE_EXTENSION);840841    // use --force to ensure that if the required version is bumped, we update it.842    // use --target-dir to ensure we have a build cache so repeated invocations aren't slow.843    // modify PATH so that cargo doesn't print a warning telling the user to modify the path.844    let mut cmd = Command::new(cargo);845    cmd.args(["install", "--locked", "--force", "--quiet"])846        .arg("--root")847        .arg(&tool_root_dir)848        .arg("--target-dir")849        .arg(tool_root_dir.join("target"))850        .arg(format!("{pkg_name}@{version}"))851        .env(852            "PATH",853            env::join_paths(854                env::split_paths(&env::var("PATH").unwrap())855                    .chain(std::iter::once(tool_bin_dir.clone())),856            )857            .expect("build dir contains invalid char"),858        );859860    // On CI, we set opt-level flag for quicker installation.861    // Since lower opt-level decreases the tool's performance,862    // we don't set this option on local.863    if is_ci {864        cmd.env("RUSTFLAGS", "-Copt-level=0");865    }866867    let cargo_exit_code = cmd.spawn()?.wait()?;868    if !cargo_exit_code.success() {869        return Err(Error::Generic("cargo install failed".to_string()));870    }871    assert!(872        matches!(bin_path.try_exists(), Ok(true)),873        "cargo install did not produce the expected binary"874    );875    eprintln!("finished building tool {bin_name}");876    Ok(bin_path)877}878879#[derive(Debug)]880enum Error {881    Io(io::Error),882    /// a is required to run b. c is extra info883    MissingReq(&'static str, &'static str, Option<String>),884    /// Tool x failed the check885    FailedCheck(&'static str),886    /// Any message, just print it887    Generic(String),888    /// Installed but wrong version889    Version {890        program: &'static str,891        required: &'static str,892        installed: String,893    },894}895896impl fmt::Display for Error {897    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {898        match self {899            Self::MissingReq(a, b, ex) => {900                write!(901                    f,902                    "{a} is required to run {b} but it could not be located. Is it installed?"903                )?;904                if let Some(s) = ex {905                    write!(f, "\n{s}")?;906                };907                Ok(())908            }909            Self::Version { program, required, installed } => write!(910                f,911                "insufficient version of '{program}' to run external tools: \912                {required} required but found {installed}",913            ),914            Self::Generic(s) => f.write_str(s),915            Self::Io(e) => write!(f, "IO error: {e}"),916            Self::FailedCheck(s) => write!(f, "checks with external tool '{s}' failed"),917        }918    }919}920921impl From<io::Error> for Error {922    fn from(value: io::Error) -> Self {923        Self::Io(value)924    }925}926927#[derive(Debug, PartialEq)]928enum ExtraCheckParseError {929    #[allow(dead_code, reason = "shown through Debug")]930    UnknownKind(String),931    #[allow(dead_code)]932    UnknownLang(String),933    UnsupportedKindForLang,934    /// Too many `:`935    TooManyParts,936    /// Tried to parse the empty string937    Empty,938    /// `auto` specified without lang part.939    AutoRequiresLang,940    /// `if-installed` specified without lang part.941    IfInstalledRequiresLang,942}943944#[derive(PartialEq, Debug)]945struct ExtraCheckArg {946    /// Only run the check if files to check have been modified.947    auto: bool,948    /// Only run the check if the requisite software is already installed.949    if_installed: bool,950    lang: ExtraCheckLang,951    /// None = run all extra checks for the given lang952    kind: Option<ExtraCheckKind>,953}954955impl ExtraCheckArg {956    fn matches(&self, lang: ExtraCheckLang, kind: ExtraCheckKind) -> bool {957        self.lang == lang && self.kind.map(|k| k == kind).unwrap_or(true)958    }959960    fn is_non_if_installed_or_matches(&self, root_path: &Path, build_dir: &Path) -> bool {961        if !self.if_installed {962            return true;963        }964965        match self.lang {966            ExtraCheckLang::Spellcheck => {967                match ensure_version(build_dir, "typos", SPELLCHECK_VER) {968                    Ok(_) => true,969                    Err(Error::Version { installed, .. }) => {970                        eprintln!(971                            "warning: the tool `typos` is detected, but version {installed} doesn't match with the expected version {SPELLCHECK_VER}"972                        );973                        false974                    }975                    _ => false,976                }977            }978            ExtraCheckLang::Shell => has_shellcheck().is_ok(),979            ExtraCheckLang::Js => {980                match self.kind {981                    Some(ExtraCheckKind::Lint) => {982                        // If Lint is enabled, check both eslint and es-check.983                        rustdoc_js::has_tool(build_dir, "eslint")984                            && rustdoc_js::has_tool(build_dir, "es-check")985                    }986                    Some(ExtraCheckKind::Typecheck) => {987                        // If Typecheck is enabled, check tsc.988                        rustdoc_js::has_tool(build_dir, "tsc")989                    }990                    None => {991                        // No kind means it will check both Lint and Typecheck.992                        rustdoc_js::has_tool(build_dir, "eslint")993                            && rustdoc_js::has_tool(build_dir, "es-check")994                            && rustdoc_js::has_tool(build_dir, "tsc")995                    }996                    Some(_) => unreachable!("js shouldn't have other type of ExtraCheckKind"),997                }998            }999            ExtraCheckLang::Py | ExtraCheckLang::Cpp => {1000                let venv_path = build_dir.join("venv");1001                let mut reqs_path = root_path.to_owned();1002                reqs_path.extend(PIP_REQ_PATH);1003                let Ok(v) = has_py_tools(&venv_path, &reqs_path) else {1004                    return false;1005                };10061007                v1008            }1009        }1010    }10111012    /// Returns `false` if this is an auto arg and the passed filename does not trigger the auto rule1013    fn is_non_auto_or_matches(&self, filepath: &str) -> bool {1014        if !self.auto {1015            return true;1016        }1017        let exts: &[&str] = match self.lang {1018            ExtraCheckLang::Py => &[".py"],1019            ExtraCheckLang::Cpp => &[".cpp"],1020            ExtraCheckLang::Shell => &[".sh"],1021            ExtraCheckLang::Js => &[".js", ".ts"],1022            ExtraCheckLang::Spellcheck => {1023                if SPELLCHECK_DIRS.iter().any(|dir| Path::new(filepath).starts_with(dir)) {1024                    return true;1025                }1026                &[]1027            }1028        };1029        exts.iter().any(|ext| filepath.ends_with(ext))1030    }10311032    fn has_supported_kind(&self) -> bool {1033        let Some(kind) = self.kind else {1034            // "run all extra checks" mode is supported for all languages.1035            return true;1036        };1037        use ExtraCheckKind::*;1038        let supported_kinds: &[_] = match self.lang {1039            ExtraCheckLang::Py => &[Fmt, Lint],1040            ExtraCheckLang::Cpp => &[Fmt],1041            ExtraCheckLang::Shell => &[Lint],1042            ExtraCheckLang::Spellcheck => &[],1043            ExtraCheckLang::Js => &[Lint, Typecheck],1044        };1045        supported_kinds.contains(&kind)1046    }1047}10481049impl FromStr for ExtraCheckArg {1050    type Err = ExtraCheckParseError;10511052    fn from_str(s: &str) -> Result<Self, Self::Err> {1053        let mut auto = false;1054        let mut if_installed = false;1055        let mut parts = s.split(':');1056        let mut first = match parts.next() {1057            Some("") | None => return Err(ExtraCheckParseError::Empty),1058            Some(part) => part,1059        };10601061        // The loop allows users to specify `auto` and `if-installed` in any order.1062        // Both auto:if-installed:<check> and if-installed:auto:<check> are valid.1063        loop {1064            match (first, auto, if_installed) {1065                ("auto", false, _) => {1066                    let Some(part) = parts.next() else {1067                        return Err(ExtraCheckParseError::AutoRequiresLang);1068                    };1069                    auto = true;1070                    first = part;1071                }1072                ("if-installed", _, false) => {1073                    let Some(part) = parts.next() else {1074                        return Err(ExtraCheckParseError::IfInstalledRequiresLang);1075                    };1076                    if_installed = true;1077                    first = part;1078                }1079                _ => break,1080            }1081        }1082        let second = parts.next();1083        if parts.next().is_some() {1084            return Err(ExtraCheckParseError::TooManyParts);1085        }1086        let arg = Self {1087            auto,1088            if_installed,1089            lang: first.parse()?,1090            kind: second.map(|s| s.parse()).transpose()?,1091        };1092        if !arg.has_supported_kind() {1093            return Err(ExtraCheckParseError::UnsupportedKindForLang);1094        }10951096        Ok(arg)1097    }1098}10991100#[derive(PartialEq, Copy, Clone, Debug)]1101enum ExtraCheckLang {1102    Py,1103    Shell,1104    Cpp,1105    Spellcheck,1106    Js,1107}11081109impl FromStr for ExtraCheckLang {1110    type Err = ExtraCheckParseError;11111112    fn from_str(s: &str) -> Result<Self, Self::Err> {1113        Ok(match s {1114            "py" => Self::Py,1115            "shell" => Self::Shell,1116            "cpp" => Self::Cpp,1117            "spellcheck" => Self::Spellcheck,1118            "js" => Self::Js,1119            _ => return Err(ExtraCheckParseError::UnknownLang(s.to_string())),1120        })1121    }1122}11231124#[derive(PartialEq, Copy, Clone, Debug)]1125enum ExtraCheckKind {1126    Lint,1127    Fmt,1128    Typecheck,1129    /// Never parsed, but used as a placeholder for1130    /// langs that never have a specific kind.1131    None,1132}11331134impl FromStr for ExtraCheckKind {1135    type Err = ExtraCheckParseError;11361137    fn from_str(s: &str) -> Result<Self, Self::Err> {1138        Ok(match s {1139            "lint" => Self::Lint,1140            "fmt" => Self::Fmt,1141            "typecheck" => Self::Typecheck,1142            _ => return Err(ExtraCheckParseError::UnknownKind(s.to_string())),1143        })1144    }1145}