// Copyright 2022 The Chromium Authors // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. use gnrt_lib::*; use crates::{ChromiumVendoredCrate, StdVendoredCrate}; use manifest::*; use std::collections::{HashMap, HashSet}; use std::fs; use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::process::{self, ExitCode}; use clap::arg; fn main() -> ExitCode { let args = clap::Command::new("gnrt") .subcommand(clap::Command::new("gen") .about("Generate GN build rules from third_party/rust crates") .arg(arg!(--"output-cargo-toml" "Output third_party/rust/Cargo.toml then exit \ immediately")) .arg(arg!(--"skip-patch" "Don't apply gnrt_build_patch after generating build files. \ Useful when updating the patch.")) .arg(arg!(--"for-std" "(WIP) Generate build files for Rust std library instead of \ third_party/rust")) ) .subcommand(clap::Command::new("download") .about("Download the crate with the given name and version to third_party/rust.") .arg(arg!([NAME] "Name of the crate to download").required(true)) .arg(arg!([VERSION] "Version of the crate to download").required(true)) .arg( arg!(--"security-critical" "Whether the crate is considered to be \ security critical." ).possible_values(["yes", "no"]).required(true) ) ) .get_matches(); let paths = paths::ChromiumPaths::new().unwrap(); match args.subcommand() { Some(("gen", args)) => { if args.is_present("for-std") { // This is not fully implemented. Currently, it will print data helpful // for development then quit. generate_for_std(&args, &paths) } else { generate_for_third_party(&args, &paths) } } Some(("download", args)) => { let security = args.value_of("security-critical").unwrap() == "yes"; let name = args.value_of("NAME").unwrap(); use std::str::FromStr; let version = semver::Version::from_str(args.value_of("VERSION").unwrap()) .expect("Invalid version specified"); download::download(name, version, security, &paths) } _ => unreachable!("Invalid subcommand"), } } fn generate_for_third_party(args: &clap::ArgMatches, paths: &paths::ChromiumPaths) -> ExitCode { let manifest_contents = String::from_utf8(fs::read(paths.third_party.join("third_party.toml")).unwrap()).unwrap(); let third_party_manifest: ThirdPartyManifest = match toml::de::from_str(&manifest_contents) { Ok(m) => m, Err(e) => { eprintln!("Failed to parse 'third_party.toml': {e}"); return ExitCode::FAILURE; } }; // Collect special fields from third_party.toml. // // TODO(crbug.com/1291994): handle visibility separately for each kind. let mut pub_deps = HashSet::::new(); let mut build_script_outputs = HashMap::>::new(); for (dep_name, dep_spec) in [ &third_party_manifest.dependency_spec.dependencies, &third_party_manifest.dependency_spec.dev_dependencies, &third_party_manifest.dependency_spec.build_dependencies, ] .into_iter() .flatten() { let (version_req, is_public, dep_outputs): (&_, bool, &[_]) = match dep_spec { Dependency::Short(version_req) => (version_req, true, &[]), Dependency::Full(dep) => ( dep.version.as_ref().unwrap(), dep.allow_first_party_usage, &dep.build_script_outputs, ), }; let epoch = crates::Epoch::from_version_req_str(&version_req.0); let crate_id = crates::ChromiumVendoredCrate { name: dep_name.clone(), epoch }; if is_public { pub_deps.insert(crate_id.clone()); } if !dep_outputs.is_empty() { build_script_outputs.insert(crate_id, dep_outputs.to_vec()); } } // Rebind as immutable. let (pub_deps, build_script_outputs) = (pub_deps, build_script_outputs); // Traverse our third-party directory to collect the set of vendored crates. // Used to generate Cargo.toml [patch] sections, and later to check against // `cargo metadata`'s dependency resolution to ensure we have all the crates // we need. We sort `crates` for a stable ordering of [patch] sections. let mut crates = crates::collect_third_party_crates(paths.third_party.clone()).unwrap(); crates.sort_unstable_by(|a, b| a.0.cmp(&b.0)); // Generate a fake root Cargo.toml for dependency resolution. let cargo_manifest = generate_fake_cargo_toml( third_party_manifest, crates.iter().map(|(c, _)| manifest::PatchSpecification { package_name: c.name.clone(), patch_name: c.patch_name(), path: c.crate_path(), }), ); if args.is_present("output-cargo-toml") { println!("{}", toml::ser::to_string(&cargo_manifest).unwrap()); return ExitCode::SUCCESS; } // Create a fake package: Cargo.toml and an empty main.rs. This allows cargo // to construct a full dependency graph as if Chrome were a cargo package. write!( io::BufWriter::new(fs::File::create(paths.third_party.join("Cargo.toml")).unwrap()), "# {}\n\n{}", AUTOGENERATED_FILE_HEADER, toml::to_string(&cargo_manifest).unwrap() ) .unwrap(); create_dirs_if_needed(&paths.third_party.join("src")).unwrap(); write!( io::BufWriter::new(fs::File::create(paths.third_party.join("src/main.rs")).unwrap()), "// {}", AUTOGENERATED_FILE_HEADER ) .unwrap(); // Run `cargo metadata` and process the output to get a list of crates we // depend on. let mut command = cargo_metadata::MetadataCommand::new(); command.current_dir(&paths.third_party); let dependencies = deps::collect_dependencies(&command.exec().unwrap(), None); // Compare cargo's dependency resolution with the crates we have on disk. We // want to ensure: // * Each resolved dependency matches with a crate we discovered (no missing // deps). // * Each discovered crate matches with a resolved dependency (no unused // crates). let mut has_error = false; let present_crates: HashSet<&ChromiumVendoredCrate> = crates.iter().map(|(c, _)| c).collect(); // Construct the set of requested third-party crates, ensuring we don't have // duplicate epochs. For example, if we resolved two versions of a // dependency with the same major version, we cannot continue. let mut req_crates = HashSet::::new(); for package in &dependencies { if !req_crates.insert(package.third_party_crate_id()) { panic!("found another requested package with the same name and epoch: {:?}", package); } } let req_crates = req_crates; for dep in dependencies.iter() { if !present_crates.contains(&dep.third_party_crate_id()) { has_error = true; println!("Missing dependency: {} {}", dep.package_name, dep.version); for edge in dep.dependency_path.iter() { println!(" {edge}"); } } else if !dep.is_local { // Transitive deps may be requested with version requirements stricter than // ours: e.g. 1.57 instead of just major version 1. If the version we have // checked in, e.g. 1.56, has the same epoch but doesn't meet the version // requirement, the symptom is Cargo will resolve the dependency to an // upstream source instead of our local path. We must detect this case to // ensure correctness. has_error = true; println!( "Resolved {} {} to an upstream source. The requested version \ likely has the same epoch as the discovered crate but \ something has a more stringent version requirement.", dep.package_name, dep.version ); println!(" Resolved version: {}", dep.version); } } for present_crate in present_crates.iter() { if !req_crates.contains(present_crate) { has_error = true; println!("Unused crate: {present_crate}"); } } if has_error { return ExitCode::FAILURE; } let build_files: HashMap = gn::build_files_from_deps( &dependencies, &paths, &crates.iter().cloned().collect(), &build_script_outputs, &pub_deps, ); // Before modifying anything make sure we have a one-to-one mapping of // discovered crates and build file data. for (crate_id, _) in build_files.iter() { // This shouldn't happen, but check anyway in case we have a strange // logic error above. assert!(present_crates.contains(&crate_id)); } for crate_id in present_crates.iter() { if !build_files.contains_key(*crate_id) { println!("Error: discovered crate {crate_id}, but no build file was generated."); has_error = true; } } if has_error { return ExitCode::FAILURE; } // Wipe all previous BUILD.gn files. If we fail, we don't want to leave a // mix of old and new build files. for build_file in crates.iter().map(|(crate_id, _)| build_file_path(crate_id, &paths)) { if build_file.exists() { fs::remove_file(&build_file).unwrap(); } } // Generate build files, wiping the previous ones so we don't have any stale // build rules. for (crate_id, _) in crates.iter() { let build_file_path = build_file_path(crate_id, &paths); let build_file_data = match build_files.get(&crate_id) { Some(build_file) => build_file, None => panic!("missing build data for {crate_id}"), }; write_build_file(&build_file_path, build_file_data).unwrap(); } // Apply patch for BUILD.gn files. let build_patch = paths.root.join(paths.third_party.join("gnrt_build_patch")); if !args.is_present("skip-patch") && build_patch.exists() { let status = process::Command::new("git") .arg("apply") .arg(build_patch) .current_dir(&paths.root) .status() .unwrap(); check_exit_status(status, "applying build patch").unwrap(); } ExitCode::SUCCESS } fn generate_for_std(_args: &clap::ArgMatches, paths: &paths::ChromiumPaths) -> ExitCode { // Run `cargo metadata` from the std package in the Rust source tree (which // is a workspace). let mut command = cargo_metadata::MetadataCommand::new(); command.current_dir(paths.rust_std); // Ensure we use exactly the dependency versions specified in the Rust // source's Cargo.lock. This is the only officially tested libstd. command.other_options(["--locked".to_string(), "--offline".to_string()]); // Compute the set of crates we need to build to build libstd. let dependencies = deps::collect_dependencies(&command.exec().unwrap(), Some(vec!["std".to_string()])); // Collect the set of third-party dependencies vendored in the Rust source // package. let vendored_crates: HashMap = crates::collect_std_vendored_crates(paths.rust_src_vendor).unwrap().into_iter().collect(); // Collect vendored dependencies, and also check that all resolved // dependencies point to our Rust source package. Build rules will be // generated for these crates separately from std, alloc, and core which // need special treatment. let src_prefix = paths.root.join(paths.rust_src); for dep in &dependencies { // Skip workspace members. They are not third-party deps in this // context. if dep.is_workspace_member { continue; } // Skip "rust-std-workspace-*" deps, which are shims to allow std to // depend on third-party crates. See // https://github.com/rust-lang/rust/tree/master/library/rustc-std-workspace-core if dep.package_name.starts_with("rustc-std-workspace-") { continue; } // Skip crates in stdarch, which is a separate workspace in the same // source tree. if dep.package_name.starts_with("std_detect") { continue; } // Skip "rustc-workspace-hack", which is irrelevant to the standard // library build. if dep.package_name.starts_with("rustc-workspace-hack") { continue; } // Only process deps with a library target: we are producing build rules // for the standard library, so transitive binary dependencies don't // make sense. let lib = match &dep.lib_target { Some(lib) => lib, None => continue, }; if !lib.root.starts_with(&src_prefix) { println!( "Found dependency that was not locally available: {} {}", dep.package_name, dep.version ); return ExitCode::FAILURE; } let (crate_id, _): (&StdVendoredCrate, _) = match vendored_crates.get_key_value(&StdVendoredCrate { name: dep.package_name.clone(), version: dep.version.clone(), // Placeholder value for lookup. is_latest: false, }) { Some(id) => id, None => { println!( "Resolved dependency does not match any vendored crate: {} {}", dep.package_name, dep.version ); return ExitCode::FAILURE; } }; // To test, print the list of needed vendored crates and the path of // dependency edges leading to them. println!("{crate_id}"); for next in &dep.dependency_path { println!(" {next}"); } } ExitCode::SUCCESS } fn build_file_path(crate_id: &ChromiumVendoredCrate, paths: &paths::ChromiumPaths) -> PathBuf { let mut path = paths.root.clone(); path.push(&paths.third_party); path.push(crate_id.build_path()); path.push("BUILD.gn"); path } fn write_build_file(path: &Path, build_file: &gn::BuildFile) -> io::Result<()> { let output_handle = fs::File::create(path).unwrap(); // Run our GN output through the official formatter. The gn invocation will // write directly to the output file. let mut formatter = process::Command::new("gn") .arg("format") .arg("--stdin") .stdin(process::Stdio::piped()) .stdout(output_handle) .spawn() .unwrap(); write!(io::BufWriter::new(formatter.stdin.take().unwrap()), "{}", build_file.display())?; let status = formatter.wait()?; check_exit_status(status, "formatting GN output") } fn check_exit_status(status: process::ExitStatus, cmd_msg: &str) -> io::Result<()> { use std::fmt::Write; if status.success() { Ok(()) } else { let mut msg: String = format!("{cmd_msg} failed with "); match status.code() { Some(code) => write!(msg, "{code}").unwrap(), None => write!(msg, "no code").unwrap(), }; Err(io::Error::new(io::ErrorKind::Other, msg)) } } fn create_dirs_if_needed(path: &Path) -> io::Result<()> { if path.is_dir() { return Ok(()); } if let Some(parent) = path.parent() { create_dirs_if_needed(parent)?; } fs::create_dir(path) } /// A message prepended to autogenerated files. Notes this tool generated it and /// not to edit directly. static AUTOGENERATED_FILE_HEADER: &'static str = "!!! DO NOT EDIT -- Autogenerated by gnrt from third_party.toml. Edit that file instead. See tools/crates/gnrt.";