From 3ec01b7693d46745c0fac413eb79065ef3712fac Mon Sep 17 00:00:00 2001 From: Jeremy Soller Date: Fri, 13 Jun 2025 12:28:55 -0600 Subject: [PATCH] Greatly increase recipe scanning performance --- Cargo.lock | 40 ++++++++++++++++++ Cargo.toml | 1 + src/bin/cook.rs | 4 +- src/bin/find_recipe.rs | 19 ++++----- src/bin/list_recipes.rs | 36 ++++++---------- src/bin/pkg_deps.rs | 4 -- src/package.rs | 4 +- src/recipe.rs | 4 +- src/recipe_find.rs | 92 ++++++++++++----------------------------- 9 files changed, 95 insertions(+), 109 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e63e8c22..4cc6bb78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -321,6 +321,16 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bstr" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "bumpalo" version = "3.17.0" @@ -951,6 +961,19 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "globset" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + [[package]] name = "gpt" version = "3.1.0" @@ -1254,6 +1277,22 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "ignore" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "indexmap" version = "2.7.1" @@ -1806,6 +1845,7 @@ name = "redox_cookbook" version = "0.1.0" dependencies = [ "blake3 1.5.3", + "ignore", "object", "pbr", "pkgar 0.1.17", diff --git a/Cargo.toml b/Cargo.toml index 44fc90a7..15608f40 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ path = "src/lib.rs" [dependencies] blake3 = "=1.5.3" # 1.5.4 is incompatible with blake3 0.3 dependency from pkgar +ignore = "0.4" object = { version = "0.36", features = ["build_core"] } pbr = "1.0.2" pkgar = { path = "pkgar/pkgar" } diff --git a/src/bin/cook.rs b/src/bin/cook.rs index bd9916cf..dca5329f 100644 --- a/src/bin/cook.rs +++ b/src/bin/cook.rs @@ -1,6 +1,6 @@ use cookbook::blake3::blake3_progress; use cookbook::package::StageToml; -use cookbook::recipe::{BuildKind, CookRecipe, PackageRecipe, Recipe, SourceRecipe}; +use cookbook::recipe::{BuildKind, CookRecipe, Recipe, SourceRecipe}; use cookbook::recipe_find::recipe_find; use std::{ collections::BTreeSet, @@ -601,7 +601,7 @@ fn build( let mut dep_pkgars = BTreeSet::new(); for dependency in recipe.build.dependencies.iter() { //TODO: sanitize name - let dependency_dir = recipe_find(dependency, Path::new("recipes"))?; + let dependency_dir = recipe_find(dependency); if dependency_dir.is_none() { return Err(format!("failed to find recipe directory '{}'", dependency)); } diff --git a/src/bin/find_recipe.rs b/src/bin/find_recipe.rs index ea84db43..92b8ec05 100644 --- a/src/bin/find_recipe.rs +++ b/src/bin/find_recipe.rs @@ -1,6 +1,5 @@ use cookbook::recipe_find::recipe_find; use std::env::args; -use std::path::Path; use std::process::exit; // use clap::Parser; @@ -12,15 +11,15 @@ fn main() { usage(); exit(2); } - let result = recipe_find(&args().last().unwrap(), Path::new("recipes")); - if result.is_err() { - eprintln!("{}", result.err().unwrap()); - exit(2); - } else if result.as_ref().unwrap().is_none() { - eprintln!("recipe {} not found", &args().last().unwrap()); + let recipe_name = &args().last().unwrap(); + match recipe_find(recipe_name) { + Some(path) => { + println!("{}", path.display()); + exit(0); + }, + None => { + eprintln!("recipe {} not found", recipe_name); exit(1); - } else { - println!("{}", result.unwrap().unwrap().display()); - exit(0); + } } } diff --git a/src/bin/list_recipes.rs b/src/bin/list_recipes.rs index 1597e4ce..81f88e47 100644 --- a/src/bin/list_recipes.rs +++ b/src/bin/list_recipes.rs @@ -1,5 +1,4 @@ use cookbook::recipe_find::list_recipes; -use std::path::Path; use std::process::exit; // use clap::Parser; @@ -8,31 +7,22 @@ fn main() { .nth(1) .map_or(false, |a| a == "-s" || a == "--short"); - let result = list_recipes(Path::new("recipes"), Default::default()); + let result = list_recipes(Default::default()); + if result.is_empty() { + eprintln!("recipes not found"); + exit(1); + } else { + for path in result { + let Some(file_name) = path.file_name() else { + continue; + }; - match result { - Ok(result) => { - if result.is_empty() { - eprintln!("recipes not found"); - exit(1); + if print_short { + println!("{}", file_name.to_string_lossy()); } else { - for path in result { - let Some(file_name) = path.file_name() else { - continue; - }; - - if print_short { - println!("{}", file_name.to_string_lossy()); - } else { - println!("{}", path.to_string_lossy()); - } - } - exit(0); + println!("{}", path.to_string_lossy()); } } - Err(error) => { - eprintln!("{error}"); - exit(2); - } + exit(0); } } diff --git a/src/bin/pkg_deps.rs b/src/bin/pkg_deps.rs index 2265cdfd..657b3039 100644 --- a/src/bin/pkg_deps.rs +++ b/src/bin/pkg_deps.rs @@ -4,10 +4,6 @@ use std::{env::args, process::ExitCode}; /// Same as `cookbook/src/bin/cook.rs`. const DEP_DEPTH: usize = 16; -fn usage() { - eprintln!("Usage: pkg_deps [package1 package2 ...]"); -} - fn main() -> ExitCode { let names = args().skip(1).collect::>(); let packages = StageToml::new_recursive(&names, DEP_DEPTH).expect("package not found"); diff --git a/src/package.rs b/src/package.rs index 15d1815f..c86a610f 100644 --- a/src/package.rs +++ b/src/package.rs @@ -1,4 +1,4 @@ -use std::{env, fs, path::Path}; +use std::{env, fs}; use crate::recipe_find::recipe_find; @@ -14,7 +14,7 @@ pub struct StageToml { impl StageToml { pub fn new(name: String) -> Result { //TODO: sanitize recipe name? - let dir = recipe_find(&name, Path::new("recipes"))?; + let dir = recipe_find(&name); if dir.is_none() { return Err(format!("failed to find recipe directory '{}'", name)); } diff --git a/src/recipe.rs b/src/recipe.rs index 92bfce44..456f6097 100644 --- a/src/recipe.rs +++ b/src/recipe.rs @@ -1,6 +1,6 @@ use std::{ fs, - path::{Path, PathBuf}, + path::PathBuf, }; use serde::{Deserialize, Serialize}; @@ -114,7 +114,7 @@ pub struct CookRecipe { impl CookRecipe { pub fn new(name: String) -> Result { //TODO: sanitize recipe name? - let dir = recipe_find(&name, Path::new("recipes"))?; + let dir = recipe_find(&name); if dir.is_none() { return Err(format!("failed to find recipe directory '{}'", name)); } diff --git a/src/recipe_find.rs b/src/recipe_find.rs index 8354264e..b0cedcbd 100644 --- a/src/recipe_find.rs +++ b/src/recipe_find.rs @@ -1,78 +1,38 @@ +use std::collections::HashMap; use std::ffi::OsStr; -use std::fs::{self}; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; +use std::sync::LazyLock; -pub fn recipe_find(recipe: &str, dir: &Path) -> Result, String> { - let mut recipe_path = None; - if !dir.is_dir() { - return Ok(None); - } - for entry in fs::read_dir(dir).map_err(|e| e.to_string())? { - let entry = entry.map_err(|e| e.to_string())?; - if entry.file_name() == OsStr::new("recipe.sh") - || entry.file_name() == OsStr::new("recipe.toml") - { - // println!("recipe is {:?}", dir.file_name()); - if dir.file_name().unwrap() != OsStr::new(recipe) { - return Ok(None); - } else { - return Ok(Some(dir.to_path_buf())); +static RECIPE_PATHS: LazyLock> = LazyLock::new(|| { + let mut recipe_paths = HashMap::new(); + for entry_res in ignore::Walk::new("recipes") { + let entry = entry_res.unwrap(); + if entry.file_name() == OsStr::new("recipe.sh") || entry.file_name() == OsStr::new("recipe.toml") { + let recipe_file = entry.path(); + let Some(recipe_dir) = recipe_file.parent() else { continue }; + let Some(recipe_name) = recipe_dir.file_name().and_then(|x| x.to_str()) else { continue }; + if let Some(other_dir) = recipe_paths.insert(recipe_name.to_string(), recipe_dir.to_path_buf()) { + panic!( + "recipe {} has two or more entries {:?}, {:?}", + recipe_name, + other_dir, + recipe_dir + ); } } } + recipe_paths +}); - for entry in fs::read_dir(dir).map_err(|e| e.to_string())? { - let entry = entry.map_err(|e| e.to_string())?; - if !entry.file_type().map_err(|e| e.to_string())?.is_dir() { - continue; - } - let found = recipe_find(recipe, entry.path().as_path())?; - if found.is_none() { - continue; - } - if recipe_path.is_none() { - recipe_path = found; - } else { - return Err(format!( - "recipe {} has two or more entries {}, {}", - recipe, - recipe_path.unwrap().display(), - found.unwrap().display() - )); - } - } - - Ok(recipe_path) +pub fn recipe_find(recipe: &str) -> Option { + RECIPE_PATHS.get(recipe).cloned() } -pub fn list_recipes(dir: &Path, prefix: PathBuf) -> Result, String> { +pub fn list_recipes(prefix: PathBuf) -> Vec { let mut recipes = Vec::::new(); - if !dir.is_dir() { - return Ok(recipes); - } - for entry in fs::read_dir(dir).map_err(|e| e.to_string())? { - let entry = entry.map_err(|e| e.to_string())?; - if entry.file_name() == OsStr::new("recipe.sh") - || entry.file_name() == OsStr::new("recipe.toml") - { - recipes.push(prefix); - return Ok(recipes); - } - } - - for entry in fs::read_dir(dir).map_err(|e| e.to_string())? { - let entry = entry.map_err(|e| e.to_string())?; - if !entry.file_type().map_err(|e| e.to_string())?.is_dir() { - continue; - } - let name = entry.file_name(); - let Some(name) = name.to_str() else { - eprintln!("invalid UTF-8 for entry {entry:?}"); - continue; - }; - let mut found = list_recipes(entry.path().as_path(), prefix.join(name))?; - recipes.append(&mut found); + for (_name, path) in RECIPE_PATHS.iter() { + recipes.push(prefix.join(path)); } recipes.sort(); - Ok(recipes) + recipes }