Greatly increase recipe scanning performance

Cargo.lock

@@ -321,6 +321,16 @@ dependencies = [
  "alloc-stdlib",
 ]
 
+[[package]]
+name = "bstr"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.17.0"
@@ -951,6 +961,19 @@ version = "0.31.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
 
+[[package]]
+name = "globset"
+version = "0.4.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54a1028dfc5f5df5da8a56a73e6c153c9a9708ec57232470703592a3f18e49f5"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "log",
+ "regex-automata",
+ "regex-syntax",
+]
+
 [[package]]
 name = "gpt"
 version = "3.1.0"
@@ -1254,6 +1277,22 @@ dependencies = [
  "icu_properties",
 ]
 
+[[package]]
+name = "ignore"
+version = "0.4.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b"
+dependencies = [
+ "crossbeam-deque",
+ "globset",
+ "log",
+ "memchr",
+ "regex-automata",
+ "same-file",
+ "walkdir",
+ "winapi-util",
+]
+
 [[package]]
 name = "indexmap"
 version = "2.7.1"
@@ -1806,6 +1845,7 @@ name = "redox_cookbook"
 version = "0.1.0"
 dependencies = [
  "blake3 1.5.3",
+ "ignore",
  "object",
  "pbr",
  "pkgar 0.1.17",

Cargo.toml

@@ -21,6 +21,7 @@ path = "src/lib.rs"
 
 [dependencies]
 blake3 = "=1.5.3" # 1.5.4 is incompatible with blake3 0.3 dependency from pkgar
+ignore = "0.4"
 object = { version = "0.36", features = ["build_core"] }
 pbr = "1.0.2"
 pkgar = { path = "pkgar/pkgar" }

src/bin/cook.rs

@@ -1,6 +1,6 @@
 use cookbook::blake3::blake3_progress;
 use cookbook::package::StageToml;
-use cookbook::recipe::{BuildKind, CookRecipe, PackageRecipe, Recipe, SourceRecipe};
+use cookbook::recipe::{BuildKind, CookRecipe, Recipe, SourceRecipe};
 use cookbook::recipe_find::recipe_find;
 use std::{
     collections::BTreeSet,
@@ -601,7 +601,7 @@ fn build(
     let mut dep_pkgars = BTreeSet::new();
     for dependency in recipe.build.dependencies.iter() {
         //TODO: sanitize name
-        let dependency_dir = recipe_find(dependency, Path::new("recipes"))?;
+        let dependency_dir = recipe_find(dependency);
         if dependency_dir.is_none() {
             return Err(format!("failed to find recipe directory '{}'", dependency));
         }

src/bin/find_recipe.rs

@@ -1,6 +1,5 @@
 use cookbook::recipe_find::recipe_find;
 use std::env::args;
-use std::path::Path;
 use std::process::exit;
 // use clap::Parser;
 
@@ -12,15 +11,15 @@ fn main() {
         usage();
         exit(2);
     }
-    let result = recipe_find(&args().last().unwrap(), Path::new("recipes"));
-    if result.is_err() {
-        eprintln!("{}", result.err().unwrap());
-        exit(2);
-    } else if result.as_ref().unwrap().is_none() {
-        eprintln!("recipe {} not found", &args().last().unwrap());
+    let recipe_name = &args().last().unwrap();
+    match recipe_find(recipe_name) {
+        Some(path) => {
+            println!("{}", path.display());
+            exit(0);
+        },
+        None => {
+        eprintln!("recipe {} not found", recipe_name);
         exit(1);
-    } else {
-        println!("{}", result.unwrap().unwrap().display());
-        exit(0);
+        }
     }
 }

src/bin/list_recipes.rs

@@ -1,5 +1,4 @@
 use cookbook::recipe_find::list_recipes;
-use std::path::Path;
 use std::process::exit;
 // use clap::Parser;
 
@@ -8,31 +7,22 @@ fn main() {
         .nth(1)
         .map_or(false, |a| a == "-s" || a == "--short");
 
-    let result = list_recipes(Path::new("recipes"), Default::default());
+    let result = list_recipes(Default::default());
+    if result.is_empty() {
+        eprintln!("recipes not found");
+        exit(1);
+    } else {
+        for path in result {
+            let Some(file_name) = path.file_name() else {
+                continue;
+            };
 
-    match result {
-        Ok(result) => {
-            if result.is_empty() {
-                eprintln!("recipes not found");
-                exit(1);
+            if print_short {
+                println!("{}", file_name.to_string_lossy());
             } else {
-                for path in result {
-                    let Some(file_name) = path.file_name() else {
-                        continue;
-                    };
-
-                    if print_short {
-                        println!("{}", file_name.to_string_lossy());
-                    } else {
-                        println!("{}", path.to_string_lossy());
-                    }
-                }
-                exit(0);
+                println!("{}", path.to_string_lossy());
             }
         }
-        Err(error) => {
-            eprintln!("{error}");
-            exit(2);
-        }
+        exit(0);
     }
 }

src/bin/pkg_deps.rs

@@ -4,10 +4,6 @@ use std::{env::args, process::ExitCode};
 /// Same as `cookbook/src/bin/cook.rs`.
 const DEP_DEPTH: usize = 16;
 
-fn usage() {
-    eprintln!("Usage: pkg_deps [package1 package2 ...]");
-}
-
 fn main() -> ExitCode {
     let names = args().skip(1).collect::<Vec<String>>();
     let packages = StageToml::new_recursive(&names, DEP_DEPTH).expect("package not found");

src/package.rs

@@ -1,4 +1,4 @@
-use std::{env, fs, path::Path};
+use std::{env, fs};
 
 use crate::recipe_find::recipe_find;
 
@@ -14,7 +14,7 @@ pub struct StageToml {
 impl StageToml {
     pub fn new(name: String) -> Result<Self, String> {
         //TODO: sanitize recipe name?
-        let dir = recipe_find(&name, Path::new("recipes"))?;
+        let dir = recipe_find(&name);
         if dir.is_none() {
             return Err(format!("failed to find recipe directory '{}'", name));
         }

src/recipe.rs

@@ -1,6 +1,6 @@
 use std::{
     fs,
-    path::{Path, PathBuf},
+    path::PathBuf,
 };
 
 use serde::{Deserialize, Serialize};
@@ -114,7 +114,7 @@ pub struct CookRecipe {
 impl CookRecipe {
     pub fn new(name: String) -> Result<Self, String> {
         //TODO: sanitize recipe name?
-        let dir = recipe_find(&name, Path::new("recipes"))?;
+        let dir = recipe_find(&name);
         if dir.is_none() {
             return Err(format!("failed to find recipe directory '{}'", name));
         }

src/recipe_find.rs

@@ -1,78 +1,38 @@
+use std::collections::HashMap;
 use std::ffi::OsStr;
-use std::fs::{self};
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
+use std::sync::LazyLock;
 
-pub fn recipe_find(recipe: &str, dir: &Path) -> Result<Option<PathBuf>, String> {
-    let mut recipe_path = None;
-    if !dir.is_dir() {
-        return Ok(None);
-    }
-    for entry in fs::read_dir(dir).map_err(|e| e.to_string())? {
-        let entry = entry.map_err(|e| e.to_string())?;
-        if entry.file_name() == OsStr::new("recipe.sh")
-            || entry.file_name() == OsStr::new("recipe.toml")
-        {
-            // println!("recipe is {:?}", dir.file_name());
-            if dir.file_name().unwrap() != OsStr::new(recipe) {
-                return Ok(None);
-            } else {
-                return Ok(Some(dir.to_path_buf()));
+static RECIPE_PATHS: LazyLock<HashMap<String, PathBuf>> = LazyLock::new(|| {
+    let mut recipe_paths = HashMap::new();
+    for entry_res in ignore::Walk::new("recipes") {
+        let entry = entry_res.unwrap();
+        if entry.file_name() == OsStr::new("recipe.sh") || entry.file_name() == OsStr::new("recipe.toml") {
+            let recipe_file = entry.path();
+            let Some(recipe_dir) = recipe_file.parent() else { continue };
+            let Some(recipe_name) = recipe_dir.file_name().and_then(|x| x.to_str()) else { continue };
+            if let Some(other_dir) = recipe_paths.insert(recipe_name.to_string(), recipe_dir.to_path_buf()) {
+                panic!(
+                    "recipe {} has two or more entries {:?}, {:?}",
+                    recipe_name,
+                    other_dir,
+                    recipe_dir
+                );
             }
         }
     }
+    recipe_paths
+});
 
-    for entry in fs::read_dir(dir).map_err(|e| e.to_string())? {
-        let entry = entry.map_err(|e| e.to_string())?;
-        if !entry.file_type().map_err(|e| e.to_string())?.is_dir() {
-            continue;
-        }
-        let found = recipe_find(recipe, entry.path().as_path())?;
-        if found.is_none() {
-            continue;
-        }
-        if recipe_path.is_none() {
-            recipe_path = found;
-        } else {
-            return Err(format!(
-                "recipe {} has two or more entries {}, {}",
-                recipe,
-                recipe_path.unwrap().display(),
-                found.unwrap().display()
-            ));
-        }
-    }
-
-    Ok(recipe_path)
+pub fn recipe_find(recipe: &str) -> Option<PathBuf> {
+    RECIPE_PATHS.get(recipe).cloned()
 }
 
-pub fn list_recipes(dir: &Path, prefix: PathBuf) -> Result<Vec<PathBuf>, String> {
+pub fn list_recipes(prefix: PathBuf) -> Vec<PathBuf> {
     let mut recipes = Vec::<PathBuf>::new();
-    if !dir.is_dir() {
-        return Ok(recipes);
-    }
-    for entry in fs::read_dir(dir).map_err(|e| e.to_string())? {
-        let entry = entry.map_err(|e| e.to_string())?;
-        if entry.file_name() == OsStr::new("recipe.sh")
-            || entry.file_name() == OsStr::new("recipe.toml")
-        {
-            recipes.push(prefix);
-            return Ok(recipes);
-        }
-    }
-
-    for entry in fs::read_dir(dir).map_err(|e| e.to_string())? {
-        let entry = entry.map_err(|e| e.to_string())?;
-        if !entry.file_type().map_err(|e| e.to_string())?.is_dir() {
-            continue;
-        }
-        let name = entry.file_name();
-        let Some(name) = name.to_str() else {
-            eprintln!("invalid UTF-8 for entry {entry:?}");
-            continue;
-        };
-        let mut found = list_recipes(entry.path().as_path(), prefix.join(name))?;
-        recipes.append(&mut found);
+    for (_name, path) in RECIPE_PATHS.iter() {
+        recipes.push(prefix.join(path));
     }
     recipes.sort();
-    Ok(recipes)
+    recipes
 }