Move recipes from pkg crate, remove blake3 pbr

src/bin/repo.rs

@@ -9,8 +9,8 @@ use cookbook::cook::package::{package, package_handle_push};
 use cookbook::cook::pty::{PtyOut, UnixSlavePty, flush_pty, setup_pty};
 use cookbook::cook::script::KILL_ALL_PID;
 use cookbook::cook::tree::{self, WalkTreeEntry};
-use cookbook::log_to_pty;
 use cookbook::recipe::{CookRecipe, recipes_flatten_package_names, recipes_mark_as_deps};
+use cookbook::{log_to_pty, staged_pkg};
 use pkg::{PackageName, PackageState};
 use ratatui::Terminal;
 use ratatui::layout::{Constraint, Direction, Layout, Position, Rect};
@@ -507,8 +507,8 @@ fn parse_args(args: Vec<String>) -> anyhow::Result<(CliConfig, CliCommand, Vec<C
                 );
             }
             let all_recipes_path = match &config.category {
-                None => pkg::recipes::list(""),
-                Some(prefix) => pkg::recipes::list("")
+                None => staged_pkg::list(""),
+                Some(prefix) => staged_pkg::list("")
                     .into_iter()
                     .filter(|p| p.starts_with(prefix))
                     .collect(),

src/bin/repo_builder.rs

@@ -1,9 +1,9 @@
-use cookbook::WALK_DEPTH;
 use cookbook::cook::ident::{get_ident, init_ident};
 use cookbook::cook::{fetch, package as cook_package};
 use cookbook::recipe::CookRecipe;
 use cookbook::web::{CliWebConfig, generate_web};
-use pkg::{Package, PackageName, recipes};
+use cookbook::{WALK_DEPTH, staged_pkg};
+use pkg::PackageName;
 use pkg::{Repository, SourceIdentifier};
 use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::env;
@@ -85,7 +85,7 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
     //
     // The following adds the package dependencies of the recipes to the repo as
     // well.
-    let (recipe_list, recipe_map) = Package::new_recursive_nonstop(target_packages, WALK_DEPTH);
+    let (recipe_list, recipe_map) = staged_pkg::new_recursive_nonstop(target_packages, WALK_DEPTH);
 
     if recipe_list.len() == 0 {
         // Fail-Safe
@@ -99,7 +99,7 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
     // === 1. Push recipes in list ===
     for recipe_toml in &recipe_list {
         let recipe = &recipe_toml.name;
-        let Some(recipe_path) = recipes::find(recipe.name()) else {
+        let Some(recipe_path) = staged_pkg::find(recipe.name()) else {
             eprintln!("recipe {} not found", recipe);
             continue;
         };
@@ -191,7 +191,7 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
             recipe
         );
 
-        let Some(recipe_path) = recipes::find(recipe.name()) else {
+        let Some(recipe_path) = staged_pkg::find(recipe.name()) else {
             eprintln!("recipe {} not found", recipe);
             continue;
         };

src/blake3.rs

@@ -1,35 +0,0 @@
-use blake3::Hasher;
-use std::{fs, io::Result, path::Path, time::Duration};
-
-use crate::progress_bar::{ProgressBar, ProgressBarRead};
-
-pub fn blake3_progress<P: AsRef<Path>>(path: P) -> Result<String> {
-    let len = fs::metadata(&path)?.len();
-
-    let mut f = fs::File::open(&path)?;
-
-    let mut pb = ProgressBar::new(len);
-    pb.message("blake3: ");
-    pb.set_max_refresh_rate(Some(Duration::new(1, 0)));
-    pb.set_units(pbr::Units::Bytes);
-
-    let mut pbr = ProgressBarRead::new(&mut pb, &mut f);
-    let hash = Hasher::new().update_reader(&mut pbr)?.finalize();
-    let res = format!("{}", hash.to_hex());
-
-    pb.finish_println("");
-
-    Ok(res)
-}
-
-pub fn blake3_silent<P: AsRef<Path>>(path: P) -> Result<String> {
-    let mut f = fs::File::open(&path)?;
-
-    let hash = Hasher::new().update_reader(&mut f)?.finalize();
-    let res = format!("{}", hash.to_hex());
-    Ok(res)
-}
-
-pub fn hash_to_hex(h: [u8; 32]) -> String {
-    format!("{}", blake3::Hash::from_bytes(h).to_hex())
-}

src/cook/fetch.rs

@@ -10,7 +10,8 @@ use crate::is_redox;
 use crate::log_to_pty;
 use crate::recipe::BuildKind;
 use crate::recipe::CookRecipe;
-use crate::{blake3, recipe::SourceRecipe};
+use crate::recipe::SourceRecipe;
+use crate::wrap_io_err;
 use pkg::SourceIdentifier;
 use pkg::net_backend::DownloadBackendWriter;
 use std::cell::RefCell;
@@ -21,20 +22,13 @@ use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::rc::Rc;
 
-pub(crate) fn get_blake3(path: &PathBuf, show_progress: bool) -> Result<String, String> {
-    if show_progress {
-        blake3::blake3_progress(&path)
-    } else {
-        blake3::blake3_silent(&path)
-    }
-    .map_err(|err| {
-        format!(
-            "failed to calculate blake3 of '{}': {}\n{:?}",
-            path.display(),
-            err,
-            err
-        )
-    })
+pub(crate) fn get_blake3(path: &PathBuf) -> crate::Result<String> {
+    let mut f = fs::File::open(&path).map_err(wrap_io_err!(path, "Opening file for blake3"))?;
+    let hash = blake3::Hasher::new()
+        .update_reader(&mut f)
+        .map_err(wrap_io_err!(path, "Reading file for blake3"))?
+        .finalize();
+    Ok(hash.to_hex().to_string())
 }
 
 pub fn fetch_offline(recipe: &CookRecipe, logger: &PtyOut) -> Result<PathBuf, String> {
@@ -86,7 +80,7 @@ pub fn fetch_offline(recipe: &CookRecipe, logger: &PtyOut) -> Result<PathBuf, St
         }) => {
             if !source_dir.is_dir() {
                 let source_tar = recipe_dir.join("source.tar");
-                let source_tar_blake3 = get_blake3(&source_tar, true && logger.is_none())?;
+                let source_tar_blake3 = get_blake3(&source_tar)?;
                 if source_tar.exists() {
                     if let Some(blake3) = blake3 {
                         if source_tar_blake3 != *blake3 {
@@ -329,7 +323,7 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result
                 if !check_source {
                     break;
                 }
-                let source_tar_blake3 = get_blake3(&source_tar, tar_updated && logger.is_none())?;
+                let source_tar_blake3 = get_blake3(&source_tar)?;
                 if let Some(blake3) = blake3 {
                     if source_tar_blake3 == *blake3 {
                         break;

src/cook/package.rs

@@ -8,7 +8,6 @@ use pkgar::ext::PackageSrcExt;
 use pkgar_core::HeaderFlags;
 
 use crate::{
-    blake3::hash_to_hex,
     config::CookConfig,
     cook::{cook_build::BuildResult, fetch, fs::*, pty::PtyOut},
     log_to_pty,
@@ -171,7 +170,9 @@ pub fn package_toml(
         };
 
         (
-            hash_to_hex(package.header().blake3),
+            blake3::Hash::from_bytes(package.header().blake3)
+                .to_hex()
+                .to_string(),
             package_size,
             storage_size,
         )

src/lib.rs

@@ -1,11 +1,9 @@
-pub mod blake3;
 pub mod config;
 pub mod cook;
 pub mod recipe;
+pub mod staged_pkg;
 pub mod web;
 
-mod progress_bar;
-
 /// Default for maximum number of levels to descend down dependencies tree.
 pub const WALK_DEPTH: usize = 16;
 

src/progress_bar.rs

@@ -1,22 +0,0 @@
-pub use pbr::ProgressBar;
-
-use std::io::{Read, Result, Write};
-
-pub struct ProgressBarRead<'p, 'r, P: Write + 'p, R: Read + 'r> {
-    pb: &'p mut ProgressBar<P>,
-    r: &'r mut R,
-}
-
-impl<'p, 'r, P: Write, R: Read> ProgressBarRead<'p, 'r, P, R> {
-    pub fn new(pb: &'p mut ProgressBar<P>, r: &'r mut R) -> ProgressBarRead<'p, 'r, P, R> {
-        ProgressBarRead { pb, r }
-    }
-}
-
-impl<'p, 'r, P: Write, R: Read> Read for ProgressBarRead<'p, 'r, P, R> {
-    fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
-        let count = self.r.read(buf)?;
-        self.pb.add(count as u64);
-        Ok(count)
-    }
-}

src/recipe.rs

@@ -5,11 +5,11 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use pkg::{PackageError, PackageName, recipes};
+use pkg::{PackageError, PackageName};
 use regex::Regex;
 use serde::{Deserialize, Serialize};
 
-use crate::{WALK_DEPTH, cook::package as cook_package};
+use crate::{WALK_DEPTH, cook::package as cook_package, staged_pkg};
 
 /// Specifies how to download the source for a recipe
 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
@@ -248,7 +248,7 @@ impl CookRecipe {
     }
 
     pub fn from_name(name: PackageName) -> Result<Self, PackageError> {
-        let dir = recipes::find(name.name())
+        let dir = staged_pkg::find(name.name())
             .ok_or_else(|| PackageError::PackageNotFound(name.clone()))?;
         let file = dir.join("recipe.toml");
         let recipe = Recipe::new(&file)?;

src/staged_pkg.rs

@@ -0,0 +1,160 @@
+use std::borrow::Cow;
+use std::collections::{BTreeMap, BTreeSet, HashMap};
+use std::ffi::OsStr;
+use std::path::{Path, PathBuf};
+use std::sync::LazyLock;
+
+use pkg::{Package, PackageError, PackageName};
+
+// This file contains code that caches recipe paths.
+
+// TODO: This file is previously resides in `pkg` crate,
+// and can actually be merged with other logic in this cookbook.
+
+static RECIPE_PATHS: LazyLock<HashMap<PackageName, PathBuf>> = LazyLock::new(|| {
+    let mut recipe_paths = HashMap::new();
+    for entry_res in ignore::Walk::new("recipes") {
+        let Ok(entry) = entry_res else {
+            continue;
+        };
+        if entry.file_name() == OsStr::new("recipe.toml") {
+            let recipe_file = entry.path();
+            let Some(recipe_dir) = recipe_file.parent() else {
+                continue;
+            };
+            let Some(recipe_name) = recipe_dir
+                .file_name()
+                .and_then(|x| x.to_str()?.try_into().ok())
+            else {
+                continue;
+            };
+            if let Some(other_dir) = recipe_paths.insert(recipe_name, recipe_dir.to_path_buf()) {
+                eprintln!(
+                    "recipe {:?} has two or more entries: {:?} replaced by {:?}",
+                    recipe_dir.file_name(),
+                    other_dir,
+                    recipe_dir,
+                );
+            }
+        }
+    }
+    recipe_paths
+});
+
+pub fn find(recipe: &str) -> Option<&'static Path> {
+    RECIPE_PATHS.get(recipe).map(PathBuf::as_path)
+}
+
+pub fn list(prefix: impl AsRef<Path>) -> BTreeSet<PathBuf> {
+    let prefix = prefix.as_ref();
+    RECIPE_PATHS
+        .values()
+        .map(|path| prefix.join(path))
+        .collect()
+}
+
+pub fn new(name: &PackageName) -> Result<Package, PackageError> {
+    let dir = find(name.name()).ok_or_else(|| PackageError::PackageNotFound(name.clone()))?;
+    from_path(dir, name.suffix())
+}
+
+pub fn from_path(dir: &Path, feature: Option<&str>) -> Result<Package, PackageError> {
+    let target = redoxer::target();
+
+    let stage_name = match feature {
+        Some(f) => Cow::Owned(format!("stage.{f}.toml")),
+        None => Cow::Borrowed("stage.toml"),
+    };
+
+    let file = dir.join("target").join(target).join(stage_name.as_ref());
+    if !file.is_file() {
+        return Err(PackageError::FileMissing(file));
+    }
+
+    let toml = std::fs::read_to_string(&file)
+        .map_err(|err| PackageError::FileError(err.raw_os_error(), file.clone()))?;
+    toml::from_str(&toml).map_err(|err| PackageError::Parse(err, Some(file)))
+}
+
+pub fn new_recursive(
+    names: &[PackageName],
+    nonstop: bool,
+    recursion: usize,
+) -> Result<Vec<Package>, PackageError> {
+    if names.len() == 0 {
+        return Ok(vec![]);
+    }
+    let (list, map) = new_recursive_nonstop(names, recursion);
+    if nonstop && list.len() > 0 {
+        Ok(list)
+    } else if !nonstop && map.len() == list.len() {
+        Ok(list)
+    } else {
+        let (_, res) = map.into_iter().find(|(_, v)| v.is_err()).unwrap();
+        Err(res.err().unwrap())
+    }
+}
+
+// list ordered success packages and map of failed packages
+// a package can be both success and failed if dependencies aren't satistied
+pub fn new_recursive_nonstop(
+    names: &[PackageName],
+    recursion: usize,
+) -> (
+    Vec<Package>,
+    BTreeMap<PackageName, Result<(), PackageError>>,
+) {
+    let mut packages = Vec::new();
+    let mut packages_map = BTreeMap::new();
+    for name in names {
+        if packages_map.contains_key(name) {
+            continue;
+        }
+
+        let package = if recursion == 0 {
+            Err(PackageError::Recursion(Default::default()))
+        } else {
+            new(name)
+        };
+
+        match package {
+            Ok(package) => {
+                let mut has_invalid_dependency = false;
+                let (dependencies, dependencies_map) =
+                    new_recursive_nonstop(&package.depends, recursion - 1);
+                for dependency in dependencies {
+                    if !packages_map.contains_key(&dependency.name) {
+                        packages_map.insert(dependency.name.clone(), Ok(()));
+                        packages.push(dependency);
+                    }
+                }
+                for (dep_name, result) in dependencies_map {
+                    if let Err(mut e) = result {
+                        if !packages_map.contains_key(&dep_name) {
+                            e.append_recursion(name);
+                            packages_map.insert(dep_name, Err(e));
+                        }
+                        has_invalid_dependency = true;
+                    }
+                }
+                // TODO: this if check is redundant
+                if !packages_map.contains_key(name) {
+                    packages_map.insert(
+                        name.clone(),
+                        if has_invalid_dependency {
+                            Err(PackageError::DependencyInvalid(name.clone()))
+                        } else {
+                            Ok(())
+                        },
+                    );
+                    packages.push(package);
+                }
+            }
+            Err(e) => {
+                packages_map.insert(name.clone(), Err(e));
+            }
+        }
+    }
+
+    (packages, packages_map)
+}

src/web.rs

@@ -8,6 +8,7 @@ use pkg::{Package, PackageName};
 
 use crate::{
     recipe::CookRecipe,
+    staged_pkg,
     web::html::{generate_html_index, generate_html_pkg},
 };
 
@@ -59,11 +60,14 @@ pub fn generate_web(all_packages: &Vec<String>, config: &CliWebConfig) {
     let mut dependents_map: HashMap<String, BTreeSet<String>> = HashMap::new();
 
     for package_name in all_packages {
-        let Some(recipe_path) = pkg::recipes::find(package_name) else {
+        let Ok(package_name) = PackageName::new(package_name) else {
             continue;
         };
-        // TODO: Package::from_path
-        let Ok(package) = Package::new(&PackageName::new(package_name).unwrap()) else {
+        let Some(recipe_path) = staged_pkg::find(package_name.name()) else {
+            continue;
+        };
+        let Ok(package) = staged_pkg::from_path(&recipe_path, package_name.suffix()) else {
+            // TODO: report failed build
             continue;
         };
         let Ok(recipe) = CookRecipe::from_path(&recipe_path, true, false) else {