diff --git a/src/bin/repo.rs b/src/bin/repo.rs index f7d394eb..6543ef09 100644 --- a/src/bin/repo.rs +++ b/src/bin/repo.rs @@ -9,8 +9,8 @@ use cookbook::cook::package::{package, package_handle_push}; use cookbook::cook::pty::{PtyOut, UnixSlavePty, flush_pty, setup_pty}; use cookbook::cook::script::KILL_ALL_PID; use cookbook::cook::tree::{self, WalkTreeEntry}; -use cookbook::log_to_pty; use cookbook::recipe::{CookRecipe, recipes_flatten_package_names, recipes_mark_as_deps}; +use cookbook::{log_to_pty, staged_pkg}; use pkg::{PackageName, PackageState}; use ratatui::Terminal; use ratatui::layout::{Constraint, Direction, Layout, Position, Rect}; @@ -507,8 +507,8 @@ fn parse_args(args: Vec) -> anyhow::Result<(CliConfig, CliCommand, Vec pkg::recipes::list(""), - Some(prefix) => pkg::recipes::list("") + None => staged_pkg::list(""), + Some(prefix) => staged_pkg::list("") .into_iter() .filter(|p| p.starts_with(prefix)) .collect(), diff --git a/src/bin/repo_builder.rs b/src/bin/repo_builder.rs index 3142166e..e6611846 100644 --- a/src/bin/repo_builder.rs +++ b/src/bin/repo_builder.rs @@ -1,9 +1,9 @@ -use cookbook::WALK_DEPTH; use cookbook::cook::ident::{get_ident, init_ident}; use cookbook::cook::{fetch, package as cook_package}; use cookbook::recipe::CookRecipe; use cookbook::web::{CliWebConfig, generate_web}; -use pkg::{Package, PackageName, recipes}; +use cookbook::{WALK_DEPTH, staged_pkg}; +use pkg::PackageName; use pkg::{Repository, SourceIdentifier}; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::env; @@ -85,7 +85,7 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> { // // The following adds the package dependencies of the recipes to the repo as // well. - let (recipe_list, recipe_map) = Package::new_recursive_nonstop(target_packages, WALK_DEPTH); + let (recipe_list, recipe_map) = staged_pkg::new_recursive_nonstop(target_packages, WALK_DEPTH); if recipe_list.len() == 0 { // Fail-Safe @@ -99,7 +99,7 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> { // === 1. Push recipes in list === for recipe_toml in &recipe_list { let recipe = &recipe_toml.name; - let Some(recipe_path) = recipes::find(recipe.name()) else { + let Some(recipe_path) = staged_pkg::find(recipe.name()) else { eprintln!("recipe {} not found", recipe); continue; }; @@ -191,7 +191,7 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> { recipe ); - let Some(recipe_path) = recipes::find(recipe.name()) else { + let Some(recipe_path) = staged_pkg::find(recipe.name()) else { eprintln!("recipe {} not found", recipe); continue; }; diff --git a/src/blake3.rs b/src/blake3.rs deleted file mode 100644 index 6ad16476..00000000 --- a/src/blake3.rs +++ /dev/null @@ -1,35 +0,0 @@ -use blake3::Hasher; -use std::{fs, io::Result, path::Path, time::Duration}; - -use crate::progress_bar::{ProgressBar, ProgressBarRead}; - -pub fn blake3_progress>(path: P) -> Result { - let len = fs::metadata(&path)?.len(); - - let mut f = fs::File::open(&path)?; - - let mut pb = ProgressBar::new(len); - pb.message("blake3: "); - pb.set_max_refresh_rate(Some(Duration::new(1, 0))); - pb.set_units(pbr::Units::Bytes); - - let mut pbr = ProgressBarRead::new(&mut pb, &mut f); - let hash = Hasher::new().update_reader(&mut pbr)?.finalize(); - let res = format!("{}", hash.to_hex()); - - pb.finish_println(""); - - Ok(res) -} - -pub fn blake3_silent>(path: P) -> Result { - let mut f = fs::File::open(&path)?; - - let hash = Hasher::new().update_reader(&mut f)?.finalize(); - let res = format!("{}", hash.to_hex()); - Ok(res) -} - -pub fn hash_to_hex(h: [u8; 32]) -> String { - format!("{}", blake3::Hash::from_bytes(h).to_hex()) -} diff --git a/src/cook/fetch.rs b/src/cook/fetch.rs index 3d61dfa0..f9a8ebfc 100644 --- a/src/cook/fetch.rs +++ b/src/cook/fetch.rs @@ -10,7 +10,8 @@ use crate::is_redox; use crate::log_to_pty; use crate::recipe::BuildKind; use crate::recipe::CookRecipe; -use crate::{blake3, recipe::SourceRecipe}; +use crate::recipe::SourceRecipe; +use crate::wrap_io_err; use pkg::SourceIdentifier; use pkg::net_backend::DownloadBackendWriter; use std::cell::RefCell; @@ -21,20 +22,13 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::rc::Rc; -pub(crate) fn get_blake3(path: &PathBuf, show_progress: bool) -> Result { - if show_progress { - blake3::blake3_progress(&path) - } else { - blake3::blake3_silent(&path) - } - .map_err(|err| { - format!( - "failed to calculate blake3 of '{}': {}\n{:?}", - path.display(), - err, - err - ) - }) +pub(crate) fn get_blake3(path: &PathBuf) -> crate::Result { + let mut f = fs::File::open(&path).map_err(wrap_io_err!(path, "Opening file for blake3"))?; + let hash = blake3::Hasher::new() + .update_reader(&mut f) + .map_err(wrap_io_err!(path, "Reading file for blake3"))? + .finalize(); + Ok(hash.to_hex().to_string()) } pub fn fetch_offline(recipe: &CookRecipe, logger: &PtyOut) -> Result { @@ -86,7 +80,7 @@ pub fn fetch_offline(recipe: &CookRecipe, logger: &PtyOut) -> Result { if !source_dir.is_dir() { let source_tar = recipe_dir.join("source.tar"); - let source_tar_blake3 = get_blake3(&source_tar, true && logger.is_none())?; + let source_tar_blake3 = get_blake3(&source_tar)?; if source_tar.exists() { if let Some(blake3) = blake3 { if source_tar_blake3 != *blake3 { @@ -329,7 +323,7 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result if !check_source { break; } - let source_tar_blake3 = get_blake3(&source_tar, tar_updated && logger.is_none())?; + let source_tar_blake3 = get_blake3(&source_tar)?; if let Some(blake3) = blake3 { if source_tar_blake3 == *blake3 { break; diff --git a/src/cook/package.rs b/src/cook/package.rs index ba8a177f..9bc86c6c 100644 --- a/src/cook/package.rs +++ b/src/cook/package.rs @@ -8,7 +8,7 @@ use pkgar::ext::PackageSrcExt; use pkgar_core::HeaderFlags; use crate::{ - blake3::hash_to_hex, + Error, config::CookConfig, cook::{cook_build::BuildResult, fetch, fs::*, pty::PtyOut}, log_to_pty, @@ -30,6 +30,7 @@ pub fn package( target_dir.join("stage.toml"), recipe, None, + None, recipe.recipe.package.dependencies.clone(), &auto_deps, )?; @@ -110,6 +111,7 @@ pub fn package( package_meta, recipe, Some((Path::new(public_path), &package_file)), + package, package_deps, &deps, )?; @@ -123,6 +125,7 @@ pub fn package_toml( toml_path: PathBuf, recipe: &CookRecipe, package_file: Option<(&Path, &PathBuf)>, + package_suffix: Option<&OptionalPackageRecipe>, mut package_deps: Vec, auto_deps: &BTreeSet, ) -> Result<(), String> { @@ -150,9 +153,13 @@ pub fn package_toml( ) })?; let package_size = mt.len(); - let storage_size = match package.header().flags.packaging() { + let header = package.header(); + let storage_size = match header.flags.packaging() { pkgar_core::Packaging::LZMA2 => { - let mut size = 0; + let mut size = header + .total_size() + .map_err(|e| Error::Pkgar(pkgar::Error::Core(e)))? + as u64; let entries = package .read_entries() .map_err(|e| format!("Unable to get lzma entry: {e}"))?; @@ -171,7 +178,9 @@ pub fn package_toml( }; ( - hash_to_hex(package.header().blake3), + blake3::Hash::from_bytes(package.header().blake3) + .to_hex() + .to_string(), package_size, storage_size, ) @@ -182,7 +191,11 @@ pub fn package_toml( let ident_source = fetch::fetch_get_source_info(recipe)?; let package = Package { - name: recipe.name.with_prefix(PackagePrefix::Any), + name: PackageName::new(get_package_name( + recipe.name.without_prefix(), + package_suffix, + )) + .unwrap(), version: recipe.guess_version().unwrap_or("TODO".into()), target: recipe.target.to_string(), blake3: hash, diff --git a/src/cook/tree.rs b/src/cook/tree.rs index a37fb29a..e811c508 100644 --- a/src/cook/tree.rs +++ b/src/cook/tree.rs @@ -149,7 +149,8 @@ pub fn walk_file_tree(dir: &PathBuf, prefix: &str, buffer: &mut String) -> std:: return Ok(0); } let fmt_err = std::io::Error::other; - let entries: Vec<_> = std::fs::read_dir(dir)?.filter_map(|e| e.ok()).collect(); + let mut entries: Vec<_> = std::fs::read_dir(dir)?.filter_map(|e| e.ok()).collect(); + entries.sort_by(|a, b| a.file_name().cmp(&b.file_name())); let mut total_size = 0; for (index, entry) in entries.iter().enumerate() { let path = entry.path(); diff --git a/src/lib.rs b/src/lib.rs index d7284a32..b11bb1c0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,9 @@ -pub mod blake3; pub mod config; pub mod cook; pub mod recipe; +pub mod staged_pkg; pub mod web; -mod progress_bar; - /// Default for maximum number of levels to descend down dependencies tree. pub const WALK_DEPTH: usize = 16; diff --git a/src/progress_bar.rs b/src/progress_bar.rs deleted file mode 100644 index ef7ce1fc..00000000 --- a/src/progress_bar.rs +++ /dev/null @@ -1,22 +0,0 @@ -pub use pbr::ProgressBar; - -use std::io::{Read, Result, Write}; - -pub struct ProgressBarRead<'p, 'r, P: Write + 'p, R: Read + 'r> { - pb: &'p mut ProgressBar

, - r: &'r mut R, -} - -impl<'p, 'r, P: Write, R: Read> ProgressBarRead<'p, 'r, P, R> { - pub fn new(pb: &'p mut ProgressBar

, r: &'r mut R) -> ProgressBarRead<'p, 'r, P, R> { - ProgressBarRead { pb, r } - } -} - -impl<'p, 'r, P: Write, R: Read> Read for ProgressBarRead<'p, 'r, P, R> { - fn read(&mut self, buf: &mut [u8]) -> Result { - let count = self.r.read(buf)?; - self.pb.add(count as u64); - Ok(count) - } -} diff --git a/src/recipe.rs b/src/recipe.rs index 03dafaea..6f20632f 100644 --- a/src/recipe.rs +++ b/src/recipe.rs @@ -5,11 +5,11 @@ use std::{ path::{Path, PathBuf}, }; -use pkg::{PackageError, PackageName, recipes}; +use pkg::{PackageError, PackageName}; use regex::Regex; use serde::{Deserialize, Serialize}; -use crate::{WALK_DEPTH, cook::package as cook_package}; +use crate::{WALK_DEPTH, cook::package as cook_package, staged_pkg}; /// Specifies how to download the source for a recipe #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] @@ -214,7 +214,7 @@ impl CookRecipe { pub fn new(name: PackageName, dir: PathBuf, mut recipe: Recipe) -> Result { let target = cook_package::package_target(&name); if name.is_host() { - let thisname = name.name(); + let thisname = name.without_host(); let fn_map = |p: PackageName| { if p.is_host() { if p.name() == thisname { None } else { Some(p) } @@ -248,7 +248,7 @@ impl CookRecipe { } pub fn from_name(name: PackageName) -> Result { - let dir = recipes::find(name.name()) + let dir = staged_pkg::find(name.name()) .ok_or_else(|| PackageError::PackageNotFound(name.clone()))?; let file = dir.join("recipe.toml"); let recipe = Recipe::new(&file)?; diff --git a/src/staged_pkg.rs b/src/staged_pkg.rs new file mode 100644 index 00000000..d7abbce0 --- /dev/null +++ b/src/staged_pkg.rs @@ -0,0 +1,160 @@ +use std::borrow::Cow; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; +use std::sync::LazyLock; + +use pkg::{Package, PackageError, PackageName}; + +// This file contains code that caches recipe paths. + +// TODO: This file is previously resides in `pkg` crate, +// and can actually be merged with other logic in this cookbook. + +static RECIPE_PATHS: LazyLock> = LazyLock::new(|| { + let mut recipe_paths = HashMap::new(); + for entry_res in ignore::Walk::new("recipes") { + let Ok(entry) = entry_res else { + continue; + }; + if entry.file_name() == OsStr::new("recipe.toml") { + let recipe_file = entry.path(); + let Some(recipe_dir) = recipe_file.parent() else { + continue; + }; + let Some(recipe_name) = recipe_dir + .file_name() + .and_then(|x| x.to_str()?.try_into().ok()) + else { + continue; + }; + if let Some(other_dir) = recipe_paths.insert(recipe_name, recipe_dir.to_path_buf()) { + eprintln!( + "recipe {:?} has two or more entries: {:?} replaced by {:?}", + recipe_dir.file_name(), + other_dir, + recipe_dir, + ); + } + } + } + recipe_paths +}); + +pub fn find(recipe: &str) -> Option<&'static Path> { + RECIPE_PATHS.get(recipe).map(PathBuf::as_path) +} + +pub fn list(prefix: impl AsRef) -> BTreeSet { + let prefix = prefix.as_ref(); + RECIPE_PATHS + .values() + .map(|path| prefix.join(path)) + .collect() +} + +pub fn new(name: &PackageName) -> Result { + let dir = find(name.name()).ok_or_else(|| PackageError::PackageNotFound(name.clone()))?; + from_path(dir, name.suffix()) +} + +pub fn from_path(dir: &Path, feature: Option<&str>) -> Result { + let target = redoxer::target(); + + let stage_name = match feature { + Some(f) => Cow::Owned(format!("stage.{f}.toml")), + None => Cow::Borrowed("stage.toml"), + }; + + let file = dir.join("target").join(target).join(stage_name.as_ref()); + if !file.is_file() { + return Err(PackageError::FileMissing(file)); + } + + let toml = std::fs::read_to_string(&file) + .map_err(|err| PackageError::FileError(err.raw_os_error(), file.clone()))?; + toml::from_str(&toml).map_err(|err| PackageError::Parse(err, Some(file))) +} + +pub fn new_recursive( + names: &[PackageName], + nonstop: bool, + recursion: usize, +) -> Result, PackageError> { + if names.len() == 0 { + return Ok(vec![]); + } + let (list, map) = new_recursive_nonstop(names, recursion); + if nonstop && list.len() > 0 { + Ok(list) + } else if !nonstop && map.len() == list.len() { + Ok(list) + } else { + let (_, res) = map.into_iter().find(|(_, v)| v.is_err()).unwrap(); + Err(res.err().unwrap()) + } +} + +/// List ordered success packages and map of failed packages. +/// A package can be both success and failed if dependencies aren't satistied. +pub fn new_recursive_nonstop( + names: &[PackageName], + recursion: usize, +) -> ( + Vec, + BTreeMap>, +) { + let mut packages = Vec::new(); + let mut packages_map = BTreeMap::new(); + for name in names { + if packages_map.contains_key(name) { + continue; + } + + let package = if recursion == 0 { + Err(PackageError::Recursion(Default::default())) + } else { + new(name) + }; + + match package { + Ok(package) => { + let mut has_invalid_dependency = false; + let (dependencies, dependencies_map) = + new_recursive_nonstop(&package.depends, recursion - 1); + for dependency in dependencies { + if !packages_map.contains_key(&dependency.name) { + packages_map.insert(dependency.name.clone(), Ok(())); + packages.push(dependency); + } + } + for (dep_name, result) in dependencies_map { + if let Err(mut e) = result { + if !packages_map.contains_key(&dep_name) { + e.append_recursion(name); + packages_map.insert(dep_name, Err(e)); + } + has_invalid_dependency = true; + } + } + // TODO: this check is redundant + if !packages_map.contains_key(name) { + packages_map.insert( + name.clone(), + if has_invalid_dependency { + Err(PackageError::DependencyInvalid(name.clone())) + } else { + Ok(()) + }, + ); + packages.push(package); + } + } + Err(e) => { + packages_map.insert(name.clone(), Err(e)); + } + } + } + + (packages, packages_map) +} diff --git a/src/web.rs b/src/web.rs index db19098e..e669bc8a 100644 --- a/src/web.rs +++ b/src/web.rs @@ -8,6 +8,7 @@ use pkg::{Package, PackageName}; use crate::{ recipe::CookRecipe, + staged_pkg, web::html::{generate_html_index, generate_html_pkg}, }; @@ -59,14 +60,17 @@ pub fn generate_web(all_packages: &Vec, config: &CliWebConfig) { let mut dependents_map: HashMap> = HashMap::new(); for package_name in all_packages { - let Some(recipe_path) = pkg::recipes::find(package_name) else { + let Ok(package_name) = PackageName::new(package_name) else { continue; }; - // TODO: Package::from_path - let Ok(package) = Package::new(&PackageName::new(package_name).unwrap()) else { + let Some(recipe_path) = staged_pkg::find(package_name.name()) else { continue; }; - let Ok(recipe) = CookRecipe::from_path(&recipe_path, true, false) else { + let Ok(mut package) = staged_pkg::from_path(&recipe_path, package_name.suffix()) else { + // TODO: report failed build + continue; + }; + let Ok(mut recipe) = CookRecipe::from_path(&recipe_path, true, false) else { continue; }; @@ -77,6 +81,11 @@ pub fn generate_web(all_packages: &Vec, config: &CliWebConfig) { .insert(package.name.to_string()); } + // TODO: temporary bug fix in the suffix lost + package.name = package_name.clone(); + // CookRecipe::from_path always have no suffix + recipe.name = package_name; + valid_packages.push((package, recipe)); } diff --git a/src/web/html.rs b/src/web/html.rs index 1f4d405e..e7905fe2 100644 --- a/src/web/html.rs +++ b/src/web/html.rs @@ -108,7 +108,7 @@ pub fn generate_html_pkg( &config.this_repo, host, &package.commit_identifier, - Some(&format!("recipes/{category}/{name}/recipe.toml")), + Some(&format!("recipes/{category}/{}/recipe.toml", name.name())), ); let short_commit = get_short_commit(&package.commit_identifier); source_html += &format!( diff --git a/src/web/style.css b/src/web/style.css index 585697e0..560735ea 100644 --- a/src/web/style.css +++ b/src/web/style.css @@ -6,8 +6,8 @@ body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; - background-color: #f9f9fb; - color: #24292e; + background-color: #eee; + color: #222; line-height: 1.6; } @@ -23,8 +23,8 @@ body { .category-title { font-size: 1.5rem; - color: #24292e; - border-bottom: 2px solid #e1e4e8; + color: #222; + border-bottom: 2px solid #eee; padding-bottom: 10px; margin-bottom: 20px; font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; @@ -38,8 +38,8 @@ body { } .package-card { - background-color: #ffffff; - border: 1px solid #e1e4e8; + background-color: #fff; + border: 1px solid #eee; border-radius: 6px; padding: 15px; margin: 10px; @@ -74,16 +74,16 @@ body { align-items: center; color: #6a737d; font-size: 0.9rem; - border-top: 1px solid #e1e4e8; + border-top: 1px solid #eee; padding-top: 10px; } .package-card .pkg-version { font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; - background-color: #f3f4f6; + background-color: #fff; padding: 3px 6px; border-radius: 4px; - color: #24292e; + color: #222; } .package-card .pkg-size { @@ -91,14 +91,14 @@ body { } a { - color: #24292e; + color: #222; text-decoration: none; - border-bottom: 1px solid #e1e4e8; + border-bottom: 1px solid #eee; } a:hover { color: #000000; - border-bottom: 1px solid #24292e; + border-bottom: 1px solid #222; } h1, h2, h3 { @@ -108,7 +108,7 @@ h1, h2, h3 { code { font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace; - background-color: #f3f4f6; + background-color: #fff; padding: 0.2em 0.4em; border-radius: 3px; font-size: 0.9em; @@ -116,7 +116,7 @@ code { .card { background-color: #ffffff; - border: 1px solid #e1e4e8; + border: 1px solid #eee; border-radius: 6px; padding: 20px; margin-bottom: 20px; @@ -124,7 +124,7 @@ code { .pkg-header, .index-header { background-color: #ffffff; - border-bottom: 1px solid #d1d5da; + border-bottom: 1px solid #ddd; padding: 40px 0; margin-bottom: 40px; text-align: center; @@ -157,19 +157,19 @@ code { } .back-link:hover { - color: #24292e; + color: #222; border: none; } .install-action { display: inline-block; - background-color: #f3f4f6; - border: 1px solid #d1d5da; + background-color: #fff; + border: 1px solid #ddd; border-radius: 6px; padding: 12px 20px; font-family: ui-monospace, SFMono-Regular, monospace; font-size: 1.1rem; - color: #24292e; + color: #222; } .install-action .prompt { @@ -220,7 +220,7 @@ table { th, td { padding: 10px 0; text-align: left; - border-bottom: 1px solid #e1e4e8; + border-bottom: 1px solid #eee; } th { @@ -241,6 +241,52 @@ th { .pkg-deps li, .pkg-dependents li { padding: 8px 0; - border-bottom: 1px solid #e1e4e8; + border-bottom: 1px solid #eee; width: 50%; } + +@media (prefers-color-scheme: dark) { + body { + background-color: #000; + color: #ccc; + } + + .package-card, .card, .pkg-header, .index-header { + background-color: #111; + border-color: #333; + } + + .category-title { + color: #f0f6fc; + border-bottom-color: #333; + } + + .package-card .pkg-stats { + color: #8b949e; + border-top-color: #333; + } + + .package-card .pkg-version, code, .install-action { + background-color: #222; + color: #cdd; + border-color: #333; + } + + a, .pkg-header h1, .back-link:hover { + color: #5af; + border-bottom-color: #333; + } + + a:hover { + color: #7cf; + border-bottom-color: #7cf; + } + + .pkg-header .version, .pkg-header .description, .back-link, .install-action .prompt, th { + color: #999; + } + + th, td, .pkg-deps li, .pkg-dependents li { + border-bottom-color: #333; + } +}