diff --git a/mk/prefix.mk b/mk/prefix.mk index f77756682..7edcfb604 100644 --- a/mk/prefix.mk +++ b/mk/prefix.mk @@ -17,7 +17,7 @@ UPSTREAM_RUSTC_VERSION=2025-11-15 export PREFIX_RUSTFLAGS=-L $(ROOT)/$(PREFIX_INSTALL)/$(TARGET)/lib export RUSTUP_TOOLCHAIN=$(ROOT)/$(PREFIX_INSTALL) export REDOXER_TOOLCHAIN=$(RUSTUP_TOOLCHAIN) -PREFIX_CONFIG=CI=1 COOKBOOK_CLEAN_BUILD=true COOKBOOK_VERBOSE=true COOKBOOK_NONSTOP=false +PREFIX_CONFIG=CI=1 COOKBOOK_CLEAN_BUILD=true COOKBOOK_CLEAN_TARGET=false COOKBOOK_VERBOSE=true COOKBOOK_NONSTOP=false prefix: $(PREFIX)/sysroot @@ -129,7 +129,7 @@ else @echo "\033[1;36;49mBuilding binutils-install\033[0m" rm -rf "$@.partial" "$@" mkdir -p "$@.partial" - export CI=1 $(PREFIX_CONFIG) COOKBOOK_HOST_SYSROOT=/usr COOKBOOK_CROSS_TARGET=$(TARGET) COOKBOOK_CROSS_GNU_TARGET=$(GNU_TARGET) && \ + export $(PREFIX_CONFIG) COOKBOOK_HOST_SYSROOT=/usr COOKBOOK_CROSS_TARGET=$(TARGET) COOKBOOK_CROSS_GNU_TARGET=$(GNU_TARGET) && \ ./target/release/repo cook host:binutils-gdb cp -r "$(BINUTILS_TARGET)/stage/usr/". "$@.partial" touch "$@.partial" diff --git a/src/bin/repo.rs b/src/bin/repo.rs index 7dcbc5f16..f1eebf228 100644 --- a/src/bin/repo.rs +++ b/src/bin/repo.rs @@ -8,7 +8,7 @@ use cookbook::cook::ident; use cookbook::cook::package::package; use cookbook::cook::pty::{PtyOut, UnixSlavePty, flush_pty, setup_pty}; use cookbook::cook::script::KILL_ALL_PID; -use cookbook::cook::tree::{WalkTreeEntry, display_tree_entry, format_size, walk_tree_entry}; +use cookbook::cook::tree::{self, WalkTreeEntry}; use cookbook::log_to_pty; use cookbook::recipe::{CookRecipe, recipes_flatten_package_names, recipes_mark_as_deps}; use pkg::PackageName; @@ -63,14 +63,15 @@ const REPO_HELP_STR: &str = r#" --repo-binary override recipes config to use repo_binary cook env and their defaults: - CI= set to any value to disable TUI - COOKBOOK_LOGS= whether to capture build logs (default is !CI) - COOKBOOK_OFFLINE=false prevent internet access if possible + CI= set to any value to disable TUI + COOKBOOK_LOGS= whether to capture build logs (default is !CI) + COOKBOOK_OFFLINE=false prevent internet access if possible ignored when command "fetch" is used - COOKBOOK_NONSTOP=false pkeep running even a recipe build failed - COOKBOOK_VERBOSE=true print success/error on each recipe - COOKBOOK_CLEAN_BUILD=false remove build directory before building - COOKBOOK_MAKE_JOBS= override build jobs count from nproc + COOKBOOK_NONSTOP=false keep running even a recipe build failed + COOKBOOK_VERBOSE=true print success/error on each recipe + COOKBOOK_CLEAN_BUILD=false remove build directory before building + COOKBOOK_CLEAN_TARGET=false remove target directory after building + COOKBOOK_MAKE_JOBS= override build jobs count from nproc "#; #[derive(Clone)] @@ -579,8 +580,7 @@ fn handle_cook( &target_dir, &recipe.name, &recipe.recipe, - config.cook.offline, - config.cook.clean_build, + &config.cook, !is_deps, logger, ) @@ -589,6 +589,24 @@ fn handle_cook( package(&recipe, &stage_dirs, &auto_deps, logger) .map_err(|err| anyhow!("failed to package: {:?}", err))?; + if config.cook.clean_target { + let stage_dirs = get_stage_dirs(&recipe.recipe.optional_packages, &target_dir); + if config.cook.verbose && stage_dirs.iter().any(|d| d.is_dir()) { + log_to_pty!(logger, "DEBUG: Listing stage files before removing them"); + } + for stage_dir in stage_dirs { + if stage_dir.is_dir() { + if config.cook.verbose { + if let Some(stage_name) = stage_dir.file_name() { + log_to_pty!(logger, "--- {}.pkgar:", stage_name.to_string_lossy()); + } + tree::walk_file_tree(&stage_dir, " ", logger)?; + } + fs::remove_dir_all(&stage_dir) + .map_err(|err| anyhow!("failed to remove stage dir: {:?}", err))?; + } + } + } Ok(()) } @@ -669,7 +687,7 @@ fn handle_push(recipes: &Vec, config: &CliConfig) -> anyhow::Result< }; if config.with_package_deps { for (i, root) in roots.iter().enumerate() { - walk_tree_entry( + tree::walk_tree_entry( &root.name, &recipe_map, "", @@ -706,7 +724,7 @@ fn handle_push(recipes: &Vec, config: &CliConfig) -> anyhow::Result< println!(""); println!( "Pushed {} of {} {}", - format_size(total_size), + tree::format_size(total_size), visited.len(), if visited.len() == 1 { "package" @@ -727,7 +745,7 @@ fn handle_tree(recipes: &Vec, _config: &CliConfig) -> anyhow::Result let roots: Vec<&CookRecipe> = recipes.iter().filter(|r| !r.is_deps).collect(); let num_roots = roots.len(); for (i, root) in roots.iter().enumerate() { - display_tree_entry( + tree::display_tree_entry( &root.name, &recipe_map, "", @@ -740,7 +758,7 @@ fn handle_tree(recipes: &Vec, _config: &CliConfig) -> anyhow::Result println!(""); println!( "Estimated image size: {} of {} {}", - format_size(total_size), + tree::format_size(total_size), visited.len(), if visited.len() == 1 { "package" diff --git a/src/config.rs b/src/config.rs index cdcc6bef6..05f1baa1f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -19,8 +19,11 @@ pub struct CookConfigOpt { /// whether to print verbose logs to certain commands /// build failure still be printed anyway pub verbose: Option, - /// whether to always clean the build directory + /// whether to always clean the build directory before building pub clean_build: Option, + /// whether to always clean the target directory after building + /// (deletes everything except pkgar files) + pub clean_target: Option, } #[derive(Debug, Default, Clone, Deserialize, PartialEq, Serialize)] @@ -32,6 +35,7 @@ pub struct CookConfig { pub nonstop: bool, pub verbose: bool, pub clean_build: bool, + pub clean_target: bool, } impl From for CookConfig { @@ -44,6 +48,7 @@ impl From for CookConfig { nonstop: value.nonstop.unwrap(), verbose: value.verbose.unwrap(), clean_build: value.clean_build.unwrap(), + clean_target: value.clean_target.unwrap(), } } } @@ -98,6 +103,9 @@ pub fn init_config() { if config.cook_opt.clean_build.is_none() { config.cook_opt.clean_build = Some(extract_env("COOKBOOK_CLEAN_BUILD", false)); } + if config.cook_opt.clean_target.is_none() { + config.cook_opt.clean_target = Some(extract_env("COOKBOOK_CLEAN_TARGET", false)); + } if config.mirrors.len() == 0 { // The GNU FTP mirror below is automatically inserted for convenience // You can choose other mirrors by setting it on cookbook.toml diff --git a/src/cook/cook_build.rs b/src/cook/cook_build.rs index a4271a1c9..2ec252e02 100644 --- a/src/cook/cook_build.rs +++ b/src/cook/cook_build.rs @@ -1,6 +1,7 @@ use pkg::package::PackageError; use pkg::{Package, PackageName}; +use crate::config::CookConfig; use crate::cook::fs::*; use crate::cook::package::{package_source_paths, package_target}; use crate::cook::pty::PtyOut; @@ -172,16 +173,15 @@ pub fn build( target_dir: &Path, name: &PackageName, recipe: &Recipe, - offline_mode: bool, - clean_build: bool, + cook_config: &CookConfig, check_source: bool, logger: &PtyOut, ) -> Result<(Vec, BTreeSet), String> { let sysroot_dir = target_dir.join("sysroot"); let toolchain_dir = target_dir.join("toolchain"); let stage_dirs = get_stage_dirs(&recipe.optional_packages, target_dir); - let cli_verbose = crate::config::get_config().cook.verbose; - let cli_jobs = crate::config::get_config().cook.jobs; + let cli_verbose = cook_config.verbose; + let cli_jobs = cook_config.jobs; if recipe.build.kind == BuildKind::None { // metapackages don't need to do anything here return Ok((stage_dirs, BTreeSet::new())); @@ -205,11 +205,24 @@ pub fn build( } } + macro_rules! make_auto_deps { + () => { + build_auto_deps( + recipe, + target_dir, + &stage_dirs, + cook_config, + dep_pkgars, + logger, + ) + }; + } + if !check_source && stage_dirs.iter().all(|dir| dir.exists()) { - let auto_deps = build_auto_deps(recipe, target_dir, &stage_dirs, dep_pkgars, logger)?; if cli_verbose { log_to_pty!(logger, "DEBUG: using cached build, not checking source"); } + let auto_deps = make_auto_deps!()?; return Ok((stage_dirs, auto_deps)); } @@ -219,6 +232,7 @@ pub fn build( source_modified = recipe_modified } } + let deps_modified = dep_pkgars .iter() .map(|(_dep, pkgar)| modified(pkgar)) @@ -230,6 +244,37 @@ pub fn build( .max() .unwrap_or(Ok(SystemTime::UNIX_EPOCH))?; + // check stage dir modified against pkgar files, any files missing will result in UNIX_EPOCH + let stage_modified = modified_all( + &stage_dirs + .iter() + .map(|p| p.with_added_extension("pkgar")) + .collect(), + modified, + ) + .unwrap_or(SystemTime::UNIX_EPOCH); + // Rebuild stage if source is newer + if stage_modified < source_modified + || stage_modified < deps_modified + || stage_modified < deps_host_modified + { + for stage_dir in &stage_dirs { + if stage_dir.is_dir() { + log_to_pty!(logger, "DEBUG: updating '{}'", stage_dir.display()); + remove_stage_dir(stage_dir)?; + } + } + } else { + if cli_verbose { + log_to_pty!(logger, "DEBUG: using cached build"); + } + if cook_config.clean_target { + // stop early otherwise we'll end up rebuilding + let auto_deps = make_auto_deps!()?; + return Ok((stage_dirs, auto_deps)); + } + } + // Rebuild sysroot if source is newer if recipe.build.kind != BuildKind::Remote { let updated = build_deps_dir( @@ -262,36 +307,17 @@ pub fn build( } } - // Rebuild stage if source is newer - if stage_dirs.iter().any(|dir| dir.is_dir()) { - let stage_modified = - modified_all(&stage_dirs, modified_dir).unwrap_or(SystemTime::UNIX_EPOCH); - if stage_modified < source_modified - || stage_modified < deps_modified - || stage_modified < deps_host_modified - { - for stage_dir in &stage_dirs { - log_to_pty!(logger, "DEBUG: updating '{}'", stage_dir.display()); - remove_stage_dir(stage_dir)?; - } - } else { - if cli_verbose { - log_to_pty!(logger, "DEBUG: using cached build"); - } - } - } - - if !stage_dirs.last().is_some_and(|dir| dir.is_dir()) { - let stage_dir = stage_dirs - .last() - .expect("Should have atleast one stage dir"); + let stage_dir = stage_dirs + .last() + .expect("Should have atleast one stage dir"); + let build_dir = get_build_dir(target_dir); + if !stage_dir.is_dir() { // Create stage.tmp let stage_dir_tmp = target_dir.join("stage.tmp"); create_dir_clean(&stage_dir_tmp)?; - // Create build, if it does not exist - let build_dir = get_build_dir(target_dir); - if clean_build || !build_dir.is_dir() { + // Create build dir, if it does not exist + if cook_config.clean_build || !build_dir.is_dir() { create_dir_clean(&build_dir)?; } @@ -307,8 +333,9 @@ pub fn build( }; if recipe.build.kind == BuildKind::Remote { - return build_remote(stage_dirs, recipe, target_dir); + return build_remote(stage_dirs, recipe, target_dir, cook_config); } + //TODO: better integration with redoxer (library instead of binary) //TODO: configurable target //TODO: Add more configurability, convert scripts to Rust? @@ -382,7 +409,7 @@ pub fn build( if cli_verbose { command.env("COOKBOOK_VERBOSE", "1"); } - if offline_mode { + if cook_config.offline { command.env("COOKBOOK_OFFLINE", "1"); } command @@ -421,8 +448,16 @@ pub fn build( rename(&stage_dir_tmp, &stage_dir)?; } - let auto_deps = build_auto_deps(recipe, target_dir, &stage_dirs, dep_pkgars, logger)?; + if cook_config.clean_target { + remove_all(&build_dir)?; + remove_all(&sysroot_dir)?; + if toolchain_dir.is_dir() { + remove_all(&toolchain_dir)?; + } + // don't remove stage dir yet + } + let auto_deps = make_auto_deps!()?; Ok((stage_dirs, auto_deps)) } @@ -536,13 +571,15 @@ fn build_auto_deps( recipe: &Recipe, target_dir: &Path, stage_dirs: &Vec, + cook_config: &CookConfig, mut dep_pkgars: BTreeSet<(PackageName, PathBuf)>, logger: &PtyOut, ) -> Result, String> { let auto_deps_path = target_dir.join("auto_deps.toml"); - if auto_deps_path.is_file() && modified(&auto_deps_path)? < modified_all(stage_dirs, modified)? - { - remove_all(&auto_deps_path)? + if auto_deps_path.is_file() && !cook_config.clean_target { + if modified(&auto_deps_path)? < modified_all(stage_dirs, modified)? { + remove_all(&auto_deps_path)? + } } let auto_deps = if auto_deps_path.exists() { @@ -572,6 +609,7 @@ pub fn build_remote( stage_dirs: Vec, recipe: &Recipe, target_dir: &Path, + cook_config: &CookConfig, ) -> Result<(Vec, BTreeSet), String> { let source_toml = target_dir.join("source.toml"); let source_pubkey = target_dir.join("id_ed25519.pub.toml"); @@ -581,6 +619,10 @@ pub fn build_remote( // declare pkg dependencies as autodeps dependency let stage_dir = &stage_dirs[i]; + if cook_config.clean_target && stage_dir.with_added_extension("pkgar").is_file() { + continue; + } + if !stage_dir.is_dir() { let (_, source_pkgar, _) = package_source_paths(package, &target_dir); let stage_dir_tmp = target_dir.join("stage.tmp"); @@ -598,9 +640,10 @@ pub fn build_remote( } let auto_deps_path = target_dir.join("auto_deps.toml"); - if auto_deps_path.is_file() && modified(&auto_deps_path)? < modified_all(&stage_dirs, modified)? - { - remove_all(&auto_deps_path)? + if auto_deps_path.is_file() && !cook_config.clean_target { + if modified(&auto_deps_path)? < modified_all(&stage_dirs, modified)? { + remove_all(&auto_deps_path)? + } } let auto_deps = if auto_deps_path.exists() { diff --git a/src/cook/package.rs b/src/cook/package.rs index 81aebdf99..0c4f29cb6 100644 --- a/src/cook/package.rs +++ b/src/cook/package.rs @@ -47,7 +47,14 @@ pub fn package( .map_err(|err| format!("failed to save pkgar secret key: {:?}", err))?; } - let stage_modified = modified_all(stage_dirs, modified_dir)?; + let Ok(stage_modified) = modified_all(stage_dirs, modified_dir) else { + // stage dirs doesn't exist, assume safe only when clean_target = true + if !crate::config::get_config().cook.clean_target { + return Err("Stage directory is not present at packaging step".into()); + } else { + return Ok(()); + } + }; let packages = recipe.recipe.get_packages_list(); diff --git a/src/cook/tree.rs b/src/cook/tree.rs index 70bacb1d3..0174a5cf3 100644 --- a/src/cook/tree.rs +++ b/src/cook/tree.rs @@ -7,7 +7,7 @@ use std::{ use anyhow::Context; use pkg::{Package, PackageName}; -use crate::recipe::CookRecipe; +use crate::{cook::pty::PtyOut, log_to_pty, recipe::CookRecipe}; pub enum WalkTreeEntry<'a> { Built(&'a PathBuf, u64), @@ -123,6 +123,45 @@ pub fn display_pkg_fn( Ok(()) } +pub fn walk_file_tree(dir: &PathBuf, prefix: &str, logger: &PtyOut) -> std::io::Result { + if !dir.is_dir() { + return Ok(0); + } + + let entries: Vec<_> = std::fs::read_dir(dir)?.filter_map(|e| e.ok()).collect(); + let mut total_size = 0; + for (index, entry) in entries.iter().enumerate() { + let path = entry.path(); + let metadata = entry.metadata()?; + let is_last = index == entries.len() - 1; + + let line_prefix = if is_last { "└── " } else { "├── " }; + let file_name = path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("Unknown"); + + if path.is_dir() { + log_to_pty!(logger, "{}{}{}/", prefix, line_prefix, file_name); + let new_prefix = format!("{}{}", prefix, if is_last { " " } else { "│ " }); + walk_file_tree(&path, &new_prefix, logger)?; + } else { + let size = metadata.len(); + total_size += size; + log_to_pty!( + logger, + "{}{}{} ({})", + prefix, + line_prefix, + file_name, + format_size(size) + ); + } + } + + Ok(total_size) +} + pub fn format_size(bytes: u64) -> String { if bytes == 0 { return "0 B".to_string();