diff --git a/src/bin/repo.rs b/src/bin/repo.rs index 6543ef09a..396761713 100644 --- a/src/bin/repo.rs +++ b/src/bin/repo.rs @@ -2,7 +2,7 @@ use ansi_to_tui::IntoText; use anyhow::{Context, anyhow, bail}; use cookbook::config::{CookConfig, get_config, init_config}; use cookbook::cook::cook_build::{build, get_stage_dirs, remove_stage_dir}; -use cookbook::cook::fetch::{fetch, fetch_offline}; +use cookbook::cook::fetch::{FetchResult, fetch, fetch_offline}; use cookbook::cook::fs::{create_target_dir, run_command}; use cookbook::cook::ident; use cookbook::cook::package::{package, package_handle_push}; @@ -334,11 +334,12 @@ fn repo_inner( CliCommand::Fetch | CliCommand::Cook => { let repo_inner_fn = move |logger: &PtyOut| -> Result { let is_cook = *command == CliCommand::Cook; - let mut cached = false; - let source_dir = handle_fetch(recipe, config, is_cook, logger)?; - if is_cook { - cached = handle_cook(recipe, config, source_dir, logger)?; - } + let fetch_result = handle_fetch(recipe, config, is_cook, logger)?; + let cached = if is_cook { + handle_cook(recipe, config, fetch_result.source_dir, logger)? + } else { + fetch_result.cached + }; Ok(cached) }; let Some(log_path) = &config.logs_dir else { @@ -699,7 +700,7 @@ fn handle_fetch( config: &CliConfig, allow_offline: bool, logger: &PtyOut, -) -> anyhow::Result { +) -> anyhow::Result { let source_dir = match config.cook.offline && allow_offline { true => fetch_offline(&recipe, logger), false => fetch(&recipe, !recipe.is_deps, logger), @@ -1177,7 +1178,7 @@ fn run_tui_cook( config: CliConfig, recipes: Vec, ) -> anyhow::Result> { - let (work_tx, work_rx) = mpsc::channel::<(CookRecipe, PathBuf)>(); + let (work_tx, work_rx) = mpsc::channel::<(CookRecipe, FetchResult)>(); let (status_tx, status_rx) = mpsc::channel::(); let running = Arc::new(AtomicBool::new(true)); @@ -1189,7 +1190,7 @@ fn run_tui_cook( let cooker_status_tx = status_tx.clone(); let cooker_prompting = prompting.clone(); let cooker_handle = thread::spawn(move || { - 'done: for (mut recipe, source_dir) in work_rx { + 'done: for (mut recipe, fetch_result) in work_rx { let name = recipe.name.clone(); let (mut stdout_writer, mut stderr_writer) = setup_logger(&cooker_status_tx, &name); let mut logger = Some((&mut stdout_writer, &mut stderr_writer)); @@ -1198,7 +1199,12 @@ fn run_tui_cook( .send(StatusUpdate::StartCook(name.clone())) .unwrap(); let _ = recipe.reload_recipe(); // reread recipe.toml in case we're retrying - let handler = handle_cook(&recipe, &cooker_config, source_dir.clone(), &logger); + let handler = handle_cook( + &recipe, + &cooker_config, + fetch_result.source_dir.clone(), + &logger, + ); if let Some(log_path) = cooker_config.logs_dir.as_ref() { if let Err(err_ctx) = &handler { log_to_pty!(&logger, "\n{:?}", err_ctx) @@ -1315,11 +1321,11 @@ fn run_tui_cook( .unwrap_or_default(); } match handler { - Ok(source_dir) => { + Ok(fetch) => { fetcher_status_tx .send(StatusUpdate::Fetched(recipe.clone())) .unwrap(); - if work_tx.send((recipe.clone(), source_dir)).is_err() { + if work_tx.send((recipe.clone(), fetch)).is_err() { // Cooker thread died break 'done; } diff --git a/src/cook/cook_build.rs b/src/cook/cook_build.rs index 0a23a002f..f6772069f 100644 --- a/src/cook/cook_build.rs +++ b/src/cook/cook_build.rs @@ -356,6 +356,7 @@ pub fn build( return build_remote(stage_dirs, recipe, target_dir, cook_config); } + let mut allow_cargo_offline = false; //TODO: better integration with redoxer (library instead of binary) //TODO: configurable target //TODO: Add more configurability, convert scripts to Rust? @@ -366,6 +367,7 @@ pub fn build( cargopackages, cargoexamples, } => { + allow_cargo_offline = true; let mut script = format!( "DYNAMIC_INIT\n{}\nCOOKBOOK_CARGO_PATH={} ", flags_fn("COOKBOOK_CARGO_FLAGS", cargoflags), @@ -455,8 +457,10 @@ pub fn build( if cli_verbose { command.env("COOKBOOK_VERBOSE", "1"); } - if cook_config.offline { + if cook_config.offline && allow_cargo_offline { command.env("COOKBOOK_OFFLINE", "1"); + } else { + command.env_remove("COOKBOOK_OFFLINE"); } if let Ok(ident_source) = fetch::fetch_get_source_info(&cook_recipe) { command.env("COOKBOOK_SOURCE_IDENT", ident_source.source_identifier); diff --git a/src/cook/fetch.rs b/src/cook/fetch.rs index f9a8ebfc1..3bd6e3379 100644 --- a/src/cook/fetch.rs +++ b/src/cook/fetch.rs @@ -1,4 +1,8 @@ +use crate::Error; +use crate::Result; +use crate::bail_other_err; use crate::config::translate_mirror; +use crate::cook::cook_build; use crate::cook::fetch_repo; use crate::cook::fetch_repo::PlainPtyCallback; use crate::cook::fs::*; @@ -12,6 +16,7 @@ use crate::recipe::BuildKind; use crate::recipe::CookRecipe; use crate::recipe::SourceRecipe; use crate::wrap_io_err; +use crate::wrap_other_err; use pkg::SourceIdentifier; use pkg::net_backend::DownloadBackendWriter; use std::cell::RefCell; @@ -22,7 +27,31 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::rc::Rc; -pub(crate) fn get_blake3(path: &PathBuf) -> crate::Result { +pub struct FetchResult { + pub source_dir: PathBuf, + pub source_ident: String, + pub cached: bool, +} + +impl FetchResult { + pub fn new(source_dir: PathBuf, ident: String, cached: bool) -> Self { + Self { + source_dir, + source_ident: ident, + cached, + } + } + + pub fn cached(source_dir: PathBuf, ident: String) -> Self { + Self { + source_dir, + source_ident: ident, + cached: true, + } + } +} + +pub(crate) fn get_blake3(path: &PathBuf) -> Result { let mut f = fs::File::open(&path).map_err(wrap_io_err!(path, "Opening file for blake3"))?; let hash = blake3::Hasher::new() .update_reader(&mut f) @@ -31,33 +60,29 @@ pub(crate) fn get_blake3(path: &PathBuf) -> crate::Result { Ok(hash.to_hex().to_string()) } -pub fn fetch_offline(recipe: &CookRecipe, logger: &PtyOut) -> Result { +pub fn fetch_offline(recipe: &CookRecipe, logger: &PtyOut) -> Result { let recipe_dir = &recipe.dir; let source_dir = recipe_dir.join("source"); match recipe.recipe.build.kind { BuildKind::None => { // the build function doesn't need source dir exists - fetch_apply_source_info(recipe, "".to_string())?; - return Ok(source_dir); + let ident = fetch_apply_source_info(recipe, "".to_string())?; + return Ok(FetchResult::cached(source_dir, ident)); } BuildKind::Remote => { - fetch_remote(recipe_dir, recipe, true, logger)?; - return Ok(source_dir); + return fetch_remote(recipe_dir, recipe, true, source_dir, logger); } _ => {} } - let ident = match &recipe.recipe.source { - Some(SourceRecipe::Path { path: _ }) | None => { - fetch(recipe, true, logger)?; - "local_source".to_string() - } + let result = match &recipe.recipe.source { + Some(SourceRecipe::Path { path: _ }) | None => fetch(recipe, true, logger)?, Some(SourceRecipe::SameAs { same_as }) => { let recipe = fetch_resolve_canon(recipe_dir, &same_as, recipe.name.is_host())?; // recursively fetch - fetch_offline(&recipe, logger)?; + let r = fetch_offline(&recipe, logger)?; fetch_make_symlink(&source_dir, &same_as)?; - fetch_get_source_info(&recipe)?.source_identifier + r } Some(SourceRecipe::Git { git: _, @@ -70,7 +95,7 @@ pub fn fetch_offline(recipe: &CookRecipe, logger: &PtyOut) -> Result { offline_check_exists(&source_dir)?; let (head_rev, _) = get_git_head_rev(&source_dir)?; - head_rev + FetchResult::cached(source_dir, head_rev) } Some(SourceRecipe::Tar { tar: _, @@ -78,82 +103,80 @@ pub fn fetch_offline(recipe: &CookRecipe, logger: &PtyOut) -> Result { - if !source_dir.is_dir() { + let ident = blake3.clone().unwrap_or("no_tar_blake3_hash_info".into()); + let cached = source_dir.is_dir(); + if !cached { let source_tar = recipe_dir.join("source.tar"); let source_tar_blake3 = get_blake3(&source_tar)?; if source_tar.exists() { if let Some(blake3) = blake3 { if source_tar_blake3 != *blake3 { - return Err(format!( - "The downloaded tar blake3 '{source_tar_blake3}' is not equal to blake3 in recipe.toml." - )); + bail_other_err!( + "The downloaded tar blake3 {source_tar_blake3:?} is not equal to blake3 in recipe.toml" + ); } create_dir(&source_dir)?; fetch_extract_tar(source_tar, &source_dir, logger)?; fetch_apply_patches(recipe_dir, patches, script, &source_dir, logger)?; } else { // need to trust this tar file - return Err(format!( - "Please add blake3 = \"{source_tar_blake3}\" to '{recipe}'", + bail_other_err!( + "Please add blake3 = {source_tar_blake3:?} to {recipe:?}", recipe = recipe_dir.join("recipe.toml").display(), - )); + ); } - } else { - offline_check_exists(&source_dir)?; } } - blake3.clone().unwrap_or("no_tar_blake3_hash_info".into()) + offline_check_exists(&source_dir)?; + FetchResult::new(source_dir, ident, cached) } }; - fetch_apply_source_info(recipe, ident)?; + fetch_apply_source_info(recipe, result.source_ident.clone())?; - Ok(source_dir) + Ok(result) } -pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result { +pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result { let recipe_dir = &recipe.dir; let source_dir = recipe_dir.join("source"); match recipe.recipe.build.kind { BuildKind::None => { // the build function doesn't need source dir exists - fetch_apply_source_info(recipe, "".to_string())?; - return Ok(source_dir); + let ident = fetch_apply_source_info(recipe, "".to_string())?; + return Ok(FetchResult::cached(source_dir, ident)); } BuildKind::Remote => { - fetch_remote(recipe_dir, recipe, false, logger)?; - return Ok(source_dir); + return fetch_remote(recipe_dir, recipe, false, source_dir, logger); } _ => {} } - let ident = match &recipe.recipe.source { + let result = match &recipe.recipe.source { Some(SourceRecipe::SameAs { same_as }) => { let recipe = fetch_resolve_canon(recipe_dir, &same_as, recipe.name.is_host())?; // recursively fetch - fetch(&recipe, check_source, logger)?; + let r = fetch(&recipe, check_source, logger)?; fetch_make_symlink(&source_dir, &same_as)?; - fetch_get_source_info(&recipe)?.source_identifier + r } Some(SourceRecipe::Path { path }) => { - if !source_dir.is_dir() || modified_dir(Path::new(&path))? > modified_dir(&source_dir)? - { + let path = Path::new(&path); + let cached = source_dir.is_dir() && modified_dir(path)? <= modified_dir(&source_dir)?; + if !cached { log_to_pty!( logger, - "[DEBUG]: {} is newer than {}", - path, + "[DEBUG]: {:?} is newer than {:?}", + path.display(), source_dir.display() ); - copy_dir_all(path, &source_dir).map_err(|e| { - format!( - "Couldn't copy source from {} to {}: {}", - path, - source_dir.display(), - e - ) - })?; + copy_dir_all(path, &source_dir).map_err(wrap_io_err!( + path, + source_dir, + "Copying source" + ))?; } - "local_source".to_string() + FetchResult::new(source_dir, "local_source".to_string(), cached) } Some(SourceRecipe::Git { git, @@ -166,7 +189,7 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result }) => { //TODO: use libgit? let shallow_clone = *shallow_clone == Some(true); - let can_skip_rebuild = if !source_dir.is_dir() { + let cached = if !source_dir.is_dir() { // Create source.tmp let source_dir_tmp = recipe_dir.join("source.tmp"); create_dir_clean(&source_dir_tmp)?; @@ -196,10 +219,10 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result true } else { if !source_dir.join(".git").is_dir() { - return Err(format!( - "'{}' is not a git repository, but recipe indicated git source", - source_dir.display(), - )); + bail_other_err!( + "{:?} is not a git repository, but recipe indicated git source", + source_dir.display() + ); } // Reset origin @@ -247,7 +270,7 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result } }; - if !can_skip_rebuild { + if !cached { if let Some(_upstream) = upstream { //TODO: set upstream URL (is this needed?) // git remote set-url upstream "$GIT_UPSTREAM" &> /dev/null || @@ -305,7 +328,7 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result } let (head_rev, _) = get_git_head_rev(&source_dir)?; - head_rev + FetchResult::new(source_dir, head_rev, cached) } Some(SourceRecipe::Tar { tar, @@ -314,6 +337,7 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result script, }) => { let source_tar = recipe_dir.join("source.tar"); + let ident = blake3.clone().unwrap_or("no_tar_blake3_hash_info".into()); let mut tar_updated = false; loop { if !source_tar.is_file() { @@ -329,9 +353,9 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result break; } if tar_updated { - return Err(format!( - "The downloaded tar blake3 '{source_tar_blake3}' is not equal to blake3 in recipe.toml" - )); + bail_other_err!( + "The downloaded tar blake3 {source_tar_blake3:?} is not equal to blake3 in recipe.toml" + ) } else { log_to_pty!( logger, @@ -350,6 +374,7 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result break; } } + let mut cached = true; if source_dir.is_dir() { if tar_updated || fetch_is_patches_newer(recipe_dir, patches, &source_dir)? { log_to_pty!( @@ -368,8 +393,9 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result // Move source.tmp to source atomically rename(&source_dir_tmp, &source_dir)?; + cached = false; } - blake3.clone().unwrap_or("no_tar_blake3_hash_info".into()) + FetchResult::new(source_dir, ident, cached) } // Local Sources None => { @@ -381,7 +407,7 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result ); create_dir(&source_dir)?; } - "local_source".into() + FetchResult::cached(source_dir, "local_source".into()) } }; @@ -392,24 +418,43 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result cargoexamples: _, } = &recipe.recipe.build.kind { - // TODO: No need to fetch if !check_source and already fetched? - fetch_cargo(&source_dir, cargopath.as_ref(), logger)?; + if fetch_will_build(recipe) { + fetch_cargo(&result.source_dir, cargopath.as_ref(), logger)?; + } } - fetch_apply_source_info(recipe, ident)?; + fetch_apply_source_info(recipe, result.source_ident.to_string())?; - Ok(source_dir) + Ok(result) } -pub(crate) fn fetch_make_symlink(source_dir: &PathBuf, same_as: &String) -> Result<(), String> { +/// This does the same check as in cook_build +fn fetch_will_build(recipe: &CookRecipe) -> bool { + let check_source = !recipe.is_deps; + if !check_source { + // there could be more check here, but it's heavy so just assume it will build + return true; + } + + let stage_dirs = + cook_build::get_stage_dirs(&recipe.recipe.optional_packages, &recipe.target_dir()); + let stage_pkgars: Vec = stage_dirs + .iter() + .map(|p| p.with_added_extension("pkgar")) + .collect(); + let stage_present = stage_pkgars.iter().all(|file| file.is_file()); + !stage_present +} + +pub(crate) fn fetch_make_symlink(source_dir: &PathBuf, same_as: &String) -> Result<()> { let target_dir = Path::new(same_as).join("source"); if !source_dir.is_symlink() { if source_dir.is_dir() { - return Err(format!( - "'{dir}' is a directory, but recipe indicated a symlink. \n\ - try removing '{dir}' if you haven't made any changes that would be lost", + bail_other_err!( + "'{dir:?}' is a directory, but recipe indicated a symlink. \n\ + try removing '{dir:?}' if you haven't made any changes that would be lost", dir = source_dir.display(), - )); + ) } std::os::unix::fs::symlink(&target_dir, source_dir).map_err(|err| { format!( @@ -428,7 +473,7 @@ pub(crate) fn fetch_resolve_canon( recipe_dir: &Path, same_as: &String, is_host: bool, -) -> Result { +) -> Result { let canon_dir = Path::new(recipe_dir).join(same_as); if canon_dir .to_str() @@ -438,20 +483,19 @@ pub(crate) fn fetch_resolve_canon( .count() > 50 { - return Err(format!("Infinite loop detected")); + bail_other_err!("Infinite loop detected"); } if !canon_dir.exists() { - return Err(format!("'{dir}' is not exists.", dir = canon_dir.display())); + bail_other_err!("{dir:?} is not exists", dir = canon_dir.display()); } - CookRecipe::from_path(canon_dir.as_path(), true, is_host) - .map_err(|e| format!("Unable to load {dir}: {e:?}", dir = canon_dir.display())) + CookRecipe::from_path(canon_dir.as_path(), true, is_host).map_err(Error::from) } pub(crate) fn fetch_extract_tar( source_tar: PathBuf, source_dir_tmp: &PathBuf, logger: &PtyOut, -) -> Result<(), String> { +) -> Result<()> { let mut command = Command::new("tar"); let verbose = crate::config::get_config().cook.verbose; if is_redox() { @@ -475,7 +519,7 @@ pub(crate) fn fetch_cargo( source_dir: &PathBuf, cargopath: Option<&String>, logger: &PtyOut, -) -> Result<(), String> { +) -> Result<()> { let mut source_dir = source_dir.clone(); if let Some(cargopath) = cargopath { source_dir = source_dir.join(cargopath); @@ -501,8 +545,9 @@ pub fn fetch_remote( recipe_dir: &Path, recipe: &CookRecipe, offline_mode: bool, + source_dir: PathBuf, logger: &PtyOut, -) -> Result<(), String> { +) -> Result { let (mut manager, repository) = fetch_repo::get_binary_repo(); let target_dir = create_target_dir(recipe_dir, recipe.target)?; if logger.is_some() { @@ -517,13 +562,14 @@ pub fn fetch_remote( .to_str() .unwrap(); + let mut result = None; + let mut cached = true; + for package in packages { let (_, source_pkgar, source_toml) = package_source_paths(package, &target_dir); let source_name = get_package_name(name, package); let Some(repo_blake3) = repository.packages.get(&source_name) else { - return Err(format!( - "Package {source_name} does not exist in server repository" - )); + bail_other_err!("Package {source_name} does not exist in server repository") }; if !offline_mode { @@ -558,6 +604,8 @@ pub fn fetch_remote( &mut writer, ) .map_err(|e| format!("Unable to download source.pkgar: {e:?}"))?; + + cached = false; } // manager.download(file, 0, dest) @@ -566,7 +614,7 @@ pub fn fetch_remote( offline_check_exists(&source_toml)?; } - // guaranteed to exist once + // guaranteed to exist once and last in iteration if package.is_none() { let pkg_toml = read_source_toml(&source_toml)?; @@ -579,13 +627,19 @@ pub fn fetch_remote( ..Default::default() }, )?; + + result = Some(FetchResult::new( + source_dir.clone(), + pkg_toml.source_identifier, + cached, + )); } } - Ok(()) + result.ok_or_else(wrap_other_err!("There's no mandatory package in remote")) } -fn read_source_toml(source_toml: &Path) -> Result { +fn read_source_toml(source_toml: &Path) -> Result { let mut file = File::open(source_toml).map_err(|e| format!("Unable to open source.toml: {e:?}"))?; let mut contents = String::new(); @@ -600,16 +654,13 @@ pub(crate) fn fetch_is_patches_newer( recipe_dir: &Path, patches: &Vec, source_dir: &PathBuf, -) -> Result { +) -> Result { // don't check source files inside as it can be mixed with user patches let source_time = modified(&source_dir)?; for patch_name in patches { let patch_file = recipe_dir.join(patch_name); if !patch_file.is_file() { - return Err(format!( - "failed to find patch file '{}'", - patch_file.display() - )); + bail_other_err!("Failed to find patch file {:?}", patch_file.display()); } let patch_time = modified(&patch_file)?; @@ -626,14 +677,11 @@ pub(crate) fn fetch_apply_patches( script: &Option, source_dir_tmp: &PathBuf, logger: &PtyOut, -) -> Result<(), String> { +) -> Result<()> { for patch_name in patches { let patch_file = recipe_dir.join(patch_name); if !patch_file.is_file() { - return Err(format!( - "failed to find patch file '{}'", - patch_file.display() - )); + bail_other_err!("Failed to find patch file {:?}", patch_file.display()); } let patch = fs::read_to_string(&patch_file).map_err(|err| { @@ -665,7 +713,7 @@ pub(crate) fn fetch_apply_patches( pub(crate) fn fetch_apply_source_info( recipe: &CookRecipe, source_identifier: String, -) -> Result<(), String> { +) -> Result { let ident = crate::cook::ident::get_ident(); let info = SourceIdentifier { commit_identifier: ident.commit.to_string(), @@ -673,20 +721,22 @@ pub(crate) fn fetch_apply_source_info( source_identifier: source_identifier, }; - fetch_apply_source_info_from_remote(&recipe, &info) + fetch_apply_source_info_from_remote(&recipe, &info)?; + + Ok(info.source_identifier) } pub(crate) fn fetch_apply_source_info_from_remote( recipe: &CookRecipe, info: &SourceIdentifier, -) -> Result<(), String> { +) -> Result<()> { let target_dir = create_target_dir(&recipe.dir, recipe.target)?; let source_toml_path = target_dir.join("source_info.toml"); serialize_and_write(&source_toml_path, &info)?; Ok(()) } -pub fn fetch_get_source_info(recipe: &CookRecipe) -> Result { +pub fn fetch_get_source_info(recipe: &CookRecipe) -> Result { let target_dir = recipe.target_dir(); let source_toml_path = target_dir.join("source_info.toml"); let toml_content = fs::read_to_string(source_toml_path) diff --git a/src/cook/fs.rs b/src/cook/fs.rs index ebe975bb8..684fbc617 100644 --- a/src/cook/fs.rs +++ b/src/cook/fs.rs @@ -235,7 +235,7 @@ pub fn run_command_stdin( pub fn serialize_and_write(file_path: &Path, content: &T) -> Result<()> { let toml_content = toml::to_string(content).map_err(|err| { wrap_other_err!( - "Failed to serialize content for '{}': {}", + "Failed to serialize content for {:?}: {}", file_path.display(), err )()