From e0c6322a88cfe4b9f146b71dd4dbe0dce7246528 Mon Sep 17 00:00:00 2001 From: Wildan M Date: Sun, 19 Apr 2026 04:48:25 +0700 Subject: [PATCH] Fix binary repo caching --- src/cook/cook_build.rs | 61 +++++++++++++++++++++++------------------- src/cook/fetch_repo.rs | 14 ++++++---- src/cook/package.rs | 46 +++++++++---------------------- 3 files changed, 54 insertions(+), 67 deletions(-) diff --git a/src/cook/cook_build.rs b/src/cook/cook_build.rs index 58127c7a5..8592f5541 100644 --- a/src/cook/cook_build.rs +++ b/src/cook/cook_build.rs @@ -264,7 +264,7 @@ pub fn build( } if recipe.build.kind == BuildKind::Remote { - return build_remote(stage_dirs, recipe, target_dir, cook_config); + return build_remote(stage_dirs, stage_pkgars, recipe, target_dir, logger); } let (sysroot_cached, toolchain_cached) = ( @@ -708,48 +708,53 @@ fn build_auto_deps( pub fn build_remote( stage_dirs: Vec, + stage_pkgars: Vec, recipe: &Recipe, target_dir: &Path, - cook_config: &CookConfig, + logger: &PtyOut, ) -> Result { let source_toml = target_dir.join("source.toml"); let source_pubkey = "build/remotes/pub_key_static.redox-os.org.toml"; + let auto_deps_path = target_dir.join("auto_deps.toml"); let packages = recipe.get_packages_list(); - for (i, package) in packages.into_iter().enumerate() { - // declare pkg dependencies as autodeps dependency - let stage_dir = &stage_dirs[i]; + let mut cached = auto_deps_path.is_file(); + for (i, package) in packages.iter().enumerate() { + let stage_pkgar = &stage_pkgars[i]; - if cook_config.clean_target && stage_dir.with_added_extension("pkgar").is_file() { - continue; + if !stage_pkgar.is_file() { + cached = false; + break; } - // TODO: Compare blake3 hashes - if !stage_dir.is_dir() { - let (_, source_pkgar, _) = package_source_paths(package, &target_dir); - let stage_dir_tmp = target_dir.join("stage.tmp"); - pkgar::extract(&source_pubkey, &source_pkgar, &stage_dir_tmp)?; - // Move stage.tmp to stage atomically - fs::rename(&stage_dir_tmp, &stage_dir)?; + let (_, source_pkgar, _) = package_source_paths(*package, &target_dir); + if fs::modified(&source_pkgar)? > fs::modified(&stage_pkgar)? { + cached = false; + break; } } - let auto_deps_path = target_dir.join("auto_deps.toml"); - if auto_deps_path.is_file() && !cook_config.clean_target { - if fs::modified(&auto_deps_path)? < fs::modified_all(&stage_dirs, fs::modified)? { - fs::remove_all(&auto_deps_path)? - } - } - - let auto_deps = if auto_deps_path.exists() { - let toml_content = - fs::read_to_string(&auto_deps_path).map_err(|_| "failed to read cached auto_deps")?; + if cached { + log_to_pty!(logger, "DEBUG: using cached build"); + let toml_content = fs::read_to_string(&auto_deps_path)?; let wrapper: AutoDeps = toml::from_str(&toml_content).map_err(|_| "failed to deserialize cached auto_deps")?; - wrapper.packages - } else { - let toml_content = - fs::read_to_string(&source_toml).map_err(|_| "failed to read source.toml")?; + return Ok(BuildResult::cached(stage_dirs, wrapper.packages)); + } + + for (i, package) in packages.into_iter().enumerate() { + let stage_dir = &stage_dirs[i]; + let (_, source_pkgar, _) = package_source_paths(package, &target_dir); + fs::create_dir_clean(stage_dir)?; + pkgar::extract(&source_pubkey, &source_pkgar, &stage_dir)?; + } + + if auto_deps_path.is_file() { + fs::remove_all(&auto_deps_path)? + } + + let auto_deps = { + let toml_content = fs::read_to_string(&source_toml)?; let pkg_toml: Package = toml::from_str(&toml_content).map_err(|_| "failed to deserialize source.toml")?; let wrapper = AutoDeps { diff --git a/src/cook/fetch_repo.rs b/src/cook/fetch_repo.rs index 799fe49be..8dcbeadb9 100644 --- a/src/cook/fetch_repo.rs +++ b/src/cook/fetch_repo.rs @@ -6,9 +6,10 @@ use std::{ time::Duration, }; +use crate::cook::fs; use pkg::{ PackageName, RemotePackage, RepoManager, Repository, - callback::{Callback, SilentCallback}, + callback::{Callback, PlainCallback, SilentCallback}, net_backend::{CurlBackend, DownloadBackend}, }; @@ -22,8 +23,8 @@ fn load_cached_repo(path: &Path) -> Option { let metadata = std::fs::metadata(path).ok()?; if !crate::config::get_config().cook.offline { - let yesterday = std::time::SystemTime::now().checked_sub(Duration::from_secs(24 * 3600))?; - if metadata.modified().ok()? < yesterday { + let stale_time = std::time::SystemTime::now().checked_sub(Duration::from_secs(8 * 3600))?; + if metadata.modified().ok()? < stale_time { // stale cache let _ = std::fs::remove_file(path); return None; @@ -50,9 +51,12 @@ fn init_binary_repo() -> (RepoManager, Repository) { let (toml_str, _) = repo .get_package_toml(&PackageName::new("repo").unwrap()) .expect("Failed to fetch repo.toml"); - Repository::from_toml(&toml_str).expect("Fetched repo.toml is invalid") + let repo = Repository::from_toml(&toml_str).expect("Fetched repo.toml is invalid"); + fs::serialize_and_write(&repo_path.join("repo.toml"), &repo).expect("Unable to save repo"); + repo }); - + // reset here to not clobber pty + repo.callback = Rc::new(RefCell::new(PlainCallback::new())); (repo, repo_toml) } diff --git a/src/cook/package.rs b/src/cook/package.rs index 822f0ef96..d94c20b43 100644 --- a/src/cook/package.rs +++ b/src/cook/package.rs @@ -13,6 +13,7 @@ use crate::{ cook::{cook_build::BuildResult, fetch, fs::*, pty::PtyOut}, log_to_pty, recipe::{BuildKind, CookRecipe, OptionalPackageRecipe}, + wrap_io_err, }; pub fn package( @@ -44,12 +45,8 @@ pub fn package( create_dir(Path::new("build"))?; } let (public_key, secret_key) = pkgar_keys::SecretKeyFile::new(); - public_key - .save(public_path) - .map_err(|err| format!("failed to save pkgar public key: {:?}", err))?; - secret_key - .save(secret_path) - .map_err(|err| format!("failed to save pkgar secret key: {:?}", err))?; + public_key.save(public_path)?; + secret_key.save(secret_path)?; } let packages = recipe.recipe.get_packages_list(); @@ -77,8 +74,7 @@ pub fn package( false => pkgar_core::Packaging::Uncompressed, }, ), - ) - .map_err(|err| format!("failed to create pkgar archive: {:?}", err))?; + )?; } let deps = if package.is_some() { @@ -89,8 +85,7 @@ pub fn package( if !package_meta.is_file() { let name = match package { - Some(p) => PackageName::new(format!("{}.{}", name.name(), p.name)) - .map_err(|e| format!("{}", e))?, + Some(p) => PackageName::new(format!("{}.{}", name.name(), p.name))?, None => name.clone(), }; let package_deps = match package { @@ -137,21 +132,10 @@ pub fn package_toml( let (hash, network_size, storage_size) = if let Some((pkey_path, archive_path)) = package_file { use pkgar_core::PackageSrc; - let pkey = pkgar_keys::PublicKeyFile::open(pkey_path) - .map_err(|e| format!("Unable to read public key: {e:?}"))? - .pkey; - let mut package = pkgar::PackageFile::new(archive_path, &pkey).map_err(|e| { - format!( - "Unable to read packaged pkgar file {}: {e:?}", - archive_path.display(), - ) - })?; - let mt = std::fs::metadata(archive_path).map_err(|e| { - format!( - "Unable to read packaged pkgar file {}: {e:?}", - archive_path.display(), - ) - })?; + let pkey = pkgar_keys::PublicKeyFile::open(pkey_path)?.pkey; + let mut package = pkgar::PackageFile::new(archive_path, &pkey)?; + let mt = std::fs::metadata(archive_path) + .map_err(wrap_io_err!(archive_path, "Reading metadata"))?; let package_size = mt.len(); let header = package.header(); let storage_size = match header.flags.packaging() { @@ -160,17 +144,11 @@ pub fn package_toml( .total_size() .map_err(|e| Error::Pkgar(pkgar::Error::Core(e)))? as u64; - let entries = package - .read_entries() - .map_err(|e| format!("Unable to get lzma entry: {e}"))?; + let entries = package.read_entries()?; for entry in entries { - let data_reader = package - .data_reader(&entry) - .map_err(|e| format!("Unable to read lzma entry: {e}"))?; + let data_reader = package.data_reader(&entry)?; size += data_reader.unpacked_size; - package - .restore_reader(data_reader.into_inner()) - .map_err(|e| format!("Unable to put lzma entry: {e}"))?; + package.restore_reader(data_reader.into_inner())?; } size }