Fix binary repo caching

src/cook/cook_build.rs

@@ -264,7 +264,7 @@ pub fn build(
     }
 
     if recipe.build.kind == BuildKind::Remote {
-        return build_remote(stage_dirs, recipe, target_dir, cook_config);
+        return build_remote(stage_dirs, stage_pkgars, recipe, target_dir, logger);
     }
 
     let (sysroot_cached, toolchain_cached) = (
@@ -708,48 +708,53 @@ fn build_auto_deps(
 
 pub fn build_remote(
     stage_dirs: Vec<PathBuf>,
+    stage_pkgars: Vec<PathBuf>,
     recipe: &Recipe,
     target_dir: &Path,
-    cook_config: &CookConfig,
+    logger: &PtyOut,
 ) -> Result<BuildResult> {
     let source_toml = target_dir.join("source.toml");
     let source_pubkey = "build/remotes/pub_key_static.redox-os.org.toml";
+    let auto_deps_path = target_dir.join("auto_deps.toml");
 
     let packages = recipe.get_packages_list();
-    for (i, package) in packages.into_iter().enumerate() {
-        // declare pkg dependencies as autodeps dependency
-        let stage_dir = &stage_dirs[i];
+    let mut cached = auto_deps_path.is_file();
+    for (i, package) in packages.iter().enumerate() {
+        let stage_pkgar = &stage_pkgars[i];
 
-        if cook_config.clean_target && stage_dir.with_added_extension("pkgar").is_file() {
-            continue;
+        if !stage_pkgar.is_file() {
+            cached = false;
+            break;
         }
 
-        // TODO: Compare blake3 hashes
-        if !stage_dir.is_dir() {
-            let (_, source_pkgar, _) = package_source_paths(package, &target_dir);
-            let stage_dir_tmp = target_dir.join("stage.tmp");
-            pkgar::extract(&source_pubkey, &source_pkgar, &stage_dir_tmp)?;
-            // Move stage.tmp to stage atomically
-            fs::rename(&stage_dir_tmp, &stage_dir)?;
+        let (_, source_pkgar, _) = package_source_paths(*package, &target_dir);
+        if fs::modified(&source_pkgar)? > fs::modified(&stage_pkgar)? {
+            cached = false;
+            break;
         }
     }
 
-    let auto_deps_path = target_dir.join("auto_deps.toml");
-    if auto_deps_path.is_file() && !cook_config.clean_target {
-        if fs::modified(&auto_deps_path)? < fs::modified_all(&stage_dirs, fs::modified)? {
-            fs::remove_all(&auto_deps_path)?
-        }
-    }
-
-    let auto_deps = if auto_deps_path.exists() {
-        let toml_content =
-            fs::read_to_string(&auto_deps_path).map_err(|_| "failed to read cached auto_deps")?;
+    if cached {
+        log_to_pty!(logger, "DEBUG: using cached build");
+        let toml_content = fs::read_to_string(&auto_deps_path)?;
         let wrapper: AutoDeps =
             toml::from_str(&toml_content).map_err(|_| "failed to deserialize cached auto_deps")?;
-        wrapper.packages
-    } else {
-        let toml_content =
-            fs::read_to_string(&source_toml).map_err(|_| "failed to read source.toml")?;
+        return Ok(BuildResult::cached(stage_dirs, wrapper.packages));
+    }
+
+    for (i, package) in packages.into_iter().enumerate() {
+        let stage_dir = &stage_dirs[i];
+        let (_, source_pkgar, _) = package_source_paths(package, &target_dir);
+        fs::create_dir_clean(stage_dir)?;
+        pkgar::extract(&source_pubkey, &source_pkgar, &stage_dir)?;
+    }
+
+    if auto_deps_path.is_file() {
+        fs::remove_all(&auto_deps_path)?
+    }
+
+    let auto_deps = {
+        let toml_content = fs::read_to_string(&source_toml)?;
         let pkg_toml: Package =
             toml::from_str(&toml_content).map_err(|_| "failed to deserialize source.toml")?;
         let wrapper = AutoDeps {

src/cook/fetch_repo.rs

@@ -6,9 +6,10 @@ use std::{
     time::Duration,
 };
 
+use crate::cook::fs;
 use pkg::{
     PackageName, RemotePackage, RepoManager, Repository,
-    callback::{Callback, SilentCallback},
+    callback::{Callback, PlainCallback, SilentCallback},
     net_backend::{CurlBackend, DownloadBackend},
 };
 
@@ -22,8 +23,8 @@ fn load_cached_repo(path: &Path) -> Option<Repository> {
     let metadata = std::fs::metadata(path).ok()?;
 
     if !crate::config::get_config().cook.offline {
-        let yesterday = std::time::SystemTime::now().checked_sub(Duration::from_secs(24 * 3600))?;
-        if metadata.modified().ok()? < yesterday {
+        let stale_time = std::time::SystemTime::now().checked_sub(Duration::from_secs(8 * 3600))?;
+        if metadata.modified().ok()? < stale_time {
             // stale cache
             let _ = std::fs::remove_file(path);
             return None;
@@ -50,9 +51,12 @@ fn init_binary_repo() -> (RepoManager, Repository) {
         let (toml_str, _) = repo
             .get_package_toml(&PackageName::new("repo").unwrap())
             .expect("Failed to fetch repo.toml");
-        Repository::from_toml(&toml_str).expect("Fetched repo.toml is invalid")
+        let repo = Repository::from_toml(&toml_str).expect("Fetched repo.toml is invalid");
+        fs::serialize_and_write(&repo_path.join("repo.toml"), &repo).expect("Unable to save repo");
+        repo
     });
-
+    // reset here to not clobber pty
+    repo.callback = Rc::new(RefCell::new(PlainCallback::new()));
     (repo, repo_toml)
 }
 

src/cook/package.rs

@@ -13,6 +13,7 @@ use crate::{
     cook::{cook_build::BuildResult, fetch, fs::*, pty::PtyOut},
     log_to_pty,
     recipe::{BuildKind, CookRecipe, OptionalPackageRecipe},
+    wrap_io_err,
 };
 
 pub fn package(
@@ -44,12 +45,8 @@ pub fn package(
             create_dir(Path::new("build"))?;
         }
         let (public_key, secret_key) = pkgar_keys::SecretKeyFile::new();
-        public_key
-            .save(public_path)
-            .map_err(|err| format!("failed to save pkgar public key: {:?}", err))?;
-        secret_key
-            .save(secret_path)
-            .map_err(|err| format!("failed to save pkgar secret key: {:?}", err))?;
+        public_key.save(public_path)?;
+        secret_key.save(secret_path)?;
     }
 
     let packages = recipe.recipe.get_packages_list();
@@ -77,8 +74,7 @@ pub fn package(
                         false => pkgar_core::Packaging::Uncompressed,
                     },
                 ),
-            )
-            .map_err(|err| format!("failed to create pkgar archive: {:?}", err))?;
+            )?;
         }
 
         let deps = if package.is_some() {
@@ -89,8 +85,7 @@ pub fn package(
 
         if !package_meta.is_file() {
             let name = match package {
-                Some(p) => PackageName::new(format!("{}.{}", name.name(), p.name))
-                    .map_err(|e| format!("{}", e))?,
+                Some(p) => PackageName::new(format!("{}.{}", name.name(), p.name))?,
                 None => name.clone(),
             };
             let package_deps = match package {
@@ -137,21 +132,10 @@ pub fn package_toml(
 
     let (hash, network_size, storage_size) = if let Some((pkey_path, archive_path)) = package_file {
         use pkgar_core::PackageSrc;
-        let pkey = pkgar_keys::PublicKeyFile::open(pkey_path)
-            .map_err(|e| format!("Unable to read public key: {e:?}"))?
-            .pkey;
-        let mut package = pkgar::PackageFile::new(archive_path, &pkey).map_err(|e| {
-            format!(
-                "Unable to read packaged pkgar file {}: {e:?}",
-                archive_path.display(),
-            )
-        })?;
-        let mt = std::fs::metadata(archive_path).map_err(|e| {
-            format!(
-                "Unable to read packaged pkgar file {}: {e:?}",
-                archive_path.display(),
-            )
-        })?;
+        let pkey = pkgar_keys::PublicKeyFile::open(pkey_path)?.pkey;
+        let mut package = pkgar::PackageFile::new(archive_path, &pkey)?;
+        let mt = std::fs::metadata(archive_path)
+            .map_err(wrap_io_err!(archive_path, "Reading metadata"))?;
         let package_size = mt.len();
         let header = package.header();
         let storage_size = match header.flags.packaging() {
@@ -160,17 +144,11 @@ pub fn package_toml(
                     .total_size()
                     .map_err(|e| Error::Pkgar(pkgar::Error::Core(e)))?
                     as u64;
-                let entries = package
-                    .read_entries()
-                    .map_err(|e| format!("Unable to get lzma entry: {e}"))?;
+                let entries = package.read_entries()?;
                 for entry in entries {
-                    let data_reader = package
-                        .data_reader(&entry)
-                        .map_err(|e| format!("Unable to read lzma entry: {e}"))?;
+                    let data_reader = package.data_reader(&entry)?;
                     size += data_reader.unpacked_size;
-                    package
-                        .restore_reader(data_reader.into_inner())
-                        .map_err(|e| format!("Unable to put lzma entry: {e}"))?;
+                    package.restore_reader(data_reader.into_inner())?;
                 }
                 size
             }