Merge branch 'binary-mirror' into 'master'

Support mirroring binary packages

See merge request redox-os/redox!2259

Cargo.lock

@@ -918,7 +918,7 @@ dependencies = [
 [[package]]
 name = "redoxer"
 version = "0.2.63"
-source = "git+https://gitlab.redox-os.org/redox-os/redoxer.git#e4c40952b1a3c2da74557c62c1e51f5c6ab699f7"
+source = "git+https://gitlab.redox-os.org/redox-os/redoxer.git#bf7488c52d63f5be2aac2bf16fd74dfb5be4baaa"
 dependencies = [
  "anyhow",
  "dirs",

src/cook/cook_build.rs

@@ -4,9 +4,11 @@ use pkgar_core::PackageSrc;
 use pkgar_keys::PublicKeyFile;
 
 use crate::config::CookConfig;
+use crate::cook::fetch_repo;
 use crate::cook::package::{package_source_paths, package_target};
 use crate::cook::{fetch, fs, pty::PtyOut, script::*};
 use crate::recipe::{AutoDeps, BuildKind, CookRecipe, OptionalPackageRecipe, Recipe};
+use std::io::Read;
 use std::{
     collections::{BTreeSet, VecDeque},
     path::{Path, PathBuf},
@@ -740,8 +742,8 @@ pub fn build_remote(
     logger: &PtyOut,
 ) -> Result<BuildResult> {
     let source_toml = target_dir.join("source.toml");
-    let source_pubkey = "build/remotes/pub_key_static.redox-os.org.toml";
     let auto_deps_path = target_dir.join("auto_deps.toml");
+    let source_pubkey = fetch_repo::get_binary_pubkey();
 
     let packages = recipe.get_packages_list();
     let mut cached = auto_deps_path.is_file();
@@ -770,7 +772,12 @@ pub fn build_remote(
         let stage_dir = &stage_dirs[i];
         let (_, source_pkgar, _) = package_source_paths(package, &target_dir);
         fs::create_dir_clean(stage_dir)?;
-        pkgar::extract(&source_pubkey, &source_pkgar, &stage_dir)?;
+        pkgar::extract(&source_pubkey, &source_pkgar, &stage_dir).map_err(|e| {
+            if matches!(e, pkgar::Error::Core(pkgar_core::Error::Dryoc(_))) {
+                hint_incorrect_pkey(logger, &source_pubkey, &source_pkgar);
+            }
+            e
+        })?
     }
 
     if auto_deps_path.is_file() {
@@ -789,6 +796,27 @@ pub fn build_remote(
     Ok(BuildResult::new(stage_dirs, auto_deps))
 }
 
+fn hint_incorrect_pkey(logger: &PtyOut, source_pubkey: &Path, source_pkgar: &Path) -> Option<()> {
+    // TODO: Move to pkgar
+    let mut file = std::fs::File::open(source_pkgar).ok()?;
+    let mut buffer = vec![0u8; pkgar_core::HEADER_SIZE];
+    file.read_exact(&mut buffer).ok()?;
+    log_to_pty!(
+        logger,
+        "DEBUG: Pkgar key mismatch, hint:\n - expected: {:?}\n - current:  {:?}",
+        blake3::Hash::from_bytes(unsafe {
+            pkgar_core::Header::new_unchecked(&buffer).ok()?.public_key
+        })
+        .to_hex()
+        .to_string(),
+        blake3::Hash::from_bytes(PublicKeyFile::open(source_pubkey).ok()?.pkey)
+            .to_hex()
+            .to_string(),
+    );
+
+    Some(())
+}
+
 #[cfg(test)]
 mod tests {
     use std::os::unix;

src/cook/fetch_repo.rs

@@ -40,16 +40,18 @@ fn init_binary_repo() -> (RepoManager, Repository) {
     let download_backend = CurlBackend::new().expect("Curl not found");
     let mut repo = RepoManager::new(callback, Box::new(download_backend));
     let target = redoxer::target();
-    repo.add_remote(crate::REMOTE_PKG_SOURCE, target)
+    let remote_source = crate::config::translate_mirror(crate::REMOTE_PKG_SOURCE);
+    repo.add_remote(&remote_source, target)
         .expect("Unable to add remote");
 
     let repo_path = PathBuf::from("build/remotes");
     repo.set_download_path(repo_path.clone());
     repo.sync_keys().expect("Unable to sync keys");
 
-    let repo_toml = load_cached_repo(&repo_path.join(format!("{target}_repo.toml")))
-        .unwrap_or_else(|| {
-            let repo = download_repo(&repo, repo_path)
+    let repo_toml =
+        load_cached_repo(&repo_path.join(format!("repo_{}_{target}.toml", repo.remotes[0])))
+            .unwrap_or_else(|| {
+                let repo = download_repo(&repo, repo_path)
                 .map_err(|e| {
                     eprintln!(
                         "Unable to load server repo.toml, all recipes will build from source: {e}"
@@ -57,8 +59,8 @@ fn init_binary_repo() -> (RepoManager, Repository) {
                     e
                 })
                 .unwrap_or_default();
-            repo
-        });
+                repo
+            });
     // reset here to not clobber pty
     repo.callback = Rc::new(RefCell::new(PlainCallback::new()));
     (repo, repo_toml)
@@ -82,6 +84,17 @@ pub fn get_binary_repo() -> (RepoManager, Repository) {
         ((*repo).clone(), repo_toml.clone())
     })
 }
+pub fn get_binary_pubkey() -> PathBuf {
+    BINARY_REPO.with(|cell| {
+        let mut opt = cell.borrow_mut();
+        if opt.is_none() {
+            *opt = Some(init_binary_repo());
+        }
+        let (repo, _) = opt.as_ref().unwrap();
+        let repo_path = PathBuf::from("build/remotes");
+        repo_path.join(format!("pub_key_{}.toml", repo.remotes[0]))
+    })
+}
 
 pub struct PlainPtyCallback {
     size: u64,

src/lib.rs

@@ -7,7 +7,8 @@ pub mod web;
 /// Default for maximum number of levels to descend down dependencies tree.
 pub const WALK_DEPTH: usize = 16;
 
-/// Default remote package source, for recipes with build type = "remote"
+/// Default remote package source, for recipes with build type = "remote".
+/// This is overridable with mirrors config (via `cookbook.toml`).
 pub const REMOTE_PKG_SOURCE: &str = "https://static.redox-os.org/pkg";
 
 pub fn is_redox() -> bool {