Implement REPO_BINARY auto update

src/bin/repo.rs

@@ -1631,8 +1631,8 @@ fn run_tui_cook(
         kill_everything();
     }
 
-    fetcher_handle.join().unwrap();
-    cooker_handle.join().unwrap();
+    let _ = fetcher_handle.join();
+    let _ = cooker_handle.join();
 
     Ok(app.dump_logs_on_exit)
 }

src/bin/repo_builder.rs

@@ -3,8 +3,8 @@ use cookbook::cook::ident::{get_ident, init_ident};
 use cookbook::cook::{fetch, package as cook_package};
 use cookbook::recipe::CookRecipe;
 use cookbook::web::{CliWebConfig, generate_web};
-use pkg::package::{Repository, SourceIdentifier};
 use pkg::{Package, PackageName, recipes};
+use pkg::{Repository, SourceIdentifier};
 use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::env;
 use std::fs::{self, File};

src/cook.rs

@@ -1,6 +1,7 @@
 // avoid confusion with build.rs
 pub mod cook_build;
 pub mod fetch;
+pub mod fetch_repo;
 pub mod fs;
 pub mod ident;
 pub mod package;

src/cook/cook_build.rs

@@ -1,4 +1,4 @@
-use pkg::package::PackageError;
+use pkg::PackageError;
 use pkg::{Package, PackageName};
 
 use crate::config::CookConfig;
@@ -555,14 +555,7 @@ fn build_deps_dir(
                 &tags_dir,
                 &dep_pkgars
                     .iter()
-                    .map(|(name, _)| {
-                        // TODO: without_host should just return as_str
-                        if name.is_host() {
-                            &name.as_str()["host:".len()..]
-                        } else {
-                            name.as_str()
-                        }
-                    })
+                    .map(|(name, _)| name.without_host())
                     .collect(),
             )?
         {
@@ -588,7 +581,7 @@ fn build_deps_dir(
 
         let pkey_path = "build/id_ed25519.pub.toml";
         for (name, archive_path) in dep_pkgars {
-            let tag_file = tags_dir.join(name.without_host().as_str());
+            let tag_file = tags_dir.join(name.without_prefix());
             fs::write(&tag_file, "")
                 .map_err(|e| format!("failed to write tag file {}: {:?}", tag_file.display(), e))?;
             pkgar::extract(pkey_path, &archive_path, deps_dir_tmp.to_str().unwrap()).map_err(
@@ -665,7 +658,7 @@ pub fn build_remote(
     cook_config: &CookConfig,
 ) -> Result<(Vec<PathBuf>, BTreeSet<PackageName>), String> {
     let source_toml = target_dir.join("source.toml");
-    let source_pubkey = target_dir.join("id_ed25519.pub.toml");
+    let source_pubkey = "build/remotes/pub_key_static.redox-os.org.toml";
 
     let packages = recipe.get_packages_list();
     for (i, package) in packages.into_iter().enumerate() {

src/cook/fetch.rs

@@ -1,7 +1,6 @@
-use pkg::package::SourceIdentifier;
-
-use crate::REMOTE_PKG_SOURCE;
 use crate::config::translate_mirror;
+use crate::cook::fetch_repo;
+use crate::cook::fetch_repo::PlainPtyCallback;
 use crate::cook::fs::*;
 use crate::cook::package::get_package_name;
 use crate::cook::package::package_source_paths;
@@ -12,11 +11,15 @@ use crate::log_to_pty;
 use crate::recipe::BuildKind;
 use crate::recipe::CookRecipe;
 use crate::{blake3, recipe::SourceRecipe};
+use pkg::SourceIdentifier;
+use pkg::net_backend::DownloadBackendWriter;
+use std::cell::RefCell;
 use std::fs;
 use std::fs::File;
 use std::io::Read;
 use std::path::{Path, PathBuf};
 use std::process::Command;
+use std::rc::Rc;
 
 pub(crate) fn get_blake3(path: &PathBuf, show_progress: bool) -> Result<String, String> {
     if show_progress {
@@ -507,34 +510,16 @@ pub(crate) fn fetch_cargo(
     Ok(())
 }
 
-fn get_remote_url(name: &str, ext: &str) -> String {
-    return format!(
-        "{}/{}/{}.{}",
-        REMOTE_PKG_SOURCE,
-        redoxer::target(),
-        name,
-        ext
-    );
-}
-
-fn get_pubkey_url() -> String {
-    return format!("{}/id_ed25519.pub.toml", REMOTE_PKG_SOURCE);
-}
-
 pub fn fetch_remote(
     recipe_dir: &Path,
     recipe: &CookRecipe,
     offline_mode: bool,
     logger: &PtyOut,
 ) -> Result<(), String> {
+    let (mut manager, repository) = fetch_repo::get_binary_repo();
     let target_dir = create_target_dir(recipe_dir, recipe.target)?;
-    let source_pubkey = target_dir.join("id_ed25519.pub.toml");
-    if !offline_mode {
-        download_wget(&get_pubkey_url(), &source_pubkey, logger)?;
-    } else {
-        offline_check_exists(&source_pubkey)?;
-    }
-
+    let writer = logger.as_ref().unwrap().1.try_clone().unwrap();
+    manager.set_callback(Rc::new(RefCell::new(PlainPtyCallback::new(writer))));
     let packages = recipe.recipe.get_packages_list();
 
     let name = recipe_dir
@@ -546,15 +531,47 @@ pub fn fetch_remote(
     for package in packages {
         let (_, source_pkgar, source_toml) = package_source_paths(package, &target_dir);
         let source_name = get_package_name(name, package);
+        let Some(repo_blake3) = repository.packages.get(&source_name) else {
+            return Err(format!(
+                "Package {source_name} does not exist in server repository"
+            ));
+        };
 
         if !offline_mode {
-            //TODO: Check freshness
-            download_wget(
-                &get_remote_url(&source_name, "pkgar"),
-                &source_pkgar,
-                logger,
-            )?;
-            download_wget(&get_remote_url(&source_name, "toml"), &source_toml, logger)?;
+            if source_toml.is_file() {
+                let pkg_toml = read_source_toml(&source_toml)?;
+                if &pkg_toml.blake3 != repo_blake3 {
+                    log_to_pty!(logger, "DEBUG: Updating source binaries");
+                    remove_all(&source_toml)?;
+                    if source_pkgar.is_file() {
+                        remove_all(&source_pkgar)?;
+                    }
+                }
+            }
+
+            if !source_toml.is_file() {
+                {
+                    let toml_file = File::create(&source_toml)
+                        .map_err(|e| format!("Unable to create source.toml: {e:?}"))?;
+                    let mut writer = DownloadBackendWriter::ToFile(toml_file);
+                    manager
+                        .download(&format!("{}.toml", &source_name), None, &mut writer)
+                        .map_err(|e| format!("Unable to download source.toml: {e:?}"))?;
+                }
+                let pkg_toml = read_source_toml(&source_toml)?;
+                let pkgar_file = File::create(&source_pkgar)
+                    .map_err(|e| format!("Unable to create source.pkgar: {e:?}"))?;
+                let mut writer = DownloadBackendWriter::ToFile(pkgar_file);
+                manager
+                    .download(
+                        &format!("{}.pkgar", &source_name),
+                        Some(pkg_toml.network_size),
+                        &mut writer,
+                    )
+                    .map_err(|e| format!("Unable to download source.pkgar: {e:?}"))?;
+            }
+
+            // manager.download(file, 0, dest)
         } else {
             offline_check_exists(&source_pkgar)?;
             offline_check_exists(&source_toml)?;
@@ -562,14 +579,7 @@ pub fn fetch_remote(
 
         // guaranteed to exist once
         if package.is_none() {
-            let mut file = File::open(&source_toml)
-                .map_err(|e| format!("Unable to open source.toml: {e:?}"))?;
-            let mut contents = String::new();
-            file.read_to_string(&mut contents)
-                .map_err(|e| format!("Unable to read source.toml: {e:?}"))?;
-
-            let pkg_toml = pkg::Package::from_toml(&contents)
-                .map_err(|e| format!("Unable to parse source.toml: {e:?}"))?;
+            let pkg_toml = read_source_toml(&source_toml)?;
 
             fetch_apply_source_info_from_remote(
                 recipe,
@@ -586,6 +596,17 @@ pub fn fetch_remote(
     Ok(())
 }
 
+fn read_source_toml(source_toml: &Path) -> Result<pkg::Package, String> {
+    let mut file =
+        File::open(source_toml).map_err(|e| format!("Unable to open source.toml: {e:?}"))?;
+    let mut contents = String::new();
+    file.read_to_string(&mut contents)
+        .map_err(|e| format!("Unable to read source.toml: {e:?}"))?;
+    let pkg_toml = pkg::Package::from_toml(&contents)
+        .map_err(|e| format!("Unable to parse source.toml: {e:?}"))?;
+    Ok(pkg_toml)
+}
+
 pub(crate) fn fetch_is_patches_newer(
     recipe_dir: &Path,
     patches: &Vec<String>,
@@ -657,7 +678,7 @@ pub(crate) fn fetch_apply_source_info(
     source_identifier: String,
 ) -> Result<(), String> {
     let ident = crate::cook::ident::get_ident();
-    let info = pkg::package::SourceIdentifier {
+    let info = SourceIdentifier {
         commit_identifier: ident.commit.to_string(),
         time_identifier: ident.time.to_string(),
         source_identifier: source_identifier,
@@ -668,7 +689,7 @@ pub(crate) fn fetch_apply_source_info(
 
 pub(crate) fn fetch_apply_source_info_from_remote(
     recipe: &CookRecipe,
-    info: &pkg::package::SourceIdentifier,
+    info: &SourceIdentifier,
 ) -> Result<(), String> {
     let target_dir = create_target_dir(&recipe.dir, recipe.target)?;
     let source_toml_path = target_dir.join("source_info.toml");

src/cook/fetch_repo.rs

@@ -0,0 +1,204 @@
+use std::{
+    cell::RefCell,
+    io::{PipeWriter, Write},
+    path::{Path, PathBuf},
+    rc::Rc,
+    time::Duration,
+};
+
+use pkg::{
+    PackageName, RemotePackage, RepoManager, Repository,
+    callback::{Callback, SilentCallback},
+    net_backend::{CurlBackend, DownloadBackend},
+};
+
+// TODO: This is a workaround, but as long as whole
+// fetch operation is in single thread, this is ok
+thread_local! {
+static BINARY_REPO: RefCell<Option<(RepoManager, Repository)>> = RefCell::new(None);
+}
+
+fn load_cached_repo(path: &Path) -> Option<Repository> {
+    let metadata = std::fs::metadata(path).ok()?;
+
+    if !crate::config::get_config().cook.offline {
+        let yesterday = std::time::SystemTime::now().checked_sub(Duration::from_secs(24 * 3600))?;
+        if metadata.modified().ok()? < yesterday {
+            // stale cache
+            let _ = std::fs::remove_file(path);
+            return None;
+        }
+    }
+
+    let toml_str = std::fs::read_to_string(path).ok()?;
+    Repository::from_toml(&toml_str).ok()
+}
+
+fn init_binary_repo() -> (RepoManager, Repository) {
+    let callback = Rc::new(RefCell::new(SilentCallback::new()));
+    let download_backend = CurlBackend::new().expect("Curl not found");
+    let mut repo = RepoManager::new(callback, Box::new(download_backend));
+
+    repo.add_remote(crate::REMOTE_PKG_SOURCE, redoxer::target())
+        .expect("Unable to add remote");
+
+    let repo_path = PathBuf::from("build/remotes");
+    repo.set_download_path(repo_path.clone());
+    repo.sync_keys().expect("Unable to sync keys");
+
+    let repo_toml = load_cached_repo(&repo_path.join("repo.toml")).unwrap_or_else(|| {
+        let (toml_str, _) = repo
+            .get_package_toml(&PackageName::new("repo").unwrap())
+            .expect("Failed to fetch repo.toml");
+        Repository::from_toml(&toml_str).expect("Fetched repo.toml is invalid")
+    });
+
+    (repo, repo_toml)
+}
+
+pub fn get_binary_repo() -> (RepoManager, Repository) {
+    BINARY_REPO.with(|cell| {
+        let mut opt = cell.borrow_mut();
+        if opt.is_none() {
+            *opt = Some(init_binary_repo());
+        }
+        let (repo, repo_toml) = opt.as_ref().unwrap();
+        ((*repo).clone(), repo_toml.clone())
+    })
+}
+
+pub struct PlainPtyCallback {
+    size: u64,
+    unknown_size: bool,
+    pos: u64,
+    fetch_processed: usize,
+    fetch_total: usize,
+    interactive: bool,
+    download_file: Option<String>,
+    pty: PipeWriter,
+}
+
+impl PlainPtyCallback {
+    pub fn new(pty: PipeWriter) -> Self {
+        Self {
+            size: 0,
+            unknown_size: false,
+            pos: 0,
+            fetch_processed: 0,
+            fetch_total: 0,
+            interactive: false,
+            download_file: None,
+            pty,
+        }
+    }
+
+    /// Set if user require to agree on terminal
+    pub fn set_interactive(&mut self, enabled: bool) {
+        self.interactive = enabled;
+    }
+
+    fn flush(&self) {
+        let _ = std::io::stderr().flush();
+    }
+
+    pub fn format_size(bytes: u64) -> String {
+        if bytes == 0 {
+            return "0 B".to_string();
+        }
+        const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"];
+        let i = (bytes as f64).log(1024.0).floor() as usize;
+        let size = bytes as f64 / 1024.0_f64.powi(i as i32);
+        format!("{:.2} {}", size, UNITS[i])
+    }
+
+    fn downloading_str(&self) -> &'static str {
+        "Downloading"
+    }
+}
+
+const RESET_LINE: &str = "\r\x1b[2K";
+
+impl Callback for PlainPtyCallback {
+    fn fetch_start(&mut self, initial_count: usize) {
+        self.fetch_total = 0;
+        self.fetch_processed = 0;
+        self.fetch_package_increment(0, initial_count);
+    }
+
+    fn fetch_package_name(&mut self, pkg_name: &PackageName) {
+        // resuming after fetch_package_increment
+        let _ = write!(&self.pty, " {}", pkg_name.as_str());
+        self.flush();
+    }
+
+    fn fetch_package_increment(&mut self, added_processed: usize, added_count: usize) {
+        self.fetch_processed += added_processed;
+        self.fetch_total += added_count;
+
+        let _ = write!(
+            &self.pty,
+            "{RESET_LINE}Fetching: [{}/{}]",
+            self.fetch_processed, self.fetch_total
+        );
+        self.flush();
+    }
+
+    fn fetch_end(&mut self) {
+        if self.fetch_processed == self.fetch_total {
+            let _ = writeln!(&self.pty, "{RESET_LINE}Fetch complete.");
+        } else {
+            let _ = writeln!(&self.pty, "{RESET_LINE}Fetch incomplete.");
+        }
+    }
+
+    fn download_start(&mut self, length: u64, file: &str) {
+        self.size = length;
+        self.unknown_size = length == 0;
+        self.pos = 0;
+        if !self.unknown_size {
+            let _ = write!(&self.pty, "{RESET_LINE}{} {file}", self.downloading_str());
+            self.download_file = Some(file.to_string());
+            self.flush();
+        }
+    }
+
+    fn download_increment(&mut self, downloaded: u64) {
+        self.pos += downloaded;
+        if self.unknown_size {
+            self.size += downloaded;
+        }
+        if self.unknown_size {
+            return;
+        }
+
+        // keep using MB for consistency
+        let pos_mb = self.pos as f64 / 1_048_576.0;
+        let size_mb = self.size as f64 / 1_048_576.0;
+        let file_name = self
+            .download_file
+            .as_ref()
+            .map(|s| s.as_str())
+            .unwrap_or("");
+        let _ = write!(
+            &self.pty,
+            "{RESET_LINE}{} {} [{:.2} MB / {:.2} MB]",
+            self.downloading_str(),
+            file_name,
+            pos_mb,
+            size_mb
+        );
+        self.flush();
+    }
+
+    fn download_end(&mut self) {
+        if !self.unknown_size {
+            let _ = writeln!(&self.pty, "");
+            self.download_file = None;
+        }
+    }
+
+    fn install_extract(&mut self, remote_pkg: &RemotePackage) {
+        let _ = writeln!(&self.pty, "Extracting {}...", remote_pkg.package.name);
+        self.flush();
+    }
+}

src/cook/package.rs

@@ -3,7 +3,7 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use pkg::{Package, PackageName};
+use pkg::{Package, PackageName, PackagePrefix};
 
 use crate::{
     blake3::hash_to_hex,
@@ -79,7 +79,7 @@ pub fn package(
         }
 
         let deps = if package.is_some() {
-            BTreeSet::from([name.without_host()])
+            BTreeSet::from([name.with_prefix(PackagePrefix::Any)])
         } else {
             auto_deps.clone()
         };
@@ -155,7 +155,7 @@ pub fn package_toml(
     let ident_source = fetch::fetch_get_source_info(recipe)?;
 
     let package = Package {
-        name: recipe.name.without_host(),
+        name: recipe.name.with_prefix(PackagePrefix::Any),
         version: package_version(&recipe.recipe),
         target: recipe.target.to_string(),
         blake3: hash,

src/recipe.rs

@@ -5,7 +5,7 @@ use std::{
     path::{Path, PathBuf},
 };
 
-use pkg::{PackageName, package::PackageError, recipes};
+use pkg::{PackageError, PackageName, recipes};
 use regex::Regex;
 use serde::{
     Deserialize, Serialize,