From 69b6b3fd5775420daf9a3cf33975f90ca7b10538 Mon Sep 17 00:00:00 2001 From: Wildan M Date: Sun, 8 Mar 2026 20:31:39 +0700 Subject: [PATCH] Implement REPO_BINARY auto update --- src/bin/repo.rs | 4 +- src/bin/repo_builder.rs | 2 +- src/cook.rs | 1 + src/cook/cook_build.rs | 15 +-- src/cook/fetch.rs | 103 ++++++++++++-------- src/cook/fetch_repo.rs | 204 ++++++++++++++++++++++++++++++++++++++++ src/cook/package.rs | 6 +- src/recipe.rs | 2 +- 8 files changed, 278 insertions(+), 59 deletions(-) create mode 100644 src/cook/fetch_repo.rs diff --git a/src/bin/repo.rs b/src/bin/repo.rs index 14742d14d..823286769 100644 --- a/src/bin/repo.rs +++ b/src/bin/repo.rs @@ -1631,8 +1631,8 @@ fn run_tui_cook( kill_everything(); } - fetcher_handle.join().unwrap(); - cooker_handle.join().unwrap(); + let _ = fetcher_handle.join(); + let _ = cooker_handle.join(); Ok(app.dump_logs_on_exit) } diff --git a/src/bin/repo_builder.rs b/src/bin/repo_builder.rs index 5264a6e44..3142166e1 100644 --- a/src/bin/repo_builder.rs +++ b/src/bin/repo_builder.rs @@ -3,8 +3,8 @@ use cookbook::cook::ident::{get_ident, init_ident}; use cookbook::cook::{fetch, package as cook_package}; use cookbook::recipe::CookRecipe; use cookbook::web::{CliWebConfig, generate_web}; -use pkg::package::{Repository, SourceIdentifier}; use pkg::{Package, PackageName, recipes}; +use pkg::{Repository, SourceIdentifier}; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::env; use std::fs::{self, File}; diff --git a/src/cook.rs b/src/cook.rs index 70a53a066..9c989215c 100644 --- a/src/cook.rs +++ b/src/cook.rs @@ -1,6 +1,7 @@ // avoid confusion with build.rs pub mod cook_build; pub mod fetch; +pub mod fetch_repo; pub mod fs; pub mod ident; pub mod package; diff --git a/src/cook/cook_build.rs b/src/cook/cook_build.rs index 3378047c4..47b02f3e0 100644 --- a/src/cook/cook_build.rs +++ b/src/cook/cook_build.rs @@ -1,4 +1,4 @@ -use pkg::package::PackageError; +use pkg::PackageError; use pkg::{Package, PackageName}; use crate::config::CookConfig; @@ -555,14 +555,7 @@ fn build_deps_dir( &tags_dir, &dep_pkgars .iter() - .map(|(name, _)| { - // TODO: without_host should just return as_str - if name.is_host() { - &name.as_str()["host:".len()..] - } else { - name.as_str() - } - }) + .map(|(name, _)| name.without_host()) .collect(), )? { @@ -588,7 +581,7 @@ fn build_deps_dir( let pkey_path = "build/id_ed25519.pub.toml"; for (name, archive_path) in dep_pkgars { - let tag_file = tags_dir.join(name.without_host().as_str()); + let tag_file = tags_dir.join(name.without_prefix()); fs::write(&tag_file, "") .map_err(|e| format!("failed to write tag file {}: {:?}", tag_file.display(), e))?; pkgar::extract(pkey_path, &archive_path, deps_dir_tmp.to_str().unwrap()).map_err( @@ -665,7 +658,7 @@ pub fn build_remote( cook_config: &CookConfig, ) -> Result<(Vec, BTreeSet), String> { let source_toml = target_dir.join("source.toml"); - let source_pubkey = target_dir.join("id_ed25519.pub.toml"); + let source_pubkey = "build/remotes/pub_key_static.redox-os.org.toml"; let packages = recipe.get_packages_list(); for (i, package) in packages.into_iter().enumerate() { diff --git a/src/cook/fetch.rs b/src/cook/fetch.rs index c55cd23ff..8f22d2128 100644 --- a/src/cook/fetch.rs +++ b/src/cook/fetch.rs @@ -1,7 +1,6 @@ -use pkg::package::SourceIdentifier; - -use crate::REMOTE_PKG_SOURCE; use crate::config::translate_mirror; +use crate::cook::fetch_repo; +use crate::cook::fetch_repo::PlainPtyCallback; use crate::cook::fs::*; use crate::cook::package::get_package_name; use crate::cook::package::package_source_paths; @@ -12,11 +11,15 @@ use crate::log_to_pty; use crate::recipe::BuildKind; use crate::recipe::CookRecipe; use crate::{blake3, recipe::SourceRecipe}; +use pkg::SourceIdentifier; +use pkg::net_backend::DownloadBackendWriter; +use std::cell::RefCell; use std::fs; use std::fs::File; use std::io::Read; use std::path::{Path, PathBuf}; use std::process::Command; +use std::rc::Rc; pub(crate) fn get_blake3(path: &PathBuf, show_progress: bool) -> Result { if show_progress { @@ -507,34 +510,16 @@ pub(crate) fn fetch_cargo( Ok(()) } -fn get_remote_url(name: &str, ext: &str) -> String { - return format!( - "{}/{}/{}.{}", - REMOTE_PKG_SOURCE, - redoxer::target(), - name, - ext - ); -} - -fn get_pubkey_url() -> String { - return format!("{}/id_ed25519.pub.toml", REMOTE_PKG_SOURCE); -} - pub fn fetch_remote( recipe_dir: &Path, recipe: &CookRecipe, offline_mode: bool, logger: &PtyOut, ) -> Result<(), String> { + let (mut manager, repository) = fetch_repo::get_binary_repo(); let target_dir = create_target_dir(recipe_dir, recipe.target)?; - let source_pubkey = target_dir.join("id_ed25519.pub.toml"); - if !offline_mode { - download_wget(&get_pubkey_url(), &source_pubkey, logger)?; - } else { - offline_check_exists(&source_pubkey)?; - } - + let writer = logger.as_ref().unwrap().1.try_clone().unwrap(); + manager.set_callback(Rc::new(RefCell::new(PlainPtyCallback::new(writer)))); let packages = recipe.recipe.get_packages_list(); let name = recipe_dir @@ -546,15 +531,47 @@ pub fn fetch_remote( for package in packages { let (_, source_pkgar, source_toml) = package_source_paths(package, &target_dir); let source_name = get_package_name(name, package); + let Some(repo_blake3) = repository.packages.get(&source_name) else { + return Err(format!( + "Package {source_name} does not exist in server repository" + )); + }; if !offline_mode { - //TODO: Check freshness - download_wget( - &get_remote_url(&source_name, "pkgar"), - &source_pkgar, - logger, - )?; - download_wget(&get_remote_url(&source_name, "toml"), &source_toml, logger)?; + if source_toml.is_file() { + let pkg_toml = read_source_toml(&source_toml)?; + if &pkg_toml.blake3 != repo_blake3 { + log_to_pty!(logger, "DEBUG: Updating source binaries"); + remove_all(&source_toml)?; + if source_pkgar.is_file() { + remove_all(&source_pkgar)?; + } + } + } + + if !source_toml.is_file() { + { + let toml_file = File::create(&source_toml) + .map_err(|e| format!("Unable to create source.toml: {e:?}"))?; + let mut writer = DownloadBackendWriter::ToFile(toml_file); + manager + .download(&format!("{}.toml", &source_name), None, &mut writer) + .map_err(|e| format!("Unable to download source.toml: {e:?}"))?; + } + let pkg_toml = read_source_toml(&source_toml)?; + let pkgar_file = File::create(&source_pkgar) + .map_err(|e| format!("Unable to create source.pkgar: {e:?}"))?; + let mut writer = DownloadBackendWriter::ToFile(pkgar_file); + manager + .download( + &format!("{}.pkgar", &source_name), + Some(pkg_toml.network_size), + &mut writer, + ) + .map_err(|e| format!("Unable to download source.pkgar: {e:?}"))?; + } + + // manager.download(file, 0, dest) } else { offline_check_exists(&source_pkgar)?; offline_check_exists(&source_toml)?; @@ -562,14 +579,7 @@ pub fn fetch_remote( // guaranteed to exist once if package.is_none() { - let mut file = File::open(&source_toml) - .map_err(|e| format!("Unable to open source.toml: {e:?}"))?; - let mut contents = String::new(); - file.read_to_string(&mut contents) - .map_err(|e| format!("Unable to read source.toml: {e:?}"))?; - - let pkg_toml = pkg::Package::from_toml(&contents) - .map_err(|e| format!("Unable to parse source.toml: {e:?}"))?; + let pkg_toml = read_source_toml(&source_toml)?; fetch_apply_source_info_from_remote( recipe, @@ -586,6 +596,17 @@ pub fn fetch_remote( Ok(()) } +fn read_source_toml(source_toml: &Path) -> Result { + let mut file = + File::open(source_toml).map_err(|e| format!("Unable to open source.toml: {e:?}"))?; + let mut contents = String::new(); + file.read_to_string(&mut contents) + .map_err(|e| format!("Unable to read source.toml: {e:?}"))?; + let pkg_toml = pkg::Package::from_toml(&contents) + .map_err(|e| format!("Unable to parse source.toml: {e:?}"))?; + Ok(pkg_toml) +} + pub(crate) fn fetch_is_patches_newer( recipe_dir: &Path, patches: &Vec, @@ -657,7 +678,7 @@ pub(crate) fn fetch_apply_source_info( source_identifier: String, ) -> Result<(), String> { let ident = crate::cook::ident::get_ident(); - let info = pkg::package::SourceIdentifier { + let info = SourceIdentifier { commit_identifier: ident.commit.to_string(), time_identifier: ident.time.to_string(), source_identifier: source_identifier, @@ -668,7 +689,7 @@ pub(crate) fn fetch_apply_source_info( pub(crate) fn fetch_apply_source_info_from_remote( recipe: &CookRecipe, - info: &pkg::package::SourceIdentifier, + info: &SourceIdentifier, ) -> Result<(), String> { let target_dir = create_target_dir(&recipe.dir, recipe.target)?; let source_toml_path = target_dir.join("source_info.toml"); diff --git a/src/cook/fetch_repo.rs b/src/cook/fetch_repo.rs new file mode 100644 index 000000000..799fe49be --- /dev/null +++ b/src/cook/fetch_repo.rs @@ -0,0 +1,204 @@ +use std::{ + cell::RefCell, + io::{PipeWriter, Write}, + path::{Path, PathBuf}, + rc::Rc, + time::Duration, +}; + +use pkg::{ + PackageName, RemotePackage, RepoManager, Repository, + callback::{Callback, SilentCallback}, + net_backend::{CurlBackend, DownloadBackend}, +}; + +// TODO: This is a workaround, but as long as whole +// fetch operation is in single thread, this is ok +thread_local! { +static BINARY_REPO: RefCell> = RefCell::new(None); +} + +fn load_cached_repo(path: &Path) -> Option { + let metadata = std::fs::metadata(path).ok()?; + + if !crate::config::get_config().cook.offline { + let yesterday = std::time::SystemTime::now().checked_sub(Duration::from_secs(24 * 3600))?; + if metadata.modified().ok()? < yesterday { + // stale cache + let _ = std::fs::remove_file(path); + return None; + } + } + + let toml_str = std::fs::read_to_string(path).ok()?; + Repository::from_toml(&toml_str).ok() +} + +fn init_binary_repo() -> (RepoManager, Repository) { + let callback = Rc::new(RefCell::new(SilentCallback::new())); + let download_backend = CurlBackend::new().expect("Curl not found"); + let mut repo = RepoManager::new(callback, Box::new(download_backend)); + + repo.add_remote(crate::REMOTE_PKG_SOURCE, redoxer::target()) + .expect("Unable to add remote"); + + let repo_path = PathBuf::from("build/remotes"); + repo.set_download_path(repo_path.clone()); + repo.sync_keys().expect("Unable to sync keys"); + + let repo_toml = load_cached_repo(&repo_path.join("repo.toml")).unwrap_or_else(|| { + let (toml_str, _) = repo + .get_package_toml(&PackageName::new("repo").unwrap()) + .expect("Failed to fetch repo.toml"); + Repository::from_toml(&toml_str).expect("Fetched repo.toml is invalid") + }); + + (repo, repo_toml) +} + +pub fn get_binary_repo() -> (RepoManager, Repository) { + BINARY_REPO.with(|cell| { + let mut opt = cell.borrow_mut(); + if opt.is_none() { + *opt = Some(init_binary_repo()); + } + let (repo, repo_toml) = opt.as_ref().unwrap(); + ((*repo).clone(), repo_toml.clone()) + }) +} + +pub struct PlainPtyCallback { + size: u64, + unknown_size: bool, + pos: u64, + fetch_processed: usize, + fetch_total: usize, + interactive: bool, + download_file: Option, + pty: PipeWriter, +} + +impl PlainPtyCallback { + pub fn new(pty: PipeWriter) -> Self { + Self { + size: 0, + unknown_size: false, + pos: 0, + fetch_processed: 0, + fetch_total: 0, + interactive: false, + download_file: None, + pty, + } + } + + /// Set if user require to agree on terminal + pub fn set_interactive(&mut self, enabled: bool) { + self.interactive = enabled; + } + + fn flush(&self) { + let _ = std::io::stderr().flush(); + } + + pub fn format_size(bytes: u64) -> String { + if bytes == 0 { + return "0 B".to_string(); + } + const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; + let i = (bytes as f64).log(1024.0).floor() as usize; + let size = bytes as f64 / 1024.0_f64.powi(i as i32); + format!("{:.2} {}", size, UNITS[i]) + } + + fn downloading_str(&self) -> &'static str { + "Downloading" + } +} + +const RESET_LINE: &str = "\r\x1b[2K"; + +impl Callback for PlainPtyCallback { + fn fetch_start(&mut self, initial_count: usize) { + self.fetch_total = 0; + self.fetch_processed = 0; + self.fetch_package_increment(0, initial_count); + } + + fn fetch_package_name(&mut self, pkg_name: &PackageName) { + // resuming after fetch_package_increment + let _ = write!(&self.pty, " {}", pkg_name.as_str()); + self.flush(); + } + + fn fetch_package_increment(&mut self, added_processed: usize, added_count: usize) { + self.fetch_processed += added_processed; + self.fetch_total += added_count; + + let _ = write!( + &self.pty, + "{RESET_LINE}Fetching: [{}/{}]", + self.fetch_processed, self.fetch_total + ); + self.flush(); + } + + fn fetch_end(&mut self) { + if self.fetch_processed == self.fetch_total { + let _ = writeln!(&self.pty, "{RESET_LINE}Fetch complete."); + } else { + let _ = writeln!(&self.pty, "{RESET_LINE}Fetch incomplete."); + } + } + + fn download_start(&mut self, length: u64, file: &str) { + self.size = length; + self.unknown_size = length == 0; + self.pos = 0; + if !self.unknown_size { + let _ = write!(&self.pty, "{RESET_LINE}{} {file}", self.downloading_str()); + self.download_file = Some(file.to_string()); + self.flush(); + } + } + + fn download_increment(&mut self, downloaded: u64) { + self.pos += downloaded; + if self.unknown_size { + self.size += downloaded; + } + if self.unknown_size { + return; + } + + // keep using MB for consistency + let pos_mb = self.pos as f64 / 1_048_576.0; + let size_mb = self.size as f64 / 1_048_576.0; + let file_name = self + .download_file + .as_ref() + .map(|s| s.as_str()) + .unwrap_or(""); + let _ = write!( + &self.pty, + "{RESET_LINE}{} {} [{:.2} MB / {:.2} MB]", + self.downloading_str(), + file_name, + pos_mb, + size_mb + ); + self.flush(); + } + + fn download_end(&mut self) { + if !self.unknown_size { + let _ = writeln!(&self.pty, ""); + self.download_file = None; + } + } + + fn install_extract(&mut self, remote_pkg: &RemotePackage) { + let _ = writeln!(&self.pty, "Extracting {}...", remote_pkg.package.name); + self.flush(); + } +} diff --git a/src/cook/package.rs b/src/cook/package.rs index 0c4f29cb6..23b929728 100644 --- a/src/cook/package.rs +++ b/src/cook/package.rs @@ -3,7 +3,7 @@ use std::{ path::{Path, PathBuf}, }; -use pkg::{Package, PackageName}; +use pkg::{Package, PackageName, PackagePrefix}; use crate::{ blake3::hash_to_hex, @@ -79,7 +79,7 @@ pub fn package( } let deps = if package.is_some() { - BTreeSet::from([name.without_host()]) + BTreeSet::from([name.with_prefix(PackagePrefix::Any)]) } else { auto_deps.clone() }; @@ -155,7 +155,7 @@ pub fn package_toml( let ident_source = fetch::fetch_get_source_info(recipe)?; let package = Package { - name: recipe.name.without_host(), + name: recipe.name.with_prefix(PackagePrefix::Any), version: package_version(&recipe.recipe), target: recipe.target.to_string(), blake3: hash, diff --git a/src/recipe.rs b/src/recipe.rs index 61792e439..003febe12 100644 --- a/src/recipe.rs +++ b/src/recipe.rs @@ -5,7 +5,7 @@ use std::{ path::{Path, PathBuf}, }; -use pkg::{PackageName, package::PackageError, recipes}; +use pkg::{PackageError, PackageName, recipes}; use regex::Regex; use serde::{ Deserialize, Serialize,