Merge branch 'expand-pkg-toml' into 'master'

Use expanded package toml and switch repo.toml to use blake3 hash

See merge request redox-os/redox!1702

Cargo.lock

@@ -2046,6 +2046,23 @@ dependencies = [
  "toml 0.8.23",
 ]
 
+[[package]]
+name = "redox-pkg"
+version = "0.2.8"
+source = "git+https://gitlab.redox-os.org/redox-os/pkgutils.git#08b71582e7fd43bf7af401cfcd5f0d402dd3f6da"
+dependencies = [
+ "anyhow",
+ "ignore",
+ "pkgar 0.1.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pkgar-core 0.1.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pkgar-keys 0.1.19 (registry+https://github.com/rust-lang/crates.io-index)",
+ "reqwest",
+ "serde",
+ "serde_derive",
+ "thiserror 1.0.69",
+ "toml 0.8.23",
+]
+
 [[package]]
 name = "redox-scheme"
 version = "0.7.0"
@@ -2072,7 +2089,7 @@ dependencies = [
  "pkgar-core 0.1.19 (git+https://gitlab.redox-os.org/redox-os/pkgar.git)",
  "pkgar-keys 0.1.19 (git+https://gitlab.redox-os.org/redox-os/pkgar.git)",
  "ratatui",
- "redox-pkg",
+ "redox-pkg 0.2.8 (git+https://gitlab.redox-os.org/redox-os/pkgutils.git)",
  "redox_installer",
  "redoxer",
  "regex",
@@ -2102,7 +2119,7 @@ dependencies = [
  "pkgar-core 0.1.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "pkgar-keys 0.1.19 (registry+https://github.com/rust-lang/crates.io-index)",
  "rand 0.8.5",
- "redox-pkg",
+ "redox-pkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "redox_liner",
  "redox_syscall",
  "redoxfs",
@@ -2161,12 +2178,12 @@ dependencies = [
 [[package]]
 name = "redoxer"
 version = "0.2.61"
-source = "git+https://gitlab.redox-os.org/redox-os/redoxer#ea965186ce02cfc01a7d2085dfd007cdc67649c6"
+source = "git+https://gitlab.redox-os.org/redox-os/redoxer.git#ea965186ce02cfc01a7d2085dfd007cdc67649c6"
 dependencies = [
  "anyhow",
  "dirs 6.0.0",
  "proc-mounts",
- "redox-pkg",
+ "redox-pkg 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "redox_installer",
  "redox_syscall",
  "redoxfs",

Cargo.toml

@@ -7,14 +7,18 @@ default-run = "repo"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
-[[bin]]
-name = "cook"
-path = "src/bin/cook.rs"
-
 [[bin]]
 name = "cookbook_redoxer"
 path = "src/bin/cookbook_redoxer.rs"
 
+[[bin]]
+name = "repo"
+path = "src/bin/repo.rs"
+
+[[bin]]
+name = "repo_builder"
+path = "src/bin/repo_builder.rs"
+
 [lib]
 name = "cookbook"
 path = "src/lib.rs"
@@ -36,9 +40,9 @@ pbr = "1.0.2"
 pkgar = { git = "https://gitlab.redox-os.org/redox-os/pkgar.git" }
 pkgar-core = { git = "https://gitlab.redox-os.org/redox-os/pkgar.git" }
 pkgar-keys = { git = "https://gitlab.redox-os.org/redox-os/pkgar.git" }
-redox-pkg = "0.2.8"
+redox-pkg = { git = "https://gitlab.redox-os.org/redox-os/pkgutils.git" }
 redox_installer = "0.2.37"
-redoxer = { git = "https://gitlab.redox-os.org/redox-os/redoxer" }
+redoxer = { git = "https://gitlab.redox-os.org/redox-os/redoxer.git" }
 regex = "1.11"
 serde = { version = "=1.0.197", features = ["derive"] }
 termion = "4"

src/bin/repo_builder.rs

@@ -1,5 +1,6 @@
 use anyhow::anyhow;
 use cookbook::WALK_DEPTH;
+use cookbook::config::{get_config, init_config};
 use pkg::{Package, PackageName, recipes};
 use std::collections::{BTreeMap, HashMap};
 use std::env;
@@ -21,17 +22,38 @@ fn is_newer(src: &Path, dst: &Path) -> bool {
     }
 }
 
+#[derive(Clone)]
+struct CliConfig {
+    repo_dir: PathBuf,
+    nonstop: bool,
+    appstream: bool,
+    recipe_list: Vec<String>,
+}
+
+impl CliConfig {
+    fn parse_args() -> Result<Self, std::io::Error> {
+        let mut args = env::args().skip(1);
+        let repo_dir = args
+            .next()
+            .expect("Usage: repo_builder <REPO_DIR> <recipe1> <recipe2> ...");
+        Ok(CliConfig {
+            repo_dir: PathBuf::from(repo_dir),
+            nonstop: get_config().cook.nonstop,
+            appstream: env::var("COOKBOOK_APPSTREAM").ok().as_deref() == Some("true"),
+            recipe_list: args.collect(),
+        })
+    }
+}
+
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    let mut args = env::args().skip(1);
+    init_config();
-    let repo_dir = args
+    let conf = CliConfig::parse_args()?;
-        .next()
+    Ok(publish_packages(&conf)?)
-        .expect("Usage: repo_builder <REPO_DIR> <recipe1> <recipe2> ...");
-    Ok(publish_packages(args.collect(), repo_dir)?)
 }
 
 // TODO: Make this callable from repo bin
-fn publish_packages(recipe_list: Vec<String>, repo_dir: String) -> anyhow::Result<()> {
+fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
-    let repo_path = Path::new(&repo_dir);
+    let repo_path = &config.repo_dir;
     if !repo_path.is_dir() {
         fs::create_dir_all(repo_path)?;
     }
@@ -42,10 +64,12 @@ fn publish_packages(recipe_list: Vec<String>, repo_dir: String) -> anyhow::Resul
     // The following adds the package dependencies of the recipes to the repo as
     // well.
     let recipe_list = Package::new_recursive(
-        &recipe_list
+        &config
+            .recipe_list
             .iter()
             .map(PackageName::new)
             .collect::<Result<Vec<_>, _>>()?,
+        config.nonstop,
         WALK_DEPTH,
     )?
     .into_iter()
@@ -63,7 +87,7 @@ fn publish_packages(recipe_list: Vec<String>, repo_dir: String) -> anyhow::Resul
         };
 
         let cookbook_recipe = Path::new(&recipe_path);
-        let target = env::var("TARGET").unwrap_or_else(|_| "x86_64-unknown-linux-gnu".into());
+        let target = redoxer::target();
         let stage_dir = cookbook_recipe.join("target").join(&target).join("stage");
 
         let pkgar_src = stage_dir.with_extension("pkgar");
@@ -90,7 +114,7 @@ fn publish_packages(recipe_list: Vec<String>, repo_dir: String) -> anyhow::Resul
     }
 
     // === 2. Optional AppStream generation ===
-    if env::var("COOKBOOK_APPSTREAM").ok().as_deref() == Some("true") {
+    if config.appstream {
         eprintln!("\x1b[01;38;5;155mrepo - generating appstream data\x1b[0m");
 
         let root = env::var("ROOT").unwrap_or_else(|_| ".".into());
@@ -167,9 +191,10 @@ fn publish_packages(recipe_list: Vec<String>, repo_dir: String) -> anyhow::Resul
         let content = fs::read_to_string(&path)?;
         let parsed: Value = toml::from_str(&content)?;
 
+        let empty_ver = Value::String("".to_string());
         let version_str = parsed
-            .get("version")
+            .get("blake3")
-            .unwrap_or(&Value::String("".to_string()))
+            .unwrap_or_else(|| parsed.get("version").unwrap_or_else(|| &empty_ver))
             .to_string(); // includes quotes
         let package_name = path.file_stem().unwrap().to_string_lossy().to_string();
         packages.insert(package_name, version_str);

src/blake3.rs

@@ -29,3 +29,7 @@ pub fn blake3_silent<P: AsRef<Path>>(path: P) -> Result<String> {
     let res = format!("{}", hash.to_hex());
     Ok(res)
 }
+
+pub fn hash_to_hex(h: [u8; 32]) -> String {
+    format!("{}", blake3::Hash::from_bytes(h).to_hex())
+}

src/cook/package.rs

@@ -1,8 +1,12 @@
-use std::{collections::BTreeSet, path::Path};
+use std::{
+    collections::BTreeSet,
+    path::{Path, PathBuf},
+};
 
 use pkg::{Package, PackageName};
 
 use crate::{
+    blake3::hash_to_hex,
     cook::{fs::*, pty::PtyOut},
     log_to_pty,
     recipe::{BuildKind, Recipe},
@@ -18,7 +22,7 @@ pub fn package(
 ) -> Result<(), String> {
     if recipe.build.kind == BuildKind::None {
         // metapackages don't have stage dir
-        package_toml(target_dir, name, recipe, auto_deps)?;
+        package_toml(target_dir, name, recipe, None, auto_deps)?;
         return Ok(());
     }
 
@@ -59,7 +63,13 @@ pub fn package(
     }
 
     if !package_meta.is_file() {
-        package_toml(target_dir, name, recipe, auto_deps)?;
+        package_toml(
+            target_dir,
+            name,
+            recipe,
+            Some((Path::new(public_path), &package_file)),
+            auto_deps,
+        )?;
     }
 
     Ok(())
@@ -69,6 +79,7 @@ pub fn package_toml(
     target_dir: &Path,
     name: &PackageName,
     recipe: &Recipe,
+    package_file: Option<(&Path, &PathBuf)>,
     auto_deps: &BTreeSet<PackageName>,
 ) -> Result<(), String> {
     let mut depends = recipe.package.dependencies.clone();
@@ -77,10 +88,37 @@ pub fn package_toml(
             depends.push(dep.clone());
         }
     }
+
+    let (hash, size) = if let Some((pkey_path, archive_path)) = package_file {
+        use pkgar_core::PackageSrc;
+        let pkey = pkgar_keys::PublicKeyFile::open(pkey_path)
+            .map_err(|e| format!("Unable to read public key: {e:?}"))?
+            .pkey;
+        let package = pkgar::PackageFile::new(archive_path, &pkey).map_err(|e| {
+            format!(
+                "Unable to read packaged pkgar file {}: {e:?}",
+                archive_path.display(),
+            )
+        })?;
+        let mt = std::fs::metadata(archive_path).map_err(|e| {
+            format!(
+                "Unable to read packaged pkgar file {}: {e:?}",
+                archive_path.display(),
+            )
+        })?;
+        (hash_to_hex(package.header().blake3), mt.len())
+    } else {
+        ("".into(), 0)
+    };
+
     let package = Package {
         name: name.clone(),
         version: package_version(recipe),
         target: redoxer::target().to_string(),
+        blake3: hash,
+        // this size will be different once pkgar supports compression
+        network_size: size,
+        storage_size: size,
         depends,
     };