MahnoKropotkinvich/redox
1
0
Fork 0
mirror of https://gitlab.redox-os.org/redox-os/redox.git synced 2026-06-19 11:24:18 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge branch 'isolate-pkg' into 'master'

Browse Source 
Various web generator improvements

See merge request redox-os/redox!2025
This commit is contained in:
Jeremy Soller 2026-03-27 07:25:15 -06:00
commit e7ebc8a674
13 changed files with 285 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/bin/repo.rs
View File

@ -9,8 +9,8 @@ use cookbook::cook::package::{package, package_handle_push};
use cookbook::cook::pty::{PtyOut, UnixSlavePty, flush_pty, setup_pty};
use cookbook::cook::script::KILL_ALL_PID;
use cookbook::cook::tree::{self, WalkTreeEntry};
use cookbook::log_to_pty;
use cookbook::recipe::{CookRecipe, recipes_flatten_package_names, recipes_mark_as_deps};
use cookbook::{log_to_pty, staged_pkg};
use pkg::{PackageName, PackageState};
use ratatui::Terminal;
use ratatui::layout::{Constraint, Direction, Layout, Position, Rect};
@ -507,8 +507,8 @@ fn parse_args(args: Vec<String>) -> anyhow::Result<(CliConfig, CliCommand, Vec<C
);
}
let all_recipes_path = match &config.category {
None => pkg::recipes::list(""),
Some(prefix) => pkg::recipes::list("")
None => staged_pkg::list(""),
Some(prefix) => staged_pkg::list("")
.into_iter()
.filter(|p| p.starts_with(prefix))
.collect(),

10
src/bin/repo_builder.rs
View File

@ -1,9 +1,9 @@
use cookbook::WALK_DEPTH;
use cookbook::cook::ident::{get_ident, init_ident};
use cookbook::cook::{fetch, package as cook_package};
use cookbook::recipe::CookRecipe;
use cookbook::web::{CliWebConfig, generate_web};
use pkg::{Package, PackageName, recipes};
use cookbook::{WALK_DEPTH, staged_pkg};
use pkg::PackageName;
use pkg::{Repository, SourceIdentifier};
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::env;
@ -85,7 +85,7 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
//
// The following adds the package dependencies of the recipes to the repo as
// well.
let (recipe_list, recipe_map) = Package::new_recursive_nonstop(target_packages, WALK_DEPTH);
let (recipe_list, recipe_map) = staged_pkg::new_recursive_nonstop(target_packages, WALK_DEPTH);
if recipe_list.len() == 0 {
// Fail-Safe
@ -99,7 +99,7 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
// === 1. Push recipes in list ===
for recipe_toml in &recipe_list {
let recipe = &recipe_toml.name;
let Some(recipe_path) = recipes::find(recipe.name()) else {
let Some(recipe_path) = staged_pkg::find(recipe.name()) else {
eprintln!("recipe {} not found", recipe);
continue;
};
@ -191,7 +191,7 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
recipe
);
let Some(recipe_path) = recipes::find(recipe.name()) else {
let Some(recipe_path) = staged_pkg::find(recipe.name()) else {
eprintln!("recipe {} not found", recipe);
continue;
};

35
src/blake3.rs
View File

@ -1,35 +0,0 @@
use blake3::Hasher;
use std::{fs, io::Result, path::Path, time::Duration};
use crate::progress_bar::{ProgressBar, ProgressBarRead};
pub fn blake3_progress<P: AsRef<Path>>(path: P) -> Result<String> {
let len = fs::metadata(&path)?.len();
let mut f = fs::File::open(&path)?;
let mut pb = ProgressBar::new(len);
pb.message("blake3: ");
pb.set_max_refresh_rate(Some(Duration::new(1, 0)));
pb.set_units(pbr::Units::Bytes);
let mut pbr = ProgressBarRead::new(&mut pb, &mut f);
let hash = Hasher::new().update_reader(&mut pbr)?.finalize();
let res = format!("{}", hash.to_hex());
pb.finish_println("");
Ok(res)
}
pub fn blake3_silent<P: AsRef<Path>>(path: P) -> Result<String> {
let mut f = fs::File::open(&path)?;
let hash = Hasher::new().update_reader(&mut f)?.finalize();
let res = format!("{}", hash.to_hex());
Ok(res)
}
pub fn hash_to_hex(h: [u8; 32]) -> String {
format!("{}", blake3::Hash::from_bytes(h).to_hex())
}

28
src/cook/fetch.rs
View File

@ -10,7 +10,8 @@ use crate::is_redox;
use crate::log_to_pty;
use crate::recipe::BuildKind;
use crate::recipe::CookRecipe;
use crate::{blake3, recipe::SourceRecipe};
use crate::recipe::SourceRecipe;
use crate::wrap_io_err;
use pkg::SourceIdentifier;
use pkg::net_backend::DownloadBackendWriter;
use std::cell::RefCell;
@ -21,20 +22,13 @@ use std::path::{Path, PathBuf};
use std::process::Command;
use std::rc::Rc;
pub(crate) fn get_blake3(path: &PathBuf, show_progress: bool) -> Result<String, String> {
if show_progress {
blake3::blake3_progress(&path)
} else {
blake3::blake3_silent(&path)
}
.map_err(|err| {
format!(
"failed to calculate blake3 of '{}': {}\n{:?}",
path.display(),
err,
err
)
})
pub(crate) fn get_blake3(path: &PathBuf) -> crate::Result<String> {
let mut f = fs::File::open(&path).map_err(wrap_io_err!(path, "Opening file for blake3"))?;
let hash = blake3::Hasher::new()
.update_reader(&mut f)
.map_err(wrap_io_err!(path, "Reading file for blake3"))?
.finalize();
Ok(hash.to_hex().to_string())
}
pub fn fetch_offline(recipe: &CookRecipe, logger: &PtyOut) -> Result<PathBuf, String> {
@ -86,7 +80,7 @@ pub fn fetch_offline(recipe: &CookRecipe, logger: &PtyOut) -> Result<PathBuf, St
}) => {
if !source_dir.is_dir() {
let source_tar = recipe_dir.join("source.tar");
let source_tar_blake3 = get_blake3(&source_tar, true && logger.is_none())?;
let source_tar_blake3 = get_blake3(&source_tar)?;
if source_tar.exists() {
if let Some(blake3) = blake3 {
if source_tar_blake3 != *blake3 {
@ -329,7 +323,7 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result
if !check_source {
break;
}
let source_tar_blake3 = get_blake3(&source_tar, tar_updated && logger.is_none())?;
let source_tar_blake3 = get_blake3(&source_tar)?;
if let Some(blake3) = blake3 {
if source_tar_blake3 == *blake3 {
break;

23
src/cook/package.rs
View File

@ -8,7 +8,7 @@ use pkgar::ext::PackageSrcExt;
use pkgar_core::HeaderFlags;
use crate::{
blake3::hash_to_hex,
Error,
config::CookConfig,
cook::{cook_build::BuildResult, fetch, fs::*, pty::PtyOut},
log_to_pty,
@ -30,6 +30,7 @@ pub fn package(
target_dir.join("stage.toml"),
recipe,
None,
None,
recipe.recipe.package.dependencies.clone(),
&auto_deps,
)?;
@ -110,6 +111,7 @@ pub fn package(
package_meta,
recipe,
Some((Path::new(public_path), &package_file)),
package,
package_deps,
&deps,
)?;
@ -123,6 +125,7 @@ pub fn package_toml(
toml_path: PathBuf,
recipe: &CookRecipe,
package_file: Option<(&Path, &PathBuf)>,
package_suffix: Option<&OptionalPackageRecipe>,
mut package_deps: Vec<PackageName>,
auto_deps: &BTreeSet<PackageName>,
) -> Result<(), String> {
@ -150,9 +153,13 @@ pub fn package_toml(
)
})?;
let package_size = mt.len();
let storage_size = match package.header().flags.packaging() {
let header = package.header();
let storage_size = match header.flags.packaging() {
pkgar_core::Packaging::LZMA2 => {
let mut size = 0;
let mut size = header
.total_size()
.map_err(|e| Error::Pkgar(pkgar::Error::Core(e)))?
as u64;
let entries = package
.read_entries()
.map_err(|e| format!("Unable to get lzma entry: {e}"))?;
@ -171,7 +178,9 @@ pub fn package_toml(
};
(
hash_to_hex(package.header().blake3),
blake3::Hash::from_bytes(package.header().blake3)
.to_hex()
.to_string(),
package_size,
storage_size,
)
@ -182,7 +191,11 @@ pub fn package_toml(
let ident_source = fetch::fetch_get_source_info(recipe)?;
let package = Package {
name: recipe.name.with_prefix(PackagePrefix::Any),
name: PackageName::new(get_package_name(
recipe.name.without_prefix(),
package_suffix,
))
.unwrap(),
version: recipe.guess_version().unwrap_or("TODO".into()),
target: recipe.target.to_string(),
blake3: hash,

3
src/cook/tree.rs
View File

@ -149,7 +149,8 @@ pub fn walk_file_tree(dir: &PathBuf, prefix: &str, buffer: &mut String) -> std::
return Ok(0);
}
let fmt_err = std::io::Error::other;
let entries: Vec<_> = std::fs::read_dir(dir)?.filter_map(|e| e.ok()).collect();
let mut entries: Vec<_> = std::fs::read_dir(dir)?.filter_map(|e| e.ok()).collect();
entries.sort_by(|a, b| a.file_name().cmp(&b.file_name()));
let mut total_size = 0;
for (index, entry) in entries.iter().enumerate() {
let path = entry.path();

4
src/lib.rs
View File

@ -1,11 +1,9 @@
pub mod blake3;
pub mod config;
pub mod cook;
pub mod recipe;
pub mod staged_pkg;
pub mod web;
mod progress_bar;
/// Default for maximum number of levels to descend down dependencies tree.
pub const WALK_DEPTH: usize = 16;

22
src/progress_bar.rs
View File

@ -1,22 +0,0 @@
pub use pbr::ProgressBar;
use std::io::{Read, Result, Write};
pub struct ProgressBarRead<'p, 'r, P: Write + 'p, R: Read + 'r> {
pb: &'p mut ProgressBar<P>,
r: &'r mut R,
}
impl<'p, 'r, P: Write, R: Read> ProgressBarRead<'p, 'r, P, R> {
pub fn new(pb: &'p mut ProgressBar<P>, r: &'r mut R) -> ProgressBarRead<'p, 'r, P, R> {
ProgressBarRead { pb, r }
}
}
impl<'p, 'r, P: Write, R: Read> Read for ProgressBarRead<'p, 'r, P, R> {
fn read(&mut self, buf: &mut [u8]) -> Result<usize> {
let count = self.r.read(buf)?;
self.pb.add(count as u64);
Ok(count)
}
}

8
src/recipe.rs
View File

@ -5,11 +5,11 @@ use std::{
path::{Path, PathBuf},
};
use pkg::{PackageError, PackageName, recipes};
use pkg::{PackageError, PackageName};
use regex::Regex;
use serde::{Deserialize, Serialize};
use crate::{WALK_DEPTH, cook::package as cook_package};
use crate::{WALK_DEPTH, cook::package as cook_package, staged_pkg};
/// Specifies how to download the source for a recipe
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
@ -214,7 +214,7 @@ impl CookRecipe {
pub fn new(name: PackageName, dir: PathBuf, mut recipe: Recipe) -> Result<Self, PackageError> {
let target = cook_package::package_target(&name);
if name.is_host() {
let thisname = name.name();
let thisname = name.without_host();
let fn_map = |p: PackageName| {
if p.is_host() {
if p.name() == thisname { None } else { Some(p) }
@ -248,7 +248,7 @@ impl CookRecipe {
}
pub fn from_name(name: PackageName) -> Result<Self, PackageError> {
let dir = recipes::find(name.name())
let dir = staged_pkg::find(name.name())
.ok_or_else(|| PackageError::PackageNotFound(name.clone()))?;
let file = dir.join("recipe.toml");
let recipe = Recipe::new(&file)?;

160
src/staged_pkg.rs Normal file
View File

@ -0,0 +1,160 @@
use std::borrow::Cow;
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::ffi::OsStr;
use std::path::{Path, PathBuf};
use std::sync::LazyLock;
use pkg::{Package, PackageError, PackageName};
// This file contains code that caches recipe paths.
// TODO: This file is previously resides in `pkg` crate,
// and can actually be merged with other logic in this cookbook.
static RECIPE_PATHS: LazyLock<HashMap<PackageName, PathBuf>> = LazyLock::new(|| {
let mut recipe_paths = HashMap::new();
for entry_res in ignore::Walk::new("recipes") {
let Ok(entry) = entry_res else {
continue;
};
if entry.file_name() == OsStr::new("recipe.toml") {
let recipe_file = entry.path();
let Some(recipe_dir) = recipe_file.parent() else {
continue;
};
let Some(recipe_name) = recipe_dir
.file_name()
.and_then(|x| x.to_str()?.try_into().ok())
else {
continue;
};
if let Some(other_dir) = recipe_paths.insert(recipe_name, recipe_dir.to_path_buf()) {
eprintln!(
"recipe {:?} has two or more entries: {:?} replaced by {:?}",
recipe_dir.file_name(),
other_dir,
recipe_dir,
);
}
}
}
recipe_paths
});
pub fn find(recipe: &str) -> Option<&'static Path> {
RECIPE_PATHS.get(recipe).map(PathBuf::as_path)
}
pub fn list(prefix: impl AsRef<Path>) -> BTreeSet<PathBuf> {
let prefix = prefix.as_ref();
RECIPE_PATHS
.values()
.map(|path| prefix.join(path))
.collect()
}
pub fn new(name: &PackageName) -> Result<Package, PackageError> {
let dir = find(name.name()).ok_or_else(|| PackageError::PackageNotFound(name.clone()))?;
from_path(dir, name.suffix())
}
pub fn from_path(dir: &Path, feature: Option<&str>) -> Result<Package, PackageError> {
let target = redoxer::target();
let stage_name = match feature {
Some(f) => Cow::Owned(format!("stage.{f}.toml")),
None => Cow::Borrowed("stage.toml"),
};
let file = dir.join("target").join(target).join(stage_name.as_ref());
if !file.is_file() {
return Err(PackageError::FileMissing(file));
}
let toml = std::fs::read_to_string(&file)
.map_err(|err| PackageError::FileError(err.raw_os_error(), file.clone()))?;
toml::from_str(&toml).map_err(|err| PackageError::Parse(err, Some(file)))
}
pub fn new_recursive(
names: &[PackageName],
nonstop: bool,
recursion: usize,
) -> Result<Vec<Package>, PackageError> {
if names.len() == 0 {
return Ok(vec![]);
}
let (list, map) = new_recursive_nonstop(names, recursion);
if nonstop && list.len() > 0 {
Ok(list)
} else if !nonstop && map.len() == list.len() {
Ok(list)
} else {
let (_, res) = map.into_iter().find(|(_, v)| v.is_err()).unwrap();
Err(res.err().unwrap())
}
}
/// List ordered success packages and map of failed packages.
/// A package can be both success and failed if dependencies aren't satistied.
pub fn new_recursive_nonstop(
names: &[PackageName],
recursion: usize,
) -> (
Vec<Package>,
BTreeMap<PackageName, Result<(), PackageError>>,
) {
let mut packages = Vec::new();
let mut packages_map = BTreeMap::new();
for name in names {
if packages_map.contains_key(name) {
continue;
}
let package = if recursion == 0 {
Err(PackageError::Recursion(Default::default()))
} else {
new(name)
};
match package {
Ok(package) => {
let mut has_invalid_dependency = false;
let (dependencies, dependencies_map) =
new_recursive_nonstop(&package.depends, recursion - 1);
for dependency in dependencies {
if !packages_map.contains_key(&dependency.name) {
packages_map.insert(dependency.name.clone(), Ok(()));
packages.push(dependency);
}
}
for (dep_name, result) in dependencies_map {
if let Err(mut e) = result {
if !packages_map.contains_key(&dep_name) {
e.append_recursion(name);
packages_map.insert(dep_name, Err(e));
}
has_invalid_dependency = true;
}
}
// TODO: this check is redundant
if !packages_map.contains_key(name) {
packages_map.insert(
name.clone(),
if has_invalid_dependency {
Err(PackageError::DependencyInvalid(name.clone()))
} else {
Ok(())
},
);
packages.push(package);
}
}
Err(e) => {
packages_map.insert(name.clone(), Err(e));
}
}
}
(packages, packages_map)
}

17
src/web.rs
View File

@ -8,6 +8,7 @@ use pkg::{Package, PackageName};
use crate::{
recipe::CookRecipe,
staged_pkg,
web::html::{generate_html_index, generate_html_pkg},
};
@ -59,14 +60,17 @@ pub fn generate_web(all_packages: &Vec<String>, config: &CliWebConfig) {
let mut dependents_map: HashMap<String, BTreeSet<String>> = HashMap::new();
for package_name in all_packages {
let Some(recipe_path) = pkg::recipes::find(package_name) else {
let Ok(package_name) = PackageName::new(package_name) else {
continue;
};
// TODO: Package::from_path
let Ok(package) = Package::new(&PackageName::new(package_name).unwrap()) else {
let Some(recipe_path) = staged_pkg::find(package_name.name()) else {
continue;
};
let Ok(recipe) = CookRecipe::from_path(&recipe_path, true, false) else {
let Ok(mut package) = staged_pkg::from_path(&recipe_path, package_name.suffix()) else {
// TODO: report failed build
continue;
};
let Ok(mut recipe) = CookRecipe::from_path(&recipe_path, true, false) else {
continue;
};
@ -77,6 +81,11 @@ pub fn generate_web(all_packages: &Vec<String>, config: &CliWebConfig) {
.insert(package.name.to_string());
}
// TODO: temporary bug fix in the suffix lost
package.name = package_name.clone();
// CookRecipe::from_path always have no suffix
recipe.name = package_name;
valid_packages.push((package, recipe));
}

2
src/web/html.rs
View File

@ -108,7 +108,7 @@ pub fn generate_html_pkg(
&config.this_repo,
host,
&package.commit_identifier,
Some(&format!("recipes/{category}/{name}/recipe.toml")),
Some(&format!("recipes/{category}/{}/recipe.toml", name.name())),
);
let short_commit = get_short_commit(&package.commit_identifier);
source_html += &format!(

88
src/web/style.css
View File

@ -6,8 +6,8 @@
body {
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
background-color: #f9f9fb;
color: #24292e;
background-color: #eee;
color: #222;
line-height: 1.6;
}
@ -23,8 +23,8 @@ body {
.category-title {
font-size: 1.5rem;
color: #24292e;
border-bottom: 2px solid #e1e4e8;
color: #222;
border-bottom: 2px solid #eee;
padding-bottom: 10px;
margin-bottom: 20px;
font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace;
@ -38,8 +38,8 @@ body {
}
.package-card {
background-color: #ffffff;
border: 1px solid #e1e4e8;
background-color: #fff;
border: 1px solid #eee;
border-radius: 6px;
padding: 15px;
margin: 10px;
@ -74,16 +74,16 @@ body {
align-items: center;
color: #6a737d;
font-size: 0.9rem;
border-top: 1px solid #e1e4e8;
border-top: 1px solid #eee;
padding-top: 10px;
}
.package-card .pkg-version {
font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace;
background-color: #f3f4f6;
background-color: #fff;
padding: 3px 6px;
border-radius: 4px;
color: #24292e;
color: #222;
}
.package-card .pkg-size {
@ -91,14 +91,14 @@ body {
}
a {
color: #24292e;
color: #222;
text-decoration: none;
border-bottom: 1px solid #e1e4e8;
border-bottom: 1px solid #eee;
}
a:hover {
color: #000000;
border-bottom: 1px solid #24292e;
border-bottom: 1px solid #222;
}
h1, h2, h3 {
@ -108,7 +108,7 @@ h1, h2, h3 {
code {
font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, "Liberation Mono", monospace;
background-color: #f3f4f6;
background-color: #fff;
padding: 0.2em 0.4em;
border-radius: 3px;
font-size: 0.9em;
@ -116,7 +116,7 @@ code {
.card {
background-color: #ffffff;
border: 1px solid #e1e4e8;
border: 1px solid #eee;
border-radius: 6px;
padding: 20px;
margin-bottom: 20px;
@ -124,7 +124,7 @@ code {
.pkg-header, .index-header {
background-color: #ffffff;
border-bottom: 1px solid #d1d5da;
border-bottom: 1px solid #ddd;
padding: 40px 0;
margin-bottom: 40px;
text-align: center;
@ -157,19 +157,19 @@ code {
}
.back-link:hover {
color: #24292e;
color: #222;
border: none;
}
.install-action {
display: inline-block;
background-color: #f3f4f6;
border: 1px solid #d1d5da;
background-color: #fff;
border: 1px solid #ddd;
border-radius: 6px;
padding: 12px 20px;
font-family: ui-monospace, SFMono-Regular, monospace;
font-size: 1.1rem;
color: #24292e;
color: #222;
}
.install-action .prompt {
@ -220,7 +220,7 @@ table {
th, td {
padding: 10px 0;
text-align: left;
border-bottom: 1px solid #e1e4e8;
border-bottom: 1px solid #eee;
}
th {
@ -241,6 +241,52 @@ th {
.pkg-deps li, .pkg-dependents li {
padding: 8px 0;
border-bottom: 1px solid #e1e4e8;
border-bottom: 1px solid #eee;
width: 50%;
}
@media (prefers-color-scheme: dark) {
body {
background-color: #000;
color: #ccc;
}
.package-card, .card, .pkg-header, .index-header {
background-color: #111;
border-color: #333;
}
.category-title {
color: #f0f6fc;
border-bottom-color: #333;
}
.package-card .pkg-stats {
color: #8b949e;
border-top-color: #333;
}
.package-card .pkg-version, code, .install-action {
background-color: #222;
color: #cdd;
border-color: #333;
}
a, .pkg-header h1, .back-link:hover {
color: #5af;
border-bottom-color: #333;
}
a:hover {
color: #7cf;
border-bottom-color: #7cf;
}
.pkg-header .version, .pkg-header .description, .back-link, .install-action .prompt, th {
color: #999;
}
th, td, .pkg-deps li, .pkg-dependents li {
border-bottom-color: #333;
}
}