MahnoKropotkinvich/redox
1
0
Fork 0
mirror of https://gitlab.redox-os.org/redox-os/redox.git synced 2026-06-17 15:34:18 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Implement recipes as host toolchain

Browse Source
This commit is contained in:
Wildan M 2025-11-29 00:02:59 -08:00
parent f4fe816850
commit 3e1595133f
No known key found for this signature in database
GPG Key ID: 01AC53185C679C79
12 changed files with 187 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -912,7 +912,7 @@ dependencies = [
[[package]]
name = "redox-pkg"
version = "0.2.8"
source = "git+https://gitlab.redox-os.org/redox-os/pkgutils.git#a3a0fabc20394b887196bb370bedf38d040d5428"
source = "git+https://gitlab.redox-os.org/redox-os/pkgutils.git#b89f08df540b597949207147e0645aed24eb2a80"
dependencies = [
"anyhow",
"ignore",

23
recipes/wip/dev/git-tools/lazygit/recipe.toml Normal file
View File

@ -0,0 +1,23 @@
#TODO: Incomplete std/syscall porting, or vendor patches
[source]
git = "https://github.com/jesseduffield/lazygit"
[build]
template = "custom"
dev-dependencies = [
"host:go"
]
script = """
export GOTOOLCHAIN=local
case "${TARGET}" in
x86_64-unknown-linux-gnu) export GOARCH=amd64 GOOS=linux;;
aarch64-unknown-linux-gnu) export GOARCH=arm64 GOOS=linux;;
i586-unknown-redox) export GOARCH=386 GOOS=redox;;
x86_64-unknown-redox) export GOARCH=amd64 GOOS=redox;;
aarch64-unknown-redox) export GOARCH=arm64 GOOS=redox;;
riscv64gc-unknown-redox) export GOARCH=riscv64 GOOS=redox;;
esac
mkdir -p $COOKBOOK_STAGE/usr/bin
go build -C ${COOKBOOK_SOURCE} -o $COOKBOOK_STAGE/usr/bin/lazygit
"""

24
recipes/wip/dev/lang/go/recipe.toml
View File

@ -21,27 +21,31 @@ fi
# Go does not support out-of-tree builds :(
rsync -a --delete "${COOKBOOK_SOURCE}/" ./
export GOOS=redox
case "${TARGET}" in
x86-unknown-redox) export GOARCH=386;;
x86_64-unknown-redox) export GOARCH=amd64;;
aarch64-unknown-redox) export GOARCH=arm64;;
riscv64-unknown-redox) export GOARCH=riscv64;;
x86_64-unknown-linux-gnu) export GOARCH=amd64 GOOS=linux;;
aarch64-unknown-linux-gnu) export GOARCH=arm64 GOOS=linux;;
i586-unknown-redox) export GOARCH=386 GOOS=redox;;
x86_64-unknown-redox) export GOARCH=amd64 GOOS=redox;;
aarch64-unknown-redox) export GOARCH=arm64 GOOS=redox;;
riscv64gc-unknown-redox) export GOARCH=riscv64 GOOS=redox;;
esac
export CGO_ENABLED=1
export CC=x86_64-unknown-redox-gcc
export CCX=x86_64-unknown-redox-g++
echo "go1.25" > VERSION # to set -trimpath
echo "go1.25.0" > VERSION # to set -trimpath
(cd ./src && bash ./make.bash)
mkdir -p "${COOKBOOK_STAGE}"/usr/bin \
"${COOKBOOK_STAGE}"/usr/lib/golang/{bin,lib,misc,pkg/include,pkg/tool,src}
rsync -a bin/redox_${GOARCH}/* "${COOKBOOK_STAGE}"/usr/lib/golang/bin/
if [ "$TARGET" = "$COOKBOOK_HOST_TARGET" ]; then
rsync -a bin/* "${COOKBOOK_STAGE}"/usr/lib/golang/bin/
else
rsync -a bin/${GOOS}_${GOARCH}/* "${COOKBOOK_STAGE}"/usr/lib/golang/bin/
fi
rsync -a lib/* "${COOKBOOK_STAGE}"/usr/lib/golang/lib/
rsync -a misc/* "${COOKBOOK_STAGE}"/usr/lib/golang/misc/
rsync -a pkg/include/* "${COOKBOOK_STAGE}"/usr/lib/golang/pkg/include/
rsync -a pkg/tool/redox_${GOARCH} "${COOKBOOK_STAGE}"/usr/lib/golang/pkg/tool/
rsync -a pkg/tool/${GOOS}_${GOARCH} "${COOKBOOK_STAGE}"/usr/lib/golang/pkg/tool/
rsync -a src/* "${COOKBOOK_STAGE}"/usr/lib/golang/src/
cat go.env > "${COOKBOOK_STAGE}"/usr/lib/golang/go.env
ln -s "../lib/golang/bin/go" "${COOKBOOK_STAGE}"/usr/bin/go

19
src/bin/repo.rs
View File

@ -4,7 +4,7 @@ use cookbook::config::{CookConfig, get_config, init_config};
use cookbook::cook::cook_build::build;
use cookbook::cook::fetch::{fetch, fetch_offline};
use cookbook::cook::fs::{create_target_dir, run_command};
use cookbook::cook::package::package;
use cookbook::cook::package::{package, package_target};
use cookbook::cook::pty::{PtyOut, UnixSlavePty, flush_pty, setup_pty};
use cookbook::cook::script::KILL_ALL_PID;
use cookbook::cook::tree::{WalkTreeEntry, display_tree_entry, format_size, walk_tree_entry};
@ -316,7 +316,8 @@ fn repo_inner(
// successful fetch is not that useful to log
if *command == CliCommand::Cook || result.is_err() {
flush_pty(&mut logger);
let log_path = log_path.join(format!("{}.log", recipe.name.as_str()));
let log_path =
log_path.join(format!("{}/{}.log", recipe.target, recipe.name.name()));
status_tx
.send(StatusUpdate::FlushLog(recipe.name.clone(), log_path))
.unwrap_or_default();
@ -344,7 +345,7 @@ fn publish_packages(recipe_names: &Vec<CookRecipe>, repo_path: &PathBuf) -> anyh
let repo_bin = env::current_exe()?.parent().unwrap().join("repo_builder");
let mut command = Command::new(repo_bin);
command
.arg(repo_path.join(redoxer::target()))
.arg(repo_path)
.args(recipe_names.iter().filter_map(|n| {
if !n.is_deps {
Some(n.name.as_str())
@ -415,7 +416,6 @@ fn parse_args(args: Vec<String>) -> anyhow::Result<(CliConfig, CliCommand, Vec<C
config.category = Some(PathBuf::from("recipes").join(c));
}
if let Some(c) = config.logs_dir.as_mut() {
*c = c.join(redoxer::target());
fs::create_dir_all(c).map_err(|e| anyhow!(e))?;
}
if override_filesystem_repo_binary {
@ -487,7 +487,7 @@ fn parse_args(args: Vec<String>) -> anyhow::Result<(CliConfig, CliCommand, Vec<C
} else {
recipe_names
.iter()
.map(|f| CookRecipe::from_name(f.as_str()).unwrap())
.map(|f| CookRecipe::from_name(f.clone()).unwrap())
.collect()
}
};
@ -570,7 +570,7 @@ fn handle_cook(
logger: &PtyOut,
) -> anyhow::Result<()> {
let recipe_dir = &recipe.dir;
let target_dir = create_target_dir(recipe_dir).map_err(|e| anyhow!(e))?;
let target_dir = create_target_dir(recipe_dir, recipe.target).map_err(|e| anyhow!(e))?;
let (stage_dir, auto_deps) = build(
recipe_dir,
&source_dir,
@ -639,7 +639,7 @@ fn handle_push(recipes: &Vec<CookRecipe>, config: &CliConfig) -> anyhow::Result<
}
WalkTreeEntry::NotBuilt => Err(anyhow!(
"Package {} has not been built",
package_name.as_str()
package_name.name()
)),
WalkTreeEntry::Deduped | WalkTreeEntry::Missing => {
return Ok(());
@ -1026,7 +1026,8 @@ fn run_tui_cook(
log_to_pty!(&logger, "\n{:?}", err_ctx)
}
flush_pty(&mut logger);
let log_path = log_path.join(format!("{}.log", name.as_str()));
let log_path =
log_path.join(format!("{}/{}.log", package_target(&name), name.name()));
cooker_status_tx
.send(StatusUpdate::FlushLog(name.clone(), log_path))
.unwrap_or_default();
@ -1127,7 +1128,7 @@ fn run_tui_cook(
log_to_pty!(&logger, "\n{:?}", err_ctx)
}
flush_pty(&mut logger);
let log_path = log_path.join(format!("{}.log", name.as_str()));
let log_path = log_path.join(format!("{}/{}.log", recipe.target, name.name()));
fetcher_status_tx
.send(StatusUpdate::FlushLog(name.clone(), log_path))
.unwrap_or_default();

14
src/bin/repo_builder.rs
View File

@ -1,6 +1,7 @@
use anyhow::anyhow;
use cookbook::WALK_DEPTH;
use cookbook::config::{get_config, init_config};
use cookbook::cook::package::package_target;
use pkg::{Package, PackageName, recipes};
use std::collections::{BTreeMap, HashMap};
use std::env;
@ -53,7 +54,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// TODO: Make this callable from repo bin
fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
let repo_path = &config.repo_dir;
let repo_path = &config.repo_dir.join(redoxer::target());
if !repo_path.is_dir() {
fs::create_dir_all(repo_path)?;
}
@ -68,12 +69,14 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
.recipe_list
.iter()
.map(PackageName::new)
// Don't publish host packages
.filter(|pkg| pkg.as_ref().is_ok_and(|p| !p.is_host()))
.collect::<Result<Vec<_>, _>>()?,
config.nonstop,
WALK_DEPTH,
)?
.into_iter()
.map(|pkg| pkg.name.as_str().to_owned())
.map(|pkg| pkg.name.clone())
.collect::<Vec<_>>();
let mut appstream_sources: HashMap<String, PathBuf> = HashMap::new();
@ -81,13 +84,12 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
// === 1. Push recipes in list ===
for recipe in &recipe_list {
let Some(recipe_path) = recipes::find(recipe) else {
let Some(recipe_path) = recipes::find(recipe.name()) else {
eprintln!("recipe {} not found", recipe);
continue;
};
let cookbook_recipe = Path::new(&recipe_path);
let target = redoxer::target();
let target = package_target(recipe);
let stage_dir = cookbook_recipe.join("target").join(&target).join("stage");
let pkgar_src = stage_dir.with_extension("pkgar");
@ -109,7 +111,7 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
}
if stage_dir.join("usr/share/metainfo").exists() {
appstream_sources.insert(recipe.clone(), stage_dir.clone());
appstream_sources.insert(recipe.name().to_string(), stage_dir.clone());
}
}

154
src/cook/cook_build.rs
View File

@ -2,6 +2,7 @@ use pkg::package::PackageError;
use pkg::{Package, PackageName};
use crate::cook::fs::*;
use crate::cook::package::package_target;
use crate::cook::pty::PtyOut;
use crate::cook::script::*;
use crate::recipe::BuildKind;
@ -174,6 +175,7 @@ pub fn build(
logger: &PtyOut,
) -> Result<(PathBuf, BTreeSet<PackageName>), String> {
let sysroot_dir = target_dir.join("sysroot");
let toolchain_dir = target_dir.join("toolchain");
let stage_dir = target_dir.join("stage");
let cli_verbose = crate::config::get_config().cook.verbose;
let cli_jobs = crate::config::get_config().cook.jobs;
@ -183,17 +185,24 @@ pub fn build(
}
let mut dep_pkgars = BTreeSet::new();
let build_deps = CookRecipe::get_build_deps_recursive(&recipe.build.dependencies, false, false)
.map_err(|e| format!("{:?}", e))?;
let mut dep_host_pkgars = BTreeSet::new();
let mut build_deps =
CookRecipe::get_build_deps_recursive(&recipe.build.dependencies, false, false)
.map_err(|e| format!("{:?}", e))?;
for dep in &recipe.build.dev_dependencies {
build_deps.push(CookRecipe::from_name(dep.clone()).map_err(|e| format!("{:?}", e))?);
}
for dependency in build_deps.iter() {
dep_pkgars.insert((
dependency.name.clone(),
dependency
.dir
.join("target")
.join(redoxer::target())
.join("stage.pkgar"),
));
let pkgar = dependency
.dir
.join("target")
.join(dependency.target)
.join("stage.pkgar");
if dependency.name.is_host() {
dep_host_pkgars.insert((dependency.name.clone(), pkgar));
} else {
dep_pkgars.insert((dependency.name.clone(), pkgar));
}
}
if stage_dir.exists() && !check_source {
@ -207,57 +216,43 @@ pub fn build(
.map(|(_dep, pkgar)| modified(pkgar))
.max()
.unwrap_or(Ok(SystemTime::UNIX_EPOCH))?;
let deps_host_modified = dep_host_pkgars
.iter()
.map(|(_dep, pkgar)| modified(pkgar))
.max()
.unwrap_or(Ok(SystemTime::UNIX_EPOCH))?;
// Rebuild sysroot if source is newer
//TODO: rebuild on recipe changes
if sysroot_dir.is_dir() {
let sysroot_modified = modified_dir(&sysroot_dir)?;
if sysroot_modified < source_modified || sysroot_modified < deps_modified {
log_to_pty!(logger, "DEBUG: updating '{}'", sysroot_dir.display());
remove_all(&sysroot_dir)?;
}
if recipe.build.kind != BuildKind::Remote {
build_deps_dir(
logger,
&sysroot_dir,
target_dir.join("sysroot.tmp"),
&dep_pkgars,
source_modified,
deps_modified,
)?;
}
if !sysroot_dir.is_dir() && recipe.build.kind != BuildKind::Remote {
// Create sysroot.tmp
let sysroot_dir_tmp = target_dir.join("sysroot.tmp");
create_dir_clean(&sysroot_dir_tmp)?;
// Make sure sysroot/usr exists
create_dir(&sysroot_dir_tmp.join("usr"))?;
for folder in &["bin", "include", "lib", "share"] {
// Make sure sysroot/usr/$folder exists
create_dir(&sysroot_dir_tmp.join("usr").join(folder))?;
// Link sysroot/$folder sysroot/usr/$folder
symlink(Path::new("usr").join(folder), &sysroot_dir_tmp.join(folder))?;
}
for (_dep, archive_path) in &dep_pkgars {
let public_path = "build/id_ed25519.pub.toml";
pkgar::extract(
public_path,
&archive_path,
sysroot_dir_tmp.to_str().unwrap(),
)
.map_err(|err| {
format!(
"failed to install '{}' in '{}': {:?}",
archive_path.display(),
sysroot_dir_tmp.display(),
err
)
})?;
}
// Move sysroot.tmp to sysroot atomically
rename(&sysroot_dir_tmp, &sysroot_dir)?;
if recipe.build.kind != BuildKind::Remote && dep_host_pkgars.len() > 0 {
build_deps_dir(
logger,
&toolchain_dir,
target_dir.join("toolchain.tmp"),
&dep_host_pkgars,
source_modified,
deps_host_modified,
)?;
}
// Rebuild stage if source is newer
//TODO: rebuild on recipe changes
if stage_dir.is_dir() {
let stage_modified = modified_dir(&stage_dir)?;
if stage_modified < source_modified || stage_modified < deps_modified {
if stage_modified < source_modified
|| stage_modified < deps_modified
|| stage_modified < deps_host_modified
{
log_to_pty!(logger, "DEBUG: updating '{}'", stage_dir.display());
remove_all(&stage_dir)?;
}
@ -324,6 +319,7 @@ pub fn build(
let cookbook_stage = stage_dir_tmp.canonicalize().unwrap();
let cookbook_source = source_dir.canonicalize().unwrap();
let cookbook_sysroot = sysroot_dir.canonicalize().unwrap();
let cookbook_toolchain = toolchain_dir.canonicalize().ok();
let bash_args = if cli_verbose { "-ex" } else { "-e" };
let mut command = if is_redox() {
let mut command = Command::new("bash");
@ -340,14 +336,18 @@ pub fn build(
command
};
command.current_dir(&cookbook_build);
command.env("TARGET", package_target(name));
command.env("COOKBOOK_BUILD", &cookbook_build);
command.env("COOKBOOK_NAME", name.as_str());
command.env("COOKBOOK_NAME", name.name());
command.env("COOKBOOK_HOST_TARGET", redoxer::host_target());
command.env("COOKBOOK_RECIPE", &cookbook_recipe);
command.env("COOKBOOK_ROOT", &cookbook_root);
command.env("COOKBOOK_STAGE", &cookbook_stage);
command.env("COOKBOOK_SOURCE", &cookbook_source);
command.env("COOKBOOK_SYSROOT", &cookbook_sysroot);
if let Some(cookbook_toolchain) = &cookbook_toolchain {
command.env("COOKBOOK_TOOLCHAIN", cookbook_toolchain);
}
command.env("COOKBOOK_MAKE_JOBS", cli_jobs.to_string());
if cli_verbose {
command.env("COOKBOOK_VERBOSE", "1");
@ -373,6 +373,56 @@ pub fn build(
Ok((stage_dir, auto_deps))
}
fn build_deps_dir(
logger: &PtyOut,
deps_dir: &PathBuf,
deps_dir_tmp: PathBuf,
dep_pkgars: &BTreeSet<(PackageName, PathBuf)>,
source_modified: SystemTime,
deps_modified: SystemTime,
) -> Result<(), String> {
if deps_dir.is_dir() {
let sysroot_modified = modified_dir(deps_dir)?;
if sysroot_modified < source_modified || sysroot_modified < deps_modified {
log_to_pty!(logger, "DEBUG: updating '{}'", deps_dir.display());
remove_all(deps_dir)?;
}
}
if !deps_dir.is_dir() {
// Create sysroot.tmp
create_dir_clean(&deps_dir_tmp)?;
// Make sure sysroot/usr exists
create_dir(&deps_dir_tmp.join("usr"))?;
for folder in &["bin", "include", "lib", "share"] {
// Make sure sysroot/usr/$folder exists
create_dir(&deps_dir_tmp.join("usr").join(folder))?;
// Link sysroot/$folder sysroot/usr/$folder
symlink(Path::new("usr").join(folder), &deps_dir_tmp.join(folder))?;
}
for (_dep, archive_path) in dep_pkgars {
let public_path = "build/id_ed25519.pub.toml";
pkgar::extract(public_path, &archive_path, deps_dir_tmp.to_str().unwrap()).map_err(
|err| {
format!(
"failed to install '{}' in '{}': {:?}",
archive_path.display(),
deps_dir_tmp.display(),
err
)
},
)?;
}
// Move sysroot.tmp to sysroot atomically
rename(&deps_dir_tmp, deps_dir)?;
}
Ok(())
}
/// Calculate automatic dependencies
fn build_auto_deps(
recipe: &Recipe,

4
src/cook/fetch.rs
View File

@ -442,7 +442,9 @@ fn get_pubkey_url() -> String {
}
pub fn fetch_remote(recipe_dir: &Path, offline_mode: bool, logger: &PtyOut) -> Result<(), String> {
let target_dir = create_target_dir(recipe_dir)?;
// TODO: allow download to host target
let target = redoxer::target();
let target_dir = create_target_dir(recipe_dir, target)?;
let name = recipe_dir
.file_name()
.ok_or("Unable to get recipe name")?

4
src/cook/fs.rs
View File

@ -36,12 +36,12 @@ pub fn create_dir_clean(dir: &Path) -> Result<(), String> {
create_dir(dir)
}
pub fn create_target_dir(recipe_dir: &Path) -> Result<PathBuf, String> {
pub fn create_target_dir(recipe_dir: &Path, target: &'static str) -> Result<PathBuf, String> {
let target_parent_dir = recipe_dir.join("target");
if !target_parent_dir.is_dir() {
create_dir(&target_parent_dir)?;
}
let target_dir = target_parent_dir.join(redoxer::target());
let target_dir = target_parent_dir.join(target);
if !target_dir.is_dir() {
create_dir(&target_dir)?;
}

12
src/cook/package.rs
View File

@ -112,9 +112,9 @@ pub fn package_toml(
};
let package = Package {
name: name.clone(),
name: PackageName::new(name.name()).unwrap(),
version: package_version(recipe),
target: redoxer::target().to_string(),
target: package_target(name).to_string(),
blake3: hash,
// this size will be different once pkgar supports compression
network_size: size,
@ -143,3 +143,11 @@ fn package_version(recipe: &Recipe) -> String {
"TODO".into()
}
}
pub fn package_target(name: &PackageName) -> &'static str {
if name.is_host() {
redoxer::host_target()
} else {
redoxer::target()
}
}

6
src/cook/script.rs
View File

@ -86,6 +86,12 @@ pub(crate) static BUILD_PRESCRIPT: &str = r#"
# Add cookbook bins to path
export PATH="${COOKBOOK_ROOT}/bin:${PATH}"
# Add toolchain dir to path if exists
if [ ! -z "${COOKBOOK_TOOLCHAIN}" ]
then
export PATH="${COOKBOOK_TOOLCHAIN}/bin:${PATH}"
fi
# This puts cargo build artifacts in the build directory
export CARGO_TARGET_DIR="${COOKBOOK_BUILD}/target"

2
src/cook/tree.rs
View File

@ -54,7 +54,7 @@ pub fn walk_tree_entry(
};
let package_dir = &cook_recipe.dir;
let target_dir = create_target_dir(package_dir).map_err(|e| anyhow!(e))?;
let target_dir = create_target_dir(package_dir, redoxer::target()).map_err(|e| anyhow!(e))?;
let pkg_path = target_dir.join("stage.pkgar");
let pkg_toml = target_dir.join("stage.toml");

14
src/recipe.rs
View File

@ -12,7 +12,7 @@ use serde::{
de::{Error as DeErrorT, value::Error as DeError},
};
use crate::WALK_DEPTH;
use crate::{WALK_DEPTH, cook::package::package_target};
/// Specifies how to download the source for a recipe
#[derive(Debug, Clone, Deserialize, PartialEq, Serialize)]
@ -189,6 +189,7 @@ pub struct CookRecipe {
pub name: PackageName,
pub dir: PathBuf,
pub recipe: Recipe,
pub target: &'static str,
/// If false, it's listed on install config
pub is_deps: bool,
}
@ -208,19 +209,18 @@ impl Recipe {
impl CookRecipe {
pub fn new(name: PackageName, dir: PathBuf, recipe: Recipe) -> Result<Self, PackageError> {
let target = package_target(&name);
Ok(Self {
name,
dir,
recipe,
target,
is_deps: false,
})
}
pub fn from_name(
name: impl TryInto<PackageName, Error = PackageError>,
) -> Result<Self, PackageError> {
let name: PackageName = name.try_into()?;
let dir = recipes::find(name.as_str())
pub fn from_name(name: PackageName) -> Result<Self, PackageError> {
let dir = recipes::find(name.name())
.ok_or_else(|| PackageError::PackageNotFound(name.clone()))?;
let file = dir.join("recipe.toml");
let recipe = Recipe::new(&file)?;
@ -255,7 +255,7 @@ impl CookRecipe {
let mut recipes = Vec::new();
for name in names {
let recipe = Self::from_name(name.as_str())?;
let recipe = Self::from_name(name.clone())?;
if recurse_build_deps {
let dependencies = Self::new_recursive(