diff --git a/Cargo.lock b/Cargo.lock index b62a0a23..80d7029f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -912,7 +912,7 @@ dependencies = [ [[package]] name = "redox-pkg" version = "0.2.8" -source = "git+https://gitlab.redox-os.org/redox-os/pkgutils.git#a3a0fabc20394b887196bb370bedf38d040d5428" +source = "git+https://gitlab.redox-os.org/redox-os/pkgutils.git#b89f08df540b597949207147e0645aed24eb2a80" dependencies = [ "anyhow", "ignore", diff --git a/recipes/wip/dev/git-tools/lazygit/recipe.toml b/recipes/wip/dev/git-tools/lazygit/recipe.toml new file mode 100644 index 00000000..0969eefb --- /dev/null +++ b/recipes/wip/dev/git-tools/lazygit/recipe.toml @@ -0,0 +1,23 @@ +#TODO: Incomplete std/syscall porting, or vendor patches +[source] +git = "https://github.com/jesseduffield/lazygit" + +[build] +template = "custom" +dev-dependencies = [ + "host:go" +] +script = """ +export GOTOOLCHAIN=local +case "${TARGET}" in + x86_64-unknown-linux-gnu) export GOARCH=amd64 GOOS=linux;; + aarch64-unknown-linux-gnu) export GOARCH=arm64 GOOS=linux;; + i586-unknown-redox) export GOARCH=386 GOOS=redox;; + x86_64-unknown-redox) export GOARCH=amd64 GOOS=redox;; + aarch64-unknown-redox) export GOARCH=arm64 GOOS=redox;; + riscv64gc-unknown-redox) export GOARCH=riscv64 GOOS=redox;; +esac + +mkdir -p $COOKBOOK_STAGE/usr/bin +go build -C ${COOKBOOK_SOURCE} -o $COOKBOOK_STAGE/usr/bin/lazygit +""" diff --git a/recipes/wip/dev/lang/go/recipe.toml b/recipes/wip/dev/lang/go/recipe.toml index 278f374b..24f3afd6 100644 --- a/recipes/wip/dev/lang/go/recipe.toml +++ b/recipes/wip/dev/lang/go/recipe.toml @@ -21,27 +21,31 @@ fi # Go does not support out-of-tree builds :( rsync -a --delete "${COOKBOOK_SOURCE}/" ./ -export GOOS=redox case "${TARGET}" in - x86-unknown-redox) export GOARCH=386;; - x86_64-unknown-redox) export GOARCH=amd64;; - aarch64-unknown-redox) export GOARCH=arm64;; - riscv64-unknown-redox) export GOARCH=riscv64;; + x86_64-unknown-linux-gnu) export GOARCH=amd64 GOOS=linux;; + aarch64-unknown-linux-gnu) export GOARCH=arm64 GOOS=linux;; + i586-unknown-redox) export GOARCH=386 GOOS=redox;; + x86_64-unknown-redox) export GOARCH=amd64 GOOS=redox;; + aarch64-unknown-redox) export GOARCH=arm64 GOOS=redox;; + riscv64gc-unknown-redox) export GOARCH=riscv64 GOOS=redox;; esac export CGO_ENABLED=1 -export CC=x86_64-unknown-redox-gcc -export CCX=x86_64-unknown-redox-g++ -echo "go1.25" > VERSION # to set -trimpath +echo "go1.25.0" > VERSION # to set -trimpath (cd ./src && bash ./make.bash) mkdir -p "${COOKBOOK_STAGE}"/usr/bin \ "${COOKBOOK_STAGE}"/usr/lib/golang/{bin,lib,misc,pkg/include,pkg/tool,src} -rsync -a bin/redox_${GOARCH}/* "${COOKBOOK_STAGE}"/usr/lib/golang/bin/ + +if [ "$TARGET" = "$COOKBOOK_HOST_TARGET" ]; then + rsync -a bin/* "${COOKBOOK_STAGE}"/usr/lib/golang/bin/ +else + rsync -a bin/${GOOS}_${GOARCH}/* "${COOKBOOK_STAGE}"/usr/lib/golang/bin/ +fi rsync -a lib/* "${COOKBOOK_STAGE}"/usr/lib/golang/lib/ rsync -a misc/* "${COOKBOOK_STAGE}"/usr/lib/golang/misc/ rsync -a pkg/include/* "${COOKBOOK_STAGE}"/usr/lib/golang/pkg/include/ -rsync -a pkg/tool/redox_${GOARCH} "${COOKBOOK_STAGE}"/usr/lib/golang/pkg/tool/ +rsync -a pkg/tool/${GOOS}_${GOARCH} "${COOKBOOK_STAGE}"/usr/lib/golang/pkg/tool/ rsync -a src/* "${COOKBOOK_STAGE}"/usr/lib/golang/src/ cat go.env > "${COOKBOOK_STAGE}"/usr/lib/golang/go.env ln -s "../lib/golang/bin/go" "${COOKBOOK_STAGE}"/usr/bin/go diff --git a/src/bin/repo.rs b/src/bin/repo.rs index 8cfed9c9..2fc9d6f5 100644 --- a/src/bin/repo.rs +++ b/src/bin/repo.rs @@ -4,7 +4,7 @@ use cookbook::config::{CookConfig, get_config, init_config}; use cookbook::cook::cook_build::build; use cookbook::cook::fetch::{fetch, fetch_offline}; use cookbook::cook::fs::{create_target_dir, run_command}; -use cookbook::cook::package::package; +use cookbook::cook::package::{package, package_target}; use cookbook::cook::pty::{PtyOut, UnixSlavePty, flush_pty, setup_pty}; use cookbook::cook::script::KILL_ALL_PID; use cookbook::cook::tree::{WalkTreeEntry, display_tree_entry, format_size, walk_tree_entry}; @@ -316,7 +316,8 @@ fn repo_inner( // successful fetch is not that useful to log if *command == CliCommand::Cook || result.is_err() { flush_pty(&mut logger); - let log_path = log_path.join(format!("{}.log", recipe.name.as_str())); + let log_path = + log_path.join(format!("{}/{}.log", recipe.target, recipe.name.name())); status_tx .send(StatusUpdate::FlushLog(recipe.name.clone(), log_path)) .unwrap_or_default(); @@ -344,7 +345,7 @@ fn publish_packages(recipe_names: &Vec, repo_path: &PathBuf) -> anyh let repo_bin = env::current_exe()?.parent().unwrap().join("repo_builder"); let mut command = Command::new(repo_bin); command - .arg(repo_path.join(redoxer::target())) + .arg(repo_path) .args(recipe_names.iter().filter_map(|n| { if !n.is_deps { Some(n.name.as_str()) @@ -415,7 +416,6 @@ fn parse_args(args: Vec) -> anyhow::Result<(CliConfig, CliCommand, Vec) -> anyhow::Result<(CliConfig, CliCommand, Vec anyhow::Result<()> { let recipe_dir = &recipe.dir; - let target_dir = create_target_dir(recipe_dir).map_err(|e| anyhow!(e))?; + let target_dir = create_target_dir(recipe_dir, recipe.target).map_err(|e| anyhow!(e))?; let (stage_dir, auto_deps) = build( recipe_dir, &source_dir, @@ -639,7 +639,7 @@ fn handle_push(recipes: &Vec, config: &CliConfig) -> anyhow::Result< } WalkTreeEntry::NotBuilt => Err(anyhow!( "Package {} has not been built", - package_name.as_str() + package_name.name() )), WalkTreeEntry::Deduped | WalkTreeEntry::Missing => { return Ok(()); @@ -1026,7 +1026,8 @@ fn run_tui_cook( log_to_pty!(&logger, "\n{:?}", err_ctx) } flush_pty(&mut logger); - let log_path = log_path.join(format!("{}.log", name.as_str())); + let log_path = + log_path.join(format!("{}/{}.log", package_target(&name), name.name())); cooker_status_tx .send(StatusUpdate::FlushLog(name.clone(), log_path)) .unwrap_or_default(); @@ -1127,7 +1128,7 @@ fn run_tui_cook( log_to_pty!(&logger, "\n{:?}", err_ctx) } flush_pty(&mut logger); - let log_path = log_path.join(format!("{}.log", name.as_str())); + let log_path = log_path.join(format!("{}/{}.log", recipe.target, name.name())); fetcher_status_tx .send(StatusUpdate::FlushLog(name.clone(), log_path)) .unwrap_or_default(); diff --git a/src/bin/repo_builder.rs b/src/bin/repo_builder.rs index 9cbbee37..58a7cbb3 100644 --- a/src/bin/repo_builder.rs +++ b/src/bin/repo_builder.rs @@ -1,6 +1,7 @@ use anyhow::anyhow; use cookbook::WALK_DEPTH; use cookbook::config::{get_config, init_config}; +use cookbook::cook::package::package_target; use pkg::{Package, PackageName, recipes}; use std::collections::{BTreeMap, HashMap}; use std::env; @@ -53,7 +54,7 @@ fn main() -> Result<(), Box> { // TODO: Make this callable from repo bin fn publish_packages(config: &CliConfig) -> anyhow::Result<()> { - let repo_path = &config.repo_dir; + let repo_path = &config.repo_dir.join(redoxer::target()); if !repo_path.is_dir() { fs::create_dir_all(repo_path)?; } @@ -68,12 +69,14 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> { .recipe_list .iter() .map(PackageName::new) + // Don't publish host packages + .filter(|pkg| pkg.as_ref().is_ok_and(|p| !p.is_host())) .collect::, _>>()?, config.nonstop, WALK_DEPTH, )? .into_iter() - .map(|pkg| pkg.name.as_str().to_owned()) + .map(|pkg| pkg.name.clone()) .collect::>(); let mut appstream_sources: HashMap = HashMap::new(); @@ -81,13 +84,12 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> { // === 1. Push recipes in list === for recipe in &recipe_list { - let Some(recipe_path) = recipes::find(recipe) else { + let Some(recipe_path) = recipes::find(recipe.name()) else { eprintln!("recipe {} not found", recipe); continue; }; - let cookbook_recipe = Path::new(&recipe_path); - let target = redoxer::target(); + let target = package_target(recipe); let stage_dir = cookbook_recipe.join("target").join(&target).join("stage"); let pkgar_src = stage_dir.with_extension("pkgar"); @@ -109,7 +111,7 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> { } if stage_dir.join("usr/share/metainfo").exists() { - appstream_sources.insert(recipe.clone(), stage_dir.clone()); + appstream_sources.insert(recipe.name().to_string(), stage_dir.clone()); } } diff --git a/src/cook/cook_build.rs b/src/cook/cook_build.rs index 6148d324..8bac2f1c 100644 --- a/src/cook/cook_build.rs +++ b/src/cook/cook_build.rs @@ -2,6 +2,7 @@ use pkg::package::PackageError; use pkg::{Package, PackageName}; use crate::cook::fs::*; +use crate::cook::package::package_target; use crate::cook::pty::PtyOut; use crate::cook::script::*; use crate::recipe::BuildKind; @@ -174,6 +175,7 @@ pub fn build( logger: &PtyOut, ) -> Result<(PathBuf, BTreeSet), String> { let sysroot_dir = target_dir.join("sysroot"); + let toolchain_dir = target_dir.join("toolchain"); let stage_dir = target_dir.join("stage"); let cli_verbose = crate::config::get_config().cook.verbose; let cli_jobs = crate::config::get_config().cook.jobs; @@ -183,17 +185,24 @@ pub fn build( } let mut dep_pkgars = BTreeSet::new(); - let build_deps = CookRecipe::get_build_deps_recursive(&recipe.build.dependencies, false, false) - .map_err(|e| format!("{:?}", e))?; + let mut dep_host_pkgars = BTreeSet::new(); + let mut build_deps = + CookRecipe::get_build_deps_recursive(&recipe.build.dependencies, false, false) + .map_err(|e| format!("{:?}", e))?; + for dep in &recipe.build.dev_dependencies { + build_deps.push(CookRecipe::from_name(dep.clone()).map_err(|e| format!("{:?}", e))?); + } for dependency in build_deps.iter() { - dep_pkgars.insert(( - dependency.name.clone(), - dependency - .dir - .join("target") - .join(redoxer::target()) - .join("stage.pkgar"), - )); + let pkgar = dependency + .dir + .join("target") + .join(dependency.target) + .join("stage.pkgar"); + if dependency.name.is_host() { + dep_host_pkgars.insert((dependency.name.clone(), pkgar)); + } else { + dep_pkgars.insert((dependency.name.clone(), pkgar)); + } } if stage_dir.exists() && !check_source { @@ -207,57 +216,43 @@ pub fn build( .map(|(_dep, pkgar)| modified(pkgar)) .max() .unwrap_or(Ok(SystemTime::UNIX_EPOCH))?; + let deps_host_modified = dep_host_pkgars + .iter() + .map(|(_dep, pkgar)| modified(pkgar)) + .max() + .unwrap_or(Ok(SystemTime::UNIX_EPOCH))?; // Rebuild sysroot if source is newer //TODO: rebuild on recipe changes - if sysroot_dir.is_dir() { - let sysroot_modified = modified_dir(&sysroot_dir)?; - if sysroot_modified < source_modified || sysroot_modified < deps_modified { - log_to_pty!(logger, "DEBUG: updating '{}'", sysroot_dir.display()); - remove_all(&sysroot_dir)?; - } + if recipe.build.kind != BuildKind::Remote { + build_deps_dir( + logger, + &sysroot_dir, + target_dir.join("sysroot.tmp"), + &dep_pkgars, + source_modified, + deps_modified, + )?; } - if !sysroot_dir.is_dir() && recipe.build.kind != BuildKind::Remote { - // Create sysroot.tmp - let sysroot_dir_tmp = target_dir.join("sysroot.tmp"); - create_dir_clean(&sysroot_dir_tmp)?; - - // Make sure sysroot/usr exists - create_dir(&sysroot_dir_tmp.join("usr"))?; - for folder in &["bin", "include", "lib", "share"] { - // Make sure sysroot/usr/$folder exists - create_dir(&sysroot_dir_tmp.join("usr").join(folder))?; - - // Link sysroot/$folder sysroot/usr/$folder - symlink(Path::new("usr").join(folder), &sysroot_dir_tmp.join(folder))?; - } - - for (_dep, archive_path) in &dep_pkgars { - let public_path = "build/id_ed25519.pub.toml"; - pkgar::extract( - public_path, - &archive_path, - sysroot_dir_tmp.to_str().unwrap(), - ) - .map_err(|err| { - format!( - "failed to install '{}' in '{}': {:?}", - archive_path.display(), - sysroot_dir_tmp.display(), - err - ) - })?; - } - - // Move sysroot.tmp to sysroot atomically - rename(&sysroot_dir_tmp, &sysroot_dir)?; + if recipe.build.kind != BuildKind::Remote && dep_host_pkgars.len() > 0 { + build_deps_dir( + logger, + &toolchain_dir, + target_dir.join("toolchain.tmp"), + &dep_host_pkgars, + source_modified, + deps_host_modified, + )?; } // Rebuild stage if source is newer //TODO: rebuild on recipe changes if stage_dir.is_dir() { let stage_modified = modified_dir(&stage_dir)?; - if stage_modified < source_modified || stage_modified < deps_modified { + if stage_modified < source_modified + || stage_modified < deps_modified + || stage_modified < deps_host_modified + { log_to_pty!(logger, "DEBUG: updating '{}'", stage_dir.display()); remove_all(&stage_dir)?; } @@ -324,6 +319,7 @@ pub fn build( let cookbook_stage = stage_dir_tmp.canonicalize().unwrap(); let cookbook_source = source_dir.canonicalize().unwrap(); let cookbook_sysroot = sysroot_dir.canonicalize().unwrap(); + let cookbook_toolchain = toolchain_dir.canonicalize().ok(); let bash_args = if cli_verbose { "-ex" } else { "-e" }; let mut command = if is_redox() { let mut command = Command::new("bash"); @@ -340,14 +336,18 @@ pub fn build( command }; command.current_dir(&cookbook_build); + command.env("TARGET", package_target(name)); command.env("COOKBOOK_BUILD", &cookbook_build); - command.env("COOKBOOK_NAME", name.as_str()); + command.env("COOKBOOK_NAME", name.name()); command.env("COOKBOOK_HOST_TARGET", redoxer::host_target()); command.env("COOKBOOK_RECIPE", &cookbook_recipe); command.env("COOKBOOK_ROOT", &cookbook_root); command.env("COOKBOOK_STAGE", &cookbook_stage); command.env("COOKBOOK_SOURCE", &cookbook_source); command.env("COOKBOOK_SYSROOT", &cookbook_sysroot); + if let Some(cookbook_toolchain) = &cookbook_toolchain { + command.env("COOKBOOK_TOOLCHAIN", cookbook_toolchain); + } command.env("COOKBOOK_MAKE_JOBS", cli_jobs.to_string()); if cli_verbose { command.env("COOKBOOK_VERBOSE", "1"); @@ -373,6 +373,56 @@ pub fn build( Ok((stage_dir, auto_deps)) } +fn build_deps_dir( + logger: &PtyOut, + deps_dir: &PathBuf, + deps_dir_tmp: PathBuf, + dep_pkgars: &BTreeSet<(PackageName, PathBuf)>, + source_modified: SystemTime, + deps_modified: SystemTime, +) -> Result<(), String> { + if deps_dir.is_dir() { + let sysroot_modified = modified_dir(deps_dir)?; + if sysroot_modified < source_modified || sysroot_modified < deps_modified { + log_to_pty!(logger, "DEBUG: updating '{}'", deps_dir.display()); + remove_all(deps_dir)?; + } + } + if !deps_dir.is_dir() { + // Create sysroot.tmp + create_dir_clean(&deps_dir_tmp)?; + + // Make sure sysroot/usr exists + create_dir(&deps_dir_tmp.join("usr"))?; + for folder in &["bin", "include", "lib", "share"] { + // Make sure sysroot/usr/$folder exists + create_dir(&deps_dir_tmp.join("usr").join(folder))?; + + // Link sysroot/$folder sysroot/usr/$folder + symlink(Path::new("usr").join(folder), &deps_dir_tmp.join(folder))?; + } + + for (_dep, archive_path) in dep_pkgars { + let public_path = "build/id_ed25519.pub.toml"; + pkgar::extract(public_path, &archive_path, deps_dir_tmp.to_str().unwrap()).map_err( + |err| { + format!( + "failed to install '{}' in '{}': {:?}", + archive_path.display(), + deps_dir_tmp.display(), + err + ) + }, + )?; + } + + // Move sysroot.tmp to sysroot atomically + rename(&deps_dir_tmp, deps_dir)?; + } + + Ok(()) +} + /// Calculate automatic dependencies fn build_auto_deps( recipe: &Recipe, diff --git a/src/cook/fetch.rs b/src/cook/fetch.rs index aff03b2a..8b8a136d 100644 --- a/src/cook/fetch.rs +++ b/src/cook/fetch.rs @@ -442,7 +442,9 @@ fn get_pubkey_url() -> String { } pub fn fetch_remote(recipe_dir: &Path, offline_mode: bool, logger: &PtyOut) -> Result<(), String> { - let target_dir = create_target_dir(recipe_dir)?; + // TODO: allow download to host target + let target = redoxer::target(); + let target_dir = create_target_dir(recipe_dir, target)?; let name = recipe_dir .file_name() .ok_or("Unable to get recipe name")? diff --git a/src/cook/fs.rs b/src/cook/fs.rs index c9385260..b89df61e 100644 --- a/src/cook/fs.rs +++ b/src/cook/fs.rs @@ -36,12 +36,12 @@ pub fn create_dir_clean(dir: &Path) -> Result<(), String> { create_dir(dir) } -pub fn create_target_dir(recipe_dir: &Path) -> Result { +pub fn create_target_dir(recipe_dir: &Path, target: &'static str) -> Result { let target_parent_dir = recipe_dir.join("target"); if !target_parent_dir.is_dir() { create_dir(&target_parent_dir)?; } - let target_dir = target_parent_dir.join(redoxer::target()); + let target_dir = target_parent_dir.join(target); if !target_dir.is_dir() { create_dir(&target_dir)?; } diff --git a/src/cook/package.rs b/src/cook/package.rs index 141b2bb9..cec7f22b 100644 --- a/src/cook/package.rs +++ b/src/cook/package.rs @@ -112,9 +112,9 @@ pub fn package_toml( }; let package = Package { - name: name.clone(), + name: PackageName::new(name.name()).unwrap(), version: package_version(recipe), - target: redoxer::target().to_string(), + target: package_target(name).to_string(), blake3: hash, // this size will be different once pkgar supports compression network_size: size, @@ -143,3 +143,11 @@ fn package_version(recipe: &Recipe) -> String { "TODO".into() } } + +pub fn package_target(name: &PackageName) -> &'static str { + if name.is_host() { + redoxer::host_target() + } else { + redoxer::target() + } +} diff --git a/src/cook/script.rs b/src/cook/script.rs index 57c06990..5b48fabb 100644 --- a/src/cook/script.rs +++ b/src/cook/script.rs @@ -86,6 +86,12 @@ pub(crate) static BUILD_PRESCRIPT: &str = r#" # Add cookbook bins to path export PATH="${COOKBOOK_ROOT}/bin:${PATH}" +# Add toolchain dir to path if exists +if [ ! -z "${COOKBOOK_TOOLCHAIN}" ] +then +export PATH="${COOKBOOK_TOOLCHAIN}/bin:${PATH}" +fi + # This puts cargo build artifacts in the build directory export CARGO_TARGET_DIR="${COOKBOOK_BUILD}/target" diff --git a/src/cook/tree.rs b/src/cook/tree.rs index 1c796488..ec4feb1a 100644 --- a/src/cook/tree.rs +++ b/src/cook/tree.rs @@ -54,7 +54,7 @@ pub fn walk_tree_entry( }; let package_dir = &cook_recipe.dir; - let target_dir = create_target_dir(package_dir).map_err(|e| anyhow!(e))?; + let target_dir = create_target_dir(package_dir, redoxer::target()).map_err(|e| anyhow!(e))?; let pkg_path = target_dir.join("stage.pkgar"); let pkg_toml = target_dir.join("stage.toml"); diff --git a/src/recipe.rs b/src/recipe.rs index 96a83051..317e1465 100644 --- a/src/recipe.rs +++ b/src/recipe.rs @@ -12,7 +12,7 @@ use serde::{ de::{Error as DeErrorT, value::Error as DeError}, }; -use crate::WALK_DEPTH; +use crate::{WALK_DEPTH, cook::package::package_target}; /// Specifies how to download the source for a recipe #[derive(Debug, Clone, Deserialize, PartialEq, Serialize)] @@ -189,6 +189,7 @@ pub struct CookRecipe { pub name: PackageName, pub dir: PathBuf, pub recipe: Recipe, + pub target: &'static str, /// If false, it's listed on install config pub is_deps: bool, } @@ -208,19 +209,18 @@ impl Recipe { impl CookRecipe { pub fn new(name: PackageName, dir: PathBuf, recipe: Recipe) -> Result { + let target = package_target(&name); Ok(Self { name, dir, recipe, + target, is_deps: false, }) } - pub fn from_name( - name: impl TryInto, - ) -> Result { - let name: PackageName = name.try_into()?; - let dir = recipes::find(name.as_str()) + pub fn from_name(name: PackageName) -> Result { + let dir = recipes::find(name.name()) .ok_or_else(|| PackageError::PackageNotFound(name.clone()))?; let file = dir.join("recipe.toml"); let recipe = Recipe::new(&file)?; @@ -255,7 +255,7 @@ impl CookRecipe { let mut recipes = Vec::new(); for name in names { - let recipe = Self::from_name(name.as_str())?; + let recipe = Self::from_name(name.clone())?; if recurse_build_deps { let dependencies = Self::new_recursive(