diff --git a/mk/prefix.mk b/mk/prefix.mk index d5555c938..96f47802c 100644 --- a/mk/prefix.mk +++ b/mk/prefix.mk @@ -47,15 +47,27 @@ else cp -r "$(PREFIX)/gcc-install/". "$@.partial" cp -r "$(PREFIX)/rust-install/". "$@.partial" cp -r "$(PREFIX)/clang-install/". "$@.partial" +ifneq ($(HOSTED_REDOX),1) rm -rf "$@.partial/$(GNU_TARGET)/include/"* cp -r "$(PREFIX)/gcc-install/$(GNU_TARGET)/include/c++" "$@.partial/$(GNU_TARGET)/include/c++" +else + rm -rf "$@.partial/include/"* + cp -r "$(PREFIX)/gcc-install/include/c++" "$@.partial/include/c++" +endif export PATH="$(ROOT)/$@.partial/bin:$$PATH" && \ export CARGO="env -u CARGO cargo" $(PREFIX_CONFIG) && \ $(REPO_BIN) cook relibc +ifneq ($(HOSTED_REDOX),1) cp -r "$(RELIBC_TARGET)/stage/usr/". "$@.partial/$(GNU_TARGET)" mkdir -p "$@.partial/$(GNU_TARGET)/usr" ln -s "../include" "$@.partial/$(GNU_TARGET)/usr/include" ln -s "../lib" "$@.partial/$(GNU_TARGET)/usr/lib" +else + cp -r "$(RELIBC_TARGET)/stage/usr/". "$@.partial" + mkdir -p "$@.partial/usr" + ln -s "../include" "$@.partial/usr/include" + ln -s "../lib" "$@.partial/usr/lib" +endif touch "$@.partial" mv "$@.partial" "$@" endif @@ -139,6 +151,8 @@ else pkgar extract --pkey $(PREFIX)/id_ed25519.pub.toml --archive "$(PREFIX)/gcc13.cxx.pkgar" "$@.partial" pkgar extract --pkey $(PREFIX)/id_ed25519.pub.toml --archive "$(PREFIX)/libgcc.pkgar" "$@.partial" pkgar extract --pkey $(PREFIX)/id_ed25519.pub.toml --archive "$(PREFIX)/libstdcxx.pkgar" "$@.partial" + mv "$@.partial/usr"/* "$@.partial" + rmdir "$@.partial/usr" touch "$@.partial" mv "$@.partial" "$@" endif @@ -151,6 +165,8 @@ else mkdir -p "$@.partial" pkgar extract --pkey $(PREFIX)/id_ed25519.pub.toml --archive "$(PREFIX)/llvm21.pkgar" "$@.partial" pkgar extract --pkey $(PREFIX)/id_ed25519.pub.toml --archive "$(PREFIX)/rust.pkgar" "$@.partial" + mv "$@.partial/usr"/* "$@.partial" + rmdir "$@.partial/usr" touch "$@.partial" mv "$@.partial" "$@" endif @@ -164,6 +180,8 @@ else pkgar extract --pkey $(PREFIX)/id_ed25519.pub.toml --archive "$(PREFIX)/llvm21.runtime.pkgar" "$@.partial" pkgar extract --pkey $(PREFIX)/id_ed25519.pub.toml --archive "$(PREFIX)/clang21.pkgar" "$@.partial" pkgar extract --pkey $(PREFIX)/id_ed25519.pub.toml --archive "$(PREFIX)/lld21.pkgar" "$@.partial" + mv "$@.partial/usr"/* "$@.partial" + rmdir "$@.partial/usr" touch "$@.partial" mv "$@.partial" "$@" endif diff --git a/recipes/dev/gcc13/recipe.toml b/recipes/dev/gcc13/recipe.toml index d3fba2838..b6fd0aa36 100644 --- a/recipes/dev/gcc13/recipe.toml +++ b/recipes/dev/gcc13/recipe.toml @@ -40,7 +40,6 @@ export CFLAGS_FOR_TARGET="${CPPFLAGS}" CXXFLAGS_FOR_TARGET="${CPPFLAGS}" LDFLAGS export CC_FOR_BUILD="$CC_WRAPPER gcc" CXX_FOR_BUILD="$CC_WRAPPER g++" unset CFLAGS CPPFLAGS LDFLAGS COOKBOOK_CONFIGURE_FLAGS+=( - --with-sysroot=/ --with-gmp="${COOKBOOK_SYSROOT}/usr" --with-mpfr="${COOKBOOK_SYSROOT}/usr" --with-mpc="${COOKBOOK_SYSROOT}/usr" diff --git a/recipes/dev/rust/config.toml b/recipes/dev/rust/config.toml index b2356b81c..918d9fee2 100644 --- a/recipes/dev/rust/config.toml +++ b/recipes/dev/rust/config.toml @@ -25,7 +25,6 @@ cc = "COOKBOOK_GNU_TARGET-gcc" cxx = "COOKBOOK_GNU_TARGET-g++" ar = "COOKBOOK_GNU_TARGET-ar" linker = "COOKBOOK_GNU_TARGET-gcc" -rpath = false crt-static = false llvm-config = "COOKBOOK_SYSROOT/bin/llvm-config" diff --git a/recipes/libs/libgcc/recipe.toml b/recipes/libs/libgcc/recipe.toml index da72a5e9d..d626d3745 100644 --- a/recipes/libs/libgcc/recipe.toml +++ b/recipes/libs/libgcc/recipe.toml @@ -1,6 +1,6 @@ [build] template = "custom" script = """ -mkdir -p "${COOKBOOK_STAGE}/lib" -cp -av ${COOKBOOK_HOST_SYSROOT}/${GNU_TARGET}/lib/libgcc_s.so* ${COOKBOOK_STAGE}/lib/ +mkdir -p "${COOKBOOK_STAGE}/usr/lib" +cp -av ${COOKBOOK_HOST_SYSROOT}/${GNU_TARGET}/lib/libgcc_s.so* ${COOKBOOK_STAGE}/usr/lib/ """ diff --git a/recipes/libs/libstdcxx/recipe.toml b/recipes/libs/libstdcxx/recipe.toml index cd9664481..80e05b0f7 100644 --- a/recipes/libs/libstdcxx/recipe.toml +++ b/recipes/libs/libstdcxx/recipe.toml @@ -1,8 +1,9 @@ [build] template = "custom" script = """ +# Avoid replacing host libstdc++ when using "host:" if [ "$TARGET" != "$COOKBOOK_HOST_TARGET" ]; then -mkdir -p "${COOKBOOK_STAGE}/lib" -cp -av ${COOKBOOK_HOST_SYSROOT}/${GNU_TARGET}/lib/libstdc++.so* ${COOKBOOK_STAGE}/lib/ +mkdir -p "${COOKBOOK_STAGE}/usr/lib" +cp -av ${COOKBOOK_HOST_SYSROOT}/${GNU_TARGET}/lib/libstdc++.so* ${COOKBOOK_STAGE}/usr/lib/ fi """ diff --git a/recipes/other/cookbook/recipe.toml b/recipes/other/cookbook/recipe.toml index 17cfee67d..285a550c4 100644 --- a/recipes/other/cookbook/recipe.toml +++ b/recipes/other/cookbook/recipe.toml @@ -27,5 +27,9 @@ dependencies = [ "pkgar", "python312", "sed", - "wget", +# toolchains are dynamically linked, so this is needed + "libgmp", + "libmpfr", + "mpc", + "curl", ] diff --git a/src/cook/cook_build.rs b/src/cook/cook_build.rs index f00e55b66..c86474298 100644 --- a/src/cook/cook_build.rs +++ b/src/cook/cook_build.rs @@ -428,7 +428,6 @@ pub fn build( let local_redoxer = Path::new("target/release/cookbook_redoxer"); let mut command = if is_redox() && !local_redoxer.is_file() { let mut command = Command::new("cookbook_redoxer"); - command.arg(bash_args); command.env("COOKBOOK_REDOXER", "cookbook_redoxer"); command } else { @@ -436,10 +435,10 @@ pub fn build( .canonicalize() .unwrap_or(PathBuf::from("/bin/false")); let mut command = Command::new(&cookbook_redoxer); - command.arg("env").arg("bash").arg(bash_args); command.env("COOKBOOK_REDOXER", &cookbook_redoxer); command }; + command.arg("env").arg("bash").arg(bash_args); command.current_dir(&cookbook_build); command.env("TARGET", package_target(name)); command.env("COOKBOOK_BUILD", &cookbook_build); diff --git a/src/cook/fetch.rs b/src/cook/fetch.rs index 3bd6e3379..c06fd83e9 100644 --- a/src/cook/fetch.rs +++ b/src/cook/fetch.rs @@ -20,6 +20,7 @@ use crate::wrap_other_err; use pkg::SourceIdentifier; use pkg::net_backend::DownloadBackendWriter; use std::cell::RefCell; +use std::collections::BTreeMap; use std::fs; use std::fs::File; use std::io::Read; @@ -209,7 +210,19 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result .arg("--also-filter-submodules"); } command.arg(&source_dir_tmp); - run_command(command, logger)?; + if let Err(e) = run_command(command, logger) { + if !is_redox() { + return Err(e); + } + // TODO: RedoxFS has a race condition problem with `--recursive` and running in multi CPU. + // It is appear that running the submodule update separately fixes it. Remove this when + // `git clone https://gitlab.redox-os.org/redox-os/relibc --recursive` proven to work in Redox OS. + let mut cmds = vec!["update", "--init"]; + if shallow_clone { + cmds.push("--filter=tree:0"); + } + manual_git_recursive_submodule(logger, &source_dir_tmp, cmds)?; + } // Move source.tmp to source atomically rename(&source_dir_tmp, &source_dir)?; @@ -309,7 +322,12 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result command.arg("-C").arg(&source_dir); command.arg("submodule").arg("sync").arg("--recursive"); - run_command(command, logger)?; + if let Err(e) = run_command(command, logger) { + if !is_redox() { + return Err(e); + } + manual_git_recursive_submodule(logger, &source_dir, vec!["sync"])?; + } // Update submodules let mut command = Command::new("git"); @@ -322,7 +340,16 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result if shallow_clone { command.arg("--filter=tree:0"); } - run_command(command, logger)?; + if let Err(e) = run_command(command, logger) { + if !is_redox() { + return Err(e); + } + let mut cmds = vec!["update", "--init"]; + if shallow_clone { + cmds.push("--filter=tree:0"); + } + manual_git_recursive_submodule(logger, &source_dir, cmds)?; + } fetch_apply_patches(recipe_dir, patches, script, &source_dir, logger)?; } @@ -428,6 +455,74 @@ pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result Ok(result) } +fn manual_git_recursive_submodule( + logger: &PtyOut, + source_dir: &PathBuf, + cmd: Vec<&str>, +) -> Result<()> { + log_to_pty!( + logger, + "Git submodule {} failed, might be caused by race condition in RedoxFS, retrying without --recursive.", + cmd[0] + ); + + let mut repo_registry: BTreeMap = BTreeMap::new(); + + loop { + let mut dirty_git = false; + + let output = Command::new("find") + .args(&[".", "-name", ".git"]) + .current_dir(&source_dir) + .output() + .map_err(wrap_io_err!("Failed to execute find"))?; + + let stdout = String::from_utf8_lossy(&output.stdout); + + for line in stdout.lines() { + let git_path = PathBuf::from(line); + if let Some(repo_root) = git_path.parent() { + let repo_root_buf = repo_root.to_path_buf(); + + if !repo_registry.contains_key(&repo_root_buf) { + repo_registry.insert(repo_root_buf.clone(), false); + dirty_git = true; + } + } + } + + if !dirty_git { + // completed + return Ok(()); + } + + let pending_repos: Vec = repo_registry + .iter() + .filter(|&(_, &synced)| !synced) + .map(|(path, _)| path.clone()) + .collect(); + + if pending_repos.is_empty() { + bail_other_err!("No pending repos but dirty"); + } + + for repo in pending_repos { + println!("==> Processing: {:?}", repo); + + let mut command = Command::new("git"); + command.arg("-C").arg(&repo).current_dir(&source_dir); + command.arg("submodule"); + + for cmd in &cmd { + command.arg(cmd); + } + run_command(command, logger)?; + + repo_registry.insert(repo, true); + } + } +} + /// This does the same check as in cook_build fn fetch_will_build(recipe: &CookRecipe) -> bool { let check_source = !recipe.is_deps;