diff --git a/mk/config.mk b/mk/config.mk index 4a650dcd..d6553a0a 100644 --- a/mk/config.mk +++ b/mk/config.mk @@ -54,11 +54,13 @@ SCCACHE_BUILD?=$(shell [ -f /run/.containerenv ] && echo 1 || echo 0) CONTAINERFILE?=podman/redox-base-containerfile # Per host variables -export NPROC=nproc +NPROC=nproc +SED=sed ifneq ($(PODMAN_BUILD),1) FSTOOLS_IN_PODMAN=0 HOST_TARGET := $(shell env -u RUSTUP_TOOLCHAIN rustc -vV | grep host | cut -d: -f2 | tr -d " ") +HOST_GNU_TARGET := $(shell gcc -dumpmachine) # x86_64 linux hosts have all toolchains ifneq ($(HOST_TARGET),x86_64-unknown-linux-gnu) ifeq ($(ARCH),aarch64) @@ -101,7 +103,8 @@ endif UNAME := $(shell uname) ifeq ($(UNAME),Darwin) FUMOUNT=umount - export NPROC=sysctl -n hw.ncpu + NPROC=sysctl -n hw.ncpu + SED=gsed VB_AUDIO=coreaudio VBM=/Applications/VirtualBox.app/Contents/MacOS/VBoxManage else ifeq ($(UNAME),FreeBSD) diff --git a/mk/prefix.mk b/mk/prefix.mk index c9d085bd..ebe52f5b 100644 --- a/mk/prefix.mk +++ b/mk/prefix.mk @@ -4,11 +4,13 @@ PREFIX=prefix/$(TARGET) PREFIX_INSTALL=$(PREFIX)/sysroot/ PREFIX_PATH=$(ROOT)/$(PREFIX_INSTALL)/bin -RELIBC_SOURCE=recipes/core/relibc/source +BINUTILS_TARGET=recipes/dev/binutils-gdb/target/$(HOST_TARGET)/$(TARGET) +LIBTOOL_TARGET=recipes/dev/libtool/target/$(HOST_TARGET) +GCC_TARGET=recipes/dev/gcc13/target/$(HOST_TARGET)/$(TARGET) +LIBSTDCXX_TARGET=recipes/libs/libstdcxx-v3/target/$(TARGET)/$(HOST_TARGET) +RELIBC_FREESTANDING_TARGET=recipes/core/relibc/target/$(TARGET)/$(HOST_TARGET) +RELIBC_TARGET=recipes/core/relibc/target/$(TARGET) -BINUTILS_BRANCH=redox-2.43.1 -GCC_BRANCH=redox-13.2.0 -LIBTOOL_VERSION=2.5.4 # official RISC-V support introduced in newer version UPSTREAM_RUSTC_VERSION=2025-11-15 @@ -16,64 +18,27 @@ export PREFIX_RUSTFLAGS=-L $(ROOT)/$(PREFIX_INSTALL)/$(TARGET)/lib export RUSTUP_TOOLCHAIN=$(ROOT)/$(PREFIX_INSTALL) export REDOXER_TOOLCHAIN=$(RUSTUP_TOOLCHAIN) -export CC= -export CXX= - -ifeq ($(TARGET),riscv64gc-unknown-redox) - GCC_ARCH?=--with-arch=rv64gc --with-abi=lp64d -else - GCC_ARCH?= -endif - -# TODO(andypython): Upstream libtool patches to remove the need to locally build libtool. -# Cannot be CI built, i.e. be a part of relibc-install.tar.gz, as the prefix has to be correctly -# set while building. Otherwise aclocal will not be able to find libtool's files. Furthermore, doing -# so would break non-podman builds (not sure if they are still supported though). prefix: $(PREFIX)/sysroot # Update relibc used for compiling and clean all statically linked recipes prefix_clean: | $(FSTOOLS_TAG) - rm -rf $(PREFIX)/relibc $(PREFIX)/sysroot $(REPO_TAG) + rm -rf $(PREFIX)/relibc-install $(PREFIX)/sysroot $(REPO_TAG) $(MAKE) c.base,base-initfs,extrautils,kernel,ion,pkgutils,redoxfs,relibc -PREFIX_STRIP=\ - mkdir -p bin libexec "$(GCC_TARGET)/bin" && \ - find bin libexec "$(GCC_TARGET)/bin" "$(GCC_TARGET)/lib" \ - -type f \ - -exec strip --strip-unneeded {} ';' \ - 2> /dev/null - -$(RELIBC_SOURCE): | $(FSTOOLS_TAG) -ifeq ($(PODMAN_BUILD),1) - $(PODMAN_RUN) make $@ -else - ./target/release/repo fetch relibc - touch $(RELIBC_SOURCE) -endif - -$(PREFIX)/relibc: | $(RELIBC_SOURCE) - mkdir -p "$(@D)" - rm -rf "$@.partial" "$@" - cp -r "$(RELIBC_SOURCE)" "$@.partial" - touch "$@.partial" - mv "$@.partial" "$@" - -$(PREFIX)/relibc-install: $(PREFIX)/relibc | $(PREFIX)/rust-install $(CONTAINER_TAG) +$(PREFIX)/relibc-install: $(PREFIX)/rust-install | $(CONTAINER_TAG) ifeq ($(PODMAN_BUILD),1) $(PODMAN_RUN) make $@ else + @echo "\033[1;36;49mBuilding relibc-install\033[0m" rm -rf "$@.partial" "$@" cp -r "$(PREFIX)/rust-install" "$@.partial" rm -rf "$@.partial/$(TARGET)/include/"* cp -r "$(PREFIX)/rust-install/$(GNU_TARGET)/include/c++" "$@.partial/$(GNU_TARGET)/include/c++" cp -r "$(PREFIX)/rust-install/lib/rustlib/$(HOST_TARGET)/lib/" "$@.partial/lib/rustlib/$(HOST_TARGET)/" - cd "$<" && \ export PATH="$(ROOT)/$@.partial/bin:$$PATH" && \ - export CARGO="env -u CARGO cargo" && \ - $(MAKE) clean && \ - $(MAKE) -j `$(NPROC)` all && \ - $(MAKE) -j `$(NPROC)` install DESTDIR="$(ROOT)/$@.partial/$(GNU_TARGET)" - cd "$@.partial" && $(PREFIX_STRIP) + export CARGO="env -u CARGO cargo" CI=1 && \ + ./target/release/repo cook relibc + cp -r "$(RELIBC_TARGET)/stage/usr/". "$@.partial/$(GNU_TARGET)" touch "$@.partial" mv "$@.partial" "$@" endif @@ -86,60 +51,30 @@ $(PREFIX)/relibc-install.tar.gz: $(PREFIX)/relibc-install --directory="$<" \ . -$(PREFIX)/libtool: | $(CONTAINER_TAG) +# TODO: move this behind PREFIX_BINARY=0 when compiled prefix has it +$(PREFIX)/libtool-install: | $(FSTOOLS_TAG) $(CONTAINER_TAG) ifeq ($(PODMAN_BUILD),1) $(PODMAN_RUN) make $@ else + @echo "\033[1;36;49mBuilding libtool-install\033[0m" rm -rf "$@.partial" "$@" mkdir -p "$@.partial" - - git clone \ - --recurse-submodules \ - --shallow-submodules \ - "https://gitlab.redox-os.org/redox-os/libtool/" \ - --branch "v$(LIBTOOL_VERSION)-redox" \ - --depth 2 \ - "$@.partial" - - touch "$@.partial" - echo $(LIBTOOL_VERSION) > $@.partial/.tarball-version - mv "$@.partial" "$@" -endif - -$(PREFIX)/libtool-build: $(PREFIX)/libtool $(PREFIX)/rust-install $(CONTAINER_TAG) -ifeq ($(PODMAN_BUILD),1) - $(PODMAN_RUN) make $@ -else - rm -rf "$@.partial" "$@" - mkdir -p "$@.partial" - PATH="$(ROOT)/$(PREFIX)/rust-install/bin:$$PATH" && \ - cd "$<" && \ - ./bootstrap \ - --skip-po \ - --force \ - --gnulib-srcdir=./gnulib - PATH="$(ROOT)/$(PREFIX)/rust-install/bin:$$PATH" && \ - cd "$@.partial" && \ - cp -r $(abspath $<)/. ./ && \ - "$(ROOT)/$ anyhow::Result<()> { fs::create_dir_all(repo_path)?; } + // Don't publish host packages + let target_packages = &config + .recipe_list + .iter() + .map(PackageName::new) + .filter(|pkg| pkg.as_ref().is_ok_and(|p| !p.is_host())) + .collect::, _>>()?; + + if target_packages.len() == 0 { + return Ok(()); + } + + // TODO: publish cross target builds? + if std::env::var("COOKBOOK_CROSS_TARGET").is_ok_and(|x| !x.is_empty()) { + return Ok(()); + } + // Runtime dependencies include both `[package.dependencies]` and dynamically // linked packages discovered by auto_deps. // // The following adds the package dependencies of the recipes to the repo as // well. - let (recipe_list, recipe_map) = Package::new_recursive_nonstop( - &config - .recipe_list - .iter() - .map(PackageName::new) - // Don't publish host packages - .filter(|pkg| pkg.as_ref().is_ok_and(|p| !p.is_host())) - .collect::, _>>()?, - WALK_DEPTH, - ); + let (recipe_list, recipe_map) = Package::new_recursive_nonstop(target_packages, WALK_DEPTH); if recipe_list.len() == 0 { // Fail-Safe diff --git a/src/config.rs b/src/config.rs index c8836c6d..cdcc6bef 100644 --- a/src/config.rs +++ b/src/config.rs @@ -19,6 +19,8 @@ pub struct CookConfigOpt { /// whether to print verbose logs to certain commands /// build failure still be printed anyway pub verbose: Option, + /// whether to always clean the build directory + pub clean_build: Option, } #[derive(Debug, Default, Clone, Deserialize, PartialEq, Serialize)] @@ -29,6 +31,7 @@ pub struct CookConfig { pub logs: bool, pub nonstop: bool, pub verbose: bool, + pub clean_build: bool, } impl From for CookConfig { @@ -40,6 +43,7 @@ impl From for CookConfig { logs: value.logs.unwrap(), nonstop: value.nonstop.unwrap(), verbose: value.verbose.unwrap(), + clean_build: value.clean_build.unwrap(), } } } @@ -91,6 +95,9 @@ pub fn init_config() { if config.cook_opt.nonstop.is_none() { config.cook_opt.nonstop = Some(extract_env("COOKBOOK_NONSTOP", false)); } + if config.cook_opt.clean_build.is_none() { + config.cook_opt.clean_build = Some(extract_env("COOKBOOK_CLEAN_BUILD", false)); + } if config.mirrors.len() == 0 { // The GNU FTP mirror below is automatically inserted for convenience // You can choose other mirrors by setting it on cookbook.toml diff --git a/src/cook/cook_build.rs b/src/cook/cook_build.rs index 3b7b9255..1b88de0a 100644 --- a/src/cook/cook_build.rs +++ b/src/cook/cook_build.rs @@ -174,6 +174,7 @@ pub fn build( recipe: &Recipe, offline_mode: bool, check_source: bool, + clean_build: bool, logger: &PtyOut, ) -> Result<(Vec, BTreeSet), String> { let sysroot_dir = target_dir.join("sysroot"); @@ -276,9 +277,8 @@ pub fn build( create_dir_clean(&stage_dir_tmp)?; // Create build, if it does not exist - //TODO: flag for clean builds where build is wiped out - let build_dir = target_dir.join("build"); - if !build_dir.is_dir() { + let build_dir = get_build_dir(target_dir); + if clean_build || !build_dir.is_dir() { create_dir_clean(&build_dir)?; } @@ -428,15 +428,33 @@ pub fn remove_stage_dir(stage_dir: &PathBuf) -> Result<(), String> { } pub fn get_stage_dirs(features: &Vec, target_dir: &Path) -> Vec { + let mut target_dir = target_dir.to_path_buf(); + if let Some(cross_target) = std::env::var("COOKBOOK_CROSS_TARGET").ok() { + if cross_target != "" { + // TODO: automatically pass COOKBOOK_CROSS_GNU_TARGET? + target_dir = target_dir.join(cross_target) + } + } let mut v = Vec::new(); for f in features { v.push(target_dir.join(format!("stage.{}", f.name))); } // intentionally added last as it contains leftover files from package features - v.push(target_dir.join(format!("stage"))); + v.push(target_dir.join("stage")); v } +pub fn get_build_dir(target_dir: &Path) -> PathBuf { + let mut target_dir = target_dir.to_path_buf(); + if let Some(cross_target) = std::env::var("COOKBOOK_CROSS_TARGET").ok() { + if cross_target != "" { + // TODO: automatically pass COOKBOOK_CROSS_GNU_TARGET? + target_dir = target_dir.join(cross_target) + } + } + target_dir.join("build") +} + fn build_deps_dir( logger: &PtyOut, deps_dir: &PathBuf, diff --git a/src/cook/fs.rs b/src/cook/fs.rs index b32558bf..a75c8bda 100644 --- a/src/cook/fs.rs +++ b/src/cook/fs.rs @@ -33,7 +33,8 @@ pub fn create_dir_clean(dir: &Path) -> Result<(), String> { if dir.is_dir() { remove_all(dir)?; } - create_dir(dir) + fs::create_dir_all(dir) + .map_err(|err| format!("failed to create '{}': {}\n{:?}", dir.display(), err, err)) } pub fn create_target_dir(recipe_dir: &Path, target: &'static str) -> Result { @@ -292,9 +293,14 @@ pub fn get_git_head_rev(dir: &PathBuf) -> Result<(String, bool), String> { let head_str = fs::read_to_string(&git_head) .map_err(|e| format!("unable to read {path}: {e}", path = git_head.display()))?; if head_str.starts_with("ref: ") { - let git_ref = dir.join(".git").join(head_str["ref: ".len()..].trim_end()); - let ref_str = fs::read_to_string(&git_ref) - .map_err(|e| format!("unable to read {path}: {e}", path = git_ref.display()))?; + let entry = head_str["ref: ".len()..].trim_end(); + let git_ref = dir.join(".git").join(entry); + let ref_str = if git_ref.is_file() { + fs::read_to_string(&git_ref) + .map_err(|e| format!("unable to read {path}: {e}", path = git_ref.display()))? + } else { + get_git_ref_entry(dir, entry)? + }; Ok((ref_str.trim().to_string(), false)) } else { Ok((head_str.trim().to_string(), true)) @@ -306,12 +312,14 @@ pub fn get_git_tag_rev(dir: &PathBuf, tag: &str) -> Result { if tag.len() == 40 && tag.chars().all(|f| f.is_ascii_hexdigit()) { return Ok(tag.to_string()); } + get_git_ref_entry(dir, &format!("refs/tags/{tag}")) +} +pub fn get_git_ref_entry(dir: &PathBuf, entry: &str) -> Result { let git_refs = dir.join(".git/packed-refs"); let refs_str = fs::read_to_string(&git_refs) .map_err(|e| format!("unable to read {path}: {e}", path = git_refs.display()))?; - let expected_comment_part = format!("refs/tags/{tag}"); for line in refs_str.lines() { - if line.contains(&expected_comment_part) { + if line.contains(entry) { let sha = line .split_whitespace() .next() @@ -321,10 +329,7 @@ pub fn get_git_tag_rev(dir: &PathBuf, tag: &str) -> Result { } } - Err(format!( - "Could not find a rev tag for {}", - expected_comment_part - )) + Err(format!("Could not find a rev for {}", entry)) } /// get commit rev after fetch diff --git a/src/cook/package.rs b/src/cook/package.rs index 3a635377..81aebdf9 100644 --- a/src/cook/package.rs +++ b/src/cook/package.rs @@ -194,7 +194,14 @@ pub fn package_stage_paths( package: Option<&OptionalPackageRecipe>, target_dir: &Path, ) -> (PathBuf, PathBuf, PathBuf) { - package_name_paths(package, target_dir, "stage") + let mut target_dir = target_dir.to_path_buf(); + if let Some(cross_target) = std::env::var("COOKBOOK_CROSS_TARGET").ok() { + if cross_target != "" { + // TODO: automatically pass COOKBOOK_CROSS_GNU_TARGET? + target_dir = target_dir.join(cross_target) + } + } + package_name_paths(package, &target_dir, "stage") } pub fn package_source_paths(