Merge branch 'servo' into 'master'

servo

See merge request redox-os/redox!1641

.gitlab-ci.yml

@@ -60,9 +60,4 @@ img:
     - |
       export PATH="$HOME/.cargo/bin:$PATH" &&
       bash podman/rustinstall.sh &&
-      PODMAN_BUILD=0 REPO_BINARY=1 COOKBOOK_VERBOSE=false make ci-img IMG_TAG=$CI_COMMIT_REF_NAME &&
-      ([ $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" ] && rm -rf build/img/* || true)
-  artifacts:
-    paths:
-      - build/img/
-    expire_in: 1 week
+      PODMAN_BUILD=0 REPO_BINARY=1 FSTOOLS_NO_MOUNT=1 COOKBOOK_VERBOSE=false make ci-img IMG_TAG=$CI_COMMIT_REF_NAME

Cargo.lock

@@ -177,9 +177,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.47"
+version = "1.2.49"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd405d82c84ff7f35739f175f67d8b9fb7687a0e84ccdc78bd3568839827cf07"
+checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215"
 dependencies = [
  "find-msvc-tools",
  "shlex",
@@ -648,9 +648,9 @@ dependencies = [
 
 [[package]]
 name = "instability"
-version = "0.3.9"
+version = "0.3.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "435d80800b936787d62688c927b6490e887c7ef5ff9ce922c6c6050fca75eb9a"
+checksum = "6778b0196eefee7df739db78758e5cf9b37412268bfa5650bfeed028aed20d9c"
 dependencies = [
  "darling",
  "indoc",
@@ -682,9 +682,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
 
 [[package]]
 name = "libc"
-version = "0.2.177"
+version = "0.2.178"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
+checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091"
 
 [[package]]
 name = "libredox"
@@ -699,9 +699,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.28"
+version = "0.4.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432"
+checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
 
 [[package]]
 name = "lru"
@@ -912,7 +912,7 @@ dependencies = [
 [[package]]
 name = "redox-pkg"
 version = "0.2.9"
-source = "git+https://gitlab.redox-os.org/redox-os/pkgutils.git#78b50760a3ee6c6752d47ec6f3511cd28b5d70de"
+source = "git+https://gitlab.redox-os.org/redox-os/pkgutils.git#cbf88111fab76b8bfd688f144f7cc000408b180e"
 dependencies = [
  "anyhow",
  "ignore",
@@ -953,7 +953,7 @@ dependencies = [
 [[package]]
 name = "redox_installer"
 version = "0.2.37"
-source = "git+https://gitlab.redox-os.org/redox-os/installer.git#4fcd97e8daf1cb5c91ba7d2e52388670475aefde"
+source = "git+https://gitlab.redox-os.org/redox-os/installer.git#990d9d343eefbcdc4cecc5f204164b69cc70de41"
 dependencies = [
  "anyhow",
  "libc",
@@ -986,7 +986,7 @@ dependencies = [
 [[package]]
 name = "redoxer"
 version = "0.2.61"
-source = "git+https://gitlab.redox-os.org/redox-os/redoxer.git#a4c4dac4550bc234aa04e30efd4058daf8377678"
+source = "git+https://gitlab.redox-os.org/redox-os/redoxer.git#676bda4514e1e74b006816eca7b461a0721feea1"
 dependencies = [
  "anyhow",
  "dirs",
@@ -1133,9 +1133,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
 
 [[package]]
 name = "simd-adler32"
-version = "0.3.7"
+version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
+checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
 
 [[package]]
 name = "simdutf8"
@@ -1206,9 +1206,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
 
 [[package]]
 name = "syn"
-version = "2.0.110"
+version = "2.0.111"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea"
+checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1523,9 +1523,9 @@ checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
 
 [[package]]
 name = "winnow"
-version = "0.7.13"
+version = "0.7.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf"
+checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829"
 dependencies = [
  "memchr",
 ]

Makefile

@@ -42,7 +42,7 @@ else
 endif # CONTAINER_TAG
 else
 ifneq ($(NOT_ON_PODMAN),1)
-	$(MAKE) c.--all
+	$(MAKE) repo_clean
 	-$(FUMOUNT) $(BUILD)/filesystem/ || true
 	-$(FUMOUNT) /tmp/redox_installer/ || true
 endif # NOT_ON_PODMAN
@@ -61,7 +61,7 @@ else
 endif # CONTAINER_TAG
 else
 ifneq ($(NOT_ON_PODMAN),1)
-	$(MAKE) u.--all
+	$(MAKE) fetch_clean
 endif # NOT_ON_PODMAN
 	$(MAKE) clean NOT_ON_PODMAN=1
 endif # PODMAN_BUILD
@@ -72,6 +72,10 @@ pull:
 
 repo: $(BUILD)/repo.tag
 
+repo_clean: c.--all
+
+fetch_clean: u.--all
+
 # Podman build recipes and vars
 include mk/podman.mk
 

config/aarch64/ci.toml

@@ -7,6 +7,7 @@ prompt = false
 
 # Package settings
 [packages]
+acid = {}
 base = {}
 base-initfs = {}
 bash = {}
@@ -33,6 +34,7 @@ gettext = {}
 git = {}
 # gdbserver = {} # wrong libc type
 # gnu-binutils = {} # bfd doesn't recognize redox
+gnu-make = {}
 hicolor-icon-theme = {}
 installer = {}
 installer-gui = {}

config/i586/ci.toml

@@ -7,6 +7,7 @@ prompt = false
 
 # Package settings
 [packages]
+acid = {}
 base = {}
 base-initfs = {}
 bash = {}
@@ -31,6 +32,7 @@ freefont = {}
 freetype2 = {}
 gettext = {}
 git = {}
+gnu-make = {}
 hicolor-icon-theme = {}
 installer = {}
 installer-gui = {}

config/redoxer.toml

@@ -9,10 +9,12 @@ ca-certificates = {}
 coreutils = {}
 extrautils = {}
 findutils = {}
+gnu-make = {}
 ion = {}
 netdb = {}
 pkgutils = {}
 relibc = {}
+sed = {}
 
 # Override to not background dhcpd
 [[files]]

config/riscv64gc/ci.toml

@@ -7,6 +7,7 @@ prompt = false
 
 # Package settings
 [packages]
+acid = {}
 base = {}
 base-initfs = {}
 bash = {}
@@ -31,6 +32,7 @@ freefont = {}
 freetype2 = {}
 gettext = {}
 git = {}
+gnu-make = {}
 hicolor-icon-theme = {}
 installer = {}
 #installer-gui = {} # redox_syscall 0.4 not working on riscv64gc?

config/x86_64/ci.toml

@@ -13,8 +13,8 @@ prompt = false
 dev-essential = {}
 dev-redox = {}
 redox-tests = {}
-#x11-minimal = { # needs all dependencies added below
-#x11-full = {} # needs all dependencies added below
+x11-minimal = {}
+x11-full = {}
 
 # Normal packages below
 
@@ -72,7 +72,7 @@ fribidi = {}
 #gawk = {} # langinfo.h
 gcc13 = {}
 gdbserver = {}
-#gdk-pixbuf = {} # shared-mime-info
+gdk-pixbuf = {}
 gears = {}
 generaluser-gs = {}
 gettext = {}
@@ -168,7 +168,7 @@ pcre = {}
 perg = {}
 periodictable = {}
 php84 = {}
-#perl5 = {} # ctermid, tempnam, ttyname
+perl5 = {}
 pixelcannon = {}
 #pixman = {} # depends on glib which does not build
 #pkgar = {} # uses virtual Cargo.toml, needs recipe update
@@ -213,7 +213,7 @@ sdl2-image = {}
 sdl2-mixer = {}
 sdl2-ttf = {}
 sed = {}
-#servo = {} # some more crates to port
+servo = {}
 shared-mime-info = {}
 shellharden = {}
 shellstorm = {}
@@ -223,7 +223,7 @@ sodium = {}
 sopwith = {}
 sqlite3 = {}
 strace = {}
-#syobonaction = {} # linking errors
+syobonaction = {}
 terminfo = {}
 #termplay = {} # backtrace cannot find link.h
 timidity = {}
@@ -235,6 +235,7 @@ uutils = {}
 vim = {}
 vttest = {}
 #vvvvvv = {} # did not compile
+webkitgtk3 = {}
 #webrender = {} # unwind
 #wesnoth = {}
 winit = {}

config/x86_64/servo.toml

@@ -0,0 +1,54 @@
+# Default build system configuration
+
+include = ["../x11.toml"]
+
+# Override the default settings here
+
+# General settings
+[general]
+# Filesystem size in MiB
+filesystem_size = 15000
+
+# Package settings
+[packages]
+# example = {}
+servo = {}
+libxcursor = {}
+
+[[files]]
+path = "/usr/bin/orbital-x11"
+data = """
+#!/usr/bin/env bash
+
+set -ex
+
+export DISPLAY=:0
+echo "Starting X server on display ${DISPLAY}..."
+X "${DISPLAY}" -verbose 6 &
+X_PID=$!
+
+# Wait for X to be ready
+sleep 3
+
+# Check if X is running
+if ! kill -0 $X_PID 2>/dev/null; then
+    echo "X server failed to start!"
+    exit 1
+fi
+
+echo "Starting window manager..."
+twm &
+
+echo "Starting xterm..."
+xterm &
+#also, half of debug options in debug help do not exist any more.
+
+# Try to force X11 backend for Servo
+echo "Starting servo with X11 backend..."
+export WINIT_UNIX_BACKEND=x11
+export LD_DEBUG=all
+export RUST_LOG=trace
+export RUST_BACKTRACE=full
+export WEBRENDER_DEBUG=1
+DISPLAY=:0 /usr/servo/servo --debug disable-share-style-cache,dump-stacking-context-tree,dump-flow-tree,dump-rule-tree,dump-style-tree,gc-profile,profile-script-events,relayout-event,trace-layout,wr-stats &
+"""

mk/config.mk

@@ -11,6 +11,9 @@ ARCH?=$(HOST_ARCH)
 BOARD?=
 ## Enable to use binary prefix (much faster)
 PREFIX_BINARY?=1
+## Enable to use up-to-date rust compiler (experimental, only available to Tier 2 targets)
+## Even more experimental, add -Zbuild-std to cookbook.toml to allow compilation to Tier 3 targets
+PREFIX_USE_UPSTREAM_RUST_COMPILER?=0
 ## Enable to use binary packages (much faster)
 REPO_BINARY?=0
 ## Name of the configuration to include in the image name e.g. desktop or server
@@ -42,6 +45,8 @@ REDOXFS_MKFS_FLAGS?=
 PODMAN_BUILD?=1
 ## Set to 1 to put filesystem tools inside podman, any other value will install it to host
 FSTOOLS_IN_PODMAN?=0
+## Set to 1 if FUSE is not available and we are running in a container
+FSTOOLS_NO_MOUNT?=0
 ## Enable sccache to speed up cargo builds
 ## only do this by default if this is inside podman
 SCCACHE_BUILD?=$(shell [ -f /run/.containerenv ] && echo 1 || echo 0)
@@ -50,7 +55,6 @@ CONTAINERFILE?=podman/redox-base-containerfile
 
 # Per host variables
 export NPROC=nproc
-export REDOX_MAKE=make
 
 ifneq ($(PODMAN_BUILD),1)
 FSTOOLS_IN_PODMAN=0
@@ -102,7 +106,6 @@ ifeq ($(UNAME),Darwin)
 	VBM=/Applications/VirtualBox.app/Contents/MacOS/VBoxManage
 else ifeq ($(UNAME),FreeBSD)
 	FUMOUNT=sudo umount
-	export REDOX_MAKE=gmake
 	VB_AUDIO=pulse # To check, will probably be OSS on most setups
 	VBM=VBoxManage
 else
@@ -120,6 +123,7 @@ endif
 # Automatic variables
 ROOT=$(CURDIR)
 export RUST_COMPILER_RT_ROOT=$(ROOT)/rust/src/llvm-project/compiler-rt
+export TESTBIN?=
 RUNNING_IN_PODMAN=$(shell [ -f /run/.containerenv ] && echo 1 || echo 0)
 ifeq ($(PODMAN_BUILD),1)
 ifeq ($(RUNNING_IN_PODMAN),1)
@@ -154,36 +158,19 @@ ifeq ($(REPO_BINARY),1)
 INSTALLER_OPTS+=--repo-binary
 COOKBOOK_OPTS+=--repo-binary
 endif
+ifeq ($(FSTOOLS_NO_MOUNT),1)
+INSTALLER_OPTS+=--no-mount
+endif
 
 REPO_TAG=$(BUILD)/repo.tag
 FSTOOLS_TAG=build/fstools.tag
 export BOARD
 
-## Cross compiler variables
-AR=$(GNU_TARGET)-gcc-ar
-AS=$(GNU_TARGET)-as
-CC=$(GNU_TARGET)-gcc
-CXX=$(GNU_TARGET)-g++
-LD=$(GNU_TARGET)-ld
-NM=$(GNU_TARGET)-gcc-nm
-OBJCOPY=$(GNU_TARGET)-objcopy
-OBJDUMP=$(GNU_TARGET)-objdump
-RANLIB=$(GNU_TARGET)-gcc-ranlib
-READELF=$(GNU_TARGET)-readelf
-STRIP=$(GNU_TARGET)-strip
-
 ifeq ($(SCCACHE_BUILD),1)
 	export CC_WRAPPER:=sccache
 	export RUSTC_WRAPPER:=$(CC_WRAPPER)
-	CC=$(CC_WRAPPER) $(GNU_TARGET)-gcc
-	CXX=$(CC_WRAPPER) $(GNU_TARGET)-g++
 endif
 
-## Rust cross compile variables
-export AR_$(subst -,_,$(TARGET)):=$(AR)
-export CC_$(subst -,_,$(TARGET)):=$(CC)
-export CXX_$(subst -,_,$(TARGET)):=$(CXX)
-
 ## If Podman is being used, a container is required
 ifeq ($(PODMAN_BUILD),1)
 CONTAINER_TAG=build/container.tag

mk/podman.mk

@@ -21,7 +21,7 @@ PODMAN_VOLUMES?=--volume $(ROOT):$(CONTAINER_WORKDIR)$(PODMAN_VOLUME_FLAG) --vol
 PODMAN_ENV?=--env PATH=/root/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin --env PODMAN_BUILD=0
 PODMAN_CONFIG?=--env ARCH=$(ARCH) --env BOARD=$(BOARD) --env CONFIG_NAME=$(CONFIG_NAME) --env FILESYSTEM_CONFIG=$(FILESYSTEM_CONFIG) --env PREFIX_BINARY=$(PREFIX_BINARY) \
                --env CI=$(CI) --env COOKBOOK_MAKE_JOBS=$(COOKBOOK_MAKE_JOBS) --env COOKBOOK_LOGS=$(COOKBOOK_LOGS) --env COOKBOOK_VERBOSE=$(COOKBOOK_VERBOSE) \
-               --env REPO_APPSTREAM=$(REPO_APPSTREAM) --env REPO_BINARY=$(REPO_BINARY) --env REPO_NONSTOP=$(REPO_NONSTOP) --env REPO_OFFLINE=$(REPO_OFFLINE)
+               --env REPO_APPSTREAM=$(REPO_APPSTREAM) --env REPO_BINARY=$(REPO_BINARY) --env REPO_NONSTOP=$(REPO_NONSTOP) --env REPO_OFFLINE=$(REPO_OFFLINE) --env TESTBIN=$(TESTBIN)
 PODMAN_OPTIONS?=--rm --workdir $(CONTAINER_WORKDIR) --interactive --tty --cap-add SYS_ADMIN --device /dev/fuse --network=host --env TERM=$(TERM)
 PODMAN_RUN?=podman run $(PODMAN_OPTIONS) $(PODMAN_VOLUMES) $(PODMAN_ENV) $(PODMAN_CONFIG) $(IMAGE_TAG)
 

mk/prefix.mk

@@ -9,6 +9,7 @@ RELIBC_SOURCE=recipes/core/relibc/source
 BINUTILS_BRANCH=redox-2.43.1
 GCC_BRANCH=redox-13.2.0
 LIBTOOL_VERSION=2.5.4
+UPSTREAM_RUSTC_VERSION="2025-10-03"
 
 export PREFIX_RUSTFLAGS=-L $(ROOT)/$(PREFIX_INSTALL)/$(TARGET)/lib
 export RUSTUP_TOOLCHAIN=$(ROOT)/$(PREFIX_INSTALL)
@@ -29,6 +30,11 @@ endif
 # so would break non-podman builds (not sure if they are still supported though).
 prefix: $(PREFIX)/sysroot
 
+# Update relibc used for compiling and clean all statically linked recipes
+prefix_clean: | $(FSTOOLS_TAG)
+	rm -rf $(PREFIX)/relibc $(PREFIX)/sysroot $(REPO_TAG)
+	$(MAKE) c.base,base-initfs,extrautils,kernel,ion,pkgutils,redoxfs,relibc
+
 PREFIX_STRIP=\
 	mkdir -p bin libexec "$(GCC_TARGET)/bin" && \
 	find bin libexec "$(GCC_TARGET)/bin" "$(GCC_TARGET)/lib" \
@@ -136,6 +142,7 @@ else
 	touch "$@"
 endif
 
+# PREFIX_BINARY ---------------------------------------------------
 ifeq ($(PREFIX_BINARY),1)
 
 $(PREFIX)/rust-install.tar.gz: | $(CONTAINER_TAG)
@@ -161,10 +168,7 @@ endif
 
 else
 
-$(ROOT)/rust/configure:
-	git submodule sync --recursive
-	git submodule update --progress --init --recursive --checkout rust
-
+# BUILD GCC ---------------------------------------------------
 PREFIX_FREESTANDING_INSTALL=$(PREFIX)/gcc-freestanding-install
 PREFIX_FREESTANDING_PATH=$(ROOT)/$(PREFIX_FREESTANDING_INSTALL)/bin
 
@@ -321,6 +325,59 @@ $(PREFIX)/gcc-install.tar.gz: $(PREFIX)/gcc-install
 		--directory="$<" \
 		.
 
+# RUST FROM UPSTREAM COMPILER ---------------------------------------------------
+ifeq ($(PREFIX_USE_UPSTREAM_RUST_COMPILER),1)
+
+$(PREFIX)/rustc-install.tar.xz:
+	mkdir -p "$(@D)"
+	wget -O $@.partial "https://static.rust-lang.org/dist/$(UPSTREAM_RUSTC_VERSION)/rustc-nightly-$(HOST_TARGET).tar.xz"
+	mv $@.partial $@
+
+$(PREFIX)/cargo-install.tar.xz:
+	mkdir -p "$(@D)"
+	wget -O $@.partial "https://static.rust-lang.org/dist/$(UPSTREAM_RUSTC_VERSION)/cargo-nightly-$(HOST_TARGET).tar.xz"
+	mv $@.partial $@
+
+$(PREFIX)/rust-std-host-install.tar.xz:
+	mkdir -p "$(@D)"
+	wget -O $@.partial "https://static.rust-lang.org/dist/$(UPSTREAM_RUSTC_VERSION)/rust-std-nightly-$(HOST_TARGET).tar.xz"
+	mv $@.partial $@
+
+$(PREFIX)/rust-std-target-install.tar.xz:
+	mkdir -p "$(@D)"
+ifeq ($(TARGET),x86_64-unknown-redox)
+	wget -O $@.partial "https://static.rust-lang.org/dist/$(UPSTREAM_RUSTC_VERSION)/rust-std-nightly-$(TARGET).tar.xz"
+	mv $@.partial $@
+else
+	touch $@
+endif
+
+$(PREFIX)/rust-src-install.tar.xz:
+	mkdir -p "$(@D)"
+	wget -O $@.partial "https://static.rust-lang.org/dist/$(UPSTREAM_RUSTC_VERSION)/rust-src-nightly.tar.xz"
+	mv $@.partial $@
+
+$(PREFIX)/rust-install: $(PREFIX)/gcc-install $(PREFIX)/rustc-install.tar.xz $(PREFIX)/cargo-install.tar.xz $(PREFIX)/rust-std-host-install.tar.xz $(PREFIX)/rust-std-target-install.tar.xz $(PREFIX)/rust-src-install.tar.xz
+	rm -rf "$@.partial" "$@"
+	mkdir -p "$@.partial"
+	cp -r "$(PREFIX)/gcc-install/". "$@.partial"
+	tar --extract --file "$(PREFIX)/rustc-install.tar.xz" -C "$@.partial" rustc-nightly-$(HOST_TARGET)/rustc/ --strip-components=2
+	tar --extract --file "$(PREFIX)/cargo-install.tar.xz" --directory "$@.partial" cargo-nightly-$(HOST_TARGET)/cargo/ --strip-components=2
+	tar --extract --file "$(PREFIX)/rust-std-host-install.tar.xz" --directory "$@.partial" rust-std-nightly-$(HOST_TARGET)/rust-std-$(HOST_TARGET)/ --strip-components=2
+	tar --extract --file "$(PREFIX)/rust-src-install.tar.xz" --directory "$@.partial" rust-src-nightly/rust-src/ --strip-components=2
+ifeq ($(TARGET),x86_64-unknown-redox)
+	tar --extract --file "$(PREFIX)/rust-std-target-install.tar.xz" --directory "$@.partial" rust-std-nightly-$(TARGET)/rust-std-$(TARGET)/ --strip-components=2
+endif
+	touch "$@.partial"
+	mv "$@.partial" "$@"
+
+# BUILD RUST ---------------------------------------------------
+else 
+
+$(ROOT)/rust/configure:
+	git submodule sync --recursive
+	git submodule update --progress --init --recursive --checkout rust
+
 $(PREFIX)/rust-install: $(ROOT)/rust/configure | $(PREFIX)/gcc-install $(PREFIX)/relibc-freestanding-install $(CONTAINER_TAG)
 ifeq ($(PODMAN_BUILD),1)
 	$(PODMAN_RUN) make $@
@@ -354,6 +411,8 @@ else
 	mv "$@.partial" "$@"
 endif
 
+endif
+
 $(PREFIX)/rust-install.tar.gz: $(PREFIX)/rust-install
 	tar \
 		--create \

mk/qemu.mk

@@ -3,6 +3,7 @@
 QEMU=SDL_VIDEO_X11_DGAMOUSE=0 qemu-system-$(QEMU_ARCH)
 QEMUFLAGS=-d guest_errors -name "Redox OS $(ARCH)"
 netboot?=no
+redoxer?=no
 VGA_SUPPORTED=no
 
 ifeq ($(ARCH),i586)
@@ -162,6 +163,17 @@ else
 	QEMUFLAGS+=-serial chardev:debug -mon chardev=debug
 endif
 
+# redoxer exit code: 51 => success, 53 => failure
+ifeq ($(redoxer),yes)
+ifeq ($(ARCH),x86_64)
+	QEMUFLAGS+=-device isa-debug-exit
+else ifeq ($(ARCH),i586)
+	QEMUFLAGS+=-device isa-debug-exit
+else ifeq ($(ARCH),aarch64)
+	QEMUFLAGS+=-semihosting-config enable=on,target=native,userspace=on
+endif
+endif
+
 ifeq ($(iommu),yes)
 	QEMUFLAGS+=-machine $(QEMU_MACHINE),iommu=on
 else

podman/redox-base-containerfile

@@ -31,6 +31,7 @@ RUN apt-get update \
         gtk-doc-tools \
         gtk-update-icon-cache \
         help2man \
+        ipxe-qemu \
         intltool \
         itstool \
         libaudiofile-dev \
@@ -63,6 +64,9 @@ RUN apt-get update \
         pkg-config \
         po4a \
         protobuf-compiler \
+        qemu-system-x86 \
+        qemu-system-arm \
+        qemu-efi-aarch64 \
         python3 \
         python3-dev \
         python3-mako \

recipes/core/base/recipe.toml

@@ -26,6 +26,7 @@ BINS=(
     alxd
     e1000d
     ihdad
+    ihdgd
     ixgbed
     pcid
     pcid-spawner

recipes/core/redoxerd/recipe.toml

@@ -1,9 +0,0 @@
-[source]
-git = "https://gitlab.redox-os.org/redox-os/redoxer.git"
-
-[build]
-template = "custom"
-script = """
-COOKBOOK_SOURCE="${COOKBOOK_SOURCE}/daemon"
-cookbook_cargo
-"""

recipes/dev/autoconf/recipe.toml

@@ -4,3 +4,6 @@ blake3 = "da1cc8af8551c343de9f42af0ae53fd7dff3623487157623892b6cd7e3bb5692"
 
 [build]
 template = "configure"
+
+[package]
+dependencies = ["perl5"]

recipes/dev/automake/recipe.toml

@@ -4,3 +4,6 @@ blake3 = "f42cfc333aaaa11f2bcb05b5b0273b8706c820c22f9ba4367f7eb920551695cd"
 
 [build]
 template = "configure"
+
+[package]
+dependencies = ["perl5"]

recipes/dev/gnu-make/recipe.toml

@@ -10,9 +10,4 @@ autotools_recursive_regenerate
 """
 
 [build]
-template = "custom"
-script = """
-DYNAMIC_INIT
-rsync -av --delete "$COOKBOOK_SOURCE/." ./
-cookbook_configure
-"""
+template = "configure"

recipes/dev/luajit/recipe.toml

@@ -7,10 +7,16 @@ template = "custom"
 script = """
 DYNAMIC_INIT
 
-rsync -av --delete "${COOKBOOK_SOURCE}/" ./
+rsync -av "${COOKBOOK_SOURCE}/" ./
+OS=$(echo "${TARGET}" | cut -d - -f3)
+
+case "${OS}" in
+    linux)  SYS=Linux;;
+    redox)  SYS=Redox;;
+esac
 
 ${COOKBOOK_MAKE} -j ${COOKBOOK_MAKE_JOBS} install \
   PREFIX="${COOKBOOK_STAGE}" \
-  TARGET_SYS='Redox' \
-  CROSS="${TARGET}-"
+  TARGET_SYS="${SYS}" \
+  CROSS="${GNU_TARGET}-"
 """

recipes/dev/redoxer/recipe.toml

@@ -0,0 +1,6 @@
+[source]
+git = "https://gitlab.redox-os.org/redox-os/redoxer"
+branch = "master"
+
+[build]
+template = "cargo"

recipes/tests/hello-redox/files/test.go

@@ -0,0 +1,7 @@
+package main
+
+import "fmt"
+
+func main() {
+    fmt.Println("Hello, Redox!")
+}

recipes/tests/hello-redox/files/test.zig

@@ -0,0 +1,5 @@
+const std = @import("std");
+
+pub fn main() !void {
+    std.debug.print("Hello, Redox!\n", .{});
+}

recipes/tests/os-test-bins/recipe.toml

@@ -0,0 +1,28 @@
+[source]
+same_as = "../os-test"
+
+[build]
+template = "custom"
+script = """
+DYNAMIC_INIT
+SRC=${COOKBOOK_SOURCE}
+DST=${COOKBOOK_STAGE}/root/os-test
+if [ -z "$TESTBIN" ]; then
+pushd ${SRC}
+for file in */*/*.c; do
+    filename="${file%.*}"
+    mkdir -p $(dirname $DST/$filename)
+    # adding "true" because compilation can fail
+    ${CC} ${CFLAGS} ${LDFLAGS} "$SRC/$file" -o "$DST/$filename" -Wall || true
+    echo "./$filename" >> $DST/run.sh
+done
+popd
+else
+    mkdir -p $(dirname $DST/$TESTBIN)
+    ${CC} ${CFLAGS} ${LDFLAGS} "$SRC/$TESTBIN.c" -o "$DST/$TESTBIN" -Wall
+fi
+
+if [ -n "TESTBIN" ]; then
+"${COOKBOOK_REDOXER}" write-exec "$DST/$TESTBIN"
+fi
+"""

recipes/tests/os-test-result/recipe.toml

@@ -0,0 +1,101 @@
+[source]
+same_as = "../os-test"
+
+[build]
+template = "custom"
+dev-dependencies = [
+    "host:redoxer",
+# allows rebuilding relibc without updating prefix
+    "relibc",
+    "gnu-grep",
+    "libarchive",
+]
+script = """
+if [ "$TARGET" = "$COOKBOOK_HOST_TARGET" ]; then
+# TODO: libc conflict on toolchain
+export LD_LIBRARY_PATH="/lib/${GNU_TARGET}:${LD_LIBRARY_PATH}"
+fi
+
+rsync -a "${COOKBOOK_SOURCE}/" ./
+os=$(echo "${TARGET}" | cut -d - -f3)
+case "$os" in
+    linux)  OS=Linux;;
+    redox)  OS=Redox;;
+esac
+
+# allows linking to relibc instead of prefix/host libc
+export CC="env LIBRARY_PATH=${COOKBOOK_SYSROOT}/lib ${GNU_TARGET}-gcc"
+export CFLAGS="\
+-nostdinc \
+-nostdlib \
+-isystem ${COOKBOOK_SYSROOT}/include \
+-static \
+--sysroot ${COOKBOOK_SYSROOT} \
+${COOKBOOK_SYSROOT}/lib/crt0.o \
+${COOKBOOK_SYSROOT}/lib/crti.o \
+${COOKBOOK_SYSROOT}/lib/crtn.o \
+${COOKBOOK_SYSROOT}/lib/libc.a \
+"
+
+make OS=${OS} CC="${CC}" CFLAGS="${CFLAGS}" CPPFLAGS= \
+        LDFLAGS= EXTRA_LDFLAGS= \
+        CC_FOR_BUILD="${CC_WRAPPER} cc" CFLAGS_FOR_BUILD= CPPFLAGS_FOR_BUILD= \
+        LDFLAGS_FOR_BUILD= -j ${COOKBOOK_MAKE_JOBS} all
+
+skips=(
+    # Skip hanging tests on Linux/Redox with relibc
+    basic/stdio/putc_unlocked
+)
+
+if [ "$OS" = "Redox" ]; then
+skips+=(
+    # Skip hanging tests on Redox with relibc
+    # https://gitlab.redox-os.org/redox-os/redox/-/issues/1752
+    basic/sys_socket/accept
+    basic/sys_socket/recv
+    basic/sys_socket/send
+    basic/sys_socket/shutdown
+    signal/ppoll-block-close
+    signal/ppoll-block-close-raise
+    signal/ppoll-block-raise
+    signal/ppoll-block-sleep-raise-write
+    signal/ppoll-block-sleep-raise
+    signal/ppoll-block-sleep-write-raise
+)
+fi
+
+for skip in "${skips[@]}"
+do
+    mkdir -p out.known/${os}/"$(dirname "${skip}")"
+    echo "skipped" > out.known/${os}/"${skip}.out"
+done
+
+cp -t out -R out.known/${os}
+
+postinstall () {
+make OS=${OS} CC_FOR_BUILD="${CC_WRAPPER} cc" \
+        CFLAGS_FOR_BUILD= CPPFLAGS_FOR_BUILD= \
+        LDFLAGS_FOR_BUILD= html json jsonl
+
+    mkdir -p ${COOKBOOK_STAGE}/share/os-test
+    cp -a out ${COOKBOOK_STAGE}/share/os-test/out
+    cp -a html ${COOKBOOK_STAGE}/share/os-test/html
+    cp -a os-test.json ${COOKBOOK_STAGE}/share/os-test/os-test.json
+    cp -a os-test.jsonl ${COOKBOOK_STAGE}/share/os-test/os-test.jsonl
+}
+
+if [ "$TARGET" = "$COOKBOOK_HOST_TARGET" ]; then
+    make test
+    postinstall
+else
+    # bash: gnu-make crashes randomly but can continue
+    # issues with multi-core and make jobs
+    # https://gitlab.redox-os.org/redox-os/relibc/-/issues/240
+    # https://gitlab.redox-os.org/redox-os/redox/-/issues/1753
+    export REDOXER_QEMU_ARGS="-smp 1"
+    # make: jobs doesn't work yet
+    echo redoxer exec --folder . --folder "${COOKBOOK_SYSROOT}/usr/:/usr" --artifact out:/root/out \
+        bash -c "until make test; do echo retrying; done"
+    postinstall
+fi
+"""

recipes/tests/relibc-tests-bins/recipe.toml

@@ -0,0 +1,45 @@
+[source]
+same_as = "../../core/relibc"
+
+[build]
+template = "custom"
+script = """
+PACKAGE_PATH="tests" cookbook_cargo
+
+DYNAMIC_INIT
+SRC=${COOKBOOK_SOURCE}/tests
+EXPECTSRC=${SRC}/expected/bins_dynamic
+DST=${COOKBOOK_STAGE}/root/relibc-tests
+CFLAGS+=" -I${SRC}"
+LDFLAGS+=" -Wl,-rpath=\\$ORIGIN"
+pushd ${SRC}
+if [ -z "$TESTBIN" ]; then
+for file in **/*.c; do
+    filename="${file%.*}"
+    mkdir -p $(dirname $DST/$filename)
+    # adding "true" because compilation can fail
+    ${CC} ${CFLAGS} ${LDFLAGS} "$SRC/$file" -o "$DST/$filename" -Wall || true
+    if [[ -f "${EXPECTSRC}/$filename.stdout" ]]; then
+        echo "relibc-tests ./$filename" >> $DST/run.sh
+    else
+        echo "relibc-tests -s./$filename" >> $DST/run.sh
+    fi
+done
+rsync -a ${EXPECTSRC} ${DST}/expected
+popd
+else
+    mkdir -p $(dirname $DST/$TESTBIN) $(dirname $DST/expected/$TESTBIN)
+    ${CC} ${CFLAGS} ${LDFLAGS} "$SRC/$TESTBIN.c" -o "$DST/$TESTBIN" -Wall
+    if [[ -f "${EXPECTSRC}/$TESTBIN.stdout" ]]; then
+        cp ${EXPECTSRC}/$TESTBIN.{stdout,stderr} $(dirname $DST/expected/$TESTBIN)
+    fi
+fi
+
+if [ -n "TESTBIN" ]; then
+if [[ -f "${EXPECTSRC}/$TESTBIN.stdout" ]]; then
+"${COOKBOOK_REDOXER}" write-exec sh -c "cd /root/relibc-tests; relibc-tests ./$TESTBIN"
+else
+"${COOKBOOK_REDOXER}" write-exec sh -c "cd /root/relibc-tests; relibc-tests -s./$TESTBIN"
+fi
+fi
+"""

recipes/tests/relibc-tests/recipe.toml

@@ -4,12 +4,6 @@ same_as = "../../core/relibc"
 [build]
 template = "custom"
 script = """
-rsync -av --delete "${COOKBOOK_SOURCE}/" ./
-pushd tests
-export CARGO_TEST="${COOKBOOK_CARGO}"
-export NATIVE_RELIBC=1 # to link against prefix
-"${COOKBOOK_MAKE}" -j "${COOKBOOK_MAKE_JOBS}" all bins_verify/relibc-tests
-popd
 mkdir -pv "${COOKBOOK_STAGE}/share/relibc"
-cp -rv "tests" "${COOKBOOK_STAGE}/share/relibc/tests"
+cp -rv "${COOKBOOK_SOURCE}/tests" "${COOKBOOK_STAGE}/share/relibc-tests"
 """

recipes/wip/dev/lang/perl5/recipe.toml

@@ -4,24 +4,24 @@
 #TODO needs further testing
 [source]
 tar = "https://www.cpan.org/src/5.0/perl-5.40.0.tar.gz"
-[build]
-template = "custom"
-dependencies = [
-    "relibc"
-]
+blake3 = "8bfcbb999e0795a64ca90e1ba7308f49c30ab2619ffa25fa425527c4bfca5e0f"
 script = """
-rsync -av --delete "${COOKBOOK_SOURCE}/" ./ #move it to the top of the script after debugging - BP
 curl -L -O --time-cond perl-cross-1.6.tar.gz https://github.com/arsv/perl-cross/releases/download/1.6/perl-cross-1.6.tar.gz
 tar --strip-components=1 -xvf perl-cross-1.6.tar.gz
-wget -O ./cnf/config.sub "https://gitlab.redox-os.org/redox-os/gnu-config/-/raw/master/config.sub?inline=false"
+GNU_CONFIG_GET ./cnf/config.sub
+"""
+[build]
+template = "custom"
+dev-dependencies = ["relibc"]
+script = """
+DYNAMIC_INIT
+rsync -av --delete "${COOKBOOK_SOURCE}/" ./
+
 #Note: since perl-cross can run only inside the source-tree (out-of-tree is bugged) it's easier to do everything in the build directory
-COOKBOOK_CONFIGURE=${COOKBOOK_BUILD}/configure
 cp ${COOKBOOK_RECIPE}/configure_tool.sh ${COOKBOOK_BUILD}/cnf/configure_tool.sh
 cp ${COOKBOOK_RECIPE}/redox ${COOKBOOK_BUILD}/cnf/hints/redox
-mkdir ${COOKBOOK_SYSROOT}/usr
-ln -s "${COOKBOOK_SYSROOT}/include" "${COOKBOOK_SYSROOT}/usr/include"
 #Note: non-standard configure, familiar flags can have different meaning!
-./configure --host-cc=gcc --host-cpp=g++ --target=x86_64-unknown-redox --sysroot=${COOKBOOK_SYSROOT} --disable-mod=Sys-Syslog
+./configure --host-cc="gcc" --host-cpp="g++" --target=${TARGET} --sysroot=${COOKBOOK_SYSROOT}
 sed -i "s/^#define Netdb_name_t.*/#define Netdb_name_t const char*/" config.h
 sed -i "s/^# HAS_NANOSLEEP.*/#define HAS_NANOSLEEP/" config.h
 sed -i "s|^/.#define I_GRP.*|#define I_GRP|" config.h

recipes/wip/games/strategy/hnefatafl-copenhagen/recipe.toml

@@ -2,8 +2,7 @@
 # 1. The CJK and runes fonts don't load.
 # 2. On a button press two characters are read in instead of one.
 # 3. Backspace does not work.
-# 4. The command line prompt prints 'deprecated: legacy path "time:4" used by /usr/games/hnefatafl-client'.
-# 5. TcpStream.shutdown(): shutdown call failed: Os { code: 38, kind: Unsupported, message: "Function not implemented" }
+# 4. TcpStream.shutdown() is not implemented.
 
 [source]
 git = "https://github.com/dcampbell24/hnefatafl"
@@ -13,15 +12,16 @@ template = "custom"
 script = """
 "${COOKBOOK_CARGO}" build \
     --manifest-path "${COOKBOOK_SOURCE}/${PACKAGE_PATH}/Cargo.toml" \
-    --bin hnefatafl-client \
     --features client \
     --release \
     --no-default-features
 
 mkdir -pv "${COOKBOOK_STAGE}/usr/bin"
-cp -v \
-    "target/${TARGET}/release/hnefatafl-client" \
-    "${COOKBOOK_STAGE}/usr/bin/hnefatafl-client"
+
+cp -v "target/${TARGET}/release/hnefatafl-ai" "${COOKBOOK_STAGE}/usr/bin/hnefatafl-ai"
+cp -v "target/${TARGET}/release/hnefatafl-client" "${COOKBOOK_STAGE}/usr/bin/hnefatafl-client"
+cp -v "target/${TARGET}/release/hnefatafl-server" "${COOKBOOK_STAGE}/usr/bin/hnefatafl-server"
+cp -v "target/${TARGET}/release/hnefatafl-text-protocol" "${COOKBOOK_STAGE}/usr/bin/hnefatafl-text-protocol"
 
 mkdir -pv "${COOKBOOK_STAGE}"/usr/games
 mkdir -pv "${COOKBOOK_STAGE}"/ui/apps
@@ -30,3 +30,10 @@ cp -rv "${COOKBOOK_SOURCE}"/icons/king_256x256.png "${COOKBOOK_STAGE}"/ui/icons/
 cp -rv "${COOKBOOK_SOURCE}"/packages/redox/manifest "${COOKBOOK_STAGE}"/ui/apps/hnefatafl-client
 mv "${COOKBOOK_STAGE}"/usr/bin/hnefatafl-client "${COOKBOOK_STAGE}"/usr/games/hnefatafl-client
 """
+
+[package]
+dependencies = [
+    "dejavu",
+    "freefont",
+    "noto-color-emoji",
+]

src/bin/cookbook_redoxer.rs

@@ -5,6 +5,14 @@ fn main() {
     // Ensure all flags go to cargo
     if args.len() >= 2 {
         args.insert(2, "--".to_string());
+        if args[1] == "write-exec" {
+            if let Ok(stage_dir) = std::env::var("COOKBOOK_STAGE") {
+                args.insert(2, format!("{}/root", stage_dir));
+                args.insert(2, "--folder".to_string());
+                args.insert(2, stage_dir);
+                args.insert(2, "--root".to_string());
+            }
+        }
     }
     redoxer::main(&args);
 }

src/bin/repo.rs

@@ -1,9 +1,10 @@
 use ansi_to_tui::IntoText;
 use anyhow::{Context, anyhow, bail};
 use cookbook::config::{CookConfig, get_config, init_config};
-use cookbook::cook::cook_build::build;
+use cookbook::cook::cook_build::{build, get_stage_dirs, remove_stage_dir};
 use cookbook::cook::fetch::{fetch, fetch_offline};
 use cookbook::cook::fs::{create_target_dir, run_command};
+use cookbook::cook::ident;
 use cookbook::cook::package::package;
 use cookbook::cook::pty::{PtyOut, UnixSlavePty, flush_pty, setup_pty};
 use cookbook::cook::script::KILL_ALL_PID;
@@ -185,6 +186,9 @@ fn main_inner() -> anyhow::Result<()> {
     }
 
     let (config, command, recipe_names) = parse_args(args)?;
+    if command.is_building() {
+        ident::init_ident();
+    }
     if command == CliCommand::Cook && config.cook.tui {
         if let Some((name, e)) = run_tui_cook(config.clone(), recipe_names.clone())? {
             let _ = stderr().write(e.as_bytes());
@@ -212,8 +216,13 @@ fn main_inner() -> anyhow::Result<()> {
                 }
             }
             Err(e) => {
-                if config.cook.nonstop && verbose {
-                    eprintln!("{:?}", e);
+                if config.cook.nonstop {
+                    if verbose {
+                        eprintln!("{:?}", e);
+                    }
+                    if let Err(e) = handle_nonstop_fail(recipe) {
+                        eprintln!("{:?}", e)
+                    };
                 }
                 print_failed(&command, &recipe.name);
                 if !config.cook.nonstop {
@@ -500,6 +509,7 @@ fn parse_args(args: Vec<String>) -> anyhow::Result<(CliConfig, CliCommand, Vec<C
     {
         let repo_binary = conf.general.repo_binary == Some(true);
         let mut last_rule = if repo_binary { "binary" } else { "source" };
+        let mut should_drop_host_packages = true;
         // Use rev() so recipes that don't listed in config is inherited from parent
         for recipe in recipes.iter_mut().rev() {
             if let Some(conf) = conf.packages.get(recipe.name.as_str()) {
@@ -512,12 +522,20 @@ fn parse_args(args: Vec<String>) -> anyhow::Result<(CliConfig, CliCommand, Vec<C
                             "source"
                         }
                     }
+                };
+                if should_drop_host_packages && (last_rule == "source" || last_rule == "local") {
+                    should_drop_host_packages = false;
                 }
             };
             recipe
                 .apply_filesystem_config(last_rule)
                 .map_err(|e| anyhow!(e))?;
         }
+        // If there's no building from source, drop all host toolchain
+        // TODO: This is more of a hack to make CI passing
+        if should_drop_host_packages && config.with_package_deps {
+            recipes = recipes.into_iter().filter(|p| !p.name.is_host()).collect();
+        }
     }
 
     if command.is_informational() {
@@ -534,10 +552,9 @@ fn handle_fetch(
     allow_offline: bool,
     logger: &PtyOut,
 ) -> anyhow::Result<PathBuf> {
-    let recipe_dir = &recipe.dir;
     let source_dir = match config.cook.offline && allow_offline {
-        true => fetch_offline(recipe_dir, &recipe.recipe, logger),
-        false => fetch(recipe_dir, &recipe.recipe, logger),
+        true => fetch_offline(&recipe, logger),
+        false => fetch(&recipe, logger),
     }
     .map_err(|e| anyhow!("failed to fetch: {:?}", e))?;
 
@@ -553,7 +570,7 @@ fn handle_cook(
 ) -> anyhow::Result<()> {
     let recipe_dir = &recipe.dir;
     let target_dir = create_target_dir(recipe_dir, recipe.target).map_err(|e| anyhow!(e))?;
-    let (stage_dir, auto_deps) = build(
+    let (stage_dirs, auto_deps) = build(
         recipe_dir,
         &source_dir,
         &target_dir,
@@ -565,19 +582,23 @@ fn handle_cook(
     )
     .map_err(|err| anyhow!("failed to build: {:?}", err))?;
 
-    package(
-        &stage_dir,
-        &target_dir,
-        &recipe.name,
-        &recipe.recipe,
-        &auto_deps,
-        logger,
-    )
-    .map_err(|err| anyhow!("failed to package: {:?}", err))?;
+    package(&recipe, &stage_dirs, &auto_deps, logger)
+        .map_err(|err| anyhow!("failed to package: {:?}", err))?;
 
     Ok(())
 }
 
+/// delete stage artifacts upon nonstop failure to let repo_builder know
+fn handle_nonstop_fail(recipe: &CookRecipe) -> anyhow::Result<()> {
+    let target_dir = recipe.target_dir();
+    let stage_dirs = get_stage_dirs(&recipe.recipe.optional_packages, &target_dir);
+    for stage_dir in stage_dirs {
+        remove_stage_dir(&stage_dir)
+            .map_err(|err| anyhow!("failed to remove stage dir: {:?}", err))?;
+    }
+    Ok(())
+}
+
 fn handle_clean(
     recipe: &CookRecipe,
     _config: &CliConfig,
@@ -1034,6 +1055,8 @@ fn run_tui_cook(
                             if cooker_prompting.load(Ordering::SeqCst) == 4 {
                                 break 'done;
                             }
+                            // TODO: where to report error?
+                            let _ = handle_nonstop_fail(&recipe);
                             break;
                         }
                         while cooker_prompting.load(Ordering::SeqCst) != 0 {
@@ -1050,6 +1073,7 @@ fn run_tui_cook(
                                 } // retry
                                 3 => {
                                     cooker_prompting.swap(0, Ordering::SeqCst);
+                                    let _ = handle_nonstop_fail(&recipe);
                                     break 'again;
                                 } // skip
                                 4 => {
@@ -1140,6 +1164,7 @@ fn run_tui_cook(
                             if fetcher_prompting.load(Ordering::SeqCst) == 4 {
                                 break 'done;
                             }
+                            let _ = handle_nonstop_fail(&recipe);
                             break;
                         }
                         while fetcher_prompting.load(Ordering::SeqCst) != 0 {
@@ -1156,6 +1181,7 @@ fn run_tui_cook(
                                 } // retry
                                 3 => {
                                     fetcher_prompting.swap(0, Ordering::SeqCst);
+                                    let _ = handle_nonstop_fail(&recipe);
                                     break 'again;
                                 } // skip
                                 4 => {

src/bin/repo_builder.rs

@@ -1,10 +1,11 @@
-use anyhow::anyhow;
+use anyhow::{anyhow, bail};
 use cookbook::WALK_DEPTH;
-use cookbook::config::{get_config, init_config};
-use cookbook::cook::package as cook_package;
+use cookbook::cook::ident::{get_ident, init_ident};
+use cookbook::cook::{fetch, package as cook_package};
 use cookbook::recipe::CookRecipe;
+use pkg::package::{Repository, SourceIdentifier};
 use pkg::{Package, PackageName, recipes};
-use std::collections::{BTreeMap, HashMap};
+use std::collections::{BTreeMap, BTreeSet, HashMap};
 use std::env;
 use std::fs::{self, File};
 use std::io::{Read, Write};
@@ -27,7 +28,6 @@ fn is_newer(src: &Path, dst: &Path) -> bool {
 #[derive(Clone)]
 struct CliConfig {
     repo_dir: PathBuf,
-    nonstop: bool,
     appstream: bool,
     recipe_list: Vec<String>,
 }
@@ -40,7 +40,6 @@ impl CliConfig {
             .expect("Usage: repo_builder <REPO_DIR> <recipe1> <recipe2> ...");
         Ok(CliConfig {
             repo_dir: PathBuf::from(repo_dir),
-            nonstop: get_config().cook.nonstop,
             appstream: env::var("COOKBOOK_APPSTREAM").ok().as_deref() == Some("true"),
             recipe_list: args.collect(),
         })
@@ -48,7 +47,7 @@ impl CliConfig {
 }
 
 fn main() -> Result<(), Box<dyn std::error::Error>> {
-    init_config();
+    init_ident();
     let conf = CliConfig::parse_args()?;
     Ok(publish_packages(&conf)?)
 }
@@ -65,7 +64,7 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
     //
     // The following adds the package dependencies of the recipes to the repo as
     // well.
-    let recipe_list = Package::new_recursive(
+    let (recipe_list, recipe_map) = Package::new_recursive_nonstop(
         &config
             .recipe_list
             .iter()
@@ -73,18 +72,21 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
             // Don't publish host packages
             .filter(|pkg| pkg.as_ref().is_ok_and(|p| !p.is_host()))
             .collect::<Result<Vec<_>, _>>()?,
-        config.nonstop,
         WALK_DEPTH,
-    )?
-    .into_iter()
-    .map(|pkg| pkg.name.clone())
-    .collect::<Vec<_>>();
+    );
+
+    if recipe_list.len() == 0 {
+        // Fail-Safe
+        bail!("Zero packages are passing the build");
+    }
 
     let mut appstream_sources: HashMap<String, PathBuf> = HashMap::new();
     let mut packages: BTreeMap<String, String> = BTreeMap::new();
+    let mut outdated_packages: BTreeMap<String, SourceIdentifier> = BTreeMap::new();
 
     // === 1. Push recipes in list ===
-    for recipe in &recipe_list {
+    for recipe_toml in &recipe_list {
+        let recipe = &recipe_toml.name;
         let Some(recipe_path) = recipes::find(recipe.name()) else {
             eprintln!("recipe {} not found", recipe);
             continue;
@@ -163,22 +165,68 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
         }
     }
 
+    // === 3. List outdated packages ===
+    for (recipe, e) in recipe_map
+        .into_iter()
+        .filter_map(|(k, v)| v.err().and_then(|e| Some((k, e))))
+    {
+        eprintln!(
+            "\x1b[0;91;49mrepo - marking {} as outdated:\x1b[0m {e}",
+            recipe
+        );
+
+        let Some(recipe_path) = recipes::find(recipe.name()) else {
+            eprintln!("recipe {} not found", recipe);
+            continue;
+        };
+        let Ok(cookbook_recipe) = CookRecipe::from_path(recipe_path, true, false) else {
+            eprintln!("recipe {} unable to read", recipe);
+            continue;
+        };
+
+        match fetch::fetch_get_source_info(&cookbook_recipe) {
+            Ok(source_ident) => {
+                outdated_packages.insert(recipe.name().to_string(), source_ident);
+            }
+            Err(e) => {
+                eprintln!(
+                    "\x1b[0;91;49m  source of {} is not identifiable:\x1b[0m {e}",
+                    recipe
+                );
+                let ident = get_ident();
+                outdated_packages.insert(
+                    recipe.name().to_string(),
+                    SourceIdentifier {
+                        source_identifier: "missing_source".to_string(),
+                        commit_identifier: ident.commit.clone(),
+                        time_identifier: ident.time.clone(),
+                    },
+                );
+            }
+        };
+    }
+
     eprintln!("\x1b[01;38;5;155mrepo - generating repo.toml\x1b[0m");
 
-    // === 3. Read and update repo.toml ===
+    // === 4. Read and update repo.toml ===
     let repo_toml_path = repo_path.join("repo.toml");
     if repo_toml_path.exists() {
         let mut file = File::open(&repo_toml_path)?;
         let mut contents = String::new();
         file.read_to_string(&mut contents)?;
 
-        let parsed: Value = toml::from_str(&contents)?;
-        if let Some(pkg_table) = parsed.get("packages").and_then(|v| v.as_table()) {
-            for (k, v) in pkg_table {
-                if let Some(s) = v.as_str() {
-                    packages.insert(k.clone(), format!("\"{}\"", s));
-                } else {
-                    packages.insert(k.clone(), v.to_string());
+        let parsed: Repository = toml::from_str(&contents)?;
+        for (k, v) in parsed.packages {
+            packages.insert(k, v);
+        }
+        if parsed.outdated_packages.len() > 0 {
+            let built_packages: BTreeSet<String> = recipe_list
+                .iter()
+                .map(|p| p.name.name().to_string())
+                .collect();
+            for (k, v) in parsed.outdated_packages {
+                if outdated_packages.contains_key(&k) || !built_packages.contains(&k) {
+                    outdated_packages.insert(k, v);
                 }
             }
         }
@@ -203,21 +251,16 @@ fn publish_packages(config: &CliConfig) -> anyhow::Result<()> {
         let version_str = parsed
             .get("blake3")
             .unwrap_or_else(|| parsed.get("version").unwrap_or_else(|| &empty_ver))
-            .to_string(); // includes quotes
+            .as_str()
+            .unwrap_or("");
         let package_name = path.file_stem().unwrap().to_string_lossy().to_string();
-        packages.insert(package_name, version_str);
-    }
-
-    // FIXME: Use proper TOML serializer
-    let mut output = String::from("[packages]\n");
-    for (name, version) in &packages {
-        output.push_str(&if name.contains('.') {
-            format!("\"{name}\" = {version}\n")
-        } else {
-            format!("{name} = {version}\n")
-        });
+        packages.insert(package_name, version_str.to_string());
     }
 
+    let output = toml::to_string(&Repository {
+        packages,
+        outdated_packages,
+    })?;
     let mut output_file = File::create(&repo_toml_path)?;
     output_file.write_all(output.as_bytes())?;
 

src/cook.rs

@@ -2,6 +2,7 @@
 pub mod cook_build;
 pub mod fetch;
 pub mod fs;
+pub mod ident;
 pub mod package;
 pub mod pty;
 pub mod script;

src/cook/cook_build.rs

@@ -262,9 +262,7 @@ pub fn build(
         {
             for stage_dir in &stage_dirs {
                 log_to_pty!(logger, "DEBUG: updating '{}'", stage_dir.display());
-                if stage_dir.is_dir() {
-                    remove_all(&stage_dir)?;
-                }
+                remove_stage_dir(stage_dir)?;
             }
         }
     }
@@ -415,7 +413,21 @@ pub fn build(
     Ok((stage_dirs, auto_deps))
 }
 
-fn get_stage_dirs(features: &Vec<OptionalPackageRecipe>, target_dir: &Path) -> Vec<PathBuf> {
+pub fn remove_stage_dir(stage_dir: &PathBuf) -> Result<(), String> {
+    if stage_dir.is_dir() {
+        remove_all(&stage_dir)?;
+    }
+    let stage_file = stage_dir.with_added_extension("pkgar");
+    if stage_file.is_file() {
+        remove_all(&stage_file)?;
+    }
+    let stage_meta = stage_dir.with_added_extension("toml");
+    Ok(if stage_meta.is_file() {
+        remove_all(&stage_meta)?;
+    })
+}
+
+pub fn get_stage_dirs(features: &Vec<OptionalPackageRecipe>, target_dir: &Path) -> Vec<PathBuf> {
     let mut v = Vec::new();
     for f in features {
         v.push(target_dir.join(format!("stage.{}", f.name)));

src/cook/fetch.rs

@@ -1,3 +1,5 @@
+use pkg::package::SourceIdentifier;
+
 use crate::REMOTE_PKG_SOURCE;
 use crate::config::translate_mirror;
 use crate::cook::fs::*;
@@ -8,9 +10,11 @@ use crate::cook::script::*;
 use crate::is_redox;
 use crate::log_to_pty;
 use crate::recipe::BuildKind;
-use crate::recipe::Recipe;
+use crate::recipe::CookRecipe;
 use crate::{blake3, recipe::SourceRecipe};
 use std::fs;
+use std::fs::File;
+use std::io::Read;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
@@ -30,27 +34,33 @@ pub(crate) fn get_blake3(path: &PathBuf, show_progress: bool) -> Result<String,
     })
 }
 
-pub fn fetch_offline(
-    recipe_dir: &Path,
-    recipe: &Recipe,
-    logger: &PtyOut,
-) -> Result<PathBuf, String> {
+pub fn fetch_offline(recipe: &CookRecipe, logger: &PtyOut) -> Result<PathBuf, String> {
+    let recipe_dir = &recipe.dir;
     let source_dir = recipe_dir.join("source");
-    if recipe.build.kind == BuildKind::None {
-        // the build function doesn't need source dir exists
-        return Ok(source_dir);
-    }
-    if recipe.build.kind == BuildKind::Remote {
-        fetch_remote(recipe_dir, recipe, true, logger)?;
-        return Ok(source_dir);
+    match recipe.recipe.build.kind {
+        BuildKind::None => {
+            // the build function doesn't need source dir exists
+            fetch_apply_source_info(recipe, "".to_string())?;
+            return Ok(source_dir);
+        }
+        BuildKind::Remote => {
+            fetch_remote(recipe_dir, recipe, true, logger)?;
+            return Ok(source_dir);
+        }
+        _ => {}
     }
 
-    match &recipe.source {
+    let ident = match &recipe.recipe.source {
         Some(SourceRecipe::Path { path: _ }) | None => {
-            return fetch(recipe_dir, recipe, logger);
+            fetch(recipe, logger)?;
+            "local_source".to_string()
         }
-        Some(SourceRecipe::SameAs { same_as: _ }) => {
-            return fetch(recipe_dir, recipe, logger);
+        Some(SourceRecipe::SameAs { same_as }) => {
+            let recipe = fetch_resolve_canon(recipe_dir, &same_as, recipe.name.is_host())?;
+            // recursively fetch
+            fetch_offline(&recipe, logger)?;
+            fetch_make_symlink(&source_dir, &same_as)?;
+            fetch_get_source_info(&recipe)?.source_identifier
         }
         Some(SourceRecipe::Git {
             git: _,
@@ -62,6 +72,8 @@ pub fn fetch_offline(
             shallow_clone: _,
         }) => {
             offline_check_exists(&source_dir)?;
+            let (head_rev, _) = get_git_head_rev(&source_dir)?;
+            head_rev
         }
         Some(SourceRecipe::Tar {
             tar: _,
@@ -93,29 +105,38 @@ pub fn fetch_offline(
                     offline_check_exists(&source_dir)?;
                 }
             }
+            blake3.clone().unwrap_or("no_tar_blake3_hash_info".into())
         }
-    }
+    };
+
+    fetch_apply_source_info(recipe, ident)?;
 
     Ok(source_dir)
 }
 
-pub fn fetch(recipe_dir: &Path, recipe: &Recipe, logger: &PtyOut) -> Result<PathBuf, String> {
+pub fn fetch(recipe: &CookRecipe, logger: &PtyOut) -> Result<PathBuf, String> {
+    let recipe_dir = &recipe.dir;
     let source_dir = recipe_dir.join("source");
-    if recipe.build.kind == BuildKind::None {
-        // the build function doesn't need source dir exists
-        return Ok(source_dir);
-    }
-    if recipe.build.kind == BuildKind::Remote {
-        fetch_remote(recipe_dir, recipe, false, logger)?;
-        return Ok(source_dir);
+    match recipe.recipe.build.kind {
+        BuildKind::None => {
+            // the build function doesn't need source dir exists
+            fetch_apply_source_info(recipe, "".to_string())?;
+            return Ok(source_dir);
+        }
+        BuildKind::Remote => {
+            fetch_remote(recipe_dir, recipe, false, logger)?;
+            return Ok(source_dir);
+        }
+        _ => {}
     }
 
-    match &recipe.source {
+    let ident = match &recipe.recipe.source {
         Some(SourceRecipe::SameAs { same_as }) => {
-            let (canon_dir, recipe) = fetch_resolve_canon(recipe_dir, &same_as)?;
+            let recipe = fetch_resolve_canon(recipe_dir, &same_as, recipe.name.is_host())?;
             // recursively fetch
-            fetch(&canon_dir, &recipe, logger)?;
+            fetch(&recipe, logger)?;
             fetch_make_symlink(&source_dir, &same_as)?;
+            fetch_get_source_info(&recipe)?.source_identifier
         }
         Some(SourceRecipe::Path { path }) => {
             if !source_dir.is_dir() || modified_dir(Path::new(&path))? > modified_dir(&source_dir)?
@@ -135,6 +156,7 @@ pub fn fetch(recipe_dir: &Path, recipe: &Recipe, logger: &PtyOut) -> Result<Path
                     )
                 })?;
             }
+            "local_source".to_string()
         }
         Some(SourceRecipe::Git {
             git,
@@ -147,7 +169,7 @@ pub fn fetch(recipe_dir: &Path, recipe: &Recipe, logger: &PtyOut) -> Result<Path
         }) => {
             //TODO: use libgit?
             let shallow_clone = *shallow_clone == Some(true);
-            if !source_dir.is_dir() {
+            let can_skip_rebuild = if !source_dir.is_dir() {
                 // Create source.tmp
                 let source_dir_tmp = recipe_dir.join("source.tmp");
                 create_dir_clean(&source_dir_tmp)?;
@@ -171,9 +193,10 @@ pub fn fetch(recipe_dir: &Path, recipe: &Recipe, logger: &PtyOut) -> Result<Path
 
                 // Move source.tmp to source atomically
                 rename(&source_dir_tmp, &source_dir)?;
+
+                false
             } else {
-                let source_git_dir = source_dir.join(".git");
-                if !source_git_dir.is_dir() {
+                if !source_dir.join(".git").is_dir() {
                     return Err(format!(
                         "'{}' is not a git repository, but recipe indicated git source",
                         source_dir.display(),
@@ -191,69 +214,100 @@ pub fn fetch(recipe_dir: &Path, recipe: &Recipe, logger: &PtyOut) -> Result<Path
                 command.arg("-C").arg(&source_dir);
                 command.arg("fetch").arg("origin");
                 run_command(command, logger)?;
-            }
 
-            if let Some(_upstream) = upstream {
-                //TODO: set upstream URL
-                // git remote set-url upstream "$GIT_UPSTREAM" &> /dev/null ||
-                // git remote add upstream "$GIT_UPSTREAM"
-                // git fetch upstream
-            }
+                let (head_rev, detached_rev) = get_git_head_rev(&source_dir)?;
+                if detached_rev {
+                    if let Some(rev) = rev
+                        && let Ok(exp_rev) = get_git_tag_rev(&source_dir, &rev)
+                    {
+                        exp_rev == head_rev
+                    } else {
+                        false
+                    }
+                } else {
+                    let (_, remote_branch, remote_name, remote_url) =
+                        get_git_remote_tracking(&source_dir)?;
+                    // TODO: how to get default branch and compare it here?
+                    if remote_name == "origin" && &remote_url == chop_dot_git(git) {
+                        match get_git_fetch_rev(&source_dir, &remote_url, &remote_branch) {
+                            Ok(fetch_rev) => fetch_rev == head_rev,
+                            Err(e) => {
+                                log_to_pty!(logger, "{}", e);
+                                false
+                            }
+                        }
+                    } else {
+                        false
+                    }
+                }
+            };
 
-            if let Some(rev) = rev {
-                // Check out specified revision
-                let mut command = Command::new("git");
-                command.arg("-C").arg(&source_dir);
-                command.arg("checkout").arg(rev);
-                run_command(command, logger)?;
-            } else if !is_redox() {
-                //If patches exists, we have to drop it
-                if patches.len() > 0 {
+            if !can_skip_rebuild {
+                if let Some(_upstream) = upstream {
+                    //TODO: set upstream URL (is this needed?)
+                    // git remote set-url upstream "$GIT_UPSTREAM" &> /dev/null ||
+                    // git remote add upstream "$GIT_UPSTREAM"
+                    // git fetch upstream
+                }
+
+                if let Some(rev) = rev {
+                    // Check out specified revision
+                    let mut command = Command::new("git");
+                    command.arg("-C").arg(&source_dir);
+                    command.arg("checkout").arg(rev);
+                    run_command(command, logger)?;
+                } else if !is_redox() {
+                    //If patches exists, we have to drop it
+                    if patches.len() > 0 {
+                        let mut command = Command::new("git");
+                        command.arg("-C").arg(&source_dir);
+                        command.arg("reset").arg("--hard");
+                        run_command(command, logger)?;
+                    }
+                    //TODO: complicated stuff to check and reset branch to origin
+                    //TODO: redox can't undestand this (got exit status 1)
+                    let mut command = Command::new("bash");
+                    command.arg("-c").arg(GIT_RESET_BRANCH);
+                    if let Some(branch) = branch {
+                        command.env("BRANCH", branch);
+                    }
+                    command.current_dir(&source_dir);
+                    run_command(command, logger)?;
+                }
+
+                if !patches.is_empty() || script.is_some() {
+                    // Hard reset
                     let mut command = Command::new("git");
                     command.arg("-C").arg(&source_dir);
                     command.arg("reset").arg("--hard");
                     run_command(command, logger)?;
                 }
-                //TODO: complicated stuff to check and reset branch to origin
-                //TODO: redox can't undestand this (got exit status 1)
-                let mut command = Command::new("bash");
-                command.arg("-c").arg(GIT_RESET_BRANCH);
-                if let Some(branch) = branch {
-                    command.env("BRANCH", branch);
-                }
-                command.current_dir(&source_dir);
-                run_command(command, logger)?;
-            }
 
-            if !patches.is_empty() || script.is_some() {
-                // Hard reset
+                // Sync submodules URL
                 let mut command = Command::new("git");
                 command.arg("-C").arg(&source_dir);
-                command.arg("reset").arg("--hard");
+                command.arg("submodule").arg("sync").arg("--recursive");
+
                 run_command(command, logger)?;
+
+                // Update submodules
+                let mut command = Command::new("git");
+                command.arg("-C").arg(&source_dir);
+                command
+                    .arg("submodule")
+                    .arg("update")
+                    .arg("--init")
+                    .arg("--recursive");
+                if shallow_clone {
+                    command.arg("--filter=tree:0");
+                }
+                run_command(command, logger)?;
+
+                fetch_apply_patches(recipe_dir, patches, script, &source_dir, logger)?;
             }
 
-            // Sync submodules URL
-            let mut command = Command::new("git");
-            command.arg("-C").arg(&source_dir);
-            command.arg("submodule").arg("sync").arg("--recursive");
-
-            run_command(command, logger)?;
-
-            // Update submodules
-            let mut command = Command::new("git");
-            command.arg("-C").arg(&source_dir);
-            command
-                .arg("submodule")
-                .arg("update")
-                .arg("--init")
-                .arg("--recursive");
-            if shallow_clone {
-                command.arg("--filter=tree:0");
-            }
-            run_command(command, logger)?;
-
-            fetch_apply_patches(recipe_dir, patches, script, &source_dir, logger)?;
+            let (head_rev, _) = get_git_head_rev(&source_dir)?;
+            head_rev
         }
         Some(SourceRecipe::Tar {
             tar,
@@ -316,6 +370,7 @@ pub fn fetch(recipe_dir: &Path, recipe: &Recipe, logger: &PtyOut) -> Result<Path
                 // Move source.tmp to source atomically
                 rename(&source_dir_tmp, &source_dir)?;
             }
+            blake3.clone().unwrap_or("no_tar_blake3_hash_info".into())
         }
         // Local Sources
         None => {
@@ -327,17 +382,20 @@ pub fn fetch(recipe_dir: &Path, recipe: &Recipe, logger: &PtyOut) -> Result<Path
                 );
                 create_dir(&source_dir)?;
             }
+            "local_source".into()
         }
-    }
+    };
 
     if let BuildKind::Cargo {
         package_path,
         cargoflags: _,
-    } = &recipe.build.kind
+    } = &recipe.recipe.build.kind
     {
         fetch_cargo(&source_dir, package_path.as_ref(), logger)?;
     }
 
+    fetch_apply_source_info(recipe, ident)?;
+
     Ok(source_dir)
 }
 
@@ -367,7 +425,8 @@ pub(crate) fn fetch_make_symlink(source_dir: &PathBuf, same_as: &String) -> Resu
 pub(crate) fn fetch_resolve_canon(
     recipe_dir: &Path,
     same_as: &String,
-) -> Result<(PathBuf, Recipe), String> {
+    is_host: bool,
+) -> Result<CookRecipe, String> {
     let canon_dir = Path::new(recipe_dir).join(same_as);
     if canon_dir
         .to_str()
@@ -382,12 +441,8 @@ pub(crate) fn fetch_resolve_canon(
     if !canon_dir.exists() {
         return Err(format!("'{dir}' is not exists.", dir = canon_dir.display()));
     }
-    let recipe_path = canon_dir.join("recipe.toml");
-    let recipe_str = fs::read_to_string(&recipe_path)
-        .map_err(|e| format!("unable to read {path}: {e}", path = recipe_path.display()))?;
-    let recipe: Recipe = toml::from_str(&recipe_str)
-        .map_err(|e| format!("Unable to parse {path}: {e}", path = recipe_path.display()))?;
-    Ok((canon_dir, recipe))
+    CookRecipe::from_path(canon_dir.as_path(), true, is_host)
+        .map_err(|e| format!("Unable to load {dir}: {e:?}", dir = canon_dir.display()))
 }
 
 pub(crate) fn fetch_extract_tar(
@@ -455,13 +510,11 @@ fn get_pubkey_url() -> String {
 
 pub fn fetch_remote(
     recipe_dir: &Path,
-    recipe: &Recipe,
+    recipe: &CookRecipe,
     offline_mode: bool,
     logger: &PtyOut,
 ) -> Result<(), String> {
-    // TODO: allow download to host target (waiting for build server to have them)
-    let target = redoxer::target();
-    let target_dir = create_target_dir(recipe_dir, target)?;
+    let target_dir = create_target_dir(recipe_dir, recipe.target)?;
     let source_pubkey = target_dir.join("id_ed25519.pub.toml");
     if !offline_mode {
         download_wget(&get_pubkey_url(), &source_pubkey, logger)?;
@@ -469,7 +522,7 @@ pub fn fetch_remote(
         offline_check_exists(&source_pubkey)?;
     }
 
-    let packages = recipe.get_packages_list();
+    let packages = recipe.recipe.get_packages_list();
 
     let name = recipe_dir
         .file_name()
@@ -493,6 +546,28 @@ pub fn fetch_remote(
             offline_check_exists(&source_pkgar)?;
             offline_check_exists(&source_toml)?;
         }
+
+        // guaranteed to exist once
+        if package.is_none() {
+            let mut file = File::open(&source_toml)
+                .map_err(|e| format!("Unable to open source.toml: {e:?}"))?;
+            let mut contents = String::new();
+            file.read_to_string(&mut contents)
+                .map_err(|e| format!("Unable to read source.toml: {e:?}"))?;
+
+            let pkg_toml = pkg::Package::from_toml(&contents)
+                .map_err(|e| format!("Unable to parse source.toml: {e:?}"))?;
+
+            fetch_apply_source_info_from_remote(
+                recipe,
+                &SourceIdentifier {
+                    commit_identifier: pkg_toml.commit_identifier.clone(),
+                    source_identifier: pkg_toml.source_identifier.clone(),
+                    time_identifier: pkg_toml.time_identifier.clone(),
+                    ..Default::default()
+                },
+            )?;
+        }
     }
 
     Ok(())
@@ -563,3 +638,37 @@ pub(crate) fn fetch_apply_patches(
         )?;
     })
 }
+
+pub(crate) fn fetch_apply_source_info(
+    recipe: &CookRecipe,
+    source_identifier: String,
+) -> Result<(), String> {
+    let ident = crate::cook::ident::get_ident();
+    let info = pkg::package::SourceIdentifier {
+        commit_identifier: ident.commit.to_string(),
+        time_identifier: ident.time.to_string(),
+        source_identifier: source_identifier,
+    };
+
+    fetch_apply_source_info_from_remote(&recipe, &info)
+}
+
+pub(crate) fn fetch_apply_source_info_from_remote(
+    recipe: &CookRecipe,
+    info: &pkg::package::SourceIdentifier,
+) -> Result<(), String> {
+    let target_dir = create_target_dir(&recipe.dir, recipe.target)?;
+    let source_toml_path = target_dir.join("source_info.toml");
+    serialize_and_write(&source_toml_path, &info)?;
+    Ok(())
+}
+
+pub fn fetch_get_source_info(recipe: &CookRecipe) -> Result<SourceIdentifier, String> {
+    let target_dir = recipe.target_dir();
+    let source_toml_path = target_dir.join("source_info.toml");
+    let toml_content = fs::read_to_string(source_toml_path)
+        .map_err(|e| format!("Unable to read source_info.toml: {:?}", e))?;
+    let parsed = toml::from_str(&toml_content)
+        .map_err(|e| format!("Unable to parse source_info.toml: {:?}", e))?;
+    Ok(parsed)
+}

src/cook/fs.rs

@@ -285,3 +285,187 @@ pub fn download_wget(url: &str, dest: &PathBuf, logger: &PtyOut) -> Result<(), S
     }
     Ok(())
 }
+
+/// get commit rev and return if it's detached or not
+pub fn get_git_head_rev(dir: &PathBuf) -> Result<(String, bool), String> {
+    let git_head = dir.join(".git/HEAD");
+    let head_str = fs::read_to_string(&git_head)
+        .map_err(|e| format!("unable to read {path}: {e}", path = git_head.display()))?;
+    if head_str.starts_with("ref: ") {
+        let git_ref = dir.join(".git").join(head_str["ref: ".len()..].trim_end());
+        let ref_str = fs::read_to_string(&git_ref)
+            .map_err(|e| format!("unable to read {path}: {e}", path = git_ref.display()))?;
+        Ok((ref_str.trim().to_string(), false))
+    } else {
+        Ok((head_str.trim().to_string(), true))
+    }
+}
+
+/// get commit from "rev" which either a full commit hash or a tag name
+pub fn get_git_tag_rev(dir: &PathBuf, tag: &str) -> Result<String, String> {
+    if tag.len() == 40 && tag.chars().all(|f| f.is_ascii_hexdigit()) {
+        return Ok(tag.to_string());
+    }
+    let git_refs = dir.join(".git/packed-refs");
+    let refs_str = fs::read_to_string(&git_refs)
+        .map_err(|e| format!("unable to read {path}: {e}", path = git_refs.display()))?;
+    let expected_comment_part = format!("refs/tags/{tag}");
+    for line in refs_str.lines() {
+        if line.contains(&expected_comment_part) {
+            let sha = line
+                .split_whitespace()
+                .next()
+                .ok_or_else(|| "packed-refs line is malformed.".to_string())?;
+
+            return Ok(sha.to_string());
+        }
+    }
+
+    Err(format!(
+        "Could not find a rev tag for {}",
+        expected_comment_part
+    ))
+}
+
+/// get commit rev after fetch
+pub fn get_git_fetch_rev(
+    dir: &PathBuf,
+    remote_url: &str,
+    remote_branch: &str,
+) -> Result<String, String> {
+    let git_fetch_head = dir.join(".git/FETCH_HEAD");
+
+    let fetch_head_content = fs::read_to_string(&git_fetch_head).map_err(|e| {
+        format!(
+            "unable to read {path}: {e}",
+            path = git_fetch_head.display()
+        )
+    })?;
+
+    let expected_comment_part = format!("branch '{}' of {}", remote_branch, remote_url);
+
+    for line in fetch_head_content.lines() {
+        if line.contains(&expected_comment_part) && !line.contains("not-for-merge") {
+            let sha = line
+                .split_whitespace()
+                .next()
+                .ok_or_else(|| "FETCH_HEAD line is malformed.".to_string())?;
+
+            return Ok(sha.to_string());
+        }
+    }
+
+    Err(format!(
+        "Could not find a fetch target for tracking {}",
+        expected_comment_part
+    ))
+}
+
+/// (local_branch_name, remote_branch, remote_name, remote_url)
+///    -> ("fix_stuff", "master", "origin", "https://gitlab.redox-os.org/willnode/redox")
+pub fn get_git_remote_tracking(dir: &PathBuf) -> Result<(String, String, String, String), String> {
+    let git_head = dir.join(".git/HEAD");
+    let git_config = dir.join(".git/config");
+
+    let head_content = fs::read_to_string(&git_head)
+        .map_err(|e| format!("unable to read {path}: {e}", path = git_head.display()))?;
+
+    if !head_content.starts_with("ref: ") {
+        let sha = head_content.trim_end().to_string();
+        return Ok((sha, "".to_string(), "".to_string(), "".to_string()));
+    }
+
+    let local_branch_path = head_content["ref: ".len()..].trim_end();
+    let local_branch_name = get_git_branch_name(local_branch_path)?;
+
+    let config_content = fs::read_to_string(&git_config)
+        .map_err(|e| format!("unable to read {path}: {e}", path = git_config.display()))?;
+
+    let branch_section = format!("[branch \"{}\"]", local_branch_name);
+    let mut remote_name: Option<String> = None;
+    let mut remote_branch: Option<String> = None;
+    let mut parsing_branch_section = false;
+
+    for line in config_content.lines().map(|l| l.trim()) {
+        if line.is_empty() {
+            continue;
+        }
+
+        if line == branch_section {
+            parsing_branch_section = true;
+            continue;
+        }
+
+        if parsing_branch_section {
+            if line.starts_with('[') {
+                break;
+            }
+            if line.starts_with("remote = ") {
+                remote_name = Some(line["remote = ".len()..].trim().to_string());
+            }
+            if line.starts_with("merge = ") {
+                remote_branch = Some(get_git_branch_name(line["merge = ".len()..].trim())?);
+            }
+        }
+    }
+
+    let remote_name_str = remote_name
+        .ok_or_else(|| format!("Branch '{}' is not tracking a remote.", local_branch_name))?;
+    let remote_branch_str = remote_branch.unwrap_or("".into());
+
+    let remote_section = format!("[remote \"{}\"]", remote_name_str);
+    let mut remote_url: Option<String> = None;
+    let mut parsing_remote_section = false;
+
+    for line in config_content.lines().map(|l| l.trim()) {
+        if line.is_empty() {
+            continue;
+        }
+
+        if line == remote_section {
+            parsing_remote_section = true;
+            continue;
+        }
+
+        if parsing_remote_section {
+            if line.starts_with('[') {
+                break;
+            }
+            if line.starts_with("url = ") {
+                let mut url = line["url = ".len()..].trim();
+                url = chop_dot_git(url);
+                remote_url = Some(url.to_string());
+            }
+        }
+    }
+
+    let remote_url_str = remote_url.ok_or_else(|| {
+        format!(
+            "Could not find URL for remote '{}' in .git/config.",
+            remote_name_str
+        )
+    })?;
+
+    Ok((
+        local_branch_name,
+        remote_branch_str,
+        remote_name_str,
+        remote_url_str,
+    ))
+}
+
+pub(crate) fn chop_dot_git(url: &str) -> &str {
+    if url.ends_with(".git") {
+        return &url[..url.len() - ".git".len()];
+    }
+    url
+}
+
+fn get_git_branch_name(local_branch_path: &str) -> Result<String, String> {
+    // TODO: incorrectly handle branch with slashes
+    Ok(local_branch_path
+        .split('/')
+        .last()
+        .ok_or_else(|| format!("Failed to parse branch name of {:?}", local_branch_path))?
+        .to_string())
+}

src/cook/ident.rs

@@ -0,0 +1,46 @@
+use std::{
+    process::{Command, Stdio},
+    sync::OnceLock,
+};
+
+#[derive(Debug, Default)]
+pub struct IdentifierConfig {
+    pub commit: String,
+    pub time: String,
+}
+
+impl IdentifierConfig {
+    fn new() -> Self {
+        let (commit, _) = crate::cook::fs::get_git_head_rev(
+            &std::env::current_dir().expect("unable to get $PWD"),
+        )
+        .unwrap_or(("".into(), false));
+        // better than importing heavy deps like chrono
+        let time = String::from_utf8_lossy(
+            &Command::new("date")
+                .arg("-u")
+                .arg("+%Y-%m-%dT%H:%M:%SZ")
+                .stdout(Stdio::piped())
+                .output()
+                .expect("Failed to get current ISO-formatted time")
+                .stdout
+                .trim_ascii(),
+        )
+        .into();
+        IdentifierConfig { commit, time }
+    }
+}
+
+static IDENTIFIER_CONFIG: OnceLock<IdentifierConfig> = OnceLock::new();
+
+pub fn get_ident() -> &'static IdentifierConfig {
+    IDENTIFIER_CONFIG
+        .get()
+        .expect("Identifier is not initialized")
+}
+
+pub fn init_ident() {
+    IDENTIFIER_CONFIG
+        .set(IdentifierConfig::new())
+        .expect("Identifier is initialized twice")
+}

src/cook/package.rs

@@ -7,27 +7,26 @@ use pkg::{Package, PackageName};
 
 use crate::{
     blake3::hash_to_hex,
-    cook::{fs::*, pty::PtyOut},
+    cook::{fetch, fs::*, pty::PtyOut},
     log_to_pty,
-    recipe::{BuildKind, OptionalPackageRecipe, Recipe},
+    recipe::{BuildKind, CookRecipe, OptionalPackageRecipe, Recipe},
 };
 
 pub fn package(
+    recipe: &CookRecipe,
     stage_dirs: &Vec<PathBuf>,
-    target_dir: &Path,
-    name: &PackageName,
-    recipe: &Recipe,
     auto_deps: &BTreeSet<PackageName>,
     logger: &PtyOut,
 ) -> Result<(), String> {
-    if recipe.build.kind == BuildKind::None {
+    let name = &recipe.name;
+    let target_dir = &recipe.target_dir();
+    if recipe.recipe.build.kind == BuildKind::None {
         // metapackages don't have stage dir and optional packages
         package_toml(
             target_dir.join("stage.toml"),
-            name,
             recipe,
             None,
-            recipe.package.dependencies.clone(),
+            recipe.recipe.package.dependencies.clone(),
             &auto_deps,
         )?;
         return Ok(());
@@ -50,7 +49,7 @@ pub fn package(
 
     let stage_modified = modified_all(stage_dirs, modified_dir)?;
 
-    let packages = recipe.get_packages_list();
+    let packages = recipe.recipe.get_packages_list();
 
     for package in packages {
         let (stage_dir, package_file, package_meta) = package_stage_paths(package, target_dir);
@@ -96,11 +95,10 @@ pub fn package(
                         }
                     })
                     .collect(),
-                None => recipe.package.dependencies.clone(),
+                None => recipe.recipe.package.dependencies.clone(),
             };
             package_toml(
                 package_meta,
-                &name,
                 recipe,
                 Some((Path::new(public_path), &package_file)),
                 package_deps,
@@ -114,8 +112,7 @@ pub fn package(
 
 pub fn package_toml(
     toml_path: PathBuf,
-    name: &PackageName,
-    recipe: &Recipe,
+    recipe: &CookRecipe,
     package_file: Option<(&Path, &PathBuf)>,
     mut package_deps: Vec<PackageName>,
     auto_deps: &BTreeSet<PackageName>,
@@ -148,15 +145,21 @@ pub fn package_toml(
         ("".into(), 0)
     };
 
+    let ident_source = fetch::fetch_get_source_info(recipe)?;
+
     let package = Package {
-        name: name.without_host(),
-        version: package_version(recipe),
-        target: package_target(name).to_string(),
+        name: recipe.name.without_host(),
+        version: package_version(&recipe.recipe),
+        target: recipe.target.to_string(),
         blake3: hash,
         // this size will be different once pkgar supports compression
         network_size: size,
         storage_size: size,
         depends: package_deps,
+        commit_identifier: ident_source.commit_identifier,
+        source_identifier: ident_source.source_identifier,
+        time_identifier: ident_source.time_identifier,
+        ..Default::default()
     };
 
     serialize_and_write(&toml_path, &package)?;