From 2507029d2668e6c520a4b017f52865bb094e6701 Mon Sep 17 00:00:00 2001 From: MJ Pooladkhay Date: Tue, 27 Oct 2026 21:05:01 +0000 Subject: [PATCH] add 'make ch' that boots Redox via Cloud Hypervisor Signed-off-by: MJ Pooladkhay --- .gitignore | 2 + Makefile | 2 + mk/ch.mk | 100 +++++++++++ mk/qemu.mk | 42 +---- mk/virtiofs.mk | 42 +++++ scripts/ch-tap-setup.sh | 365 ++++++++++++++++++++++++++++++++++++++++ scripts/passt-run.sh | 96 +++++++++++ scripts/virtiofs-run.sh | 122 ++++++++------ 8 files changed, 679 insertions(+), 92 deletions(-) create mode 100644 mk/ch.mk create mode 100644 mk/virtiofs.mk create mode 100755 scripts/ch-tap-setup.sh create mode 100755 scripts/passt-run.sh diff --git a/.gitignore b/.gitignore index 628d9bba1..cf80a1bcb 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ source.tar source.tar.tmp target wget-log +# Cloud Hypervisor firmware, supplied by the user (see mk/ch.mk) +CLOUDHV.fd diff --git a/Makefile b/Makefile index 0b65e7b0d..120cdd010 100644 --- a/Makefile +++ b/Makefile @@ -96,8 +96,10 @@ include mk/repo.mk include mk/disk.mk # Emulation recipes +include mk/virtiofs.mk include mk/qemu.mk include mk/virtualbox.mk +include mk/ch.mk # CI include mk/ci.mk diff --git a/mk/ch.mk b/mk/ch.mk new file mode 100644 index 000000000..14c00a349 --- /dev/null +++ b/mk/ch.mk @@ -0,0 +1,100 @@ +# Configuration file for Cloud Hypervisor +# +# make ch # boot $(DISK), serial console on this terminal +# make ch share=... # also share host directories, see mk/virtiofs.mk +# make ch net=tap # use the tap device $(CH_TAP) instead of passt +# make ch net=no # no network device +# make ch-shutdown # stop the VM, from another terminal +# +# x86_64 only. The guest boots through the CLOUDHV.fd UEFI firmware +# (https://github.com/cloud-hypervisor/edk2/releases), taken from the repo +# root unless CH_FIRMWARE is set. The disks are virtio-blk. +# +# The serial console puts this terminal in raw mode, so Ctrl-C goes to Redox. +# Redox cannot power off Cloud Hypervisor by itself yet: use make ch-shutdown. +# +# The network is passt by default: a user-mode network stack, started by +# scripts/passt-run.sh, that answers the guest's DHCP and DNS and carries its +# traffic through ordinary host sockets. No tap, root or firewall changes are +# needed. The guest gets the host's address, and nothing reaches it from +# outside unless forwarded. CH_PASST_FLAGS passes options to passt, e.g. +# "-t 8022:22" (host port 8022 to guest port 22) or +# "--pcap $XDG_RUNTIME_DIR/redox.pcap" (capture the traffic). +# +# net=tap uses the tap device $(CH_TAP) instead: a real link that the host +# can reach on any port and watch with tcpdump. Creating it needs root: +# scripts/ch-tap-setup.sh (firewalld only) creates it and makes the host +# the guest's DHCP server and NAT gateway. + +CH?=$(shell command -v cloud-hypervisor || command -v ch) +CH_REMOTE?=ch-remote +CH_FIRMWARE?=$(wildcard CLOUDHV.fd) +CH_SMP?=4 +CH_MEM?=2048 +CH_API_SOCK?=$(BUILD)/ch.sock +CH_LOG?=$(BUILD)/ch.log +CH_TAP?=redox0 +CH_MAC?=52:54:00:12:34:56 +# Not in $(BUILD): SELinux limits passt to $XDG_RUNTIME_DIR and /tmp. +CH_PASST_SOCK?=$(or $(XDG_RUNTIME_DIR),/tmp)/redox-passt-$(ARCH)-$(CONFIG_NAME).sock +CH_PASST_FLAGS?= +CHFLAGS?=-v --log-file $(CH_LOG) + +CHFLAGS+=--cpus boot=$(CH_SMP) +CHFLAGS+=--firmware $(CH_FIRMWARE) +CHFLAGS+=--serial tty --console off +CHFLAGS+=--api-socket path=$(CH_API_SOCK) +CHFLAGS+= \ + --disk path=$(DISK),image_type=raw,serial=NVME_SERIAL \ + --disk path=$(EXTRA_DISK),image_type=raw,serial=NVME_EXTRA + +PASST_RUN= + +ifeq ($(net),tap) + # The Redox virtio-net driver only acknowledges VIRTIO_NET_F_MAC at the moment. + CHFLAGS+=--net tap=$(CH_TAP),mac=$(CH_MAC),offload_tso=off,offload_ufo=off,offload_csum=off +else ifneq ($(net),no) + CHFLAGS+=--net vhost_user=true,socket=$(CH_PASST_SOCK),mac=$(CH_MAC) + PASST_RUN=scripts/passt-run.sh "$(CH_PASST_SOCK)" $(CH_PASST_FLAGS) -- +endif + +ch_tap_found=$(shell ip link show "$(CH_TAP)" >/dev/null 2>&1 && echo yes) + +# vhost-user backends (virtiofsd, passt) need shared guest RAM. +ifneq ($(strip $(share))$(PASST_RUN),) + CHFLAGS+=--memory size=$(CH_MEM)M,shared=on +else + CHFLAGS+=--memory size=$(CH_MEM)M +endif + +ifneq ($(strip $(share)),) + CHFLAGS+=$(foreach i,$(share_idx),--fs tag=$(call vfs_tag,$(i)),socket=$(call vfs_sock,$(i))) +endif + +.PHONY: ch ch-check ch-deps ch-shutdown + +ch-check: + $(if $(filter x86_64,$(ARCH)),,$(error Cloud Hypervisor: only ARCH=x86_64 is supported)) + $(if $(CH),,$(error cloud-hypervisor not found. Get cloud-hypervisor-static from https://github.com/cloud-hypervisor/cloud-hypervisor/releases and put it in PATH or set CH=/path/to/cloud-hypervisor)) + $(if $(wildcard $(CH_FIRMWARE)),,$(error CLOUDHV.fd not found. Get it from https://github.com/cloud-hypervisor/edk2/releases and put it in the repo root or set CH_FIRMWARE=/path/to/CLOUDHV.fd)) + $(if $(filter tap,$(net)),$(if $(ch_tap_found),,$(error no tap device $(CH_TAP). Create it with scripts/ch-tap-setup.sh or leave out net=tap to use passt))) + +ch-deps: $(DISK) $(EXTRA_DISK) + +ch: ch-check ch-deps +ifneq ($(wildcard $(MOUNT_DIR)),) + @echo "Disk is mounted. Unmounting the disk..." + $(MAKE) unmount +endif + @if $(CH_REMOTE) --api-socket "$(CH_API_SOCK)" ping >/dev/null 2>&1; then \ + echo "ch: a VM is already running on $(CH_API_SOCK); stop it with make ch-shutdown" >&2; \ + exit 1; \ + fi +# A leftover socket file makes Cloud Hypervisor fail with "Address in use". + rm -f "$(CH_API_SOCK)" + $(VIRTIOFS_RUN) $(PASST_RUN) $(CH) $(CHFLAGS) + +ch-shutdown: + $(if $(shell command -v $(CH_REMOTE)),,$(error ch-remote not found. Get ch-remote-static from https://github.com/cloud-hypervisor/cloud-hypervisor/releases and put it in PATH or set CH_REMOTE=/path/to/ch-remote)) +# shutdown alone would leave Cloud Hypervisor running. + $(CH_REMOTE) --api-socket "$(CH_API_SOCK)" shutdown-vmm diff --git a/mk/qemu.mk b/mk/qemu.mk index f6f4cac3b..f1760e544 100644 --- a/mk/qemu.mk +++ b/mk/qemu.mk @@ -189,43 +189,8 @@ else QEMUFLAGS+=-machine $(QEMU_MACHINE) endif -# Share host directories with the guest through virtio-fs. -# make qemu share=/path/to/dir # one share, tag is "host" by default -# make qemu share="tag=/path/to/dir" # one share, tag is specified as "tag" -# make qemu share=/path/a share_tag=tag # one share, tag is specified as "tag" -# make qemu share="tag_1=/path/a tag_2=/path/b" # several shares: tag=dir ... -# make qemu share=redox # this build tree, tag "redox" -# make qemu share="redox=/other/path" # ordinary share, tag "redox" -# -# Each share gets its own host-side virtiofsd daemon (started and stopped by -# scripts/virtiofs-run.sh), a vhost-user-fs-pci device, and a virtio-fsd scheme -# process on Redox. In the guest, a share appears at /scheme/virtio-fsd.. -# -# WARNING: share=redox exposes the whole build tree to the guest, including -# the harddrive.img Redox is booted from, .git and .config. Writing to them -# from Redox can corrupt the build system. -share?= -share_tag?=host -share_sockdir?=$(BUILD) -QEMU_RUN= +# virtio-fs shares (share=...): the list is parsed in mk/virtiofs.mk. ifneq ($(strip $(share)),) - share_norm:=$(foreach s,$(share),$(if $(filter redox,$(s)),redox=$(ROOT),$(s))) - # Normalise to a list of tag=dir entries. - ifeq ($(words $(share_norm)),1) - share_list:=$(if $(findstring =,$(share_norm)),$(share_norm),$(share_tag)=$(share_norm)) - else - share_list:=$(share_norm) - share_bare:=$(foreach s,$(share_list),$(if $(findstring =,$(s)),,$(s))) - ifneq ($(strip $(share_bare)),) - # Keep the next line free of leading tabs: make would read it as a recipe. -$(error share: every entry must be tag=dir when sharing several directories (bad: $(share_bare)); directories containing spaces are not supported) - endif - endif - share_idx:=$(shell seq 1 $(words $(share_list))) - vfs_entry=$(word $(1),$(share_list)) - vfs_tag=$(firstword $(subst =, ,$(call vfs_entry,$(1)))) - vfs_sock=$(share_sockdir)/virtiofsd-$(1).sock - # vhost-user needs guest RAM to be shared with the virtiofsd processes, and # the NUMA memdev must be exactly as large as -m. One backend serves all # shares. @@ -236,7 +201,6 @@ $(error share: every entry must be tag=dir when sharing several directories (bad endif QEMUFLAGS+=-numa node,memdev=mem QEMUFLAGS+=$(foreach i,$(share_idx),-chardev socket,id=vfs$(i),path=$(call vfs_sock,$(i)) -device vhost-user-fs-pci,queue-size=1024,chardev=vfs$(i),tag=$(call vfs_tag,$(i))) - QEMU_RUN=scripts/virtiofs-run.sh $(foreach i,$(share_idx),--share "$(call vfs_sock,$(i))" "$(call vfs_entry,$(i))") -- endif ifeq ($(audio),no) @@ -433,9 +397,9 @@ $(BUILD)/qemu_uboot.rom: qemu: qemu-deps ifeq ($(wildcard $(MOUNT_DIR)),) - $(QEMU_RUN) $(QEMU) $(QEMUFLAGS) + $(VIRTIOFS_RUN) $(QEMU) $(QEMUFLAGS) else @echo "\033[1;38;5;196mDisk is mounted. Unmounting the disk...\033[0m"; $(MAKE) unmount - $(QEMU_RUN) $(QEMU) $(QEMUFLAGS) + $(VIRTIOFS_RUN) $(QEMU) $(QEMUFLAGS) endif diff --git a/mk/virtiofs.mk b/mk/virtiofs.mk new file mode 100644 index 000000000..f86517bfa --- /dev/null +++ b/mk/virtiofs.mk @@ -0,0 +1,42 @@ +# Share host directories with the guest through virtio-fs (QEMU and Cloud Hypervisor). +# make qemu share=/path/to/dir # one share, tag is "host" by default +# make qemu share="tag=/path/to/dir" # one share, tag is specified as "tag" +# make qemu share=/path/a share_tag=tag # one share, tag is specified as "tag" +# make qemu share="tag_1=/path/a tag_2=/path/b" # several shares: tag=dir ... +# make qemu share=redox # this build tree, tag "redox" +# make qemu share="redox=/other/path" # ordinary share, tag "redox" +# make ch share=... # the same, on Cloud Hypervisor +# +# Each share gets its own host-side virtiofsd daemon (started and stopped by +# scripts/virtiofs-run.sh), a vhost-user-fs device, and a virtio-fsd scheme +# process on Redox. In the guest, a share appears at /scheme/virtio-fsd.. +# The emulator flags for each share are added in mk/qemu.mk and mk/ch.mk. +# +# WARNING: share=redox exposes the whole build tree to the guest, including +# the harddrive.img Redox is booted from, .git and .config. Writing to them +# from Redox can corrupt the build system. + +share?= +share_tag?=host +share_sockdir?=$(BUILD) +VIRTIOFS_RUN= + +ifneq ($(strip $(share)),) + share_norm:=$(foreach s,$(share),$(if $(filter redox,$(s)),redox=$(ROOT),$(s))) + ifeq ($(words $(share_norm)),1) + share_list:=$(if $(findstring =,$(share_norm)),$(share_norm),$(share_tag)=$(share_norm)) + else + share_list:=$(share_norm) + share_bare:=$(foreach s,$(share_list),$(if $(findstring =,$(s)),,$(s))) + ifneq ($(strip $(share_bare)),) + # Keep the next line free of leading tabs: make would read it as a recipe. +$(error share: every entry must be tag=dir when sharing several directories (bad: $(share_bare)); directories containing spaces are not supported) + endif + endif + share_idx:=$(shell seq 1 $(words $(share_list))) + vfs_entry=$(word $(1),$(share_list)) + vfs_tag=$(firstword $(subst =, ,$(call vfs_entry,$(1)))) + vfs_sock=$(share_sockdir)/virtiofsd-$(1).sock + + VIRTIOFS_RUN=scripts/virtiofs-run.sh $(foreach i,$(share_idx),--share "$(call vfs_sock,$(i))" "$(call vfs_entry,$(i))") -- +endif diff --git a/scripts/ch-tap-setup.sh b/scripts/ch-tap-setup.sh new file mode 100755 index 000000000..07a5faa75 --- /dev/null +++ b/scripts/ch-tap-setup.sh @@ -0,0 +1,365 @@ +#!/usr/bin/env bash +# Host setup for `make ch net=tap`: tap redox0, IPv4 forwarding, firewalld NAT +# and dnsmasq DHCP/DNS. Needs firewalld and dnsmasq. Re-run after a reboot. +# As root without sudo, set TAP_USER=. + +set -euo pipefail + +# Debian does not put the sbin directories in a normal user's PATH. +PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin" + +TAP=redox0 +GUEST_MAC=52:54:00:12:34:56 +TAP_ADDR=192.168.249.1 +TAP_CIDR="${TAP_ADDR}/24" +DHCP_FIRST=192.168.249.10 +DHCP_LAST=192.168.249.50 +DHCP_RANGE="${DHCP_FIRST},${DHCP_LAST},255.255.255.0,12h" +ZONE=chnet +POLICY=chnet-nat +SYSCTL_DROPIN=/etc/sysctl.d/90-redox-tap.conf +DNSMASQ_PID=/run/redox0-dnsmasq.pid +WAIT_TIMEOUT=3 # seconds + +step() { + printf '\n==> %s\n' "$*" +} + +info() { + printf ' %s\n' "$*" +} + +die() { + echo "error: $*" >&2 + exit 1 +} + +as_root() { + if [[ "${EUID}" -eq 0 ]]; then + "$@" + else + sudo "$@" + fi +} + +wait_until() { + local deadline=$((SECONDS + $1)) + shift + until "$@"; do + ((SECONDS < deadline)) || return 1 + sleep 0.1 + done +} + +# Only tun/tap devices have tun_flags. IFF_TAP is 0x0002, IFF_UP is 0x1, +# owner -1 means none. +tap_exists() { + [[ -e "/sys/class/net/${TAP}" ]] +} + +is_tap() { + [[ -r "/sys/class/net/${TAP}/tun_flags" ]] \ + && (($(<"/sys/class/net/${TAP}/tun_flags") & 0x0002)) +} + +tap_owner() { + cat "/sys/class/net/${TAP}/owner" +} + +link_is_up() { + (($(<"/sys/class/net/${TAP}/flags") & 0x1)) +} + +tap_addrs() { + ip -o -4 addr show dev "${TAP}" | awk '{ print $4 }' +} + +has_addr() { + tap_addrs | grep -qx "${TAP_CIDR}" +} + +forwarding_on() { + [[ "$(sysctl -n net.ipv4.ip_forward)" == "1" ]] +} + +forwarding_persisted_by() { + grep -RlsE '^[[:space:]]*net\.ipv4\.ip_forward[[:space:]]*=[[:space:]]*1[[:space:]]*(#.*)?$' \ + /etc/sysctl.conf /etc/sysctl.d /usr/lib/sysctl.d | head -n 1 +} + +FW_CHANGED=0 + +fw_ensure_exists() { + local kind="$1" name="$2" list + case "${kind}" in + zone) list=--get-zones ;; + policy) list=--get-policies ;; + esac + if firewall-cmd --permanent "${list}" | tr ' ' '\n' | grep -qx "${name}"; then + info "${kind} ${name}: exists" + else + info "${kind} ${name}: creating" + as_root firewall-cmd --permanent "--new-${kind}=${name}" >/dev/null + FW_CHANGED=1 + fi +} + +fw_ensure() { + local what="$1" scope="$2" query="$3" add="$4" + if firewall-cmd --permanent "${scope}" "${query}" >/dev/null; then + info "${what}: already set" + else + info "${what}: adding" + as_root firewall-cmd --permanent "${scope}" "${add}" >/dev/null + FW_CHANGED=1 + fi +} + +# --get-target is permanent-only. The runtime value is the target: line. +policy_accepts() { + [[ "$(firewall-cmd --policy="${POLICY}" --list-all | awk '$1 == "target:" { print $2 }')" == "ACCEPT" ]] +} + +dnsmasq_pids() { + pgrep -f "(^|/)dnsmasq .*--interface=${TAP}( |$)" || true +} + +# dnsmasq binds DHCP to 0.0.0.0%redox0:67 but DNS to 192.168.249.1:53. +dhcp_listening() { + ss -H -uln | awk -v t="${TAP}" '$4 ~ ("%" t ":67$") { f = 1 } END { exit !f }' +} + +dns_listening() { + ss -H -uln | awk -v a="${TAP_ADDR}" -v t="${TAP}" \ + '$4 == (a ":53") || $4 == (a "%" t ":53") { f = 1 } END { exit !f }' +} + +dnsmasq_answering() { + dhcp_listening && dns_listening +} + +dhcp_port_free() { + ! dhcp_listening +} + +preflight() { + if [[ -z "${TAP_USER:-}" ]]; then + if [[ "${EUID}" -eq 0 ]]; then + TAP_USER="${SUDO_USER:-}" + else + TAP_USER="$(id -un)" + fi + fi + if [[ -z "${TAP_USER}" || "${TAP_USER}" == "root" ]]; then + die "run this as the user that starts cloud-hypervisor (sudo from that user is fine), or set TAP_USER=." + fi + TAP_UID="$(id -u "${TAP_USER}")" || die "no such user: ${TAP_USER}" + + if ! command -v firewall-cmd >/dev/null 2>&1 || ! firewall-cmd --state >/dev/null 2>&1; then + die "firewalld is not running. This script only supports firewalld; without it, use plain 'make ch', whose passt network needs no tap." + fi + local cmd + for cmd in ip sysctl dnsmasq ss pgrep; do + command -v "${cmd}" >/dev/null 2>&1 || die "missing command: ${cmd}" + done + + echo "Setting up ${TAP} for 'make ch net=tap': host ${TAP_CIDR}, guest DHCP pool ${DHCP_FIRST}-${DHCP_LAST}." + if [[ "${EUID}" -ne 0 ]]; then + echo "Privileged steps run through sudo, which may ask you to authenticate." + fi +} + +setup_tap() { + step "Tap device ${TAP}" + TAP_IS_NEW=0 + if tap_exists; then + is_tap || die "${TAP} exists but is not a tap device. Remove or rename it first." + local owner + owner="$(tap_owner)" + if [[ "${owner}" == "${TAP_UID}" ]]; then + info "exists and is owned by ${TAP_USER} (uid ${TAP_UID}), keeping it" + return + fi + if [[ "${owner}" == "-1" ]]; then + info "exists but has no owner, not ${TAP_USER} (uid ${TAP_UID}): deleting it" + else + info "exists but is owned by uid ${owner}, not ${TAP_USER} (uid ${TAP_UID}): deleting it" + fi + as_root ip link del "${TAP}" + else + info "does not exist" + fi + info "creating it, owned by ${TAP_USER} so cloud-hypervisor can open it without root" + as_root ip tuntap add "${TAP}" mode tap user "${TAP_USER}" + TAP_IS_NEW=1 +} + +setup_address() { + step "Host address ${TAP_CIDR} on ${TAP} (the guest's gateway and DNS server)" + local addr + for addr in $(tap_addrs); do + [[ "${addr}" == "${TAP_CIDR}" ]] || die "${TAP} already has ${addr}; this script expects ${TAP_CIDR}." + done + if has_addr; then + info "already set" + else + info "adding it" + as_root ip addr add "${TAP_CIDR}" dev "${TAP}" + fi + if link_is_up; then + info "link is already up" + else + info "setting the link up" + as_root ip link set "${TAP}" up + fi +} + +setup_forwarding() { + step "IPv4 forwarding (routes guest packets out of the host)" + if forwarding_on; then + info "net.ipv4.ip_forward is already 1" + else + info "setting net.ipv4.ip_forward=1" + as_root sysctl -q -w net.ipv4.ip_forward=1 + fi + + local persist + persist="$(forwarding_persisted_by || true)" + if [[ -n "${persist}" ]]; then + info "kept across reboots by ${persist}" + else + info "writing ${SYSCTL_DROPIN} so it survives a reboot" + printf 'net.ipv4.ip_forward = 1\n' | as_root tee "${SYSCTL_DROPIN}" >/dev/null + fi +} + +setup_firewalld() { + step "firewalld zone ${ZONE} (lets the guest reach dnsmasq on the host)" + fw_ensure_exists zone "${ZONE}" + fw_ensure "interface ${TAP}" --zone="${ZONE}" --query-interface="${TAP}" --add-interface="${TAP}" + fw_ensure "service dhcp (UDP 67)" --zone="${ZONE}" --query-service=dhcp --add-service=dhcp + fw_ensure "service dns (port 53)" --zone="${ZONE}" --query-service=dns --add-service=dns + + step "firewalld policy ${POLICY} (forwards and masquerades guest traffic to the outside)" + fw_ensure_exists policy "${POLICY}" + fw_ensure "ingress zone ${ZONE}" --policy="${POLICY}" --query-ingress-zone="${ZONE}" --add-ingress-zone="${ZONE}" + fw_ensure "egress zone ANY" --policy="${POLICY}" --query-egress-zone=ANY --add-egress-zone=ANY + if [[ "$(firewall-cmd --permanent --policy="${POLICY}" --get-target)" == "ACCEPT" ]]; then + info "target ACCEPT: already set" + else + info "target ACCEPT: setting" + as_root firewall-cmd --permanent --policy="${POLICY}" --set-target=ACCEPT >/dev/null + FW_CHANGED=1 + fi + fw_ensure "masquerade" --policy="${POLICY}" --query-masquerade --add-masquerade + + step "Applying firewalld settings" + if [[ "${FW_CHANGED}" -eq 1 ]]; then + info "reloading firewalld to apply the permanent changes" + as_root firewall-cmd --reload >/dev/null + else + info "nothing changed, no reload needed" + fi + # firewalld does not always attach an interface that appeared after it + # started. + if firewall-cmd --zone="${ZONE}" --query-interface="${TAP}" >/dev/null; then + info "${TAP} is in zone ${ZONE} at runtime" + else + info "attaching ${TAP} to zone ${ZONE} at runtime (firewalld missed the new interface)" + as_root firewall-cmd --zone="${ZONE}" --add-interface="${TAP}" >/dev/null + fi +} + +# dnsmasq --bind-interfaces keeps the old interface index, so a new tap needs +# a new dnsmasq. +setup_dnsmasq() { + step "dnsmasq on ${TAP} (DHCP ${DHCP_FIRST}-${DHCP_LAST}, gateway and DNS ${TAP_ADDR})" + local pids pid + mapfile -t pids < <(dnsmasq_pids) + + if [[ "${TAP_IS_NEW}" -eq 0 ]] && dnsmasq_answering; then + info "already answering DHCP and DNS (pid ${pids[*]:-unknown}), keeping it" + return + fi + if [[ "${TAP_IS_NEW}" -eq 0 ]]; then + info "dnsmasq is not answering both DHCP and DNS on ${TAP}" + elif [[ ${#pids[@]} -gt 0 ]]; then + info "${TAP} was recreated, so dnsmasq (pid ${pids[*]}) is bound to the old interface" + fi + + for pid in "${pids[@]}"; do + info "stopping dnsmasq pid ${pid}" + as_root kill "${pid}" || true + done + if ! wait_until "${WAIT_TIMEOUT}" dhcp_port_free; then + ss -uln | grep "${TAP}" >&2 || true + die "UDP 67 on ${TAP} is still in use." + fi + + info "starting dnsmasq (pid file ${DNSMASQ_PID})" + # --conf-file=/dev/null: /etc/dnsmasq.conf must not add other interfaces. + as_root dnsmasq \ + --conf-file=/dev/null \ + --interface="${TAP}" \ + --bind-interfaces \ + --except-interface=lo \ + --pid-file="${DNSMASQ_PID}" \ + --dhcp-range="${DHCP_RANGE}" \ + --dhcp-option=3,"${TAP_ADDR}" \ + --dhcp-option=6,"${TAP_ADDR}" + wait_until "${WAIT_TIMEOUT}" dnsmasq_answering || true +} + +CHECK_FAILED=0 + +check() { + local what="$1" + shift + if "$@" >/dev/null 2>&1; then + info "ok ${what}" + else + info "FAILED ${what}" + CHECK_FAILED=1 + fi +} + +run_checks() { + step "Checking" + check "${TAP} has ${TAP_CIDR}" has_addr + check "net.ipv4.ip_forward is 1" forwarding_on + check "${TAP} is in zone ${ZONE}" firewall-cmd --zone="${ZONE}" --query-interface="${TAP}" + check "zone ${ZONE} allows dhcp" firewall-cmd --zone="${ZONE}" --query-service=dhcp + check "zone ${ZONE} allows dns" firewall-cmd --zone="${ZONE}" --query-service=dns + check "policy ${POLICY} target is ACCEPT" policy_accepts + check "policy ${POLICY} masquerades" firewall-cmd --policy="${POLICY}" --query-masquerade + check "dnsmasq answers DHCP on ${TAP} (UDP 67)" dhcp_listening + check "dnsmasq answers DNS on ${TAP_ADDR} (UDP 53)" dns_listening + + if [[ "${CHECK_FAILED}" -ne 0 ]]; then + echo >&2 + echo "Sockets on ${TAP}:" >&2 + ss -uln | grep -E "%${TAP}:|${TAP_ADDR}:" >&2 || true + die "setup is incomplete, see the FAILED lines above." + fi +} + +print_summary() { + echo + echo "${TAP} is ready. It shows NO-CARRIER until cloud-hypervisor opens it; that is normal." + echo "Start Redox with 'make ch net=tap'. It adds:" + echo " --net tap=${TAP},mac=${GUEST_MAC},offload_tso=off,offload_ufo=off,offload_csum=off" + echo "Then optionally run 'dhcpd -v' in Redox and watch leases happen on the host with 'journalctl -t dnsmasq-dhcp -f'." +} + +main() { + preflight + setup_tap + setup_address + setup_forwarding + setup_firewalld + setup_dnsmasq + run_checks + print_summary +} + +main "$@" diff --git a/scripts/passt-run.sh b/scripts/passt-run.sh new file mode 100755 index 000000000..ea32f0484 --- /dev/null +++ b/scripts/passt-run.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Runs a command next to a passt vhost-user network backend, for `make ch`. +# passt-run.sh .sock [passt options...] -- +# Under SELinux, passt may only use sockets in $XDG_RUNTIME_DIR or /tmp. + +set -euo pipefail + +STARTUP_TIMEOUT=5 # seconds + +die() { + echo "passt: $*" >&2 + exit 1 +} + +usage() { + echo "usage: $0 .sock [passt options...] -- " >&2 + exit 2 +} + +[[ $# -ge 1 ]] || usage +SOCK="$1" +shift +PASST_ARGS=() +while [[ $# -gt 0 && "$1" != "--" ]]; do + PASST_ARGS+=("$1") + shift +done +[[ $# -ge 2 ]] || usage +shift # the "--" + +[[ "${SOCK}" == *.sock ]] || die "socket '${SOCK}' must end in .sock" +[[ ${#SOCK} -lt 108 ]] || die "socket path too long (${#SOCK} bytes): ${SOCK}" +LOG="${SOCK%.sock}.log" +REPAIR_SOCK="${SOCK}.repair" + +PASST_BIN="${PASST:-passt}" +command -v "${PASST_BIN}" >/dev/null 2>&1 \ + || die "passt not found. Install the passt package (dnf, apt and pacman all call it passt) or set PASST=/path/to/passt" +"${PASST_BIN}" --help 2>&1 | grep -q -- '--vhost-user' \ + || die "${PASST_BIN} has no --vhost-user mode, it is too old. Install a passt from 2025 or later." + +socket_in_use() { + ss -Hxl | awk -v sock="${SOCK}" '$5 == sock { found = 1 } END { exit !found }' +} + +if socket_in_use; then + die "${SOCK} is in use by another passt. Stop that VM first, or choose another socket." +fi + +PASST_PID= + +cleanup() { + trap - EXIT INT TERM + if [[ -n "${PASST_PID}" ]]; then + kill "${PASST_PID}" 2>/dev/null || true + wait "${PASST_PID}" 2>/dev/null || true + fi + rm -f "${SOCK}" "${REPAIR_SOCK}" +} + +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +start_passt() { + mkdir -p "$(dirname "${SOCK}")" + rm -f "${SOCK}" "${REPAIR_SOCK}" + "${PASST_BIN}" --vhost-user --foreground --one-off --socket "${SOCK}" "${PASST_ARGS[@]}" \ + "${LOG}" 2>&1 & + PASST_PID=$! +} + +start_failed() { + if [[ -s "${LOG}" ]]; then + # After a bad option passt prints its whole usage text. + sed '/^Usage:/,$d' "${LOG}" | tail -n 20 >&2 + die "$1; see ${LOG}" + fi + die "$1 and logged nothing. If SELinux is enforcing, put the socket in \$XDG_RUNTIME_DIR or /tmp." +} + +wait_for_socket() { + local deadline=$((SECONDS + STARTUP_TIMEOUT)) + until [[ -S "${SOCK}" ]]; do + kill -0 "${PASST_PID}" 2>/dev/null || start_failed "passt exited" + ((SECONDS < deadline)) || start_failed "passt did not create ${SOCK} within ${STARTUP_TIMEOUT}s" + sleep 0.1 + done +} + +start_passt +wait_for_socket +echo "passt: user-mode network at ${SOCK} (log: ${LOG})" >&2 + +# Not exec'd, so cleanup runs when the command ends. +"$@" diff --git a/scripts/virtiofs-run.sh b/scripts/virtiofs-run.sh index 1d8bcebb9..16ca5b753 100755 --- a/scripts/virtiofs-run.sh +++ b/scripts/virtiofs-run.sh @@ -1,56 +1,63 @@ #!/usr/bin/env bash -# Run a command (normally QEMU) next to one virtiofsd per shared host directory. -# -# virtiofs-run.sh --share = [--share ...] -- -# -# Used by `make qemu share=...` (see mk/qemu.mk). Each virtiofsd is started in -# the background and logs to .log. All of them are stopped -# again when the command exits, fails to start, or is interrupted. -# -# virtiofsd is taken from $VIRTIOFSD, PATH, /usr/libexec, /usr/lib/qemu or -# ~/.cache/virtiofsd-redox. podman_bootstrap.sh installs it. +# Runs a command next to one virtiofsd per shared directory, for +# `make qemu share=...` and `make ch share=...`. +# virtiofs-run.sh --share .sock = [--share ...] -- set -euo pipefail +STARTUP_TIMEOUT=5 # seconds + die() { echo "virtiofs: $*" >&2 exit 1 } usage() { - echo "usage: $0 --share = [--share ...] -- " >&2 + echo "usage: $0 --share .sock = [--share ...] -- " >&2 exit 2 } SOCKS=() -SPECS=() +TAGS=() +DIRS=() while [[ "${1:-}" == --share ]]; do [[ $# -ge 3 ]] || usage + [[ "$3" == *=* ]] || die "share '$3': expected =" SOCKS+=("$2") - SPECS+=("$3") + TAGS+=("${3%%=*}") + DIRS+=("${3#*=}") shift 3 done -[[ ${#SOCKS[@]} -gt 0 && "${1:-}" == -- && $# -gt 1 ]] || usage -shift +[[ ${#SOCKS[@]} -gt 0 && "${1:-}" == -- && $# -ge 2 ]] || usage +shift # the "--" -# Validate everything before starting anything, so a mistake leaves nothing running -seen_tags=" " -for i in "${!SOCKS[@]}"; do - sock="${SOCKS[i]}" spec="${SPECS[i]}" - [[ "${spec}" == *=* ]] || die "share '${spec}': expected =" - tag="${spec%%=*}" dir="${spec#*=}" +# virtiofsd holds a lock on .pid for as long as it runs. +share_in_use() { + [[ -e "$1.pid" ]] && ! flock -n "$1.pid" true +} + +declare -A SEEN_TAGS=() +validate_share() { + local sock="${SOCKS[$1]}" tag="${TAGS[$1]}" dir="${DIRS[$1]}" + [[ -n "${tag}" ]] || die "share '=${dir}': empty tag" [[ "${tag}" =~ ^[A-Za-z0-9._-]+$ ]] \ - || die "tag '${tag}': only letters, digits, '.', '_' and '-' are allowed (is '${spec}' a directory with an '=' in it? use tag=dir)" + || die "tag '${tag}': only letters, digits, '.', '_' and '-' are allowed (is '${tag}=${dir}' a directory with an '=' in it? use tag=dir)" [[ ${#tag} -le 36 ]] || die "tag '${tag}': longer than 36 bytes" - [[ "${seen_tags}" != *" ${tag} "* ]] || die "tag '${tag}' is used twice" - seen_tags+="${tag} " + [[ -z "${SEEN_TAGS[${tag}]:-}" ]] || die "tag '${tag}' is used twice" + SEEN_TAGS[${tag}]=1 [[ -n "${dir}" ]] || die "share '${tag}': empty directory" [[ "${dir}" != *[[:space:]]* ]] \ || die "share '${tag}': directories containing whitespace are not supported ('${dir}')" [[ "${sock}" == *.sock ]] || die "socket '${sock}' must end in .sock" - # AF_UNIX sun_path is 108 bytes including the NUL [[ ${#sock} -lt 108 ]] \ || die "socket path too long (${#sock} bytes): ${sock}; use a shorter share_sockdir" + if share_in_use "${sock}"; then + die "${sock} is in use by another virtiofsd. Stop that VM first, or use another share_sockdir." + fi +} + +for i in "${!SOCKS[@]}"; do + validate_share "${i}" done find_virtiofsd() { @@ -66,32 +73,33 @@ VIRTIOFSD_BIN="$(find_virtiofsd)" \ || die "virtiofsd not found. Install it (e.g. 'dnf install virtiofsd' or 'pacman -S virtiofsd'), run ./podman_bootstrap.sh -d, or set VIRTIOFSD=/path/to/virtiofsd" PIDS=() + cleanup() { trap - EXIT INT TERM if [[ ${#PIDS[@]} -gt 0 ]]; then kill "${PIDS[@]}" 2>/dev/null || true wait "${PIDS[@]}" 2>/dev/null || true fi - rm -f "${SOCKS[@]}" "${SOCKS[@]/%/.pid}" + local sock + for sock in "${SOCKS[@]}"; do + rm -f "${sock}" "${sock}.pid" + done } + trap cleanup EXIT trap 'exit 130' INT trap 'exit 143' TERM -resolved_dirs=" " -for i in "${!SOCKS[@]}"; do - sock="${SOCKS[i]}" spec="${SPECS[i]}" - mkdir -p "${spec#*=}" "$(dirname "${sock}")" - dir="$(cd "${spec#*=}" && pwd)" - if [[ "${resolved_dirs}" == *" ${dir} "* ]]; then +declare -A SHARED_DIRS=() +start_virtiofsd() { + local sock="${SOCKS[$1]}" dir + mkdir -p "${DIRS[$1]}" "$(dirname "${sock}")" + dir="$(cd "${DIRS[$1]}" && pwd)" + if [[ -n "${SHARED_DIRS[${dir}]:-}" ]]; then echo "virtiofs: warning: ${dir} is shared more than once; the instances do not share caches" >&2 fi - resolved_dirs+="${dir} " - rm -f "${sock}" "${sock}.pid" + SHARED_DIRS[${dir}]=1 - # The guest root maps to the invoking user, so files created in the guest - # are owned by you on the host. - # --sandbox none avoids needing root or user namespaces. "${VIRTIOFSD_BIN}" \ --socket-path="${sock}" \ --shared-dir "${dir}" \ @@ -101,24 +109,32 @@ for i in "${!SOCKS[@]}"; do --translate-gid="map:0:$(id -g):1" \ "${sock%.sock}.log" 2>&1 & PIDS+=("$!") -done +} -# Wait (up to ~5 seconds per share) until every socket exists -for i in "${!SOCKS[@]}"; do - sock="${SOCKS[i]}" tag="${SPECS[i]%%=*}" log="${SOCKS[i]%.sock}.log" - for _ in $(seq 50); do - [[ -S "${sock}" ]] && break - kill -0 "${PIDS[i]}" 2>/dev/null || break - sleep 0.1 +start_failed() { + local log="${SOCKS[$1]%.sock}.log" + tail -n 20 "${log}" >&2 || true + die "virtiofsd for '${TAGS[$1]}' $2; see ${log}" +} + +wait_for_sockets() { + local deadline=$((SECONDS + STARTUP_TIMEOUT)) i + for i in "${!SOCKS[@]}"; do + until [[ -S "${SOCKS[$i]}" ]]; do + kill -0 "${PIDS[$i]}" 2>/dev/null || start_failed "${i}" "exited" + ((SECONDS < deadline)) || start_failed "${i}" "did not create ${SOCKS[$i]} within ${STARTUP_TIMEOUT}s" + sleep 0.1 + done done - if [[ ! -S "${sock}" ]]; then - echo "virtiofs: virtiofsd for '${tag}' did not create ${sock}; see ${log}" >&2 - tail -n 20 "${log}" >&2 || true - exit 1 - fi - echo "virtiofs: sharing '${tag}' = ${SPECS[i]#*=} (log: ${log})" >&2 +} + +for i in "${!SOCKS[@]}"; do + start_virtiofsd "${i}" +done +wait_for_sockets +for i in "${!SOCKS[@]}"; do + echo "virtiofs: sharing '${TAGS[$i]}' = ${DIRS[$i]} (log: ${SOCKS[$i]%.sock}.log)" >&2 done -# Run the command as a child, not with exec, so that the EXIT trap still runs -# and stops the virtiofsd processes once the command is done. +# Not exec'd, so cleanup runs when the command ends. "$@"