diff --git a/mk/config.mk b/mk/config.mk
index d985c3964..8b1b3ed12 100644
--- a/mk/config.mk
+++ b/mk/config.mk
@@ -61,6 +61,7 @@ ifneq ($(PODMAN_BUILD),1)
 FSTOOLS_IN_PODMAN=0
 HOST_TARGET := $(shell env -u RUSTUP_TOOLCHAIN rustc -vV | grep host | cut -d: -f2 | tr -d " ")
 # x86_64 linux hosts have all toolchains
+ifeq ($(PREFIX_BINARY),1)
 ifneq ($(HOST_TARGET),x86_64-unknown-linux-gnu)
 	ifeq ($(ARCH),aarch64)
 		# aarch64 linux hosts have aarch64 toolchain
@@ -74,6 +75,7 @@ ifneq ($(HOST_TARGET),x86_64-unknown-linux-gnu)
 	endif
 endif
 endif
+endif
 
 ifeq ($(SCCACHE_BUILD),1)
 ifeq (,$(shell command -v sccache))