Automatic unfetch for tar recipes

src/bin/cook.rs

@@ -1,630 +1,24 @@
-use cookbook::blake3::blake3_progress;
-use cookbook::config::{init_config, translate_mirror};
-use cookbook::recipe::{AutoDeps, BuildKind, CookRecipe, Recipe, SourceRecipe};
+use cookbook::config::init_config;
+use cookbook::cook::fetch::*;
+use cookbook::cook::fs::*;
+use cookbook::cook::script::SHARED_PRESCRIPT;
+use cookbook::recipe::{AutoDeps, BuildKind, CookRecipe, Recipe};
 use pkg::package::Package;
 use pkg::{recipes, PackageName};
-use serde::Serialize;
 use std::collections::VecDeque;
 use std::convert::TryInto;
 use std::{
     collections::BTreeSet,
     env, fs,
-    io::{self, Write},
     path::{Path, PathBuf},
-    process::{self, Command, Stdio},
+    process::{self, Command},
     str,
     time::SystemTime,
 };
 use termion::{color, style};
-use walkdir::{DirEntry, WalkDir};
 
 use cookbook::{is_redox, WALK_DEPTH};
 
-fn remove_all(path: &Path) -> Result<(), String> {
-    if path.is_dir() {
-        fs::remove_dir_all(path)
-    } else {
-        fs::remove_file(path)
-    }
-    .map_err(|err| format!("failed to remove '{}': {}\n{:?}", path.display(), err, err))
-}
-
-fn create_dir(dir: &Path) -> Result<(), String> {
-    fs::create_dir(dir)
-        .map_err(|err| format!("failed to create '{}': {}\n{:?}", dir.display(), err, err))
-}
-
-fn create_dir_clean(dir: &Path) -> Result<(), String> {
-    if dir.is_dir() {
-        remove_all(dir)?;
-    }
-    create_dir(dir)
-}
-
-fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {
-    fs::create_dir_all(&dst)?;
-    for entry in fs::read_dir(src)? {
-        let entry = entry?;
-        let ty = entry.file_type()?;
-        if ty.is_dir() {
-            copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?;
-        } else {
-            fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?;
-        }
-    }
-    Ok(())
-}
-
-fn symlink(original: impl AsRef<Path>, link: impl AsRef<Path>) -> Result<(), String> {
-    std::os::unix::fs::symlink(&original, &link).map_err(|err| {
-        format!(
-            "failed to symlink '{}' to '{}': {}\n{:?}",
-            original.as_ref().display(),
-            link.as_ref().display(),
-            err,
-            err
-        )
-    })
-}
-
-fn modified(path: &Path) -> Result<SystemTime, String> {
-    let metadata = fs::metadata(path).map_err(|err| {
-        format!(
-            "failed to get metadata of '{}': {}\n{:#?}",
-            path.display(),
-            err,
-            err
-        )
-    })?;
-    metadata.modified().map_err(|err| {
-        format!(
-            "failed to get modified time of '{}': {}\n{:#?}",
-            path.display(),
-            err,
-            err
-        )
-    })
-}
-
-fn modified_dir_inner<F: FnMut(&DirEntry) -> bool>(
-    dir: &Path,
-    filter: F,
-) -> io::Result<SystemTime> {
-    let mut newest = fs::metadata(dir)?.modified()?;
-    for entry_res in WalkDir::new(dir).into_iter().filter_entry(filter) {
-        let entry = entry_res?;
-        let modified = entry.metadata()?.modified()?;
-        if modified > newest {
-            newest = modified;
-        }
-    }
-    Ok(newest)
-}
-
-fn modified_dir(dir: &Path) -> Result<SystemTime, String> {
-    modified_dir_inner(dir, |_| true).map_err(|err| {
-        format!(
-            "failed to get modified time of '{}': {}\n{:#?}",
-            dir.display(),
-            err,
-            err
-        )
-    })
-}
-
-fn modified_dir_ignore_git(dir: &Path) -> Result<SystemTime, String> {
-    modified_dir_inner(dir, |entry| {
-        entry
-            .file_name()
-            .to_str()
-            .map(|s| s != ".git")
-            .unwrap_or(true)
-    })
-    .map_err(|err| {
-        format!(
-            "failed to get modified time of '{}': {}\n{:#?}",
-            dir.display(),
-            err,
-            err
-        )
-    })
-}
-
-fn rename(src: &Path, dst: &Path) -> Result<(), String> {
-    fs::rename(src, dst).map_err(|err| {
-        format!(
-            "failed to rename '{}' to '{}': {}\n{:?}",
-            src.display(),
-            dst.display(),
-            err,
-            err
-        )
-    })
-}
-
-fn run_command(mut command: process::Command) -> Result<(), String> {
-    let status = command
-        .status()
-        .map_err(|err| format!("failed to run {:?}: {}\n{:#?}", command, err, err))?;
-
-    if !status.success() {
-        return Err(format!(
-            "failed to run {:?}: exited with status {}",
-            command, status
-        ));
-    }
-
-    Ok(())
-}
-
-fn run_command_stdin(mut command: process::Command, stdin_data: &[u8]) -> Result<(), String> {
-    command.stdin(Stdio::piped());
-
-    let mut child = command
-        .spawn()
-        .map_err(|err| format!("failed to spawn {:?}: {}\n{:#?}", command, err, err))?;
-
-    if let Some(ref mut stdin) = child.stdin {
-        stdin.write_all(stdin_data).map_err(|err| {
-            format!(
-                "failed to write stdin of {:?}: {}\n{:#?}",
-                command, err, err
-            )
-        })?;
-    } else {
-        return Err(format!("failed to find stdin of {:?}", command));
-    }
-
-    let status = child
-        .wait()
-        .map_err(|err| format!("failed to run {:?}: {}\n{:#?}", command, err, err))?;
-
-    if !status.success() {
-        return Err(format!(
-            "failed to run {:?}: exited with status {}",
-            command, status
-        ));
-    }
-
-    Ok(())
-}
-
-fn serialize_and_write<T: Serialize>(file_path: &Path, content: &T) -> Result<(), String> {
-    let toml_content = toml::to_string(content).map_err(|err| {
-        format!(
-            "Failed to serialize content for '{}': {}",
-            file_path.display(),
-            err
-        )
-    })?;
-
-    fs::write(file_path, toml_content)
-        .map_err(|err| format!("Failed to write to file '{}': {}", file_path.display(), err))?;
-    Ok(())
-}
-
-static SHARED_PRESCRIPT: &str = r#"
-# Build dynamically
-function DYNAMIC_INIT {
-    COOKBOOK_AUTORECONF="autoreconf"
-    autotools_recursive_regenerate() {
-        for f in $(find . -name configure.ac -o -name configure.in -type f | sort); do
-            echo "* autotools regen in '$(dirname $f)'..."
-            ( cd "$(dirname "$f")" && "${COOKBOOK_AUTORECONF}" -fvi "$@" -I${COOKBOOK_HOST_SYSROOT}/share/aclocal )
-        done
-    }
-
-    if [ "${TARGET}" != "x86_64-unknown-redox" ]
-    then
-        echo "WARN: ${TARGET} does not support dynamic linking." >&2
-        return
-    fi
-
-    echo "DEBUG: Program is being compiled dynamically."
-
-    COOKBOOK_CONFIGURE_FLAGS=(
-        --host="${GNU_TARGET}"
-        --prefix="/usr"
-        --enable-shared
-        --disable-static
-    )
-
-    COOKBOOK_CMAKE_FLAGS=(
-        -DBUILD_SHARED_LIBS=True
-        -DENABLE_SHARED=True
-        -DENABLE_STATIC=False
-    )
-
-    COOKBOOK_MESON_FLAGS=(
-        --buildtype release
-        --wrap-mode nofallback
-        --strip
-        -Ddefault_library=shared
-        -Dprefix=/usr
-    )
-
-    # TODO: check paths for spaces
-    export LDFLAGS="-Wl,-rpath-link,${COOKBOOK_SYSROOT}/lib -L${COOKBOOK_SYSROOT}/lib"
-    export RUSTFLAGS="-C target-feature=-crt-static"
-    export COOKBOOK_DYNAMIC=1
-}
-
-# Build both dynamically and statically
-function DYNAMIC_STATIC_INIT {
-    DYNAMIC_INIT
-    if [ "${COOKBOOK_DYNAMIC}" == "1" ]
-    then
-        COOKBOOK_CONFIGURE_FLAGS=(
-            --host="${GNU_TARGET}"
-            --prefix="/usr"
-            --enable-shared
-            --enable-static
-        )
-
-        COOKBOOK_CMAKE_FLAGS=(
-            -DBUILD_SHARED_LIBS=True
-            -DENABLE_SHARED=True
-            -DENABLE_STATIC=True
-        )
-
-        COOKBOOK_MESON_FLAGS=(
-            --buildtype release
-            --wrap-mode nofallback
-            --strip
-            -Ddefault_library=both
-            -Dprefix=/usr
-        )
-    fi
-}
-
-function GNU_CONFIG_GET {
-  wget -O "$1" "https://gitlab.redox-os.org/redox-os/gnu-config/-/raw/master/config.sub?inline=false"
-}
-"#;
-
-fn fetch_offline(recipe_dir: &Path, source: &Option<SourceRecipe>) -> Result<PathBuf, String> {
-    let source_dir = recipe_dir.join("source");
-    match source {
-        Some(SourceRecipe::SameAs { same_as: _ }) | Some(SourceRecipe::Path { path: _ }) | None => {
-            return fetch(recipe_dir, source);
-        }
-        Some(SourceRecipe::Git {
-            git: _,
-            upstream: _,
-            branch: _,
-            rev: _,
-            patches: _,
-            script: _,
-            shallow_clone: _,
-        })
-        | Some(SourceRecipe::Tar {
-            tar: _,
-            blake3: _,
-            patches: _,
-            script: _,
-        }) => {
-            if !source_dir.is_dir() {
-                return Err(format!(
-                    "'{dir}' is not exist and unable to continue in offline mode",
-                    dir = source_dir.display(),
-                ));
-            }
-        }
-    }
-
-    Ok(source_dir)
-}
-
-fn fetch(recipe_dir: &Path, source: &Option<SourceRecipe>) -> Result<PathBuf, String> {
-    let source_dir = recipe_dir.join("source");
-    match source {
-        Some(SourceRecipe::SameAs { same_as }) => {
-            if !source_dir.is_symlink() {
-                if source_dir.is_dir() {
-                    return Err(format!(
-                        "'{dir}' is a directory, but recipe indicated a symlink. \n\
-                        try removing '{dir}' if you haven't made any changes that would be lost",
-                        dir = source_dir.display(),
-                    ));
-                }
-                let original = Path::new(same_as).join("source");
-                std::os::unix::fs::symlink(&original, &source_dir).map_err(|err| {
-                    format!(
-                        "failed to symlink '{}' to '{}': {}\n{:?}",
-                        original.display(),
-                        source_dir.display(),
-                        err,
-                        err
-                    )
-                })?;
-            }
-        }
-        Some(SourceRecipe::Path { path }) => {
-            if !source_dir.is_dir() || modified_dir(Path::new(path))? > modified_dir(&source_dir)? {
-                eprintln!("[DEBUG]: {} is newer than {}", path, source_dir.display());
-                copy_dir_all(path, &source_dir).map_err(|e| {
-                    format!(
-                        "Couldn't copy source from {} to {}: {}",
-                        path,
-                        source_dir.display(),
-                        e
-                    )
-                })?;
-            }
-        }
-        Some(SourceRecipe::Git {
-            git,
-            upstream,
-            branch,
-            rev,
-            patches,
-            script,
-            shallow_clone,
-        }) => {
-            //TODO: use libgit?
-            let shallow_clone = *shallow_clone == Some(true);
-            if !source_dir.is_dir() {
-                // Create source.tmp
-                let source_dir_tmp = recipe_dir.join("source.tmp");
-                create_dir_clean(&source_dir_tmp)?;
-
-                // Clone the repository to source.tmp
-                let mut command = Command::new("git");
-                command
-                    .arg("clone")
-                    .arg("--recursive")
-                    .arg(translate_mirror(git));
-                if let Some(branch) = branch {
-                    command.arg("--branch").arg(branch);
-                }
-                if shallow_clone {
-                    command.arg("--depth").arg("1").arg("--shallow-submodules");
-                }
-                command.arg(&source_dir_tmp);
-                run_command(command)?;
-
-                // Move source.tmp to source atomically
-                rename(&source_dir_tmp, &source_dir)?;
-            } else if !shallow_clone {
-                // Don't let this code reset the origin for the cookbook repo
-                let source_git_dir = source_dir.join(".git");
-                if !source_git_dir.is_dir() {
-                    return Err(format!(
-                        "'{}' is not a git repository, but recipe indicated git source",
-                        source_dir.display(),
-                    ));
-                }
-
-                // Reset origin
-                let mut command = Command::new("git");
-                command.arg("-C").arg(&source_dir);
-                command.arg("remote").arg("set-url").arg("origin").arg(git);
-                run_command(command)?;
-
-                // Fetch origin
-                let mut command = Command::new("git");
-                command.arg("-C").arg(&source_dir);
-                command.arg("fetch").arg("origin");
-                run_command(command)?;
-            }
-
-            if let Some(_upstream) = upstream {
-                //TODO: set upstream URL
-                // git remote set-url upstream "$GIT_UPSTREAM" &> /dev/null ||
-                // git remote add upstream "$GIT_UPSTREAM"
-                // git fetch upstream
-            }
-
-            if let Some(rev) = rev {
-                // Check out specified revision
-                let mut command = Command::new("git");
-                command.arg("-C").arg(&source_dir);
-                command.arg("checkout").arg(rev);
-                run_command(command)?;
-            } else if !shallow_clone && !is_redox() {
-                //TODO: complicated stuff to check and reset branch to origin
-                //TODO: redox can't undestand this (got exit status 1)
-                let mut command = Command::new("bash");
-                command.arg("-c").arg(
-                    r#"
-ORIGIN_BRANCH="$(git branch --remotes | grep '^  origin/HEAD -> ' | cut -d ' ' -f 5-)"
-if [ -n "$BRANCH" ]
-then
-    ORIGIN_BRANCH="origin/$BRANCH"
-fi
-
-if [ "$(git rev-parse HEAD)" != "$(git rev-parse $ORIGIN_BRANCH)" ]
-then
-    git checkout -B "$(echo "$ORIGIN_BRANCH" | cut -d / -f 2-)" "$ORIGIN_BRANCH"
-fi"#,
-                );
-                if let Some(branch) = branch {
-                    command.env("BRANCH", branch);
-                }
-                command.current_dir(&source_dir);
-                run_command(command)?;
-            }
-
-            if !patches.is_empty() || script.is_some() {
-                // Hard reset
-                let mut command = Command::new("git");
-                command.arg("-C").arg(&source_dir);
-                command.arg("reset").arg("--hard");
-                run_command(command)?;
-            }
-
-            if !shallow_clone {
-                // Sync submodules URL
-                let mut command = Command::new("git");
-                command.arg("-C").arg(&source_dir);
-                command.arg("submodule").arg("sync").arg("--recursive");
-                run_command(command)?;
-
-                // Update submodules
-                let mut command = Command::new("git");
-                command.arg("-C").arg(&source_dir);
-                command
-                    .arg("submodule")
-                    .arg("update")
-                    .arg("--init")
-                    .arg("--recursive");
-                run_command(command)?;
-            }
-
-            // Apply patches
-            for patch_name in patches {
-                let patch_file = recipe_dir.join(patch_name);
-                if !patch_file.is_file() {
-                    return Err(format!(
-                        "failed to find patch file '{}'",
-                        patch_file.display()
-                    ));
-                }
-
-                let patch = fs::read_to_string(&patch_file).map_err(|err| {
-                    format!(
-                        "failed to read patch file '{}': {}\n{:#?}",
-                        patch_file.display(),
-                        err,
-                        err
-                    )
-                })?;
-
-                let mut command = Command::new("patch");
-                command.arg("--forward");
-                command.arg("--batch");
-                command.arg("--directory").arg(&source_dir);
-                command.arg("--strip=1");
-                run_command_stdin(command, patch.as_bytes())?;
-            }
-
-            // Run source script
-            if let Some(script) = script {
-                let mut command = Command::new("bash");
-                command.arg("-ex");
-                command.current_dir(&source_dir);
-                run_command_stdin(command, format!("{SHARED_PRESCRIPT}\n{script}").as_bytes())?;
-            }
-        }
-        Some(SourceRecipe::Tar {
-            tar,
-            blake3,
-            patches,
-            script,
-        }) => {
-            if !source_dir.is_dir() {
-                // Download tar
-                //TODO: replace wget
-                let source_tar = recipe_dir.join("source.tar");
-                if !source_tar.is_file() {
-                    let source_tar_tmp = recipe_dir.join("source.tar.tmp");
-
-                    let mut command = Command::new("wget");
-                    command.arg(translate_mirror(tar));
-                    command.arg("--continue").arg("-O").arg(&source_tar_tmp);
-                    run_command(command)?;
-
-                    // Move source.tar.tmp to source.tar atomically
-                    rename(&source_tar_tmp, &source_tar)?;
-                }
-
-                // Calculate blake3
-                let source_tar_blake3 = blake3_progress(&source_tar).map_err(|err| {
-                    format!(
-                        "failed to calculate blake3 of '{}': {}\n{:?}",
-                        source_tar.display(),
-                        err,
-                        err
-                    )
-                })?;
-                if let Some(blake3) = blake3 {
-                    // Check if it matches recipe
-                    if &source_tar_blake3 != blake3 {
-                        return Err(format!(
-                            "calculated blake3 '{}' does not match recipe blake3 '{}'",
-                            source_tar_blake3, blake3
-                        ));
-                    }
-                } else {
-                    //TODO: set blake3 hash on the recipe with something like "cook fix"
-                    eprintln!(
-                        "WARNING: set blake3 for '{}' to '{}'",
-                        source_tar.display(),
-                        source_tar_blake3
-                    );
-                }
-
-                // Create source.tmp
-                let source_dir_tmp = recipe_dir.join("source.tmp");
-                create_dir_clean(&source_dir_tmp)?;
-
-                // Extract tar to source.tmp
-                //TODO: use tar crate (how to deal with compression?)
-                let mut command = Command::new("tar");
-                if is_redox() {
-                    command.arg("xvf");
-                } else {
-                    command.arg("--extract");
-                    command.arg("--verbose");
-                    command.arg("--file");
-                }
-                command.arg(&source_tar);
-                command.arg("--directory").arg(&source_dir_tmp);
-                command.arg("--strip-components").arg("1");
-                run_command(command)?;
-
-                // Apply patches
-                for patch_name in patches {
-                    let patch_file = recipe_dir.join(patch_name);
-                    if !patch_file.is_file() {
-                        return Err(format!(
-                            "failed to find patch file '{}'",
-                            patch_file.display()
-                        ));
-                    }
-
-                    let patch = fs::read_to_string(&patch_file).map_err(|err| {
-                        format!(
-                            "failed to read patch file '{}': {}\n{:#?}",
-                            patch_file.display(),
-                            err,
-                            err
-                        )
-                    })?;
-
-                    let mut command = Command::new("patch");
-                    command.arg("--directory").arg(&source_dir_tmp);
-                    command.arg("--strip=1");
-                    run_command_stdin(command, patch.as_bytes())?;
-                }
-
-                // Run source script
-                if let Some(script) = script {
-                    let mut command = Command::new("bash");
-                    command.arg("-ex");
-                    command.current_dir(&source_dir_tmp);
-                    run_command_stdin(command, format!("{SHARED_PRESCRIPT}\n{script}").as_bytes())?;
-                }
-
-                // Move source.tmp to source atomically
-                rename(&source_dir_tmp, &source_dir)?;
-            }
-        }
-        // Local Sources
-        None => {
-            if !source_dir.is_dir() {
-                eprintln!(
-                    "WARNING: Recipe without source section expected source dir at '{}'",
-                    source_dir.display(),
-                );
-                create_dir(&source_dir)?;
-            }
-        }
-    }
-
-    Ok(source_dir)
-}
-
 fn auto_deps(
     stage_dir: &Path,
     dep_pkgars: &BTreeSet<(PackageName, PathBuf)>,

src/cook/fetch.rs

@@ -0,0 +1,417 @@
+use crate::config::translate_mirror;
+use crate::cook::fs::*;
+use crate::cook::script::*;
+use crate::is_redox;
+use crate::recipe::Recipe;
+use crate::{blake3::blake3_progress, recipe::SourceRecipe};
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+pub(crate) fn get_blake3(path: &PathBuf) -> Result<String, String> {
+    blake3_progress(&path).map_err(|err| {
+        format!(
+            "failed to calculate blake3 of '{}': {}\n{:?}",
+            path.display(),
+            err,
+            err
+        )
+    })
+}
+
+pub fn fetch_offline(recipe_dir: &Path, source: &Option<SourceRecipe>) -> Result<PathBuf, String> {
+    let source_dir = recipe_dir.join("source");
+    match source {
+        Some(SourceRecipe::Path { path: _ }) | None => {
+            return fetch(recipe_dir, source);
+        }
+        Some(SourceRecipe::SameAs { same_as: _ }) => {
+            return fetch(recipe_dir, source);
+        }
+        Some(SourceRecipe::Git {
+            git: _,
+            upstream: _,
+            branch: _,
+            rev: _,
+            patches: _,
+            script: _,
+            shallow_clone: _,
+        }) => {
+            if !source_dir.is_dir() {
+                return Err(format!(
+                    "'{dir}' is not exist and unable to continue in offline mode",
+                    dir = source_dir.display(),
+                ));
+            }
+        }
+        Some(SourceRecipe::Tar {
+            tar: _,
+            blake3,
+            patches,
+            script,
+        }) => {
+            if !source_dir.is_dir() {
+                let source_tar = recipe_dir.join("source.tar");
+                let source_tar_blake3 = get_blake3(&source_tar)?;
+                if source_tar.exists() {
+                    if let Some(blake3) = blake3 {
+                        if source_tar_blake3 != *blake3 {
+                            return Err(format!("The downloaded tar blake3 is not match and unable to continue in offline mode."));
+                        }
+                        fetch_extract_tar(source_tar, &source_dir)?;
+                        fetch_apply_patches(recipe_dir, patches, script, &source_dir)?;
+                    } else {
+                        // need to trust this tar file
+                        return Err(format!(
+                            "Please add blake3 = \"{source_tar_blake3}\" to '{recipe}'",
+                            recipe = recipe_dir.join("recipe.toml").display(),
+                        ));
+                    }
+                } else {
+                    return Err(format!(
+                        "'{dir}' is not exist and unable to continue in offline mode",
+                        dir = source_dir.display(),
+                    ));
+                }
+            }
+        }
+    }
+
+    Ok(source_dir)
+}
+
+pub fn fetch(recipe_dir: &Path, source: &Option<SourceRecipe>) -> Result<PathBuf, String> {
+    let source_dir = recipe_dir.join("source");
+    match source {
+        Some(SourceRecipe::SameAs { same_as }) => {
+            let (canon_dir, recipe) = fetch_resolve_canon(recipe_dir, same_as)?;
+            // recursively fetch
+            fetch(&canon_dir, &recipe.source)?;
+            fetch_make_symlink(&source_dir, same_as)?;
+        }
+        Some(SourceRecipe::Path { path }) => {
+            if !source_dir.is_dir() || modified_dir(Path::new(path))? > modified_dir(&source_dir)? {
+                eprintln!("[DEBUG]: {} is newer than {}", path, source_dir.display());
+                copy_dir_all(path, &source_dir).map_err(|e| {
+                    format!(
+                        "Couldn't copy source from {} to {}: {}",
+                        path,
+                        source_dir.display(),
+                        e
+                    )
+                })?;
+            }
+        }
+        Some(SourceRecipe::Git {
+            git,
+            upstream,
+            branch,
+            rev,
+            patches,
+            script,
+            shallow_clone,
+        }) => {
+            //TODO: use libgit?
+            let shallow_clone = *shallow_clone == Some(true);
+            if !source_dir.is_dir() {
+                // Create source.tmp
+                let source_dir_tmp = recipe_dir.join("source.tmp");
+                create_dir_clean(&source_dir_tmp)?;
+
+                // Clone the repository to source.tmp
+                let mut command = Command::new("git");
+                command
+                    .arg("clone")
+                    .arg("--recursive")
+                    .arg(translate_mirror(git));
+                if let Some(branch) = branch {
+                    command.arg("--branch").arg(branch);
+                }
+                if shallow_clone {
+                    command.arg("--depth").arg("1").arg("--shallow-submodules");
+                }
+                command.arg(&source_dir_tmp);
+                run_command(command)?;
+
+                // Move source.tmp to source atomically
+                rename(&source_dir_tmp, &source_dir)?;
+            } else if !shallow_clone {
+                // Don't let this code reset the origin for the cookbook repo
+                let source_git_dir = source_dir.join(".git");
+                if !source_git_dir.is_dir() {
+                    return Err(format!(
+                        "'{}' is not a git repository, but recipe indicated git source",
+                        source_dir.display(),
+                    ));
+                }
+
+                // Reset origin
+                let mut command = Command::new("git");
+                command.arg("-C").arg(&source_dir);
+                command.arg("remote").arg("set-url").arg("origin").arg(git);
+                run_command(command)?;
+
+                // Fetch origin
+                let mut command = Command::new("git");
+                command.arg("-C").arg(&source_dir);
+                command.arg("fetch").arg("origin");
+                run_command(command)?;
+            }
+
+            if let Some(_upstream) = upstream {
+                //TODO: set upstream URL
+                // git remote set-url upstream "$GIT_UPSTREAM" &> /dev/null ||
+                // git remote add upstream "$GIT_UPSTREAM"
+                // git fetch upstream
+            }
+
+            if let Some(rev) = rev {
+                // Check out specified revision
+                let mut command = Command::new("git");
+                command.arg("-C").arg(&source_dir);
+                command.arg("checkout").arg(rev);
+                run_command(command)?;
+            } else if !shallow_clone && !is_redox() {
+                //TODO: complicated stuff to check and reset branch to origin
+                //TODO: redox can't undestand this (got exit status 1)
+                let mut command = Command::new("bash");
+                command.arg("-c").arg(GIT_RESET_BRANCH);
+                if let Some(branch) = branch {
+                    command.env("BRANCH", branch);
+                }
+                command.current_dir(&source_dir);
+                run_command(command)?;
+            }
+
+            if !patches.is_empty() || script.is_some() {
+                // Hard reset
+                let mut command = Command::new("git");
+                command.arg("-C").arg(&source_dir);
+                command.arg("reset").arg("--hard");
+                run_command(command)?;
+            }
+
+            if !shallow_clone {
+                // Sync submodules URL
+                let mut command = Command::new("git");
+                command.arg("-C").arg(&source_dir);
+                command.arg("submodule").arg("sync").arg("--recursive");
+                run_command(command)?;
+
+                // Update submodules
+                let mut command = Command::new("git");
+                command.arg("-C").arg(&source_dir);
+                command
+                    .arg("submodule")
+                    .arg("update")
+                    .arg("--init")
+                    .arg("--recursive");
+                run_command(command)?;
+            }
+
+            fetch_apply_patches(recipe_dir, patches, script, &source_dir)?;
+        }
+        Some(SourceRecipe::Tar {
+            tar,
+            blake3,
+            patches,
+            script,
+        }) => {
+            let source_tar = recipe_dir.join("source.tar");
+            let mut tar_updated = false;
+            while {
+                if tar_updated {
+                    return Err(format!("The downloaded tar blake3 is not match"));
+                }
+                if !source_tar.is_file() {
+                    tar_updated = true;
+                    //TODO: replace wget
+                    if !source_tar.is_file() {
+                        let source_tar_tmp = recipe_dir.join("source.tar.tmp");
+
+                        let mut command = Command::new("wget");
+                        command.arg(translate_mirror(tar));
+                        command.arg("--continue").arg("-O").arg(&source_tar_tmp);
+                        run_command(command)?;
+
+                        // Move source.tar.tmp to source.tar atomically
+                        rename(&source_tar_tmp, &source_tar)?;
+                    }
+                }
+                let source_tar_blake3 = get_blake3(&source_tar)?;
+                if let Some(blake3) = blake3 {
+                    if source_tar_blake3 != *blake3 {
+                        remove_all(&source_tar)?;
+                        true
+                    } else {
+                        false
+                    }
+                } else {
+                    //TODO: set blake3 hash on the recipe with something like "cook fix"
+                    eprintln!(
+                        "WARNING: set blake3 for '{}' to '{}'",
+                        source_tar.display(),
+                        source_tar_blake3
+                    );
+                    false
+                }
+            } {}
+            if source_dir.is_dir() {
+                if tar_updated || fetch_is_patches_newer(recipe_dir, patches, &source_dir)? {
+                    remove_all(&source_dir)?
+                }
+            }
+            if !source_dir.is_dir() {
+                // Create source.tmp
+                let source_dir_tmp = recipe_dir.join("source.tmp");
+                create_dir_clean(&source_dir_tmp)?;
+                fetch_extract_tar(source_tar, &source_dir_tmp)?;
+                fetch_apply_patches(recipe_dir, patches, script, &source_dir_tmp)?;
+
+                // Move source.tmp to source atomically
+                rename(&source_dir_tmp, &source_dir)?;
+            }
+        }
+        // Local Sources
+        None => {
+            if !source_dir.is_dir() {
+                eprintln!(
+                    "WARNING: Recipe without source section expected source dir at '{}'",
+                    source_dir.display(),
+                );
+                create_dir(&source_dir)?;
+            }
+        }
+    }
+
+    Ok(source_dir)
+}
+
+pub(crate) fn fetch_make_symlink(source_dir: &PathBuf, same_as: &String) -> Result<(), String> {
+    let target_dir = Path::new(same_as).join("source");
+    if !source_dir.is_symlink() {
+        if source_dir.is_dir() {
+            return Err(format!(
+                "'{dir}' is a directory, but recipe indicated a symlink. \n\
+                        try removing '{dir}' if you haven't made any changes that would be lost",
+                dir = source_dir.display(),
+            ));
+        }
+        std::os::unix::fs::symlink(&target_dir, source_dir).map_err(|err| {
+            format!(
+                "failed to symlink '{}' to '{}': {}\n{:?}",
+                target_dir.display(),
+                source_dir.display(),
+                err,
+                err
+            )
+        })?;
+    }
+    Ok(())
+}
+
+pub(crate) fn fetch_resolve_canon(
+    recipe_dir: &Path,
+    same_as: &String,
+) -> Result<(PathBuf, Recipe), String> {
+    let canon_dir = Path::new(recipe_dir).join(same_as);
+    if canon_dir
+        .to_str()
+        .unwrap()
+        .chars()
+        .filter(|c| *c == '/')
+        .count()
+        > 50
+    {
+        return Err(format!("Infinite loop detected"));
+    }
+    if !canon_dir.exists() {
+        return Err(format!("'{dir}' is not exists.", dir = canon_dir.display()));
+    }
+    let recipe_path = canon_dir.join("recipe.toml");
+    let recipe_str = fs::read_to_string(&recipe_path)
+        .map_err(|e| format!("unable to read {path}: {e}", path = recipe_path.display()))?;
+    let recipe: Recipe = toml::from_str(&recipe_str)
+        .map_err(|e| format!("Unable to parse {path}: {e}", path = recipe_path.display()))?;
+    Ok((canon_dir, recipe))
+}
+
+pub(crate) fn fetch_extract_tar(
+    source_tar: PathBuf,
+    source_dir_tmp: &PathBuf,
+) -> Result<(), String> {
+    let mut command = Command::new("tar");
+    if is_redox() {
+        command.arg("xvf");
+    } else {
+        command.arg("--extract");
+        command.arg("--verbose");
+        command.arg("--file");
+    }
+    command.arg(&source_tar);
+    command.arg("--directory").arg(source_dir_tmp);
+    command.arg("--strip-components").arg("1");
+    run_command(command)?;
+    Ok(())
+}
+
+pub(crate) fn fetch_is_patches_newer(
+    recipe_dir: &Path,
+    patches: &Vec<String>,
+    source_dir: &PathBuf,
+) -> Result<bool, String> {
+    // don't check source files inside as it can be mixed with user patches
+    let source_time = modified(&source_dir)?;
+    for patch_name in patches {
+        let patch_file = recipe_dir.join(patch_name);
+        if !patch_file.is_file() {
+            return Err(format!(
+                "failed to find patch file '{}'",
+                patch_file.display()
+            ));
+        }
+
+        let patch_time = modified(&patch_file)?;
+        if patch_time > source_time {
+            return Ok(true);
+        }
+    }
+    return Ok(false);
+}
+
+pub(crate) fn fetch_apply_patches(
+    recipe_dir: &Path,
+    patches: &Vec<String>,
+    script: &Option<String>,
+    source_dir_tmp: &PathBuf,
+) -> Result<(), String> {
+    for patch_name in patches {
+        let patch_file = recipe_dir.join(patch_name);
+        if !patch_file.is_file() {
+            return Err(format!(
+                "failed to find patch file '{}'",
+                patch_file.display()
+            ));
+        }
+
+        let patch = fs::read_to_string(&patch_file).map_err(|err| {
+            format!(
+                "failed to read patch file '{}': {}\n{:#?}",
+                patch_file.display(),
+                err,
+                err
+            )
+        })?;
+
+        let mut command = Command::new("patch");
+        command.arg("--directory").arg(source_dir_tmp);
+        command.arg("--strip=1");
+        run_command_stdin(command, patch.as_bytes())?;
+    }
+    Ok(if let Some(script) = script {
+        let mut command = Command::new("bash");
+        command.arg("-ex");
+        command.current_dir(source_dir_tmp);
+        run_command_stdin(command, format!("{SHARED_PRESCRIPT}\n{script}").as_bytes())?;
+    })
+}

src/cook/fs.rs

@@ -0,0 +1,194 @@
+use serde::Serialize;
+use std::{
+    fs,
+    io::{self, Write},
+    path::Path,
+    process::{self, Stdio},
+    time::SystemTime,
+};
+use walkdir::{DirEntry, WalkDir};
+
+//TODO: pub(crate) for all of these functions
+
+pub fn remove_all(path: &Path) -> Result<(), String> {
+    if path.is_dir() {
+        fs::remove_dir_all(path)
+    } else {
+        fs::remove_file(path)
+    }
+    .map_err(|err| format!("failed to remove '{}': {}\n{:?}", path.display(), err, err))
+}
+
+pub fn create_dir(dir: &Path) -> Result<(), String> {
+    fs::create_dir(dir)
+        .map_err(|err| format!("failed to create '{}': {}\n{:?}", dir.display(), err, err))
+}
+
+pub fn create_dir_clean(dir: &Path) -> Result<(), String> {
+    if dir.is_dir() {
+        remove_all(dir)?;
+    }
+    create_dir(dir)
+}
+
+pub fn copy_dir_all(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> io::Result<()> {
+    fs::create_dir_all(&dst)?;
+    for entry in fs::read_dir(src)? {
+        let entry = entry?;
+        let ty = entry.file_type()?;
+        if ty.is_dir() {
+            copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?;
+        } else {
+            fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?;
+        }
+    }
+    Ok(())
+}
+
+pub fn symlink(original: impl AsRef<Path>, link: impl AsRef<Path>) -> Result<(), String> {
+    std::os::unix::fs::symlink(&original, &link).map_err(|err| {
+        format!(
+            "failed to symlink '{}' to '{}': {}\n{:?}",
+            original.as_ref().display(),
+            link.as_ref().display(),
+            err,
+            err
+        )
+    })
+}
+
+pub fn modified(path: &Path) -> Result<SystemTime, String> {
+    let metadata = fs::metadata(path).map_err(|err| {
+        format!(
+            "failed to get metadata of '{}': {}\n{:#?}",
+            path.display(),
+            err,
+            err
+        )
+    })?;
+    metadata.modified().map_err(|err| {
+        format!(
+            "failed to get modified time of '{}': {}\n{:#?}",
+            path.display(),
+            err,
+            err
+        )
+    })
+}
+
+pub fn modified_dir_inner<F: FnMut(&DirEntry) -> bool>(
+    dir: &Path,
+    filter: F,
+) -> io::Result<SystemTime> {
+    let mut newest = fs::metadata(dir)?.modified()?;
+    for entry_res in WalkDir::new(dir).into_iter().filter_entry(filter) {
+        let entry = entry_res?;
+        let modified = entry.metadata()?.modified()?;
+        if modified > newest {
+            newest = modified;
+        }
+    }
+    Ok(newest)
+}
+
+pub fn modified_dir(dir: &Path) -> Result<SystemTime, String> {
+    modified_dir_inner(dir, |_| true).map_err(|err| {
+        format!(
+            "failed to get modified time of '{}': {}\n{:#?}",
+            dir.display(),
+            err,
+            err
+        )
+    })
+}
+
+pub fn modified_dir_ignore_git(dir: &Path) -> Result<SystemTime, String> {
+    modified_dir_inner(dir, |entry| {
+        entry
+            .file_name()
+            .to_str()
+            .map(|s| s != ".git")
+            .unwrap_or(true)
+    })
+    .map_err(|err| {
+        format!(
+            "failed to get modified time of '{}': {}\n{:#?}",
+            dir.display(),
+            err,
+            err
+        )
+    })
+}
+
+pub fn rename(src: &Path, dst: &Path) -> Result<(), String> {
+    fs::rename(src, dst).map_err(|err| {
+        format!(
+            "failed to rename '{}' to '{}': {}\n{:?}",
+            src.display(),
+            dst.display(),
+            err,
+            err
+        )
+    })
+}
+
+pub fn run_command(mut command: process::Command) -> Result<(), String> {
+    let status = command
+        .status()
+        .map_err(|err| format!("failed to run {:?}: {}\n{:#?}", command, err, err))?;
+
+    if !status.success() {
+        return Err(format!(
+            "failed to run {:?}: exited with status {}",
+            command, status
+        ));
+    }
+
+    Ok(())
+}
+
+pub fn run_command_stdin(mut command: process::Command, stdin_data: &[u8]) -> Result<(), String> {
+    command.stdin(Stdio::piped());
+
+    let mut child = command
+        .spawn()
+        .map_err(|err| format!("failed to spawn {:?}: {}\n{:#?}", command, err, err))?;
+
+    if let Some(ref mut stdin) = child.stdin {
+        stdin.write_all(stdin_data).map_err(|err| {
+            format!(
+                "failed to write stdin of {:?}: {}\n{:#?}",
+                command, err, err
+            )
+        })?;
+    } else {
+        return Err(format!("failed to find stdin of {:?}", command));
+    }
+
+    let status = child
+        .wait()
+        .map_err(|err| format!("failed to run {:?}: {}\n{:#?}", command, err, err))?;
+
+    if !status.success() {
+        return Err(format!(
+            "failed to run {:?}: exited with status {}",
+            command, status
+        ));
+    }
+
+    Ok(())
+}
+
+pub fn serialize_and_write<T: Serialize>(file_path: &Path, content: &T) -> Result<(), String> {
+    let toml_content = toml::to_string(content).map_err(|err| {
+        format!(
+            "Failed to serialize content for '{}': {}",
+            file_path.display(),
+            err
+        )
+    })?;
+
+    fs::write(file_path, toml_content)
+        .map_err(|err| format!("Failed to write to file '{}': {}", file_path.display(), err))?;
+    Ok(())
+}

src/cook/mod.rs

@@ -0,0 +1,6 @@
+pub mod fetch;
+pub mod fs;
+pub mod script;
+//TODO: Move rest of cook functions here in the next refactor
+//pub mod build;
+//pub mod package;

src/cook/script.rs

@@ -0,0 +1,91 @@
+//TODO: pub(crate)
+pub static SHARED_PRESCRIPT: &str = r#"
+# Build dynamically
+function DYNAMIC_INIT {
+    COOKBOOK_AUTORECONF="autoreconf"
+    autotools_recursive_regenerate() {
+        for f in $(find . -name configure.ac -o -name configure.in -type f | sort); do
+            echo "* autotools regen in '$(dirname $f)'..."
+            ( cd "$(dirname "$f")" && "${COOKBOOK_AUTORECONF}" -fvi "$@" -I${COOKBOOK_HOST_SYSROOT}/share/aclocal )
+        done
+    }
+
+    if [ "${TARGET}" != "x86_64-unknown-redox" ]
+    then
+        echo "WARN: ${TARGET} does not support dynamic linking." >&2
+        return
+    fi
+
+    echo "DEBUG: Program is being compiled dynamically."
+
+    COOKBOOK_CONFIGURE_FLAGS=(
+        --host="${GNU_TARGET}"
+        --prefix="/usr"
+        --enable-shared
+        --disable-static
+    )
+
+    COOKBOOK_CMAKE_FLAGS=(
+        -DBUILD_SHARED_LIBS=True
+        -DENABLE_SHARED=True
+        -DENABLE_STATIC=False
+    )
+
+    COOKBOOK_MESON_FLAGS=(
+        --buildtype release
+        --wrap-mode nofallback
+        --strip
+        -Ddefault_library=shared
+        -Dprefix=/usr
+    )
+
+    # TODO: check paths for spaces
+    export LDFLAGS="-Wl,-rpath-link,${COOKBOOK_SYSROOT}/lib -L${COOKBOOK_SYSROOT}/lib"
+    export RUSTFLAGS="-C target-feature=-crt-static"
+    export COOKBOOK_DYNAMIC=1
+}
+
+# Build both dynamically and statically
+function DYNAMIC_STATIC_INIT {
+    DYNAMIC_INIT
+    if [ "${COOKBOOK_DYNAMIC}" == "1" ]
+    then
+        COOKBOOK_CONFIGURE_FLAGS=(
+            --host="${GNU_TARGET}"
+            --prefix="/usr"
+            --enable-shared
+            --enable-static
+        )
+
+        COOKBOOK_CMAKE_FLAGS=(
+            -DBUILD_SHARED_LIBS=True
+            -DENABLE_SHARED=True
+            -DENABLE_STATIC=True
+        )
+
+        COOKBOOK_MESON_FLAGS=(
+            --buildtype release
+            --wrap-mode nofallback
+            --strip
+            -Ddefault_library=both
+            -Dprefix=/usr
+        )
+    fi
+}
+
+function GNU_CONFIG_GET {
+  wget -O "$1" "https://gitlab.redox-os.org/redox-os/gnu-config/-/raw/master/config.sub?inline=false"
+}
+"#;
+
+pub(crate) static GIT_RESET_BRANCH: &str = r#"
+ORIGIN_BRANCH="$(git branch --remotes | grep '^  origin/HEAD -> ' | cut -d ' ' -f 5-)"
+if [ -n "$BRANCH" ]
+then
+    ORIGIN_BRANCH="origin/$BRANCH"
+fi
+
+if [ "$(git rev-parse HEAD)" != "$(git rev-parse $ORIGIN_BRANCH)" ]
+then
+    git checkout -B "$(echo "$ORIGIN_BRANCH" | cut -d / -f 2-)" "$ORIGIN_BRANCH"
+fi"#;

src/lib.rs

@@ -1,5 +1,6 @@
 pub mod blake3;
 pub mod config;
+pub mod cook;
 pub mod recipe;
 
 mod progress_bar;