diff --git a/src/bin/cook.rs b/src/bin/cook.rs index 45787cc0c..90315b4ee 100644 --- a/src/bin/cook.rs +++ b/src/bin/cook.rs @@ -1,630 +1,24 @@ -use cookbook::blake3::blake3_progress; -use cookbook::config::{init_config, translate_mirror}; -use cookbook::recipe::{AutoDeps, BuildKind, CookRecipe, Recipe, SourceRecipe}; +use cookbook::config::init_config; +use cookbook::cook::fetch::*; +use cookbook::cook::fs::*; +use cookbook::cook::script::SHARED_PRESCRIPT; +use cookbook::recipe::{AutoDeps, BuildKind, CookRecipe, Recipe}; use pkg::package::Package; use pkg::{recipes, PackageName}; -use serde::Serialize; use std::collections::VecDeque; use std::convert::TryInto; use std::{ collections::BTreeSet, env, fs, - io::{self, Write}, path::{Path, PathBuf}, - process::{self, Command, Stdio}, + process::{self, Command}, str, time::SystemTime, }; use termion::{color, style}; -use walkdir::{DirEntry, WalkDir}; use cookbook::{is_redox, WALK_DEPTH}; -fn remove_all(path: &Path) -> Result<(), String> { - if path.is_dir() { - fs::remove_dir_all(path) - } else { - fs::remove_file(path) - } - .map_err(|err| format!("failed to remove '{}': {}\n{:?}", path.display(), err, err)) -} - -fn create_dir(dir: &Path) -> Result<(), String> { - fs::create_dir(dir) - .map_err(|err| format!("failed to create '{}': {}\n{:?}", dir.display(), err, err)) -} - -fn create_dir_clean(dir: &Path) -> Result<(), String> { - if dir.is_dir() { - remove_all(dir)?; - } - create_dir(dir) -} - -fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> io::Result<()> { - fs::create_dir_all(&dst)?; - for entry in fs::read_dir(src)? { - let entry = entry?; - let ty = entry.file_type()?; - if ty.is_dir() { - copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?; - } else { - fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?; - } - } - Ok(()) -} - -fn symlink(original: impl AsRef, link: impl AsRef) -> Result<(), String> { - std::os::unix::fs::symlink(&original, &link).map_err(|err| { - format!( - "failed to symlink '{}' to '{}': {}\n{:?}", - original.as_ref().display(), - link.as_ref().display(), - err, - err - ) - }) -} - -fn modified(path: &Path) -> Result { - let metadata = fs::metadata(path).map_err(|err| { - format!( - "failed to get metadata of '{}': {}\n{:#?}", - path.display(), - err, - err - ) - })?; - metadata.modified().map_err(|err| { - format!( - "failed to get modified time of '{}': {}\n{:#?}", - path.display(), - err, - err - ) - }) -} - -fn modified_dir_inner bool>( - dir: &Path, - filter: F, -) -> io::Result { - let mut newest = fs::metadata(dir)?.modified()?; - for entry_res in WalkDir::new(dir).into_iter().filter_entry(filter) { - let entry = entry_res?; - let modified = entry.metadata()?.modified()?; - if modified > newest { - newest = modified; - } - } - Ok(newest) -} - -fn modified_dir(dir: &Path) -> Result { - modified_dir_inner(dir, |_| true).map_err(|err| { - format!( - "failed to get modified time of '{}': {}\n{:#?}", - dir.display(), - err, - err - ) - }) -} - -fn modified_dir_ignore_git(dir: &Path) -> Result { - modified_dir_inner(dir, |entry| { - entry - .file_name() - .to_str() - .map(|s| s != ".git") - .unwrap_or(true) - }) - .map_err(|err| { - format!( - "failed to get modified time of '{}': {}\n{:#?}", - dir.display(), - err, - err - ) - }) -} - -fn rename(src: &Path, dst: &Path) -> Result<(), String> { - fs::rename(src, dst).map_err(|err| { - format!( - "failed to rename '{}' to '{}': {}\n{:?}", - src.display(), - dst.display(), - err, - err - ) - }) -} - -fn run_command(mut command: process::Command) -> Result<(), String> { - let status = command - .status() - .map_err(|err| format!("failed to run {:?}: {}\n{:#?}", command, err, err))?; - - if !status.success() { - return Err(format!( - "failed to run {:?}: exited with status {}", - command, status - )); - } - - Ok(()) -} - -fn run_command_stdin(mut command: process::Command, stdin_data: &[u8]) -> Result<(), String> { - command.stdin(Stdio::piped()); - - let mut child = command - .spawn() - .map_err(|err| format!("failed to spawn {:?}: {}\n{:#?}", command, err, err))?; - - if let Some(ref mut stdin) = child.stdin { - stdin.write_all(stdin_data).map_err(|err| { - format!( - "failed to write stdin of {:?}: {}\n{:#?}", - command, err, err - ) - })?; - } else { - return Err(format!("failed to find stdin of {:?}", command)); - } - - let status = child - .wait() - .map_err(|err| format!("failed to run {:?}: {}\n{:#?}", command, err, err))?; - - if !status.success() { - return Err(format!( - "failed to run {:?}: exited with status {}", - command, status - )); - } - - Ok(()) -} - -fn serialize_and_write(file_path: &Path, content: &T) -> Result<(), String> { - let toml_content = toml::to_string(content).map_err(|err| { - format!( - "Failed to serialize content for '{}': {}", - file_path.display(), - err - ) - })?; - - fs::write(file_path, toml_content) - .map_err(|err| format!("Failed to write to file '{}': {}", file_path.display(), err))?; - Ok(()) -} - -static SHARED_PRESCRIPT: &str = r#" -# Build dynamically -function DYNAMIC_INIT { - COOKBOOK_AUTORECONF="autoreconf" - autotools_recursive_regenerate() { - for f in $(find . -name configure.ac -o -name configure.in -type f | sort); do - echo "* autotools regen in '$(dirname $f)'..." - ( cd "$(dirname "$f")" && "${COOKBOOK_AUTORECONF}" -fvi "$@" -I${COOKBOOK_HOST_SYSROOT}/share/aclocal ) - done - } - - if [ "${TARGET}" != "x86_64-unknown-redox" ] - then - echo "WARN: ${TARGET} does not support dynamic linking." >&2 - return - fi - - echo "DEBUG: Program is being compiled dynamically." - - COOKBOOK_CONFIGURE_FLAGS=( - --host="${GNU_TARGET}" - --prefix="/usr" - --enable-shared - --disable-static - ) - - COOKBOOK_CMAKE_FLAGS=( - -DBUILD_SHARED_LIBS=True - -DENABLE_SHARED=True - -DENABLE_STATIC=False - ) - - COOKBOOK_MESON_FLAGS=( - --buildtype release - --wrap-mode nofallback - --strip - -Ddefault_library=shared - -Dprefix=/usr - ) - - # TODO: check paths for spaces - export LDFLAGS="-Wl,-rpath-link,${COOKBOOK_SYSROOT}/lib -L${COOKBOOK_SYSROOT}/lib" - export RUSTFLAGS="-C target-feature=-crt-static" - export COOKBOOK_DYNAMIC=1 -} - -# Build both dynamically and statically -function DYNAMIC_STATIC_INIT { - DYNAMIC_INIT - if [ "${COOKBOOK_DYNAMIC}" == "1" ] - then - COOKBOOK_CONFIGURE_FLAGS=( - --host="${GNU_TARGET}" - --prefix="/usr" - --enable-shared - --enable-static - ) - - COOKBOOK_CMAKE_FLAGS=( - -DBUILD_SHARED_LIBS=True - -DENABLE_SHARED=True - -DENABLE_STATIC=True - ) - - COOKBOOK_MESON_FLAGS=( - --buildtype release - --wrap-mode nofallback - --strip - -Ddefault_library=both - -Dprefix=/usr - ) - fi -} - -function GNU_CONFIG_GET { - wget -O "$1" "https://gitlab.redox-os.org/redox-os/gnu-config/-/raw/master/config.sub?inline=false" -} -"#; - -fn fetch_offline(recipe_dir: &Path, source: &Option) -> Result { - let source_dir = recipe_dir.join("source"); - match source { - Some(SourceRecipe::SameAs { same_as: _ }) | Some(SourceRecipe::Path { path: _ }) | None => { - return fetch(recipe_dir, source); - } - Some(SourceRecipe::Git { - git: _, - upstream: _, - branch: _, - rev: _, - patches: _, - script: _, - shallow_clone: _, - }) - | Some(SourceRecipe::Tar { - tar: _, - blake3: _, - patches: _, - script: _, - }) => { - if !source_dir.is_dir() { - return Err(format!( - "'{dir}' is not exist and unable to continue in offline mode", - dir = source_dir.display(), - )); - } - } - } - - Ok(source_dir) -} - -fn fetch(recipe_dir: &Path, source: &Option) -> Result { - let source_dir = recipe_dir.join("source"); - match source { - Some(SourceRecipe::SameAs { same_as }) => { - if !source_dir.is_symlink() { - if source_dir.is_dir() { - return Err(format!( - "'{dir}' is a directory, but recipe indicated a symlink. \n\ - try removing '{dir}' if you haven't made any changes that would be lost", - dir = source_dir.display(), - )); - } - let original = Path::new(same_as).join("source"); - std::os::unix::fs::symlink(&original, &source_dir).map_err(|err| { - format!( - "failed to symlink '{}' to '{}': {}\n{:?}", - original.display(), - source_dir.display(), - err, - err - ) - })?; - } - } - Some(SourceRecipe::Path { path }) => { - if !source_dir.is_dir() || modified_dir(Path::new(path))? > modified_dir(&source_dir)? { - eprintln!("[DEBUG]: {} is newer than {}", path, source_dir.display()); - copy_dir_all(path, &source_dir).map_err(|e| { - format!( - "Couldn't copy source from {} to {}: {}", - path, - source_dir.display(), - e - ) - })?; - } - } - Some(SourceRecipe::Git { - git, - upstream, - branch, - rev, - patches, - script, - shallow_clone, - }) => { - //TODO: use libgit? - let shallow_clone = *shallow_clone == Some(true); - if !source_dir.is_dir() { - // Create source.tmp - let source_dir_tmp = recipe_dir.join("source.tmp"); - create_dir_clean(&source_dir_tmp)?; - - // Clone the repository to source.tmp - let mut command = Command::new("git"); - command - .arg("clone") - .arg("--recursive") - .arg(translate_mirror(git)); - if let Some(branch) = branch { - command.arg("--branch").arg(branch); - } - if shallow_clone { - command.arg("--depth").arg("1").arg("--shallow-submodules"); - } - command.arg(&source_dir_tmp); - run_command(command)?; - - // Move source.tmp to source atomically - rename(&source_dir_tmp, &source_dir)?; - } else if !shallow_clone { - // Don't let this code reset the origin for the cookbook repo - let source_git_dir = source_dir.join(".git"); - if !source_git_dir.is_dir() { - return Err(format!( - "'{}' is not a git repository, but recipe indicated git source", - source_dir.display(), - )); - } - - // Reset origin - let mut command = Command::new("git"); - command.arg("-C").arg(&source_dir); - command.arg("remote").arg("set-url").arg("origin").arg(git); - run_command(command)?; - - // Fetch origin - let mut command = Command::new("git"); - command.arg("-C").arg(&source_dir); - command.arg("fetch").arg("origin"); - run_command(command)?; - } - - if let Some(_upstream) = upstream { - //TODO: set upstream URL - // git remote set-url upstream "$GIT_UPSTREAM" &> /dev/null || - // git remote add upstream "$GIT_UPSTREAM" - // git fetch upstream - } - - if let Some(rev) = rev { - // Check out specified revision - let mut command = Command::new("git"); - command.arg("-C").arg(&source_dir); - command.arg("checkout").arg(rev); - run_command(command)?; - } else if !shallow_clone && !is_redox() { - //TODO: complicated stuff to check and reset branch to origin - //TODO: redox can't undestand this (got exit status 1) - let mut command = Command::new("bash"); - command.arg("-c").arg( - r#" -ORIGIN_BRANCH="$(git branch --remotes | grep '^ origin/HEAD -> ' | cut -d ' ' -f 5-)" -if [ -n "$BRANCH" ] -then - ORIGIN_BRANCH="origin/$BRANCH" -fi - -if [ "$(git rev-parse HEAD)" != "$(git rev-parse $ORIGIN_BRANCH)" ] -then - git checkout -B "$(echo "$ORIGIN_BRANCH" | cut -d / -f 2-)" "$ORIGIN_BRANCH" -fi"#, - ); - if let Some(branch) = branch { - command.env("BRANCH", branch); - } - command.current_dir(&source_dir); - run_command(command)?; - } - - if !patches.is_empty() || script.is_some() { - // Hard reset - let mut command = Command::new("git"); - command.arg("-C").arg(&source_dir); - command.arg("reset").arg("--hard"); - run_command(command)?; - } - - if !shallow_clone { - // Sync submodules URL - let mut command = Command::new("git"); - command.arg("-C").arg(&source_dir); - command.arg("submodule").arg("sync").arg("--recursive"); - run_command(command)?; - - // Update submodules - let mut command = Command::new("git"); - command.arg("-C").arg(&source_dir); - command - .arg("submodule") - .arg("update") - .arg("--init") - .arg("--recursive"); - run_command(command)?; - } - - // Apply patches - for patch_name in patches { - let patch_file = recipe_dir.join(patch_name); - if !patch_file.is_file() { - return Err(format!( - "failed to find patch file '{}'", - patch_file.display() - )); - } - - let patch = fs::read_to_string(&patch_file).map_err(|err| { - format!( - "failed to read patch file '{}': {}\n{:#?}", - patch_file.display(), - err, - err - ) - })?; - - let mut command = Command::new("patch"); - command.arg("--forward"); - command.arg("--batch"); - command.arg("--directory").arg(&source_dir); - command.arg("--strip=1"); - run_command_stdin(command, patch.as_bytes())?; - } - - // Run source script - if let Some(script) = script { - let mut command = Command::new("bash"); - command.arg("-ex"); - command.current_dir(&source_dir); - run_command_stdin(command, format!("{SHARED_PRESCRIPT}\n{script}").as_bytes())?; - } - } - Some(SourceRecipe::Tar { - tar, - blake3, - patches, - script, - }) => { - if !source_dir.is_dir() { - // Download tar - //TODO: replace wget - let source_tar = recipe_dir.join("source.tar"); - if !source_tar.is_file() { - let source_tar_tmp = recipe_dir.join("source.tar.tmp"); - - let mut command = Command::new("wget"); - command.arg(translate_mirror(tar)); - command.arg("--continue").arg("-O").arg(&source_tar_tmp); - run_command(command)?; - - // Move source.tar.tmp to source.tar atomically - rename(&source_tar_tmp, &source_tar)?; - } - - // Calculate blake3 - let source_tar_blake3 = blake3_progress(&source_tar).map_err(|err| { - format!( - "failed to calculate blake3 of '{}': {}\n{:?}", - source_tar.display(), - err, - err - ) - })?; - if let Some(blake3) = blake3 { - // Check if it matches recipe - if &source_tar_blake3 != blake3 { - return Err(format!( - "calculated blake3 '{}' does not match recipe blake3 '{}'", - source_tar_blake3, blake3 - )); - } - } else { - //TODO: set blake3 hash on the recipe with something like "cook fix" - eprintln!( - "WARNING: set blake3 for '{}' to '{}'", - source_tar.display(), - source_tar_blake3 - ); - } - - // Create source.tmp - let source_dir_tmp = recipe_dir.join("source.tmp"); - create_dir_clean(&source_dir_tmp)?; - - // Extract tar to source.tmp - //TODO: use tar crate (how to deal with compression?) - let mut command = Command::new("tar"); - if is_redox() { - command.arg("xvf"); - } else { - command.arg("--extract"); - command.arg("--verbose"); - command.arg("--file"); - } - command.arg(&source_tar); - command.arg("--directory").arg(&source_dir_tmp); - command.arg("--strip-components").arg("1"); - run_command(command)?; - - // Apply patches - for patch_name in patches { - let patch_file = recipe_dir.join(patch_name); - if !patch_file.is_file() { - return Err(format!( - "failed to find patch file '{}'", - patch_file.display() - )); - } - - let patch = fs::read_to_string(&patch_file).map_err(|err| { - format!( - "failed to read patch file '{}': {}\n{:#?}", - patch_file.display(), - err, - err - ) - })?; - - let mut command = Command::new("patch"); - command.arg("--directory").arg(&source_dir_tmp); - command.arg("--strip=1"); - run_command_stdin(command, patch.as_bytes())?; - } - - // Run source script - if let Some(script) = script { - let mut command = Command::new("bash"); - command.arg("-ex"); - command.current_dir(&source_dir_tmp); - run_command_stdin(command, format!("{SHARED_PRESCRIPT}\n{script}").as_bytes())?; - } - - // Move source.tmp to source atomically - rename(&source_dir_tmp, &source_dir)?; - } - } - // Local Sources - None => { - if !source_dir.is_dir() { - eprintln!( - "WARNING: Recipe without source section expected source dir at '{}'", - source_dir.display(), - ); - create_dir(&source_dir)?; - } - } - } - - Ok(source_dir) -} - fn auto_deps( stage_dir: &Path, dep_pkgars: &BTreeSet<(PackageName, PathBuf)>, diff --git a/src/cook/fetch.rs b/src/cook/fetch.rs new file mode 100644 index 000000000..dc44bb314 --- /dev/null +++ b/src/cook/fetch.rs @@ -0,0 +1,417 @@ +use crate::config::translate_mirror; +use crate::cook::fs::*; +use crate::cook::script::*; +use crate::is_redox; +use crate::recipe::Recipe; +use crate::{blake3::blake3_progress, recipe::SourceRecipe}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +pub(crate) fn get_blake3(path: &PathBuf) -> Result { + blake3_progress(&path).map_err(|err| { + format!( + "failed to calculate blake3 of '{}': {}\n{:?}", + path.display(), + err, + err + ) + }) +} + +pub fn fetch_offline(recipe_dir: &Path, source: &Option) -> Result { + let source_dir = recipe_dir.join("source"); + match source { + Some(SourceRecipe::Path { path: _ }) | None => { + return fetch(recipe_dir, source); + } + Some(SourceRecipe::SameAs { same_as: _ }) => { + return fetch(recipe_dir, source); + } + Some(SourceRecipe::Git { + git: _, + upstream: _, + branch: _, + rev: _, + patches: _, + script: _, + shallow_clone: _, + }) => { + if !source_dir.is_dir() { + return Err(format!( + "'{dir}' is not exist and unable to continue in offline mode", + dir = source_dir.display(), + )); + } + } + Some(SourceRecipe::Tar { + tar: _, + blake3, + patches, + script, + }) => { + if !source_dir.is_dir() { + let source_tar = recipe_dir.join("source.tar"); + let source_tar_blake3 = get_blake3(&source_tar)?; + if source_tar.exists() { + if let Some(blake3) = blake3 { + if source_tar_blake3 != *blake3 { + return Err(format!("The downloaded tar blake3 is not match and unable to continue in offline mode.")); + } + fetch_extract_tar(source_tar, &source_dir)?; + fetch_apply_patches(recipe_dir, patches, script, &source_dir)?; + } else { + // need to trust this tar file + return Err(format!( + "Please add blake3 = \"{source_tar_blake3}\" to '{recipe}'", + recipe = recipe_dir.join("recipe.toml").display(), + )); + } + } else { + return Err(format!( + "'{dir}' is not exist and unable to continue in offline mode", + dir = source_dir.display(), + )); + } + } + } + } + + Ok(source_dir) +} + +pub fn fetch(recipe_dir: &Path, source: &Option) -> Result { + let source_dir = recipe_dir.join("source"); + match source { + Some(SourceRecipe::SameAs { same_as }) => { + let (canon_dir, recipe) = fetch_resolve_canon(recipe_dir, same_as)?; + // recursively fetch + fetch(&canon_dir, &recipe.source)?; + fetch_make_symlink(&source_dir, same_as)?; + } + Some(SourceRecipe::Path { path }) => { + if !source_dir.is_dir() || modified_dir(Path::new(path))? > modified_dir(&source_dir)? { + eprintln!("[DEBUG]: {} is newer than {}", path, source_dir.display()); + copy_dir_all(path, &source_dir).map_err(|e| { + format!( + "Couldn't copy source from {} to {}: {}", + path, + source_dir.display(), + e + ) + })?; + } + } + Some(SourceRecipe::Git { + git, + upstream, + branch, + rev, + patches, + script, + shallow_clone, + }) => { + //TODO: use libgit? + let shallow_clone = *shallow_clone == Some(true); + if !source_dir.is_dir() { + // Create source.tmp + let source_dir_tmp = recipe_dir.join("source.tmp"); + create_dir_clean(&source_dir_tmp)?; + + // Clone the repository to source.tmp + let mut command = Command::new("git"); + command + .arg("clone") + .arg("--recursive") + .arg(translate_mirror(git)); + if let Some(branch) = branch { + command.arg("--branch").arg(branch); + } + if shallow_clone { + command.arg("--depth").arg("1").arg("--shallow-submodules"); + } + command.arg(&source_dir_tmp); + run_command(command)?; + + // Move source.tmp to source atomically + rename(&source_dir_tmp, &source_dir)?; + } else if !shallow_clone { + // Don't let this code reset the origin for the cookbook repo + let source_git_dir = source_dir.join(".git"); + if !source_git_dir.is_dir() { + return Err(format!( + "'{}' is not a git repository, but recipe indicated git source", + source_dir.display(), + )); + } + + // Reset origin + let mut command = Command::new("git"); + command.arg("-C").arg(&source_dir); + command.arg("remote").arg("set-url").arg("origin").arg(git); + run_command(command)?; + + // Fetch origin + let mut command = Command::new("git"); + command.arg("-C").arg(&source_dir); + command.arg("fetch").arg("origin"); + run_command(command)?; + } + + if let Some(_upstream) = upstream { + //TODO: set upstream URL + // git remote set-url upstream "$GIT_UPSTREAM" &> /dev/null || + // git remote add upstream "$GIT_UPSTREAM" + // git fetch upstream + } + + if let Some(rev) = rev { + // Check out specified revision + let mut command = Command::new("git"); + command.arg("-C").arg(&source_dir); + command.arg("checkout").arg(rev); + run_command(command)?; + } else if !shallow_clone && !is_redox() { + //TODO: complicated stuff to check and reset branch to origin + //TODO: redox can't undestand this (got exit status 1) + let mut command = Command::new("bash"); + command.arg("-c").arg(GIT_RESET_BRANCH); + if let Some(branch) = branch { + command.env("BRANCH", branch); + } + command.current_dir(&source_dir); + run_command(command)?; + } + + if !patches.is_empty() || script.is_some() { + // Hard reset + let mut command = Command::new("git"); + command.arg("-C").arg(&source_dir); + command.arg("reset").arg("--hard"); + run_command(command)?; + } + + if !shallow_clone { + // Sync submodules URL + let mut command = Command::new("git"); + command.arg("-C").arg(&source_dir); + command.arg("submodule").arg("sync").arg("--recursive"); + run_command(command)?; + + // Update submodules + let mut command = Command::new("git"); + command.arg("-C").arg(&source_dir); + command + .arg("submodule") + .arg("update") + .arg("--init") + .arg("--recursive"); + run_command(command)?; + } + + fetch_apply_patches(recipe_dir, patches, script, &source_dir)?; + } + Some(SourceRecipe::Tar { + tar, + blake3, + patches, + script, + }) => { + let source_tar = recipe_dir.join("source.tar"); + let mut tar_updated = false; + while { + if tar_updated { + return Err(format!("The downloaded tar blake3 is not match")); + } + if !source_tar.is_file() { + tar_updated = true; + //TODO: replace wget + if !source_tar.is_file() { + let source_tar_tmp = recipe_dir.join("source.tar.tmp"); + + let mut command = Command::new("wget"); + command.arg(translate_mirror(tar)); + command.arg("--continue").arg("-O").arg(&source_tar_tmp); + run_command(command)?; + + // Move source.tar.tmp to source.tar atomically + rename(&source_tar_tmp, &source_tar)?; + } + } + let source_tar_blake3 = get_blake3(&source_tar)?; + if let Some(blake3) = blake3 { + if source_tar_blake3 != *blake3 { + remove_all(&source_tar)?; + true + } else { + false + } + } else { + //TODO: set blake3 hash on the recipe with something like "cook fix" + eprintln!( + "WARNING: set blake3 for '{}' to '{}'", + source_tar.display(), + source_tar_blake3 + ); + false + } + } {} + if source_dir.is_dir() { + if tar_updated || fetch_is_patches_newer(recipe_dir, patches, &source_dir)? { + remove_all(&source_dir)? + } + } + if !source_dir.is_dir() { + // Create source.tmp + let source_dir_tmp = recipe_dir.join("source.tmp"); + create_dir_clean(&source_dir_tmp)?; + fetch_extract_tar(source_tar, &source_dir_tmp)?; + fetch_apply_patches(recipe_dir, patches, script, &source_dir_tmp)?; + + // Move source.tmp to source atomically + rename(&source_dir_tmp, &source_dir)?; + } + } + // Local Sources + None => { + if !source_dir.is_dir() { + eprintln!( + "WARNING: Recipe without source section expected source dir at '{}'", + source_dir.display(), + ); + create_dir(&source_dir)?; + } + } + } + + Ok(source_dir) +} + +pub(crate) fn fetch_make_symlink(source_dir: &PathBuf, same_as: &String) -> Result<(), String> { + let target_dir = Path::new(same_as).join("source"); + if !source_dir.is_symlink() { + if source_dir.is_dir() { + return Err(format!( + "'{dir}' is a directory, but recipe indicated a symlink. \n\ + try removing '{dir}' if you haven't made any changes that would be lost", + dir = source_dir.display(), + )); + } + std::os::unix::fs::symlink(&target_dir, source_dir).map_err(|err| { + format!( + "failed to symlink '{}' to '{}': {}\n{:?}", + target_dir.display(), + source_dir.display(), + err, + err + ) + })?; + } + Ok(()) +} + +pub(crate) fn fetch_resolve_canon( + recipe_dir: &Path, + same_as: &String, +) -> Result<(PathBuf, Recipe), String> { + let canon_dir = Path::new(recipe_dir).join(same_as); + if canon_dir + .to_str() + .unwrap() + .chars() + .filter(|c| *c == '/') + .count() + > 50 + { + return Err(format!("Infinite loop detected")); + } + if !canon_dir.exists() { + return Err(format!("'{dir}' is not exists.", dir = canon_dir.display())); + } + let recipe_path = canon_dir.join("recipe.toml"); + let recipe_str = fs::read_to_string(&recipe_path) + .map_err(|e| format!("unable to read {path}: {e}", path = recipe_path.display()))?; + let recipe: Recipe = toml::from_str(&recipe_str) + .map_err(|e| format!("Unable to parse {path}: {e}", path = recipe_path.display()))?; + Ok((canon_dir, recipe)) +} + +pub(crate) fn fetch_extract_tar( + source_tar: PathBuf, + source_dir_tmp: &PathBuf, +) -> Result<(), String> { + let mut command = Command::new("tar"); + if is_redox() { + command.arg("xvf"); + } else { + command.arg("--extract"); + command.arg("--verbose"); + command.arg("--file"); + } + command.arg(&source_tar); + command.arg("--directory").arg(source_dir_tmp); + command.arg("--strip-components").arg("1"); + run_command(command)?; + Ok(()) +} + +pub(crate) fn fetch_is_patches_newer( + recipe_dir: &Path, + patches: &Vec, + source_dir: &PathBuf, +) -> Result { + // don't check source files inside as it can be mixed with user patches + let source_time = modified(&source_dir)?; + for patch_name in patches { + let patch_file = recipe_dir.join(patch_name); + if !patch_file.is_file() { + return Err(format!( + "failed to find patch file '{}'", + patch_file.display() + )); + } + + let patch_time = modified(&patch_file)?; + if patch_time > source_time { + return Ok(true); + } + } + return Ok(false); +} + +pub(crate) fn fetch_apply_patches( + recipe_dir: &Path, + patches: &Vec, + script: &Option, + source_dir_tmp: &PathBuf, +) -> Result<(), String> { + for patch_name in patches { + let patch_file = recipe_dir.join(patch_name); + if !patch_file.is_file() { + return Err(format!( + "failed to find patch file '{}'", + patch_file.display() + )); + } + + let patch = fs::read_to_string(&patch_file).map_err(|err| { + format!( + "failed to read patch file '{}': {}\n{:#?}", + patch_file.display(), + err, + err + ) + })?; + + let mut command = Command::new("patch"); + command.arg("--directory").arg(source_dir_tmp); + command.arg("--strip=1"); + run_command_stdin(command, patch.as_bytes())?; + } + Ok(if let Some(script) = script { + let mut command = Command::new("bash"); + command.arg("-ex"); + command.current_dir(source_dir_tmp); + run_command_stdin(command, format!("{SHARED_PRESCRIPT}\n{script}").as_bytes())?; + }) +} diff --git a/src/cook/fs.rs b/src/cook/fs.rs new file mode 100644 index 000000000..0fa51ecfd --- /dev/null +++ b/src/cook/fs.rs @@ -0,0 +1,194 @@ +use serde::Serialize; +use std::{ + fs, + io::{self, Write}, + path::Path, + process::{self, Stdio}, + time::SystemTime, +}; +use walkdir::{DirEntry, WalkDir}; + +//TODO: pub(crate) for all of these functions + +pub fn remove_all(path: &Path) -> Result<(), String> { + if path.is_dir() { + fs::remove_dir_all(path) + } else { + fs::remove_file(path) + } + .map_err(|err| format!("failed to remove '{}': {}\n{:?}", path.display(), err, err)) +} + +pub fn create_dir(dir: &Path) -> Result<(), String> { + fs::create_dir(dir) + .map_err(|err| format!("failed to create '{}': {}\n{:?}", dir.display(), err, err)) +} + +pub fn create_dir_clean(dir: &Path) -> Result<(), String> { + if dir.is_dir() { + remove_all(dir)?; + } + create_dir(dir) +} + +pub fn copy_dir_all(src: impl AsRef, dst: impl AsRef) -> io::Result<()> { + fs::create_dir_all(&dst)?; + for entry in fs::read_dir(src)? { + let entry = entry?; + let ty = entry.file_type()?; + if ty.is_dir() { + copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()))?; + } else { + fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?; + } + } + Ok(()) +} + +pub fn symlink(original: impl AsRef, link: impl AsRef) -> Result<(), String> { + std::os::unix::fs::symlink(&original, &link).map_err(|err| { + format!( + "failed to symlink '{}' to '{}': {}\n{:?}", + original.as_ref().display(), + link.as_ref().display(), + err, + err + ) + }) +} + +pub fn modified(path: &Path) -> Result { + let metadata = fs::metadata(path).map_err(|err| { + format!( + "failed to get metadata of '{}': {}\n{:#?}", + path.display(), + err, + err + ) + })?; + metadata.modified().map_err(|err| { + format!( + "failed to get modified time of '{}': {}\n{:#?}", + path.display(), + err, + err + ) + }) +} + +pub fn modified_dir_inner bool>( + dir: &Path, + filter: F, +) -> io::Result { + let mut newest = fs::metadata(dir)?.modified()?; + for entry_res in WalkDir::new(dir).into_iter().filter_entry(filter) { + let entry = entry_res?; + let modified = entry.metadata()?.modified()?; + if modified > newest { + newest = modified; + } + } + Ok(newest) +} + +pub fn modified_dir(dir: &Path) -> Result { + modified_dir_inner(dir, |_| true).map_err(|err| { + format!( + "failed to get modified time of '{}': {}\n{:#?}", + dir.display(), + err, + err + ) + }) +} + +pub fn modified_dir_ignore_git(dir: &Path) -> Result { + modified_dir_inner(dir, |entry| { + entry + .file_name() + .to_str() + .map(|s| s != ".git") + .unwrap_or(true) + }) + .map_err(|err| { + format!( + "failed to get modified time of '{}': {}\n{:#?}", + dir.display(), + err, + err + ) + }) +} + +pub fn rename(src: &Path, dst: &Path) -> Result<(), String> { + fs::rename(src, dst).map_err(|err| { + format!( + "failed to rename '{}' to '{}': {}\n{:?}", + src.display(), + dst.display(), + err, + err + ) + }) +} + +pub fn run_command(mut command: process::Command) -> Result<(), String> { + let status = command + .status() + .map_err(|err| format!("failed to run {:?}: {}\n{:#?}", command, err, err))?; + + if !status.success() { + return Err(format!( + "failed to run {:?}: exited with status {}", + command, status + )); + } + + Ok(()) +} + +pub fn run_command_stdin(mut command: process::Command, stdin_data: &[u8]) -> Result<(), String> { + command.stdin(Stdio::piped()); + + let mut child = command + .spawn() + .map_err(|err| format!("failed to spawn {:?}: {}\n{:#?}", command, err, err))?; + + if let Some(ref mut stdin) = child.stdin { + stdin.write_all(stdin_data).map_err(|err| { + format!( + "failed to write stdin of {:?}: {}\n{:#?}", + command, err, err + ) + })?; + } else { + return Err(format!("failed to find stdin of {:?}", command)); + } + + let status = child + .wait() + .map_err(|err| format!("failed to run {:?}: {}\n{:#?}", command, err, err))?; + + if !status.success() { + return Err(format!( + "failed to run {:?}: exited with status {}", + command, status + )); + } + + Ok(()) +} + +pub fn serialize_and_write(file_path: &Path, content: &T) -> Result<(), String> { + let toml_content = toml::to_string(content).map_err(|err| { + format!( + "Failed to serialize content for '{}': {}", + file_path.display(), + err + ) + })?; + + fs::write(file_path, toml_content) + .map_err(|err| format!("Failed to write to file '{}': {}", file_path.display(), err))?; + Ok(()) +} diff --git a/src/cook/mod.rs b/src/cook/mod.rs new file mode 100644 index 000000000..e50800384 --- /dev/null +++ b/src/cook/mod.rs @@ -0,0 +1,6 @@ +pub mod fetch; +pub mod fs; +pub mod script; +//TODO: Move rest of cook functions here in the next refactor +//pub mod build; +//pub mod package; diff --git a/src/cook/script.rs b/src/cook/script.rs new file mode 100644 index 000000000..aa531d699 --- /dev/null +++ b/src/cook/script.rs @@ -0,0 +1,91 @@ +//TODO: pub(crate) +pub static SHARED_PRESCRIPT: &str = r#" +# Build dynamically +function DYNAMIC_INIT { + COOKBOOK_AUTORECONF="autoreconf" + autotools_recursive_regenerate() { + for f in $(find . -name configure.ac -o -name configure.in -type f | sort); do + echo "* autotools regen in '$(dirname $f)'..." + ( cd "$(dirname "$f")" && "${COOKBOOK_AUTORECONF}" -fvi "$@" -I${COOKBOOK_HOST_SYSROOT}/share/aclocal ) + done + } + + if [ "${TARGET}" != "x86_64-unknown-redox" ] + then + echo "WARN: ${TARGET} does not support dynamic linking." >&2 + return + fi + + echo "DEBUG: Program is being compiled dynamically." + + COOKBOOK_CONFIGURE_FLAGS=( + --host="${GNU_TARGET}" + --prefix="/usr" + --enable-shared + --disable-static + ) + + COOKBOOK_CMAKE_FLAGS=( + -DBUILD_SHARED_LIBS=True + -DENABLE_SHARED=True + -DENABLE_STATIC=False + ) + + COOKBOOK_MESON_FLAGS=( + --buildtype release + --wrap-mode nofallback + --strip + -Ddefault_library=shared + -Dprefix=/usr + ) + + # TODO: check paths for spaces + export LDFLAGS="-Wl,-rpath-link,${COOKBOOK_SYSROOT}/lib -L${COOKBOOK_SYSROOT}/lib" + export RUSTFLAGS="-C target-feature=-crt-static" + export COOKBOOK_DYNAMIC=1 +} + +# Build both dynamically and statically +function DYNAMIC_STATIC_INIT { + DYNAMIC_INIT + if [ "${COOKBOOK_DYNAMIC}" == "1" ] + then + COOKBOOK_CONFIGURE_FLAGS=( + --host="${GNU_TARGET}" + --prefix="/usr" + --enable-shared + --enable-static + ) + + COOKBOOK_CMAKE_FLAGS=( + -DBUILD_SHARED_LIBS=True + -DENABLE_SHARED=True + -DENABLE_STATIC=True + ) + + COOKBOOK_MESON_FLAGS=( + --buildtype release + --wrap-mode nofallback + --strip + -Ddefault_library=both + -Dprefix=/usr + ) + fi +} + +function GNU_CONFIG_GET { + wget -O "$1" "https://gitlab.redox-os.org/redox-os/gnu-config/-/raw/master/config.sub?inline=false" +} +"#; + +pub(crate) static GIT_RESET_BRANCH: &str = r#" +ORIGIN_BRANCH="$(git branch --remotes | grep '^ origin/HEAD -> ' | cut -d ' ' -f 5-)" +if [ -n "$BRANCH" ] +then + ORIGIN_BRANCH="origin/$BRANCH" +fi + +if [ "$(git rev-parse HEAD)" != "$(git rev-parse $ORIGIN_BRANCH)" ] +then + git checkout -B "$(echo "$ORIGIN_BRANCH" | cut -d / -f 2-)" "$ORIGIN_BRANCH" +fi"#; diff --git a/src/lib.rs b/src/lib.rs index cfad4bc49..cef82ad06 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,6 @@ pub mod blake3; pub mod config; +pub mod cook; pub mod recipe; mod progress_bar;