Merge branch 'speed-rebuild' into 'master'

Few improvements to make rebuild quick

See merge request redox-os/redox!1849

src/bin/repo.rs

@@ -293,7 +293,7 @@ fn repo_inner(
                 let is_cook = *command == CliCommand::Cook;
                 let source_dir = handle_fetch(recipe, config, is_cook, logger)?;
                 if is_cook {
-                    handle_cook(recipe, config, source_dir, recipe.is_deps, logger)?;
+                    handle_cook(recipe, config, source_dir, logger)?;
                 }
                 Ok(())
             };
@@ -630,7 +630,7 @@ fn handle_fetch(
 ) -> anyhow::Result<PathBuf> {
     let source_dir = match config.cook.offline && allow_offline {
         true => fetch_offline(&recipe, logger),
-        false => fetch(&recipe, logger),
+        false => fetch(&recipe, !recipe.is_deps, logger),
     }
     .map_err(|e| anyhow!("failed to fetch: {:?}", e))?;
 
@@ -641,7 +641,6 @@ fn handle_cook(
     recipe: &CookRecipe,
     config: &CliConfig,
     source_dir: PathBuf,
-    is_deps: bool,
     logger: &PtyOut,
 ) -> anyhow::Result<()> {
     let recipe_dir = &recipe.dir;
@@ -653,7 +652,7 @@ fn handle_cook(
         &recipe.name,
         &recipe.recipe,
         &config.cook,
-        !is_deps,
+        !recipe.is_deps,
         logger,
     )
     .map_err(|err| anyhow!("failed to build: {:?}", err))?;
@@ -1099,7 +1098,6 @@ fn run_tui_cook(
     let cooker_handle = thread::spawn(move || {
         'done: for (mut recipe, source_dir) in work_rx {
             let name = recipe.name.clone();
-            let is_deps = recipe.is_deps;
             let (mut stdout_writer, mut stderr_writer) = setup_logger(&cooker_status_tx, &name);
             let mut logger = Some((&mut stdout_writer, &mut stderr_writer));
             'again: loop {
@@ -1107,13 +1105,7 @@ fn run_tui_cook(
                     .send(StatusUpdate::StartCook(name.clone()))
                     .unwrap();
                 let _ = recipe.reload_recipe(); // reread recipe.toml in case we're retrying
-                let handler = handle_cook(
-                    &recipe,
-                    &cooker_config,
-                    source_dir.clone(),
-                    is_deps,
-                    &logger,
-                );
+                let handler = handle_cook(&recipe, &cooker_config, source_dir.clone(), &logger);
                 if let Some(log_path) = cooker_config.logs_dir.as_ref() {
                     if let Err(err_ctx) = &handler {
                         log_to_pty!(&logger, "\n{:?}", err_ctx)

src/cook/cook_build.rs

@@ -180,6 +180,10 @@ pub fn build(
     let sysroot_dir = target_dir.join("sysroot");
     let toolchain_dir = target_dir.join("toolchain");
     let stage_dirs = get_stage_dirs(&recipe.optional_packages, target_dir);
+    let stage_pkgars: Vec<PathBuf> = stage_dirs
+        .iter()
+        .map(|p| p.with_added_extension("pkgar"))
+        .collect();
     let cli_verbose = cook_config.verbose;
     let cli_jobs = cook_config.jobs;
     if recipe.build.kind == BuildKind::None {
@@ -218,7 +222,7 @@ pub fn build(
         };
     }
 
-    if !check_source && stage_dirs.iter().all(|dir| dir.exists()) {
+    if !check_source && stage_pkgars.iter().all(|file| file.is_file()) {
         if cli_verbose {
             log_to_pty!(logger, "DEBUG: using cached build, not checking source");
         }
@@ -245,14 +249,7 @@ pub fn build(
         .unwrap_or(Ok(SystemTime::UNIX_EPOCH))?;
 
     // check stage dir modified against pkgar files, any files missing will result in UNIX_EPOCH
-    let stage_modified = modified_all(
-        &stage_dirs
-            .iter()
-            .map(|p| p.with_added_extension("pkgar"))
-            .collect(),
-        modified,
-    )
-    .unwrap_or(SystemTime::UNIX_EPOCH);
+    let stage_modified = modified_all(&stage_pkgars, modified).unwrap_or(SystemTime::UNIX_EPOCH);
     // Rebuild stage if source is newer
     if stage_modified < source_modified
         || stage_modified < deps_modified

src/cook/fetch.rs

@@ -52,7 +52,7 @@ pub fn fetch_offline(recipe: &CookRecipe, logger: &PtyOut) -> Result<PathBuf, St
 
     let ident = match &recipe.recipe.source {
         Some(SourceRecipe::Path { path: _ }) | None => {
-            fetch(recipe, logger)?;
+            fetch(recipe, true, logger)?;
             "local_source".to_string()
         }
         Some(SourceRecipe::SameAs { same_as }) => {
@@ -114,7 +114,7 @@ pub fn fetch_offline(recipe: &CookRecipe, logger: &PtyOut) -> Result<PathBuf, St
     Ok(source_dir)
 }
 
-pub fn fetch(recipe: &CookRecipe, logger: &PtyOut) -> Result<PathBuf, String> {
+pub fn fetch(recipe: &CookRecipe, check_source: bool, logger: &PtyOut) -> Result<PathBuf, String> {
     let recipe_dir = &recipe.dir;
     let source_dir = recipe_dir.join("source");
     match recipe.recipe.build.kind {
@@ -134,7 +134,7 @@ pub fn fetch(recipe: &CookRecipe, logger: &PtyOut) -> Result<PathBuf, String> {
         Some(SourceRecipe::SameAs { same_as }) => {
             let recipe = fetch_resolve_canon(recipe_dir, &same_as, recipe.name.is_host())?;
             // recursively fetch
-            fetch(&recipe, logger)?;
+            fetch(&recipe, check_source, logger)?;
             fetch_make_symlink(&source_dir, &same_as)?;
             fetch_get_source_info(&recipe)?.source_identifier
         }
@@ -195,6 +195,8 @@ pub fn fetch(recipe: &CookRecipe, logger: &PtyOut) -> Result<PathBuf, String> {
                 rename(&source_dir_tmp, &source_dir)?;
 
                 false
+            } else if !check_source {
+                true
             } else {
                 if !source_dir.join(".git").is_dir() {
                     return Err(format!(
@@ -323,28 +325,29 @@ pub fn fetch(recipe: &CookRecipe, logger: &PtyOut) -> Result<PathBuf, String> {
         }) => {
             let source_tar = recipe_dir.join("source.tar");
             let mut tar_updated = false;
-            while {
+            loop {
                 if !source_tar.is_file() {
                     tar_updated = true;
                     download_wget(&tar, &source_tar, logger)?;
                 }
+                if !check_source {
+                    break;
+                }
                 let source_tar_blake3 = get_blake3(&source_tar, tar_updated && logger.is_none())?;
                 if let Some(blake3) = blake3 {
-                    if source_tar_blake3 != *blake3 {
-                        if tar_updated {
-                            return Err(format!(
-                                "The downloaded tar blake3 '{source_tar_blake3}' is not equal to blake3 in recipe.toml"
-                            ));
-                        } else {
-                            log_to_pty!(
-                                logger,
-                                "DEBUG: source tar blake3 is different and need redownload"
-                            );
-                            remove_all(&source_tar)?;
-                        }
-                        true
+                    if source_tar_blake3 == *blake3 {
+                        break;
+                    }
+                    if tar_updated {
+                        return Err(format!(
+                            "The downloaded tar blake3 '{source_tar_blake3}' is not equal to blake3 in recipe.toml"
+                        ));
                     } else {
-                        false
+                        log_to_pty!(
+                            logger,
+                            "DEBUG: source tar blake3 is different and need redownload"
+                        );
+                        remove_all(&source_tar)?;
                     }
                 } else {
                     //TODO: set blake3 hash on the recipe with something like "cook fix"
@@ -354,9 +357,9 @@ pub fn fetch(recipe: &CookRecipe, logger: &PtyOut) -> Result<PathBuf, String> {
                         source_tar.display(),
                         source_tar_blake3
                     );
-                    false
+                    break;
                 }
-            } {}
+            }
             if source_dir.is_dir() {
                 if tar_updated || fetch_is_patches_newer(recipe_dir, patches, &source_dir)? {
                     log_to_pty!(
@@ -397,6 +400,7 @@ pub fn fetch(recipe: &CookRecipe, logger: &PtyOut) -> Result<PathBuf, String> {
         cargoflags: _,
     } = &recipe.recipe.build.kind
     {
+        // TODO: No need to fetch if !check_source and already fetched?
         fetch_cargo(&source_dir, package_path.as_ref(), logger)?;
     }
 

src/cook/pty.rs

@@ -62,9 +62,8 @@ pub fn flush_pty(logger: &mut PtyOut) {
     };
     // Not sure if flush actually working
     let _ = pty.flush();
-    std::thread::sleep(Duration::from_millis(100));
+    std::thread::sleep(Duration::from_millis(10));
     let _ = file.flush();
-    std::thread::sleep(Duration::from_millis(100));
 }
 
 pub fn spawn_to_pipe(command: &mut Command, stdout_pipe: &PtyOut) -> Result<Child, Error> {