[WIP] Parallel builds and installs

Signed-off-by: trivernis <trivernis@protonmail.com>

src/builder/makepkg.rs

@@ -0,0 +1,134 @@
+use std::fmt::Debug;
+use std::path::{Path, PathBuf};
+
+use crate::internal::{
+    commands::ShellCommand,
+    error::{AppError, AppResult},
+};
+
+#[derive(Default, Debug, Clone)]
+pub struct MakePkgBuilder {
+    directory: PathBuf,
+    clean: bool,
+    no_deps: bool,
+    install: bool,
+    no_build: bool,
+    no_confirm: bool,
+    as_deps: bool,
+    skip_pgp: bool,
+    needed: bool,
+    no_prepare: bool,
+}
+
+impl MakePkgBuilder {
+    /// Sets the working directory
+    pub fn directory<D: AsRef<Path>>(mut self, dir: D) -> Self {
+        self.directory = dir.as_ref().into();
+
+        self
+    }
+
+    pub fn clean(mut self, clean: bool) -> Self {
+        self.clean = clean;
+
+        self
+    }
+
+    pub fn no_deps(mut self, no_deps: bool) -> Self {
+        self.no_deps = no_deps;
+
+        self
+    }
+
+    pub fn no_build(mut self, no_build: bool) -> Self {
+        self.no_build = no_build;
+
+        self
+    }
+
+    pub fn no_prepare(mut self, no_prepare: bool) -> Self {
+        self.no_prepare = no_prepare;
+
+        self
+    }
+
+    /// Mark packages as non-explicitly installed
+    pub fn as_deps(mut self, as_deps: bool) -> Self {
+        self.as_deps = as_deps;
+
+        self
+    }
+
+    /// Skip PGP signature checks
+    pub fn skip_pgp(mut self, skip: bool) -> Self {
+        self.skip_pgp = skip;
+
+        self
+    }
+
+    /// Do not reinstall up to date packages
+    pub fn needed(mut self, needed: bool) -> Self {
+        self.needed = needed;
+
+        self
+    }
+
+    /// Executes the makepkg command
+    #[tracing::instrument(level = "trace")]
+    pub async fn run(self) -> AppResult<()> {
+        let mut command = ShellCommand::makepkg().working_dir(self.directory);
+
+        if self.clean {
+            command = command.arg("-c");
+        }
+        if self.no_deps {
+            command = command.arg("-d")
+        }
+        if self.install {
+            command = command.arg("-c");
+        }
+        if self.no_build {
+            command = command.arg("-o");
+        }
+        if self.no_confirm {
+            command = command.arg("--noconfirm")
+        }
+        if self.as_deps {
+            command = command.arg("--asdeps")
+        }
+        if self.skip_pgp {
+            command = command.arg("--skippgp")
+        }
+        if self.needed {
+            command = command.arg("--needed");
+        }
+        if self.no_prepare {
+            command = command.arg("--noprepare")
+        }
+
+        let output = command.wait_with_output().await?;
+
+        if output.status.success() {
+            Ok(())
+        } else {
+            Err(AppError::Other(output.stderr))
+        }
+    }
+
+    #[tracing::instrument(level = "trace")]
+    pub async fn package_list<D: AsRef<Path> + Debug>(dir: D) -> AppResult<Vec<PathBuf>> {
+        let result = ShellCommand::makepkg()
+            .working_dir(dir.as_ref())
+            .arg("--packagelist")
+            .wait_with_output()
+            .await?;
+
+        if result.status.success() {
+            let packages = result.stdout.lines().map(PathBuf::from).collect();
+
+            Ok(packages)
+        } else {
+            Err(AppError::Other(result.stderr))
+        }
+    }
+}

src/builder/mod.rs

@@ -1,2 +1,3 @@
 pub mod git;
+pub mod makepkg;
 pub mod pacman;

src/builder/pacman.rs

@@ -1,8 +1,11 @@
+use std::path::{Path, PathBuf};
+
 use crate::internal::{commands::ShellCommand, error::AppResult, structs::Options};
 
 #[derive(Debug, Default)]
 pub struct PacmanInstallBuilder {
     packages: Vec<String>,
+    files: Vec<PathBuf>,
     as_deps: bool,
     no_confirm: bool,
 }
@@ -21,6 +24,13 @@ impl PacmanInstallBuilder {
         self
     }
 
+    pub fn files<I: IntoIterator<Item = T>, T: AsRef<Path>>(mut self, files: I) -> Self {
+        let mut files = files.into_iter().map(|f| f.as_ref().into()).collect();
+        self.files.append(&mut files);
+
+        self
+    }
+
     pub fn no_confirm(mut self, no_confirm: bool) -> Self {
         self.no_confirm = no_confirm;
 
@@ -36,7 +46,13 @@ impl PacmanInstallBuilder {
 
     #[tracing::instrument(level = "debug")]
     pub async fn install(self) -> AppResult<()> {
-        let mut command = ShellCommand::pacman().elevated().arg("-S").arg("--needed");
+        let mut command = ShellCommand::pacman().elevated();
+
+        if !self.packages.is_empty() {
+            command = command.arg("-S");
+        } else if !self.files.is_empty() {
+            command = command.arg("-U");
+        }
 
         if self.no_confirm {
             command = command.arg("--noconfirm")
@@ -46,7 +62,12 @@ impl PacmanInstallBuilder {
             command = command.arg("--asdeps")
         }
 
-        command.args(self.packages).wait_success().await
+        command
+            .arg("--needed")
+            .args(self.packages)
+            .args(self.files)
+            .wait_success()
+            .await
     }
 }
 

src/internal/commands.rs

@@ -1,4 +1,5 @@
 use std::ffi::{OsStr, OsString};
+use std::path::{Path, PathBuf};
 use std::process::{ExitStatus, Stdio};
 use tokio::process::{Child, Command};
 
@@ -17,6 +18,7 @@ pub struct ShellCommand {
     command: String,
     args: Vec<OsString>,
     elevated: bool,
+    working_dir: Option<PathBuf>,
 }
 
 impl ShellCommand {
@@ -55,6 +57,7 @@ impl ShellCommand {
             command: command.to_string(),
             args: Vec::new(),
             elevated: false,
+            working_dir: None,
         }
     }
 
@@ -77,6 +80,12 @@ impl ShellCommand {
         self
     }
 
+    pub fn working_dir<D: AsRef<Path>>(mut self, dir: D) -> Self {
+        self.working_dir = Some(dir.as_ref().into());
+
+        self
+    }
+
     /// Runs the command with sudo
     pub fn elevated(mut self) -> Self {
         self.elevated = true;
@@ -117,25 +126,30 @@ impl ShellCommand {
     }
 
     fn spawn(self, piped: bool) -> AppResult<Child> {
+        tracing::debug!("Running {} {:?}", self.command, self.args);
+
         let (stdout, stderr) = if piped {
             (Stdio::piped(), Stdio::piped())
         } else {
             (Stdio::inherit(), Stdio::inherit())
         };
-        let child = if self.elevated {
-            Command::new("sudo")
-                .arg(self.command)
-                .args(self.args)
-                .stdout(stdout)
-                .stderr(stderr)
-                .spawn()?
+        let mut command = if self.elevated {
+            let mut cmd = Command::new("sudo");
+            cmd.arg(self.command);
+
+            cmd
         } else {
             Command::new(self.command)
-                .args(self.args)
-                .stdout(stdout)
-                .stderr(stderr)
-                .spawn()?
         };
+        if let Some(dir) = self.working_dir {
+            command.current_dir(dir);
+        }
+
+        let child = command
+            .args(self.args)
+            .stdout(stdout)
+            .stderr(stderr)
+            .spawn()?;
 
         Ok(child)
     }

src/internal/error.rs

@@ -14,6 +14,7 @@ pub enum AppError {
     Other(String),
     Rpc(aur_rpc::error::RPCError),
     NonZeroExit,
+    BuildStepViolation,
 }
 
 impl Display for AppError {
@@ -23,6 +24,7 @@ impl Display for AppError {
             AppError::Rpc(e) => Display::fmt(e, f),
             AppError::Other(s) => Display::fmt(s, f),
             AppError::NonZeroExit => Display::fmt("exited with non zero code", f),
+            AppError::BuildStepViolation => Display::fmt("AUR build violated build steps", f),
         }
     }
 }

src/operations/aur_install.rs

@@ -2,9 +2,11 @@ use async_recursion::async_recursion;
 use aur_rpc::PackageInfo;
 use crossterm::style::Stylize;
 use futures::future;
-use std::path::PathBuf;
+use std::collections::HashSet;
+use std::path::{Path, PathBuf};
 
 use crate::builder::git::{GitCloneBuilder, GitPullBuilder};
+use crate::builder::makepkg::MakePkgBuilder;
 use crate::builder::pacman::PacmanInstallBuilder;
 use crate::internal::dependencies::DependencyInformation;
 use crate::internal::error::{AppError, AppResult, SilentUnwrap};
@@ -13,6 +15,53 @@ use crate::internal::utils::get_cache_dir;
 use crate::logging::get_logger;
 use crate::{crash, Options};
 
+#[derive(Debug)]
+pub struct BuildContext {
+    pub package: PackageInfo,
+    pub step: BuildStep,
+}
+
+#[derive(Debug)]
+pub enum BuildStep {
+    Download,
+    Build(BuildPath),
+    Install(PackageArchives),
+    Done,
+}
+
+#[derive(Debug)]
+pub struct BuildPath(pub PathBuf);
+
+#[derive(Debug)]
+pub struct PackageArchives(pub Vec<PathBuf>);
+
+impl From<PackageInfo> for BuildContext {
+    fn from(package: PackageInfo) -> Self {
+        Self {
+            package,
+            step: BuildStep::Download,
+        }
+    }
+}
+
+impl BuildContext {
+    pub fn build_path(&self) -> AppResult<&Path> {
+        if let BuildStep::Build(path) = &self.step {
+            Ok(&path.0)
+        } else {
+            Err(AppError::BuildStepViolation)
+        }
+    }
+
+    pub fn packages(&self) -> AppResult<&Vec<PathBuf>> {
+        if let BuildStep::Install(pkgs) = &self.step {
+            Ok(&pkgs.0)
+        } else {
+            Err(AppError::BuildStepViolation)
+        }
+    }
+}
+
 /// Installs a given list of packages from the aur
 #[tracing::instrument(level = "trace")]
 #[async_recursion]
@@ -45,16 +94,10 @@ pub async fn aur_install(packages: Vec<String>, options: Options) {
         );
     }
 
-    pb.finish_with_message("Found all packages in the aur");
-
-    get_logger().new_multi_progress();
+    get_logger().reset_output_type();
 
-    future::try_join_all(package_info.iter().map(download_aur_source))
-        .await
-        .unwrap();
+    pb.finish_with_message("Found all packages in the aur");
 
-    get_logger().reset_output_type();
-    tracing::info!("All sources are ready.");
     get_logger().new_multi_progress();
 
     let dependencies = future::try_join_all(package_info.iter().map(|pkg| async {
@@ -66,41 +109,118 @@ pub async fn aur_install(packages: Vec<String>, options: Options) {
     .await
     .silent_unwrap(AppExitCode::RpcError);
 
+    get_logger().new_multi_progress();
+
+    let contexts = future::try_join_all(
+        package_info
+            .into_iter()
+            .map(BuildContext::from)
+            .map(download_aur_source),
+    )
+    .await
+    .unwrap();
+
+    get_logger().reset_output_type();
+    tracing::info!("All sources are ready.");
+
     let aur_build_dependencies: Vec<PackageInfo> = dependencies
         .iter()
         .flat_map(|d| d.make_depends.aur.clone())
         .collect();
 
-    let repo_build_dependencies: Vec<String> = dependencies
+    let repo_dependencies: HashSet<String> = dependencies
         .iter()
-        .flat_map(|d| d.make_depends.repo.clone())
+        .flat_map(|d| {
+            let mut repo_deps = d.make_depends.repo.clone();
+            repo_deps.append(&mut d.depends.repo.clone());
+
+            repo_deps
+        })
         .collect();
 
     get_logger().reset_output_type();
 
-    tracing::info!("Installing repo build dependencies");
-    PacmanInstallBuilder::default()
-        .as_deps(true)
-        .packages(repo_build_dependencies)
-        .install()
-        .await
-        .silent_unwrap(AppExitCode::PacmanError);
+    if !repo_dependencies.is_empty() {
+        tracing::info!("Installing repo dependencies");
+        PacmanInstallBuilder::default()
+            .as_deps(true)
+            .packages(repo_dependencies)
+            .install()
+            .await
+            .silent_unwrap(AppExitCode::PacmanError);
+    }
 
-    tracing::info!(
-        "Installing {} build dependencies from the aur",
-        aur_build_dependencies.len()
-    );
+    if !aur_build_dependencies.is_empty() {
+        tracing::info!(
+            "Installing {} build dependencies from the aur",
+            aur_build_dependencies.len()
+        );
+        install_aur_build_dependencies(aur_build_dependencies)
+            .await
+            .unwrap();
+    }
+
+    tracing::info!("Installing {} packages", contexts.len());
+
+    build_and_install(
+        contexts,
+        MakePkgBuilder::default(),
+        PacmanInstallBuilder::default(),
+    )
+    .await
+    .silent_unwrap(AppExitCode::MakePkgError);
+}
+
+async fn install_aur_build_dependencies(deps: Vec<PackageInfo>) -> AppResult<()> {
     get_logger().new_multi_progress();
 
-    future::try_join_all(aur_build_dependencies.iter().map(download_aur_source))
-        .await
-        .unwrap();
+    let dep_contexts = future::try_join_all(
+        deps.into_iter()
+            .map(BuildContext::from)
+            .map(download_aur_source),
+    )
+    .await?;
+
+    get_logger().reset_output_type();
+
+    build_and_install(
+        dep_contexts,
+        MakePkgBuilder::default().as_deps(true),
+        PacmanInstallBuilder::default().as_deps(true),
+    )
+    .await?;
+
+    Ok(())
+}
+
+#[tracing::instrument(level = "trace")]
+async fn build_and_install(
+    ctxs: Vec<BuildContext>,
+    make_opts: MakePkgBuilder,
+    install_opts: PacmanInstallBuilder,
+) -> AppResult<()> {
+    tracing::info!("Building packages");
+    get_logger().new_multi_progress();
+    let ctxs = future::try_join_all(
+        ctxs.into_iter()
+            .map(|ctx| build_package(ctx, make_opts.clone())),
+    )
+    .await
+    .silent_unwrap(AppExitCode::MakePkgError);
+    get_logger().reset_output_type();
+
+    tracing::info!("Built {} packages", ctxs.len());
+    tracing::info!("Installing packages...");
+
+    install_packages(ctxs, install_opts).await?;
+
+    Ok(())
 }
 
 #[tracing::instrument(level = "trace", skip_all)]
-async fn download_aur_source(info: &PackageInfo) -> AppResult<PathBuf> {
+async fn download_aur_source(mut ctx: BuildContext) -> AppResult<BuildContext> {
     let pb = get_logger().new_progress_spinner();
-    let pkg_name = &info.metadata.name;
+    let pkg_name = &ctx.package.metadata.name;
     pb.set_message(format!("{pkg_name}: Downloading sources"));
 
     let cache_dir = get_cache_dir();
@@ -121,8 +241,59 @@ async fn download_aur_source(info: &PackageInfo) -> AppResult<PathBuf> {
             .await?;
 
         pb.set_message(format!("{pkg_name}: Downloading and extracting files"));
+
+        MakePkgBuilder::default()
+            .directory(&pkg_dir)
+            .no_build(true)
+            .no_deps(true)
+            .no_prepare(true)
+            .skip_pgp(true)
+            .run()
+            .await?;
     }
-    pb.finish_with_message(format!("{pkg_name} is ready to build"));
+    pb.finish_with_message(format!("{pkg_name}: Downloaded!"));
+    ctx.step = BuildStep::Build(BuildPath(pkg_dir));
+
+    Ok(ctx)
+}
+
+async fn build_package(
+    mut ctx: BuildContext,
+    make_opts: MakePkgBuilder,
+) -> AppResult<BuildContext> {
+    let pb = get_logger().new_progress_spinner();
+    let pkg_name = &ctx.package.metadata.name;
+    let build_path = ctx.build_path()?;
+    pb.set_message(format!("{pkg_name}: Building Package"));
+
+    make_opts
+        .directory(build_path)
+        .clean(true)
+        .no_deps(true)
+        .skip_pgp(true)
+        .needed(true)
+        .run()
+        .await?;
+
+    let packages = MakePkgBuilder::package_list(build_path).await?;
+    pb.finish_with_message(format!("{pkg_name}: Built!"));
+    ctx.step = BuildStep::Install(PackageArchives(packages));
+
+    Ok(ctx)
+}
+
+async fn install_packages(
+    mut ctxs: Vec<BuildContext>,
+    install_opts: PacmanInstallBuilder,
+) -> AppResult<Vec<BuildContext>> {
+    let mut packages = Vec::new();
+
+    for ctx in &mut ctxs {
+        packages.append(&mut ctx.packages()?.clone());
+        ctx.step = BuildStep::Done;
+    }
+
+    install_opts.files(packages).install().await?;
 
-    Ok(pkg_dir)
+    Ok(ctxs)
 }