From e0f58e1aba37b964b5e46c9c9717f1e4a1b3339d Mon Sep 17 00:00:00 2001 From: trivernis Date: Tue, 30 Aug 2022 21:49:03 +0200 Subject: [PATCH] [WIP] Parallel builds and installs Signed-off-by: trivernis --- src/builder/makepkg.rs | 134 ++++++++++++++++++++ src/builder/mod.rs | 1 + src/builder/pacman.rs | 25 +++- src/internal/commands.rs | 36 ++++-- src/internal/error.rs | 2 + src/operations/aur_install.rs | 229 +++++++++++++++++++++++++++++----- 6 files changed, 385 insertions(+), 42 deletions(-) create mode 100644 src/builder/makepkg.rs diff --git a/src/builder/makepkg.rs b/src/builder/makepkg.rs new file mode 100644 index 0000000..9bee86d --- /dev/null +++ b/src/builder/makepkg.rs @@ -0,0 +1,134 @@ +use std::fmt::Debug; +use std::path::{Path, PathBuf}; + +use crate::internal::{ + commands::ShellCommand, + error::{AppError, AppResult}, +}; + +#[derive(Default, Debug, Clone)] +pub struct MakePkgBuilder { + directory: PathBuf, + clean: bool, + no_deps: bool, + install: bool, + no_build: bool, + no_confirm: bool, + as_deps: bool, + skip_pgp: bool, + needed: bool, + no_prepare: bool, +} + +impl MakePkgBuilder { + /// Sets the working directory + pub fn directory>(mut self, dir: D) -> Self { + self.directory = dir.as_ref().into(); + + self + } + + pub fn clean(mut self, clean: bool) -> Self { + self.clean = clean; + + self + } + + pub fn no_deps(mut self, no_deps: bool) -> Self { + self.no_deps = no_deps; + + self + } + + pub fn no_build(mut self, no_build: bool) -> Self { + self.no_build = no_build; + + self + } + + pub fn no_prepare(mut self, no_prepare: bool) -> Self { + self.no_prepare = no_prepare; + + self + } + + /// Mark packages as non-explicitly installed + pub fn as_deps(mut self, as_deps: bool) -> Self { + self.as_deps = as_deps; + + self + } + + /// Skip PGP signature checks + pub fn skip_pgp(mut self, skip: bool) -> Self { + self.skip_pgp = skip; + + self + } + + /// Do not reinstall up to date packages + pub fn needed(mut self, needed: bool) -> Self { + self.needed = needed; + + self + } + + /// Executes the makepkg command + #[tracing::instrument(level = "trace")] + pub async fn run(self) -> AppResult<()> { + let mut command = ShellCommand::makepkg().working_dir(self.directory); + + if self.clean { + command = command.arg("-c"); + } + if self.no_deps { + command = command.arg("-d") + } + if self.install { + command = command.arg("-c"); + } + if self.no_build { + command = command.arg("-o"); + } + if self.no_confirm { + command = command.arg("--noconfirm") + } + if self.as_deps { + command = command.arg("--asdeps") + } + if self.skip_pgp { + command = command.arg("--skippgp") + } + if self.needed { + command = command.arg("--needed"); + } + if self.no_prepare { + command = command.arg("--noprepare") + } + + let output = command.wait_with_output().await?; + + if output.status.success() { + Ok(()) + } else { + Err(AppError::Other(output.stderr)) + } + } + + #[tracing::instrument(level = "trace")] + pub async fn package_list + Debug>(dir: D) -> AppResult> { + let result = ShellCommand::makepkg() + .working_dir(dir.as_ref()) + .arg("--packagelist") + .wait_with_output() + .await?; + + if result.status.success() { + let packages = result.stdout.lines().map(PathBuf::from).collect(); + + Ok(packages) + } else { + Err(AppError::Other(result.stderr)) + } + } +} diff --git a/src/builder/mod.rs b/src/builder/mod.rs index cb71352..e3233d0 100644 --- a/src/builder/mod.rs +++ b/src/builder/mod.rs @@ -1,2 +1,3 @@ pub mod git; +pub mod makepkg; pub mod pacman; diff --git a/src/builder/pacman.rs b/src/builder/pacman.rs index f7a48b3..4822c25 100644 --- a/src/builder/pacman.rs +++ b/src/builder/pacman.rs @@ -1,8 +1,11 @@ +use std::path::{Path, PathBuf}; + use crate::internal::{commands::ShellCommand, error::AppResult, structs::Options}; #[derive(Debug, Default)] pub struct PacmanInstallBuilder { packages: Vec, + files: Vec, as_deps: bool, no_confirm: bool, } @@ -21,6 +24,13 @@ impl PacmanInstallBuilder { self } + pub fn files, T: AsRef>(mut self, files: I) -> Self { + let mut files = files.into_iter().map(|f| f.as_ref().into()).collect(); + self.files.append(&mut files); + + self + } + pub fn no_confirm(mut self, no_confirm: bool) -> Self { self.no_confirm = no_confirm; @@ -36,7 +46,13 @@ impl PacmanInstallBuilder { #[tracing::instrument(level = "debug")] pub async fn install(self) -> AppResult<()> { - let mut command = ShellCommand::pacman().elevated().arg("-S").arg("--needed"); + let mut command = ShellCommand::pacman().elevated(); + + if !self.packages.is_empty() { + command = command.arg("-S"); + } else if !self.files.is_empty() { + command = command.arg("-U"); + } if self.no_confirm { command = command.arg("--noconfirm") @@ -46,7 +62,12 @@ impl PacmanInstallBuilder { command = command.arg("--asdeps") } - command.args(self.packages).wait_success().await + command + .arg("--needed") + .args(self.packages) + .args(self.files) + .wait_success() + .await } } diff --git a/src/internal/commands.rs b/src/internal/commands.rs index b8e86ad..6af8a0a 100644 --- a/src/internal/commands.rs +++ b/src/internal/commands.rs @@ -1,4 +1,5 @@ use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; use std::process::{ExitStatus, Stdio}; use tokio::process::{Child, Command}; @@ -17,6 +18,7 @@ pub struct ShellCommand { command: String, args: Vec, elevated: bool, + working_dir: Option, } impl ShellCommand { @@ -55,6 +57,7 @@ impl ShellCommand { command: command.to_string(), args: Vec::new(), elevated: false, + working_dir: None, } } @@ -77,6 +80,12 @@ impl ShellCommand { self } + pub fn working_dir>(mut self, dir: D) -> Self { + self.working_dir = Some(dir.as_ref().into()); + + self + } + /// Runs the command with sudo pub fn elevated(mut self) -> Self { self.elevated = true; @@ -117,25 +126,30 @@ impl ShellCommand { } fn spawn(self, piped: bool) -> AppResult { + tracing::debug!("Running {} {:?}", self.command, self.args); + let (stdout, stderr) = if piped { (Stdio::piped(), Stdio::piped()) } else { (Stdio::inherit(), Stdio::inherit()) }; - let child = if self.elevated { - Command::new("sudo") - .arg(self.command) - .args(self.args) - .stdout(stdout) - .stderr(stderr) - .spawn()? + let mut command = if self.elevated { + let mut cmd = Command::new("sudo"); + cmd.arg(self.command); + + cmd } else { Command::new(self.command) - .args(self.args) - .stdout(stdout) - .stderr(stderr) - .spawn()? }; + if let Some(dir) = self.working_dir { + command.current_dir(dir); + } + + let child = command + .args(self.args) + .stdout(stdout) + .stderr(stderr) + .spawn()?; Ok(child) } diff --git a/src/internal/error.rs b/src/internal/error.rs index 0b0ee81..442ee80 100644 --- a/src/internal/error.rs +++ b/src/internal/error.rs @@ -14,6 +14,7 @@ pub enum AppError { Other(String), Rpc(aur_rpc::error::RPCError), NonZeroExit, + BuildStepViolation, } impl Display for AppError { @@ -23,6 +24,7 @@ impl Display for AppError { AppError::Rpc(e) => Display::fmt(e, f), AppError::Other(s) => Display::fmt(s, f), AppError::NonZeroExit => Display::fmt("exited with non zero code", f), + AppError::BuildStepViolation => Display::fmt("AUR build violated build steps", f), } } } diff --git a/src/operations/aur_install.rs b/src/operations/aur_install.rs index 286c952..1d08558 100644 --- a/src/operations/aur_install.rs +++ b/src/operations/aur_install.rs @@ -2,9 +2,11 @@ use async_recursion::async_recursion; use aur_rpc::PackageInfo; use crossterm::style::Stylize; use futures::future; -use std::path::PathBuf; +use std::collections::HashSet; +use std::path::{Path, PathBuf}; use crate::builder::git::{GitCloneBuilder, GitPullBuilder}; +use crate::builder::makepkg::MakePkgBuilder; use crate::builder::pacman::PacmanInstallBuilder; use crate::internal::dependencies::DependencyInformation; use crate::internal::error::{AppError, AppResult, SilentUnwrap}; @@ -13,6 +15,53 @@ use crate::internal::utils::get_cache_dir; use crate::logging::get_logger; use crate::{crash, Options}; +#[derive(Debug)] +pub struct BuildContext { + pub package: PackageInfo, + pub step: BuildStep, +} + +#[derive(Debug)] +pub enum BuildStep { + Download, + Build(BuildPath), + Install(PackageArchives), + Done, +} + +#[derive(Debug)] +pub struct BuildPath(pub PathBuf); + +#[derive(Debug)] +pub struct PackageArchives(pub Vec); + +impl From for BuildContext { + fn from(package: PackageInfo) -> Self { + Self { + package, + step: BuildStep::Download, + } + } +} + +impl BuildContext { + pub fn build_path(&self) -> AppResult<&Path> { + if let BuildStep::Build(path) = &self.step { + Ok(&path.0) + } else { + Err(AppError::BuildStepViolation) + } + } + + pub fn packages(&self) -> AppResult<&Vec> { + if let BuildStep::Install(pkgs) = &self.step { + Ok(&pkgs.0) + } else { + Err(AppError::BuildStepViolation) + } + } +} + /// Installs a given list of packages from the aur #[tracing::instrument(level = "trace")] #[async_recursion] @@ -45,16 +94,10 @@ pub async fn aur_install(packages: Vec, options: Options) { ); } - pb.finish_with_message("Found all packages in the aur"); - - get_logger().new_multi_progress(); + get_logger().reset_output_type(); - future::try_join_all(package_info.iter().map(download_aur_source)) - .await - .unwrap(); + pb.finish_with_message("Found all packages in the aur"); - get_logger().reset_output_type(); - tracing::info!("All sources are ready."); get_logger().new_multi_progress(); let dependencies = future::try_join_all(package_info.iter().map(|pkg| async { @@ -66,41 +109,118 @@ pub async fn aur_install(packages: Vec, options: Options) { .await .silent_unwrap(AppExitCode::RpcError); + get_logger().new_multi_progress(); + + let contexts = future::try_join_all( + package_info + .into_iter() + .map(BuildContext::from) + .map(download_aur_source), + ) + .await + .unwrap(); + + get_logger().reset_output_type(); + tracing::info!("All sources are ready."); + let aur_build_dependencies: Vec = dependencies .iter() .flat_map(|d| d.make_depends.aur.clone()) .collect(); - let repo_build_dependencies: Vec = dependencies + let repo_dependencies: HashSet = dependencies .iter() - .flat_map(|d| d.make_depends.repo.clone()) + .flat_map(|d| { + let mut repo_deps = d.make_depends.repo.clone(); + repo_deps.append(&mut d.depends.repo.clone()); + + repo_deps + }) .collect(); get_logger().reset_output_type(); - tracing::info!("Installing repo build dependencies"); - PacmanInstallBuilder::default() - .as_deps(true) - .packages(repo_build_dependencies) - .install() - .await - .silent_unwrap(AppExitCode::PacmanError); + if !repo_dependencies.is_empty() { + tracing::info!("Installing repo dependencies"); + PacmanInstallBuilder::default() + .as_deps(true) + .packages(repo_dependencies) + .install() + .await + .silent_unwrap(AppExitCode::PacmanError); + } - tracing::info!( - "Installing {} build dependencies from the aur", - aur_build_dependencies.len() - ); + if !aur_build_dependencies.is_empty() { + tracing::info!( + "Installing {} build dependencies from the aur", + aur_build_dependencies.len() + ); + install_aur_build_dependencies(aur_build_dependencies) + .await + .unwrap(); + } + + tracing::info!("Installing {} packages", contexts.len()); + + build_and_install( + contexts, + MakePkgBuilder::default(), + PacmanInstallBuilder::default(), + ) + .await + .silent_unwrap(AppExitCode::MakePkgError); +} + +async fn install_aur_build_dependencies(deps: Vec) -> AppResult<()> { get_logger().new_multi_progress(); - future::try_join_all(aur_build_dependencies.iter().map(download_aur_source)) - .await - .unwrap(); + let dep_contexts = future::try_join_all( + deps.into_iter() + .map(BuildContext::from) + .map(download_aur_source), + ) + .await?; + + get_logger().reset_output_type(); + + build_and_install( + dep_contexts, + MakePkgBuilder::default().as_deps(true), + PacmanInstallBuilder::default().as_deps(true), + ) + .await?; + + Ok(()) +} + +#[tracing::instrument(level = "trace")] +async fn build_and_install( + ctxs: Vec, + make_opts: MakePkgBuilder, + install_opts: PacmanInstallBuilder, +) -> AppResult<()> { + tracing::info!("Building packages"); + get_logger().new_multi_progress(); + let ctxs = future::try_join_all( + ctxs.into_iter() + .map(|ctx| build_package(ctx, make_opts.clone())), + ) + .await + .silent_unwrap(AppExitCode::MakePkgError); + get_logger().reset_output_type(); + + tracing::info!("Built {} packages", ctxs.len()); + tracing::info!("Installing packages..."); + + install_packages(ctxs, install_opts).await?; + + Ok(()) } #[tracing::instrument(level = "trace", skip_all)] -async fn download_aur_source(info: &PackageInfo) -> AppResult { +async fn download_aur_source(mut ctx: BuildContext) -> AppResult { let pb = get_logger().new_progress_spinner(); - let pkg_name = &info.metadata.name; + let pkg_name = &ctx.package.metadata.name; pb.set_message(format!("{pkg_name}: Downloading sources")); let cache_dir = get_cache_dir(); @@ -121,8 +241,59 @@ async fn download_aur_source(info: &PackageInfo) -> AppResult { .await?; pb.set_message(format!("{pkg_name}: Downloading and extracting files")); + + MakePkgBuilder::default() + .directory(&pkg_dir) + .no_build(true) + .no_deps(true) + .no_prepare(true) + .skip_pgp(true) + .run() + .await?; } - pb.finish_with_message(format!("{pkg_name} is ready to build")); + pb.finish_with_message(format!("{pkg_name}: Downloaded!")); + ctx.step = BuildStep::Build(BuildPath(pkg_dir)); + + Ok(ctx) +} + +async fn build_package( + mut ctx: BuildContext, + make_opts: MakePkgBuilder, +) -> AppResult { + let pb = get_logger().new_progress_spinner(); + let pkg_name = &ctx.package.metadata.name; + let build_path = ctx.build_path()?; + pb.set_message(format!("{pkg_name}: Building Package")); + + make_opts + .directory(build_path) + .clean(true) + .no_deps(true) + .skip_pgp(true) + .needed(true) + .run() + .await?; + + let packages = MakePkgBuilder::package_list(build_path).await?; + pb.finish_with_message(format!("{pkg_name}: Built!")); + ctx.step = BuildStep::Install(PackageArchives(packages)); + + Ok(ctx) +} + +async fn install_packages( + mut ctxs: Vec, + install_opts: PacmanInstallBuilder, +) -> AppResult> { + let mut packages = Vec::new(); + + for ctx in &mut ctxs { + packages.append(&mut ctx.packages()?.clone()); + ctx.step = BuildStep::Done; + } + + install_opts.files(packages).install().await?; - Ok(pkg_dir) + Ok(ctxs) }