Massive malachite internal rework

main
Michal 2 years ago
parent a68d201bba
commit 06e906d2ef
No known key found for this signature in database
GPG Key ID: A6A1A4DCB22279B9

2
Cargo.lock generated

@ -4,7 +4,7 @@ version = 3
[[package]]
name = "Malachite"
version = "1.3.0"
version = "1.4.0"
dependencies = [
"clap",
"colored",

@ -4,7 +4,7 @@ name = "test"
repo = [
"1::amethyst",
"1::jade",
"2::notop-git",
"2::notop-git!",
"3::slippy-rb"
]

@ -13,6 +13,14 @@ pub struct Args {
/// Complete operations without prompting user
#[clap(long = "noconfirm", global(true), action = ArgAction::SetTrue)]
pub no_confirm: bool,
/// Excludes packages from given operation, if applicable
#[clap(short = 'x', long = "exclude", action = ArgAction::Append, takes_value = true)]
pub exclude: Vec<String>,
/// Operates on all packages in the workspace, if applicable
#[clap(long = "all", action = ArgAction::SetTrue, conflicts_with = "package(s)")]
pub all: bool,
}
#[derive(Debug, Clone, Subcommand)]
@ -24,30 +32,18 @@ pub enum Operation {
#[clap(name = "package(s)", action = ArgAction::Append, index = 1)]
packages: Vec<String>,
/// Builds all packages in mlc.toml (except if -x is specified)
#[clap(long = "all", takes_value = false, action = ArgAction::SetTrue, conflicts_with = "package(s)")]
all: bool,
/// Excludes packages from given operation
#[clap(short = 'x', long = "exclude", action = ArgAction::Append, takes_value = true)]
exclude: Vec<String>,
/// Does not regenerate repository after building given package(s)
#[clap(short = 'n', long = "no-regen", action = ArgAction::SetTrue)]
no_regen: bool,
},
/// Generates repository from built packages
#[clap(name = "repo-gen", aliases = & ["r"])]
/// Generates Pacman repository from built packages
#[clap(name = "repo-gen", aliases = & ["repo", "r"])]
RepoGen,
/// Prunes duplicate packages from the repository
#[clap(name = "prune", aliases = & ["p"])]
Prune,
/// Clones all git repositories from mlc.toml branching from current directory
#[clap(name = "init", aliases = & ["i"])]
Init,
#[clap(name = "clone", aliases = & ["c"])]
Clone,
/// Pulls in git repositories from mlc.toml branching from current directory
#[clap(name = "pull", aliases = & ["u"])]
@ -55,17 +51,9 @@ pub enum Operation {
/// The packages to operate on
#[clap(name = "package(s)", help = "The packages to operate on", action = ArgAction::Append, index = 1)]
packages: Vec<String>,
/// Pulls from all git repositories from mlc.toml branching from current directory
#[clap(long = "all", action = ArgAction::SetTrue, conflicts_with = "package(s)")]
all: bool,
/// Excludes packages from given operation
#[clap(short = 'x', long = "exclude", action = ArgAction::Append, takes_value = true)]
exclude: Vec<String>,
},
/// Create and/or open local config file
#[clap(name = "config", aliases = & ["c"])]
#[clap(name = "config", aliases = & ["conf"])]
Config,
}

@ -4,6 +4,7 @@ pub enum AppExitCode {
PkgNotFound = 3,
InvalidMode = 4,
DirNotEmpty = 5,
DirNotGit = 6,
RepoNotFound = 6,
ConfigNotFound = 7,
NoPkgs = 8,
}

@ -3,13 +3,22 @@ use serde_derive::Deserialize;
#[derive(Debug, Deserialize)]
pub struct Config {
pub mode: String,
pub sign: bool,
pub name: Option<String>,
pub repo: Vec<String>,
pub repo: Vec<Repo>,
}
#[derive(Debug, Deserialize)]
pub struct Repo {
pub name: String,
pub url: String,
pub priority: usize,
}
#[derive(Debug, Deserialize)]
pub struct UnexpandedConfig {
pub mode: String,
pub sign: bool,
pub name: Option<String>,
pub repo: Vec<String>,
pub urls: Vec<String>,
@ -20,3 +29,9 @@ pub struct SplitRepo {
pub indx: usize,
pub name: String,
}
#[derive(Debug)]
pub struct ErroredPackage {
pub name: String,
pub code: i32,
}

@ -36,6 +36,8 @@ fn main() {
.unwrap();
}
let exclude = &args.exclude;
if Path::exists("../.git".as_ref()) {
info!("Parent directory is a git directory, pulling latest mlc.toml. It is advised you run mlc pull/update in all malachite directories");
let dir = env::current_dir().unwrap();
@ -49,17 +51,12 @@ fn main() {
env::set_current_dir(dir).unwrap();
}
match args.subcommand.unwrap_or(Operation::Init) {
Operation::Init => operations::init(),
match args.subcommand.unwrap_or(Operation::Clone) {
Operation::Clone => operations::clone(),
Operation::Build {
packages,
exclude,
no_regen,
..
} => operations::build(packages, exclude, no_regen),
Operation::Pull {
packages, exclude, ..
} => operations::pull(packages, exclude),
packages, no_regen, ..
} => operations::build(packages, exclude.to_vec(), no_regen),
Operation::Pull { packages, .. } => operations::pull(packages, exclude.to_vec()),
Operation::RepoGen => {
let config = read_cfg();
if config.mode != "repository" {
@ -71,7 +68,6 @@ fn main() {
info!("Generating repository: {}", config.name.unwrap());
repository::generate();
}
Operation::Prune => operations::prune(),
Operation::Config => operations::config(),
}
}

@ -1,60 +1,72 @@
use crate::internal::structs::ErroredPackage;
use crate::internal::AppExitCode;
use crate::repository::generate;
use crate::{crash, info, repository, workspace};
pub fn build(packages: Vec<String>, exclude: Vec<String>, no_regen: bool) {
let all = packages.is_empty();
// Read config struct from mlc.toml
let config = workspace::read_cfg();
let all = packages.is_empty();
let mut repos: Vec<String> = vec![];
for r in config.repo {
let split = r.split('/').collect::<Vec<&str>>();
let a = split.last().unwrap();
repos.push(a.parse().unwrap());
}
if exclude.is_empty() {
// Get list of repos and subtract exclude
let mut repos: Vec<String> = config.repo.iter().map(|x| x.name.clone()).collect();
if !exclude.is_empty() {
for ex in exclude {
repos.retain(|x| *x != ex);
}
}
let mut errored: Vec<String> = vec![];
for pkg in packages {
if !repos.contains(&pkg) {
// If packages is not empty and all isn't specified, build specifed packages
let mut errored: Vec<ErroredPackage> = vec![];
if !packages.is_empty() && !all {
for pkg in &packages {
if !repos.contains(pkg) {
crash!(
AppExitCode::PkgNotFound,
"Package {} not found in repos in mlc.toml",
"Package repo {} not found in in mlc.toml",
pkg
);
} else {
let code = repository::build(&pkg);
let code = repository::build(pkg, config.sign);
if code != 0 {
errored.push(pkg);
let error = ErroredPackage { name: pkg.to_string(), code };
errored.push(error);
}
}
}
}
// If all is specified, attempt to build a package from all repos
if all {
for pkg in repos {
let code = repository::build(&pkg);
let code = repository::build(&pkg, config.sign);
if code != 0 {
errored.push(pkg);
let error = ErroredPackage { name: pkg, code };
errored.push(error);
}
}
generate();
}
// If all is not specified, but packages is empty, crash
if !all && packages.is_empty() {
crash!(AppExitCode::NoPkgs, "No packages specified");
}
// If no_regen is passed, do not generate a repository
if !no_regen {
repository::generate();
}
// Map errored packages to a string for display
let error_strings: Vec<String> = errored
.iter()
.map(|x| format!("{}: Returned {}", x.name, x.code))
.collect();
// If errored is not empty, let the user know which packages failed
if !errored.is_empty() {
info!(
"The following packages build jobs returned a non-zero exit code: {}",
errored.join(" ")
error_strings.join("\n")
)
}
}

@ -0,0 +1,49 @@
use crate::{info, workspace};
use std::process::Command;
pub fn clone() {
// Read config struct from mlc.toml
let config = workspace::read_cfg();
let repos = &config.repo;
// Get a vector of all files/dirs in the current directory, excluding config file
let dir_paths = std::fs::read_dir("./").unwrap();
let mut dirs = dir_paths
.map(|x| x.unwrap().path().display().to_string())
.collect::<Vec<String>>();
dirs.retain(|x| *x != "./mlc.toml");
// Creates a vector of the difference between cloned repos and repos defined in config
let mut repo_diff = vec![];
for repo in repos {
let name = &repo.name;
if !dirs.contains(name) {
repo_diff.push(repo);
}
}
// Diff logic
if repo_diff.is_empty() {
// No diff, do nothing
info!("All repos are already cloned");
} else {
// This is just for pretty display purposes
let display = repo_diff
.iter()
.map(|x| x.name.to_string())
.collect::<Vec<String>>()
.join(" ");
info!("New/missing repos to clone: {}", display);
// Clone all diff repos
for r in repo_diff {
info!("Cloning ({} mode): {}", config.mode, r.name);
Command::new("git")
.args(&["clone", &r.url, &r.name])
.spawn()
.unwrap()
.wait()
.unwrap();
}
}
}

@ -5,9 +5,12 @@ use std::process::Command;
use crate::create_config;
pub fn config() {
// Generate new config file if not already present
if !Path::exists("mlc.toml".as_ref()) {
create_config();
}
// Open config file in user's editor of choice
let editor = env::var("EDITOR").unwrap_or_else(|_| "nano".to_string());
Command::new(editor)
.arg("mlc.toml")

@ -1,58 +0,0 @@
use std::process::Command;
use crate::internal::AppExitCode;
use crate::{crash, info, workspace};
pub fn init() {
let config = workspace::read_cfg();
if config.mode == "workspace" || config.mode == "repository" {
let dirs_raw = Command::new("ls").arg("-1").output().unwrap().stdout;
let dirs_string = String::from_utf8_lossy(&dirs_raw);
let mut dirs = dirs_string.lines().collect::<Vec<&str>>();
dirs.retain(|x| *x != "mlc.toml");
dirs.sort_unstable();
let repos = &config.repo;
let mut repos = repos
.iter()
.map(|x| x.split('/').last().unwrap())
.collect::<Vec<&str>>();
repos.sort_unstable();
let mut diff = repos.clone();
diff.retain(|x| !dirs.contains(x));
let mut diff_matches = vec![];
for &x in &diff {
for y in config.repo.iter() {
if x == y.split('/').last().unwrap() {
diff_matches.push(y);
}
}
}
if diff.is_empty() {
info!("All repos are already cloned");
} else {
info!("New/missing repos to clone: {}", diff.join(", "));
for r in diff_matches {
info!(
"Cloning ({} mode): {}",
config.mode,
r.split('/').last().unwrap()
);
Command::new("git")
.args(&["clone", r])
.spawn()
.unwrap()
.wait()
.unwrap();
}
}
} else {
crash!(AppExitCode::InvalidMode, "Invalid mode in mlc.toml");
}
}

@ -1,11 +1,9 @@
pub use build::*;
pub use clone::*;
pub use config::*;
pub use init::*;
pub use prune::*;
pub use pull::*;
mod build;
mod clone;
mod config;
mod init;
mod prune;
mod pull;

@ -1,56 +0,0 @@
use std::fs;
use std::process::Command;
use crate::{info, read_cfg};
pub fn prune() {
let config = read_cfg();
let mut packages = vec![];
for untrimmed_repo in &config.repo {
pub fn trim_repo(a: String) -> String {
(a.split('/')
.map(|s| s.to_string())
.collect::<Vec<String>>()
.last()
.unwrap())
.to_string()
}
packages.push(trim_repo(untrimmed_repo.to_string()));
}
let mut packages_to_del = vec![];
for pkg in packages {
let dups = Command::new("bash")
.args(&[
"-c",
&format!(
"ls out/{}*.tar.* -w 1 | grep .sig | sed 's/.sig//g' | sort -r",
pkg
),
])
.output()
.unwrap()
.stdout
.to_ascii_lowercase();
let duplicates = String::from_utf8_lossy(&dups);
let duplicates_lines = duplicates.lines().collect::<Vec<&str>>();
let variable_hell = duplicates_lines.iter().skip(1).collect::<Vec<&&str>>();
if !variable_hell.is_empty() {
for var in variable_hell {
packages_to_del.push(var.to_string());
}
}
}
if !packages_to_del.is_empty() {
info!("Pruning duplicates: {}", packages_to_del.join(", "));
}
for pkg in packages_to_del {
fs::remove_file(&pkg).unwrap();
fs::remove_file(format!("{}.sig", &pkg)).unwrap();
}
}

@ -1,40 +1,57 @@
use crate::{internal::AppExitCode, crash};
use std::env;
use std::process::Command;
use crate::info;
fn do_the_pulling(packages: Vec<String>) {
for dir in packages {
let current_dir = env::current_dir().unwrap();
info!("Entering working directory: {}", dir);
env::set_current_dir(dir).unwrap();
fn do_the_pulling(repos: Vec<String>) {
for repo in repos {
// Set root dir to return after each git pull
let root_dir = env::current_dir().unwrap();
info!("Entering working directory: {}", &repo);
env::set_current_dir(repo).unwrap();
Command::new("git")
.arg("pull")
.spawn()
.unwrap()
.wait()
.unwrap();
env::set_current_dir(current_dir).unwrap();
// Return to root dir
env::set_current_dir(root_dir).unwrap();
}
}
pub fn pull(packages: Vec<String>, exclude: Vec<String>) {
// If no packages are specified, imply all
let all = packages.is_empty();
if all {
let stdout = Command::new("ls").arg("-1").output().unwrap().stdout;
let dirs_string = String::from_utf8_lossy(&stdout);
let mut dirs = dirs_string.lines().collect::<Vec<&str>>();
// Read repos from config file
let repos = crate::workspace::read_cfg()
.repo
.iter()
.map(|x| x.name.clone())
.collect::<Vec<String>>();
dirs.retain(|x| *x != "mlc.toml");
for x in exclude {
dirs.retain(|y| *y != x);
}
// Set repos_applicable for next function
let mut repos_applicable = if all {
repos
} else {
packages
};
let dirs_mapped = dirs.iter().map(|x| x.to_string()).collect();
// Subtract exclude from repos_applicable
if !exclude.is_empty() {
for ex in exclude {
repos_applicable.retain(|x| *x != ex);
}
}
do_the_pulling(dirs_mapped);
} else {
do_the_pulling(packages);
// If all is not specified and packages is empty, crash
if repos_applicable.is_empty() {
crash!(AppExitCode::NoPkgs, "No packages specified");
}
// Pull!
do_the_pulling(repos_applicable);
}

@ -6,25 +6,31 @@ use std::path::Path;
use crate::crash;
use crate::internal::AppExitCode;
const DEFAULT_CONFIG: &str = r#"# either "repository" or "workspace"
const DEFAULT_CONFIG: &str = r#"
# Either "repository" or "workspace"
mode = ""
# only required when in repository mode, decides what to call the repository and relevant files
# Only required when in repository mode, decides what to call the repository and relevant files
name = ""
# Only required when in repository mode, decides whether to PGP sign built packages
sign = true
# an array of git repos to clone from, formatted url_index::repo_name, e.g. if you had urls = [ "https://example.com/%repo%" ], 1::package would expand to https://example.com/package
# An array of Git repositories to clone from, formatted url_index::repo_name(!)
# e.g. if you had URLs = [ "https://example.com/%repo%.git" ], 1::package would expand to https://example.com/package.git
# Repository mode only: Depending on the number of "!"s appended to the name, the priority of the package will be determined. More "!"s = higher priority = built first.
repo = [
"",
""
]
# an array of urls to clone from, in the format https://example.com/%repo% (the %repo% is NOT optional)
# An array of URLs to clone from, in the format https://example.com/%repo% (the %repo% is NOT optional and will be replaced with the name of the repository)
urls = [
"",
""
]"#;
pub fn create_config() {
// Ensure current directory is empty
if env::current_dir()
.unwrap()
.read_dir()
@ -37,6 +43,8 @@ pub fn create_config() {
"Directory is not empty, please only create a repository in an empty directory"
);
}
// If config file exists, create it
if !Path::exists("mlc.toml".as_ref()) {
let mut file = File::create("mlc.toml").unwrap();
file.write_all(DEFAULT_CONFIG.as_ref()).unwrap();

@ -5,28 +5,37 @@ use std::{env, fs};
use crate::crash;
use crate::internal::AppExitCode;
pub fn build(pkg: &str) -> i32 {
pub fn build(pkg: &str, sign: bool) -> i32 {
// Set root dir to return after build
let dir = env::current_dir().unwrap();
// Create out dir if not already present
if !Path::exists("out".as_ref()) {
fs::create_dir_all("out").unwrap();
}
// If directory is not found, crash
if !Path::exists(pkg.as_ref()) {
crash!(
AppExitCode::DirNotGit,
"Git directory for {} not found, aborting",
AppExitCode::RepoNotFound,
"Repo for {} not found, aborting",
pkg
);
}
// Enter build directory
env::set_current_dir(pkg).unwrap();
// Build each package
let a = Command::new("makepkg")
.args(&["-sf", "--skippgpcheck", "--sign", "--noconfirm"])
.args(&["-sf", "--skippgpcheck", if sign { "--sign" } else {"--nosign"}, "--noconfirm"])
.spawn()
.unwrap()
.wait()
.unwrap();
// Copy built package to out dir
Command::new("bash")
.args(&["-c", "cp *.pkg.tar* ../out/"])
.spawn()
@ -34,7 +43,9 @@ pub fn build(pkg: &str) -> i32 {
.wait()
.unwrap();
// Return to root dir
env::set_current_dir(dir).unwrap();
// Return exit code
a.code().unwrap()
}

@ -5,15 +5,21 @@ use std::{env, fs};
use crate::workspace::read_cfg;
pub fn generate() {
// Read config struct from mlc.toml
let config = read_cfg();
// Get repository name from config
let name = config.name.unwrap();
// If repository exists, delete it
if Path::exists(name.as_ref()) {
fs::remove_dir_all(&name).unwrap();
}
// Create or recreate repository directory
fs::create_dir_all(&name).unwrap();
// Copy out packages to repository directory
Command::new("bash")
.args(&["-c", &format!("cp -v out/* {}/", &name)])
.spawn()
@ -21,17 +27,19 @@ pub fn generate() {
.wait()
.unwrap();
// Enter repository directory
env::set_current_dir(&name).unwrap();
let db = format!("{}.db", &name);
let files = format!("{}.files", &name);
// Create repo.db and repo.files using repo-add
Command::new("bash")
.args(&[
"-c",
&format!(
"repo-add {}.tar.gz *.pkg.tar.zst; repo-add {}.tar.gz *.pkg.tar.xz",
db, db
"GLOBIGNORE=\"*.sig\" repo-add {}.tar.gz *.pkg.tar.*",
db
),
])
.spawn()
@ -39,13 +47,7 @@ pub fn generate() {
.wait()
.unwrap();
Command::new("bash")
.args(&["-c", &format!("rm {}.{{db,files}}", &name)])
.spawn()
.unwrap()
.wait()
.unwrap();
// Replace repo.{db,files}.tar.gz with just repo.{db,files}
Command::new("bash")
.args(&[
"-c",

@ -2,10 +2,11 @@ use std::fs;
use std::path::Path;
use crate::crash;
use crate::internal::structs::{Config, SplitRepo, UnexpandedConfig};
use crate::internal::structs::{Config, Repo, SplitRepo, UnexpandedConfig};
use crate::internal::AppExitCode;
pub fn read_cfg() -> Config {
// Crash if mlc.toml doesn't exist
if !Path::exists("mlc.toml".as_ref()) {
crash!(
AppExitCode::ConfigNotFound,
@ -13,32 +14,48 @@ pub fn read_cfg() -> Config {
)
}
// Reading the config file to an UnexpandedConfig struct
let file = fs::read_to_string("mlc.toml").unwrap();
let config: UnexpandedConfig = toml::from_str(&file).unwrap();
let mut trimmed_urls: Vec<String> = vec![];
let mut expanded_repos: Vec<String> = vec![];
for url in config.urls {
let a = url.split("%repo%").collect::<Vec<&str>>()[0];
let mut b = vec![a.to_string()];
trimmed_urls.append(&mut b);
// Crash if incorrect mode is set
if config.mode != "workspace" && config.mode != "repository" {
crash!(
AppExitCode::InvalidMode,
"Invalid mode in mlc.toml, must be either \"repository\" or \"workspace\""
);
}
let config_repos = config.repo;
for x in config_repos {
let mut expanded_repos: Vec<Repo> = vec![];
// Parsing repos from the config file
for x in config.repo {
// Splits the repo name and index inta a SplitRepo struct
let split: Vec<&str> = x.split("::").collect();
let sr_struct = SplitRepo {
let split_struct = SplitRepo {
indx: split[0].parse().unwrap(),
name: split[1].parse().unwrap(),
};
let index = sr_struct.indx;
let expanded = format!("{}{}", trimmed_urls[index - 1], sr_struct.name);
expanded_repos.push(expanded);
// Parses all necessary values for expanding the repo to a Repo struct
let index = split_struct.indx;
let name = split_struct.name.replace('!', "");
let url = config.urls[index - 1].replace("%repo%", &split_struct.name);
let priority = &split_struct.name.matches('!').count();
// Creates and pushes Repo struct to expanded_repos
let repo = Repo {
name,
url,
priority: *priority,
};
expanded_repos.push(repo);
}
// Returns parsed config file
Config {
mode: config.mode,
sign: config.sign,
name: config.name,
repo: expanded_repos,
}

Loading…
Cancel
Save