Merge branch 'finetune-chrooting' into 'main'

Fineune some aspect of chrooting (and test this thing)

See merge request crystal/software/tourmaline!2
main
Julius Riegel 1 year ago
commit 5c6ea5e9a4

754
Cargo.lock generated

File diff suppressed because it is too large Load Diff

@ -16,25 +16,25 @@ path = "src/main.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
clap = { version = "4.1.6", features = ["derive"] }
clap = { version = "4.1.8", features = ["derive"] }
color-eyre = "0.6.2"
dotenv = "0.15.0"
embed-nu = "0.5.1"
embed-nu = "0.5.4"
lazy_static = "1.4.0"
libc = "0.2.139"
libc = "0.2.140"
miette = { version = "5.5.0", features = ["fancy"] }
paste = "1.0.11"
paste = "1.0.12"
rusty-value = { version = "0.6.0", features = ["derive", "json"] }
serde = { version = "1.0.152", features = ["derive"] }
serde_json = "1.0.93"
serde = { version = "1.0.155", features = ["derive"] }
serde_json = "1.0.94"
sys-mount = "2.0.2"
thiserror = "1.0.38"
tokio = { version = "1.25.0", features = ["rt", "io-std", "io-util", "process", "time", "macros", "tracing", "fs"] }
thiserror = "1.0.39"
tokio = { version = "1.26.0", features = ["rt", "io-std", "io-util", "process", "time", "macros", "tracing", "fs"] }
toml = "0.7.2"
tracing = "0.1.37"
tracing-subscriber = "0.3.16"
tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
valico = "3.6.1"
[build-dependencies]
cargo_toml = "0.15.2"
serde = { version = "1.0.152", features = ["derive"] }
serde = { version = "1.0.155", features = ["derive"] }

@ -1,3 +1,4 @@
let RUN_IN_CHROOT = true;
# Applies all system changes of `configure-locale`
def main [cfg] {
echo "Executing up task `configure-locale` with config" $cfg

@ -1,3 +1,4 @@
let RUN_IN_CHROOT = true;
# Applies all system changes of `configure-network`
def main [cfg] {
echo "Executing up task `configure-network` with config" $cfg

@ -1,3 +1,4 @@
let RUN_IN_CHROOT = true;
# Applies all system changes of `configure-unakite`
def main [cfg] {
echo "Executing up task `configure-unakite` with config" $cfg

@ -1,4 +1,86 @@
module utils {
export def is_ssd [device: string] {
$device =~ '^/dev/(nvme|mmcblk)'
}
}
module auto_partition {
use utils
export def efi [device: string] {
info "Creating efi partitions"
efi_layout $device
if utils is_ssd $device {
debug "Creating file systems for ssd"
efi_create_fs_ssd $device
} else {
debug "Creating file systems for hdd"
efi_create_fs_hdd $device
}
}
def efi_layout [device: string] {
run parted -s $device mklabel gpt
debug "Partition table created"
run parted -s $device mkpart fat32 0 300
debug "EFI partition created"
run parted -s $device mkpart primary btrfs 512MIB 100%
debug "Root partition created"
}
def efi_create_fs_ssd [device: string] {
}
def efi_create_fs_hdd [device: string] {
let boot_part = $"($device)1";
let root_part = $"($device)2"
run mkfs.vfat -F32 $boot_part
run mkfs.btrfs -f $root_part
run mount $root_part /mnt
with-cwd /mnt {
run btrfs subvolume create @
run btrfs subvolume create @home
}
run umount $root_part
run mount $root_part /mnt -o subvol=@
mkdir /mnt/boot/efi
mkdir /mnt/home
run mount $root_part /mnt/home -o subvol=@home
run mount $boot_part /mnt/boot/efi
}
export def bios [device: string] {
debug "Creating bios partitions"
bios_layout $device
}
def bios_layout [device: string] {
parted -s $device mklabel msdos
parted -s $device mkpart primary ext4 1MIB 512MIB
parted -s $device mkpart primary btrfs 512MIB 100%
}
}
# Applies all system changes of `create-partitions`
def main [cfg] {
echo "Executing up task `create-partitions` with config" $cfg
debug $"Creating partitions with config ($cfg)"
if $cfg.partitions == "Auto" {
info "Creating partitions automatically..."
use auto_partition
if $cfg.efi_partition {
auto_partition efi $cfg.device
} else {
auto_partition bios $cfg.device
}
} else {
info "Creating partitions manually"
}
info "Partitions created!"
}

@ -1,5 +1,69 @@
let REQUIRES_CHROOT = true;
def install_base_packages [] {
(
run pacstrap /mnt
base
linux-firmware
systemd-sysvcompat
networkmanager
man-db
man-pages
texinfo
nano
sudo
curl
archlinux-keyring
# crystal base
crystal-core
crystal-branding
# crystal extras
crystal-first-setup
neofetch
btrfs-progs
which
base-devel
# chaotic aur
chaotic-keyring
chaotic-mirrorlist
# fonts
noto-fonts
noto-fonts-emoji
noto-fonts-cjk
noto-fonts-extra
ttf-nerd-fonts-symbols-common
ttf-firacode-nerd
ttf-liberation
# audio
pipewire
pipewire-pulse
pipewire-alsa
pipewire-jack
wireplumber
helvum
# utils
xterm
cups
cups-pdf
bluez
bluez-cups
ntfs-3g
bash-completion
zsh-completions
)
}
# Applies all system changes of `install-base`
def main [cfg] {
echo "Executing up task `install-base` with config" $cfg
debug $"installing base with config ($cfg)"
mkdir /mnt/etc
install_base_packages
cp /etc/pacman.conf /mnt/etc/pacman.conf
run bash -c 'genfstab -U /mnt >> /mnt/etc/fstab'
info "Base packages installed!"
}

@ -1,4 +1,41 @@
let RUN_IN_CHROOT = true;
def install_grub_pkgs [] {
(run pacman -S --noconfirm
grub
efibootmgr
crystal-grub-theme
os-prober
crystal-branding
)
}
def install_grub_efi [location: string] {
debug $"Installing grub to ($location)"
install_grub_pkgs
(run grub-install
--target=x86_64-efi
--efi-directory $location
--bootloader-id=crystal
--removable
)
(run grub-install
--target=x86_64-efi
--efi-directory $location
--bootloader-id=crystal
)
"\nGRUB_THEME=\"/usr/share/grub/themes/crystal/theme.txt\"" | save --append /etc/default/grub
run grub-mkconfig -o /boot/grub/grub.cfg
}
# Applies all system changes of `install-bootloader`
def main [cfg] {
echo "Executing up task `install-bootloader` with config" $cfg
debug "Installing bootloader with config ($cfg)"
if $cfg.preset == "GrubEfi" {
install_grub_efi $cfg.location
} else {
error make {msg: "Not implemented"}
}
}

@ -1,3 +1,4 @@
let RUN_IN_CHROOT = true;
# Applies all system changes of `install-desktop`
def main [cfg] {
echo "Executing up task `install-desktop` with config" $cfg

@ -1,3 +1,4 @@
let RUN_IN_CHROOT = true;
# Applies all system changes of `install-extra-packages`
def main [cfg] {
echo "Executing up task `install-extra-packages` with config" $cfg

@ -1,3 +1,4 @@
let RUN_IN_CHROOT = true;
# Applies all system changes of `install-flatpak`
def main [cfg] {
echo "Executing up task `install-flatpak` with config" $cfg

@ -1,4 +1,27 @@
let SUPPORTED_KERNELS = ["linux", "linux-zen", "linus-hardened", "linux-lts"];
# Applies all system changes of `install-kernels`
def main [cfg] {
echo "Executing up task `install-kernels` with config" $cfg
debug $"installing kernels with config ($cfg)"
mut kernel = $cfg.default
if $kernel not-in $SUPPORTED_KERNELS {
warn $"Unsupported kernel ($kernel). Defaulting to 'linux' kernel"
$kernel = "linux"
}
run pacstrap /mnt $kernel
debug "Installing additional kernels"
$cfg.additional | each {|$k|
debug $"installing ($k)"
if $k in $SUPPORTED_KERNELS {
run pacstrap /mnt $k
} else {
warn $"Unsupported kernel ($k)"
}
}
info "Kernels installed!"
}

@ -1,3 +1,4 @@
let RUN_IN_CHROOT = true;
# Applies all system changes of `install-timeshift`
def main [cfg] {
echo "Executing up task `install-timeshift` with config" $cfg

@ -1,3 +1,4 @@
let RUN_IN_CHROOT = true;
# Applies all system changes of `install-zramd`
def main [cfg] {
echo "Executing up task `install-zramd` with config" $cfg

@ -1,3 +1,4 @@
let RUN_IN_CHROOT = true;
# Applies all system changes of `setup-root-user`
def main [cfg] {
echo "Executing up task `setup-root-user` with config" $cfg

@ -1,5 +1,5 @@
let RUN_IN_CHROOT = true;
# Applies all system changes of `setup-users`
def main [cfg] {
echo "Executing up task `setup-users` with config" $cfg
echo $TRM_CONFIG
}

@ -15,6 +15,9 @@ const VERSION: &str = concat!(
pub struct Args {
#[command(subcommand)]
pub command: Command,
#[arg(long)]
pub verbose: bool,
}
#[derive(Debug, Clone, Subcommand)]

@ -4,6 +4,8 @@ use miette::{Context, IntoDiagnostic, Result};
use rusty_value::into_json::{EnumRepr, IntoJson, IntoJsonOptions};
use tokio::fs;
use tourmaline::{distro::OSConfig, generate_script_files};
use tracing::metadata::LevelFilter;
use tracing_subscriber::fmt::format::FmtSpan;
mod args;
@ -14,6 +16,12 @@ async fn main() -> miette::Result<()> {
let _ = dotenv::dotenv();
let args = Args::parse();
if args.verbose {
init_tracing(LevelFilter::DEBUG);
} else {
init_tracing(LevelFilter::INFO);
}
match args.command {
Command::InstallFromConfig(args) => install_from_config(args).await,
Command::GenerateScripts(args) => generate_scripts(args).await,
@ -51,3 +59,12 @@ async fn generate_empty_config(args: CreateEmptyConfigArgs) -> Result<()> {
Ok(())
}
fn init_tracing(max_level: LevelFilter) {
tracing_subscriber::fmt::SubscriberBuilder::default()
.with_max_level(max_level)
.with_writer(std::io::stderr)
.with_span_events(FmtSpan::NEW | FmtSpan::CLOSE)
.compact()
.init();
}

@ -144,7 +144,7 @@ impl Mapping {
.await
.map_err(|e| ChrootError::Link(src.to_owned(), e))?;
Ok(MappingHandle::Link(LinkDrop {
path: src.to_owned(),
path: dst.to_owned(),
}))
}
@ -155,9 +155,11 @@ impl Mapping {
.map_err(|e| ChrootError::Copy(src.to_owned(), e))?;
}
fs::copy(src, dst)
.await
.map_err(|e| ChrootError::Copy(src.to_owned(), e))?;
if src.exists() {
fs::copy(src, dst)
.await
.map_err(|e| ChrootError::Copy(src.to_owned(), e))?;
}
Ok(MappingHandle::None)
}

@ -0,0 +1,36 @@
use embed_nu::{
nu_protocol::{engine::Command, Signature, SyntaxShape},
CallExt, PipelineData,
};
#[derive(Clone)]
pub struct DebugCommand;
impl Command for DebugCommand {
fn name(&self) -> &str {
"debug"
}
fn signature(&self) -> embed_nu::nu_protocol::Signature {
Signature::new("debug")
.rest("rest", SyntaxShape::Any, "the message to print")
.category(embed_nu::nu_protocol::Category::Custom("Tourmaline".into()))
}
fn usage(&self) -> &str {
"Prints the given message and values with debug severity (only when --verbose is passed to tourmaline)"
}
fn run(
&self,
engine_state: &embed_nu::nu_protocol::engine::EngineState,
stack: &mut embed_nu::nu_protocol::engine::Stack,
call: &embed_nu::nu_protocol::ast::Call,
_input: embed_nu::PipelineData,
) -> Result<embed_nu::PipelineData, embed_nu::nu_protocol::ShellError> {
let args: Vec<String> = call.rest(engine_state, stack, 0)?;
tracing::debug!("{}", args.join(" "));
Ok(PipelineData::empty())
}
}

@ -0,0 +1,36 @@
use embed_nu::{
nu_protocol::{engine::Command, Signature, SyntaxShape},
CallExt, PipelineData,
};
#[derive(Clone)]
pub struct InfoCommand;
impl Command for InfoCommand {
fn name(&self) -> &str {
"info"
}
fn signature(&self) -> embed_nu::nu_protocol::Signature {
Signature::new("info")
.rest("rest", SyntaxShape::Any, "the message to print")
.category(embed_nu::nu_protocol::Category::Custom("Tourmaline".into()))
}
fn usage(&self) -> &str {
"Prints the given message and values with info severity"
}
fn run(
&self,
engine_state: &embed_nu::nu_protocol::engine::EngineState,
stack: &mut embed_nu::nu_protocol::engine::Stack,
call: &embed_nu::nu_protocol::ast::Call,
_input: embed_nu::PipelineData,
) -> Result<embed_nu::PipelineData, embed_nu::nu_protocol::ShellError> {
let args: Vec<String> = call.rest(engine_state, stack, 0)?;
tracing::info!("{}", args.join(" "));
Ok(PipelineData::empty())
}
}

@ -0,0 +1,10 @@
mod debug;
mod info;
mod run;
mod warn;
mod with_cwd;
pub use debug::*;
pub use info::*;
pub use run::*;
pub use warn::*;
pub use with_cwd::*;

@ -0,0 +1,72 @@
use std::{
io::Write,
process::{Command, Stdio},
};
use embed_nu::{
nu_protocol::{ShellError, Signature, SyntaxShape},
CallExt, PipelineData,
};
#[derive(Clone)]
pub struct RunCommand;
impl embed_nu::nu_protocol::engine::Command for RunCommand {
fn name(&self) -> &str {
"run"
}
fn signature(&self) -> embed_nu::nu_protocol::Signature {
Signature::new("run")
.required("executable", SyntaxShape::String, "The executable to run")
.rest(
"rest",
SyntaxShape::String,
"the args for the given command",
)
.allows_unknown_args()
.category(embed_nu::nu_protocol::Category::Custom("Tourmalin".into()))
}
fn usage(&self) -> &str {
"run <executable> [...<args>]"
}
fn run(
&self,
engine_state: &embed_nu::nu_protocol::engine::EngineState,
stack: &mut embed_nu::nu_protocol::engine::Stack,
call: &embed_nu::nu_protocol::ast::Call,
input: embed_nu::PipelineData,
) -> Result<embed_nu::PipelineData, embed_nu::nu_protocol::ShellError> {
let executable: String = call.req(engine_state, stack, 0)?;
let args: Vec<String> = call.rest(engine_state, stack, 1)?;
let pwd = stack
.get_env_var(engine_state, "PWD")
.and_then(|v| v.as_string().ok())
.unwrap_or_else(|| std::env::var("PWD").unwrap_or_else(|_| "/".into()));
tracing::debug!("Running {executable} {}", args.join(" "));
let mut cmd = Command::new(&executable)
.current_dir(pwd)
.args(args)
.stdout(Stdio::inherit())
.stderr(Stdio::inherit())
.stdin(Stdio::piped())
.spawn()?;
let mut stdin = cmd.stdin.take().unwrap();
stdin.write_all(input.collect_string_strict(call.span())?.0.as_bytes())?;
if cmd.wait()?.success() {
Ok(PipelineData::empty())
} else {
Err(ShellError::ExternalCommand(
executable,
String::from("Is it written correctly?"),
call.span(),
))
}
}
}

@ -0,0 +1,36 @@
use embed_nu::{
nu_protocol::{engine::Command, Signature, SyntaxShape},
CallExt, PipelineData,
};
#[derive(Clone)]
pub struct WarnCommand;
impl Command for WarnCommand {
fn name(&self) -> &str {
"warn"
}
fn signature(&self) -> embed_nu::nu_protocol::Signature {
Signature::new("warn")
.rest("rest", SyntaxShape::Any, "the warning to print")
.category(embed_nu::nu_protocol::Category::Custom("Tourmaline".into()))
}
fn usage(&self) -> &str {
"Prints the given message and values with warning severity"
}
fn run(
&self,
engine_state: &embed_nu::nu_protocol::engine::EngineState,
stack: &mut embed_nu::nu_protocol::engine::Stack,
call: &embed_nu::nu_protocol::ast::Call,
_input: embed_nu::PipelineData,
) -> Result<embed_nu::PipelineData, embed_nu::nu_protocol::ShellError> {
let args: Vec<String> = call.rest(engine_state, stack, 0)?;
tracing::warn!("{}", args.join(" "));
Ok(PipelineData::empty())
}
}

@ -0,0 +1,102 @@
use embed_nu::{
nu_engine::eval_block_with_early_return,
nu_protocol::{engine::Closure, Signature, Span, SyntaxShape},
CallExt, NewEmpty, Value,
};
#[derive(Clone)]
pub struct WithCwdCommand;
impl embed_nu::nu_protocol::engine::Command for WithCwdCommand {
fn name(&self) -> &str {
"with-cwd"
}
fn signature(&self) -> embed_nu::nu_protocol::Signature {
Signature::new("with-cwd")
.required(
"dir",
SyntaxShape::String,
"The directory to run the closure in",
)
.required("closure", SyntaxShape::Any, "The closure to run")
.rest("rest", SyntaxShape::Any, "The parameters for the closure")
.category(embed_nu::nu_protocol::Category::Custom("Tourmalin".into()))
}
fn usage(&self) -> &str {
"with-cwd <path> <block> [<args>...]"
}
fn run(
&self,
engine_state: &embed_nu::nu_protocol::engine::EngineState,
stack: &mut embed_nu::nu_protocol::engine::Stack,
call: &embed_nu::nu_protocol::ast::Call,
input: embed_nu::PipelineData,
) -> Result<embed_nu::PipelineData, embed_nu::nu_protocol::ShellError> {
let path: String = call.req(engine_state, stack, 0)?;
let block: Closure = call.req(engine_state, stack, 1)?;
let block_args: Vec<Value> = call.rest(engine_state, stack, 2)?;
let block = engine_state.get_block(block.block_id);
let old_cwd = engine_state.get_env_var("PWD");
tracing::debug!("Executing block with CWD {path}");
stack.add_env_var("PWD".into(), Value::string(path, Span::empty()));
let params: Vec<_> = block
.signature
.required_positional
.iter()
.chain(block.signature.optional_positional.iter())
.collect();
for param in params.iter().zip(&block_args) {
if let Some(var_id) = param.0.var_id {
stack.add_var(var_id, param.1.clone())
}
}
if let Some(param) = &block.signature.rest_positional {
if block_args.len() > params.len() {
let mut args = vec![];
for r in block_args.into_iter().skip(params.len()) {
args.push(r);
}
let span = if let Some(arg) = args.first() {
arg.span()?
} else {
call.head
};
stack.add_var(
param
.var_id
.expect("Internal error: rest positional parameter lacks var_id"),
Value::List { vals: args, span },
)
}
}
let result = eval_block_with_early_return(
engine_state,
stack,
block,
input,
call.redirect_stdout,
call.redirect_stdout,
);
stack.add_env_var(
"PWD".into(),
old_cwd.cloned().unwrap_or_else(|| {
std::env::current_dir()
.map(|d| Value::string(d.to_string_lossy(), Span::empty()))
.unwrap_or_else(|_| Value::nothing(Span::empty()))
}),
);
result
}
}

@ -6,6 +6,8 @@ use std::fs;
use crate::{distro::OSConfig, error::ScriptError, utils::CFG_PATH};
use miette::{Context, IntoDiagnostic, Result};
use super::commands::{DebugCommand, InfoCommand, RunCommand, WarnCommand, WithCwdCommand};
#[derive(Clone)]
pub struct ExecBuilder {
task_config: embed_nu::Value,
@ -21,15 +23,21 @@ impl ExecBuilder {
let script_contents = Self::get_script_contents(&script)?;
let mut ctx = embed_nu::Context::builder()
.with_command_groups(CommandGroupConfig::default().all_groups(true))
.into_diagnostic()?
.with_command_groups(CommandGroupConfig::default().all_groups(true))?
.add_command(RunCommand)?
.add_command(WarnCommand)?
.add_command(InfoCommand)?
.add_command(DebugCommand)?
.add_command(WithCwdCommand)?
.add_env_var("PWD", std::env::var("PWD").unwrap_or(String::from("/")))
.add_env_var(
"PATH",
std::env::var("PATH").unwrap_or(String::from("/usr/bin")),
)
.add_parent_env_vars()
.add_var("TRM_CONFIG", os_config)
.into_diagnostic()?
.add_script(script_contents)
.into_diagnostic()?
.build()
.into_diagnostic()?;
.add_var("TRM_CONFIG", os_config)?
.add_script(script_contents)?
.build()?;
if !ctx.has_fn("main") {
Err(ScriptError::MissingMain(script).into())
} else {
@ -39,14 +47,11 @@ impl ExecBuilder {
#[tracing::instrument(level = "trace", skip_all)]
pub fn exec(mut self) -> Result<()> {
let pipeline = self
.ctx
.call_fn(
"main",
vec![Argument::Positional(self.task_config.into_expression())],
)
.into_diagnostic()?;
self.ctx.print_pipeline_stderr(pipeline).into_diagnostic()?;
let pipeline = self.ctx.call_fn(
"main",
vec![Argument::Positional(self.task_config.into_expression())],
)?;
self.ctx.print_pipeline_stderr(pipeline)?;
Ok(())
}
@ -54,7 +59,7 @@ impl ExecBuilder {
/// Returns if the script needs to be run inside the new root
pub fn requires_chroot(&self) -> bool {
self.ctx
.get_var("run_in_chroot")
.get_var("RUN_IN_CHROOT")
.and_then(|v| v.as_bool().ok())
.unwrap_or(false)
}

@ -5,6 +5,7 @@ use crate::distro::OSConfig;
use self::{base_task::BaseTask, custom_task::CustomTask, exec_builder::ExecBuilder};
pub mod base_task;
mod chrooting;
mod commands;
pub mod custom_task;
pub mod exec_builder;
pub mod task_executor;

@ -63,12 +63,20 @@ impl TaskExecutor {
#[tracing::instrument(level = "trace", skip_all)]
pub async fn execute(&mut self) -> Result<()> {
self.tasks.sort_by(Task::compare);
let chroot = Chroot::create(&*ROOT_MNT).await?;
let mut chroot = None;
for task in &self.tasks {
if let Some(up_task) = task.up(&self.os_config)? {
if up_task.requires_chroot() {
chroot.run(|| up_task.exec()).await.unwrap()??;
if chroot.is_none() {
chroot = Some(Chroot::create(&*ROOT_MNT).await?);
}
chroot
.as_ref()
.unwrap()
.run(|| up_task.exec())
.await
.unwrap()??;
} else {
up_task.exec()?;
}

Loading…
Cancel
Save