From 9d4ed2dfb50ec08ebd828cd625f2151e5202f82d Mon Sep 17 00:00:00 2001 From: trivernis Date: Sat, 10 Apr 2021 10:22:09 +0200 Subject: [PATCH] Add qalc command and improve README Signed-off-by: trivernis --- Cargo.lock | 67 ------------------- Cargo.toml | 2 - README.md | 58 +++++++++++++++-- src/commands/misc/mod.rs | 4 +- src/commands/misc/qalc.rs | 18 ++++++ src/commands/music/mod.rs | 10 ++- src/providers/mod.rs | 1 + src/providers/music/lyrics.rs | 1 + src/providers/music/mod.rs | 104 +----------------------------- src/providers/music/queue.rs | 11 ++-- src/providers/music/youtube_dl.rs | 87 +++++++++++++++++++++++++ src/providers/qalc.rs | 8 +++ src/utils/mod.rs | 24 +++++++ 13 files changed, 207 insertions(+), 188 deletions(-) create mode 100644 src/commands/misc/qalc.rs create mode 100644 src/providers/music/youtube_dl.rs create mode 100644 src/providers/qalc.rs diff --git a/Cargo.lock b/Cargo.lock index 7bcc2ba..d3ecd2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -15,12 +15,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "ahash" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" - [[package]] name = "aho-corasick" version = "0.7.15" @@ -474,18 +468,6 @@ dependencies = [ "num-traits 0.1.43", ] -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - [[package]] name = "fern" version = "0.6.0" @@ -720,18 +702,6 @@ name = "hashbrown" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" -dependencies = [ - "ahash", -] - -[[package]] -name = "hashlink" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8" -dependencies = [ - "hashbrown", -] [[package]] name = "hermit-abi" @@ -971,16 +941,6 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56d855069fafbb9b344c0f962150cd2c1187975cb1c22c1522c240d8c4986714" -[[package]] -name = "libsqlite3-sys" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d31059f22935e6c31830db5249ba2b7ecd54fd73a9909286f0a67aa55c2fbd" -dependencies = [ - "pkg-config", - "vcpkg", -] - [[package]] name = "lock_api" version = "0.4.3" @@ -1611,21 +1571,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "rusqlite" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38ee71cbab2c827ec0ac24e76f82eca723cee92c509a65f67dee393c25112" -dependencies = [ - "bitflags 1.2.1", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "memchr", - "smallvec", -] - [[package]] name = "rustc-serialize" version = "0.3.24" @@ -1782,16 +1727,6 @@ dependencies = [ "syn", ] -[[package]] -name = "serde_rusqlite" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1023b4e2409dc2daa72310e80f81627d7e499416f00d1d1d6a1f3f5eac133b14" -dependencies = [ - "rusqlite", - "serde", -] - [[package]] name = "serde_urlencoded" version = "0.7.0" @@ -2131,11 +2066,9 @@ dependencies = [ "rand 0.8.3", "regex", "reqwest", - "rusqlite", "serde", "serde_derive", "serde_json", - "serde_rusqlite", "serenity", "songbird", "sysinfo", diff --git a/Cargo.toml b/Cargo.toml index 14d6e70..74e82e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,8 +10,6 @@ edition = "2018" serenity = "0.10.5" dotenv = "0.15.0" tokio = { version = "1.4.0", features = ["macros", "rt-multi-thread"] } -serde_rusqlite = "0.26.0" -rusqlite = "0.24" serde_derive = "1.0.125" serde = "1.0.125" thiserror = "1.0.24" diff --git a/README.md b/README.md index 2628477..af57864 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,62 @@ -# 2B meets Rust - -Don't expect the code quality to be acceptable. - +

+2B (Tobi) Discord bot. +

+

+A rust rewrite of the originally js 2B bot. +

## Current feature set - minecraft information - playing music from youtube +- miscellaneous commands + +## System Dependencies + +The bot depends on a few programs to be installed on the system. + +### Data Storage + +- [postgresql](https://www.postgresql.org/) + + +### Music + +- [FFmpeg](https://github.com/FFmpeg/FFmpeg) +- [youtube-dl](https://github.com/ytdl-org/youtube-dl) + + +### Misc Commands + +- [qalculate](https://github.com/Qalculate/libqalculate) + + +## API Dependencies + +The bot depends on the following APIs + +- [Discord](https://discord.com/developers/applications): It's a discord bot... +- [Spotify](https://developer.spotify.com/documentation/web-api/): To fetch song names to be searched on youtube for music playback +- [lyrics.ohv](https://lyricsovh.docs.apiary.io): To fetch lyrics for playing songs + + +## Dev Dependencies + +- Rust +- Other stuff that you have to figure out yourself because it just works for me + + +## Configuration + +The bot reads all required configuration values from the environment or optionally from a .env file. +The required values are: +- `BOT_TOKEN` (required): Discord bot token +- `BOT_OWNER` (required): Discord UserID of the bot owner +- `DATABASE_URL` (required): Connection uri to the postgres database in the schema `postgres://myuser:mypassword@localhost:5432/database` +- `SPOTIFY_CLIENT_ID` (required): Spotify API Client ID +- `SPOTIFY_CLIENT_SECRET` (required): Spotify API Client Secret +- `BOT_PREFIX` (optional): The prefix of the bot. Defaults to `~` if not set. +- `LOG_DIR` (optional): Directory to store log files in. Defaults to `logs` in the cwd. ## License diff --git a/src/commands/misc/mod.rs b/src/commands/misc/mod.rs index 2c1a64f..03b48eb 100644 --- a/src/commands/misc/mod.rs +++ b/src/commands/misc/mod.rs @@ -2,6 +2,7 @@ use serenity::framework::standard::macros::group; use pekofy::PEKOFY_COMMAND; use ping::PING_COMMAND; +use qalc::QALC_COMMAND; use shutdown::SHUTDOWN_COMMAND; use stats::STATS_COMMAND; use time::TIME_COMMAND; @@ -10,11 +11,12 @@ use timezones::TIMEZONES_COMMAND; pub(crate) mod help; mod pekofy; mod ping; +mod qalc; mod shutdown; mod stats; mod time; mod timezones; #[group] -#[commands(ping, stats, shutdown, pekofy, time, timezones)] +#[commands(ping, stats, shutdown, pekofy, time, timezones, qalc)] pub struct Misc; diff --git a/src/commands/misc/qalc.rs b/src/commands/misc/qalc.rs new file mode 100644 index 0000000..e682764 --- /dev/null +++ b/src/commands/misc/qalc.rs @@ -0,0 +1,18 @@ +use crate::providers::qalc; +use serenity::client::Context; +use serenity::framework::standard::macros::command; +use serenity::framework::standard::{Args, CommandResult}; +use serenity::model::channel::Message; + +#[command] +#[description("Calculates an expression")] +#[min_args(1)] +#[usage("")] +#[example("1 * 1 + 1 / sqrt(2)")] +async fn qalc(ctx: &Context, msg: &Message, args: Args) -> CommandResult { + let expression = args.message(); + let result = qalc::qalc(expression).await?; + msg.reply(ctx, format!("`{}`", result)).await?; + + Ok(()) +} diff --git a/src/commands/music/mod.rs b/src/commands/music/mod.rs index cd09ad4..bcf1462 100644 --- a/src/commands/music/mod.rs +++ b/src/commands/music/mod.rs @@ -1,9 +1,7 @@ use std::sync::Arc; use crate::providers::music::queue::{MusicQueue, Song}; -use crate::providers::music::{ - get_video_information, get_videos_for_playlist, search_video_information, -}; +use crate::providers::music::youtube_dl; use crate::utils::context_data::{DatabaseContainer, Store}; use crate::utils::error::{BotError, BotResult}; use regex::Regex; @@ -299,7 +297,7 @@ async fn get_songs_for_query(ctx: &Context, msg: &Message, query: &str) -> BotRe if YOUTUBE_URL_REGEX.is_match(&query) { log::debug!("Query is youtube video or playlist"); // try fetching the url as a playlist - songs = get_videos_for_playlist(&query) + songs = youtube_dl::get_videos_for_playlist(&query) .await? .into_iter() .map(Song::from) @@ -308,7 +306,7 @@ async fn get_songs_for_query(ctx: &Context, msg: &Message, query: &str) -> BotRe // if no songs were found fetch the song as a video if songs.len() == 0 { log::debug!("Query is youtube video"); - let mut song: Song = get_video_information(&query).await?.into(); + let mut song: Song = youtube_dl::get_video_information(&query).await?.into(); added_one_msg(&ctx, msg, &mut song).await?; songs.push(song); } else { @@ -333,7 +331,7 @@ async fn get_songs_for_query(ctx: &Context, msg: &Message, query: &str) -> BotRe songs.push(song); } else { log::debug!("Query is a youtube search"); - let mut song: Song = search_video_information(query.clone()) + let mut song: Song = youtube_dl::search_video_information(query.clone()) .await? .ok_or(BotError::Msg(format!("Noting found for {}", query)))? .into(); diff --git a/src/providers/mod.rs b/src/providers/mod.rs index 26428cb..b26a4ed 100644 --- a/src/providers/mod.rs +++ b/src/providers/mod.rs @@ -1,2 +1,3 @@ pub(crate) mod music; +pub(crate) mod qalc; pub(crate) mod settings; diff --git a/src/providers/music/lyrics.rs b/src/providers/music/lyrics.rs index 4d9d306..5359d20 100644 --- a/src/providers/music/lyrics.rs +++ b/src/providers/music/lyrics.rs @@ -4,6 +4,7 @@ use serde_derive::Deserialize; const API_ENDPOINT: &str = "https://api.lyrics.ovh/v1/"; +/// Returns the lyrics of a song pub async fn get_lyrics(artist: &str, title: &str) -> BotResult> { lazy_static::lazy_static! { static ref DOUBLE_LB_REGEX: Regex = Regex::new(r"\n\n").unwrap(); } log::debug!("Requesting lyrics for '{}' by '{}'", title, artist); diff --git a/src/providers/music/mod.rs b/src/providers/music/mod.rs index 16e337c..7c5a6e7 100644 --- a/src/providers/music/mod.rs +++ b/src/providers/music/mod.rs @@ -1,107 +1,5 @@ -use crate::providers::music::queue::Song; -use crate::providers::music::responses::{PlaylistEntry, VideoInformation}; -use crate::utils::error::BotResult; -use futures::future::BoxFuture; -use futures::FutureExt; -use std::process::Stdio; -use std::sync::atomic::{AtomicU8, Ordering}; -use std::sync::Arc; -use std::time::Duration; -use tokio::io::AsyncReadExt; -use tokio::process::Command; - pub(crate) mod lyrics; pub(crate) mod queue; pub(crate) mod responses; pub(crate) mod spotify; - -static THREAD_LIMIT: u8 = 64; - -/// Returns a list of youtube videos for a given url -pub(crate) async fn get_videos_for_playlist(url: &str) -> BotResult> { - log::debug!("Getting playlist information for {}", url); - let output = - youtube_dl(&["--no-warnings", "--flat-playlist", "--dump-json", "-i", url]).await?; - - let videos = output - .lines() - .filter_map(|l| serde_json::from_str::(l).ok()) - .collect(); - - Ok(videos) -} - -/// Returns information for a single video by using youtube-dl -pub(crate) async fn get_video_information(url: &str) -> BotResult { - log::debug!("Fetching information for '{}'", url); - let output = youtube_dl(&["--no-warnings", "--dump-json", "-i", url]).await?; - - let information = serde_json::from_str(&*output)?; - - Ok(information) -} - -/// Searches for a video -pub(crate) async fn search_video_information(query: String) -> BotResult> { - log::debug!("Searching for video '{}'", query); - let output = youtube_dl(&[ - "--no-warnings", - "--dump-json", - "-i", - format!("ytsearch:\"{}\"", query).as_str(), - ]) - .await?; - let information = serde_json::from_str(&*output)?; - - Ok(information) -} - -/// Searches songs on youtube in parallel -#[allow(dead_code)] -async fn parallel_search_youtube(song_names: Vec) -> Vec { - let search_futures: Vec>>> = song_names - .into_iter() - .map(|s| search_video_information(s).boxed()) - .collect(); - let information: Vec>> = - futures::future::join_all(search_futures).await; - information - .into_iter() - .filter_map(|i| i.ok().and_then(|s| s).map(Song::from)) - .collect() -} - -/// Executes youtube-dl asynchronously -/// An atomic U8 is used to control the number of parallel processes -/// to avoid using too much memory -async fn youtube_dl(args: &[&str]) -> BotResult { - lazy_static::lazy_static! { static ref THREAD_LOCK: Arc = Arc::new(AtomicU8::new(0)); } - log::trace!("Running youtube-dl with args {:?}", args); - - while THREAD_LOCK.load(Ordering::SeqCst) >= THREAD_LIMIT { - tokio::time::sleep(Duration::from_millis(100)).await; - } - THREAD_LOCK.fetch_add(1, Ordering::Relaxed); - - let ytdl = Command::new("youtube-dl") - .args(args) - .stdout(Stdio::piped()) - .spawn() - .map_err(|e| { - THREAD_LOCK.fetch_sub(1, Ordering::Relaxed); - e - })?; - let mut output = String::new(); - ytdl.stdout - .unwrap() - .read_to_string(&mut output) - .await - .map_err(|e| { - THREAD_LOCK.fetch_sub(1, Ordering::Relaxed); - e - })?; - log::trace!("youtube-dl response is {}", output); - THREAD_LOCK.fetch_sub(1, Ordering::Relaxed); - - Ok(output) -} +pub(crate) mod youtube_dl; diff --git a/src/providers/music/queue.rs b/src/providers/music/queue.rs index 712439c..90972a4 100644 --- a/src/providers/music/queue.rs +++ b/src/providers/music/queue.rs @@ -4,7 +4,7 @@ use songbird::tracks::TrackHandle; use crate::messages::music::NowPlayingMessage; use crate::providers::music::responses::{PlaylistEntry, VideoInformation}; -use crate::providers::music::search_video_information; +use crate::providers::music::youtube_dl; use crate::utils::shuffle_vec_deque; use aspotify::{Track, TrackSimplified}; @@ -110,10 +110,11 @@ impl Song { Some(url) } else { log::debug!("Lazy fetching video for title"); - let information = search_video_information(format!("{} - {}", self.author, self.title)) - .await - .ok() - .and_then(|i| i)?; + let information = + youtube_dl::search_video_information(format!("{} - {}", self.author, self.title)) + .await + .ok() + .and_then(|i| i)?; self.url = Some(information.webpage_url.clone()); self.thumbnail = information.thumbnail; self.author = information.uploader; diff --git a/src/providers/music/youtube_dl.rs b/src/providers/music/youtube_dl.rs new file mode 100644 index 0000000..1cd6299 --- /dev/null +++ b/src/providers/music/youtube_dl.rs @@ -0,0 +1,87 @@ +use crate::providers::music::queue::Song; +use crate::providers::music::responses::{PlaylistEntry, VideoInformation}; +use crate::utils::error::BotResult; +use crate::utils::run_command_async; +use futures::future::BoxFuture; +use futures::FutureExt; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +static THREAD_LIMIT: u8 = 64; + +/// Returns a list of youtube videos for a given url +pub(crate) async fn get_videos_for_playlist(url: &str) -> BotResult> { + log::debug!("Getting playlist information for {}", url); + let output = + youtube_dl(&["--no-warnings", "--flat-playlist", "--dump-json", "-i", url]).await?; + + let videos = output + .lines() + .filter_map(|l| serde_json::from_str::(l).ok()) + .collect(); + + Ok(videos) +} + +/// Returns information for a single video by using youtube-dl +pub(crate) async fn get_video_information(url: &str) -> BotResult { + log::debug!("Fetching information for '{}'", url); + let output = youtube_dl(&["--no-warnings", "--dump-json", "-i", url]).await?; + + let information = serde_json::from_str(&*output)?; + + Ok(information) +} + +/// Searches for a video +pub(crate) async fn search_video_information(query: String) -> BotResult> { + log::debug!("Searching for video '{}'", query); + let output = youtube_dl(&[ + "--no-warnings", + "--dump-json", + "-i", + format!("ytsearch:\"{}\"", query).as_str(), + ]) + .await?; + let information = serde_json::from_str(&*output)?; + + Ok(information) +} + +/// Searches songs on youtube in parallel +#[allow(dead_code)] +async fn parallel_search_youtube(song_names: Vec) -> Vec { + let search_futures: Vec>>> = song_names + .into_iter() + .map(|s| search_video_information(s).boxed()) + .collect(); + let information: Vec>> = + futures::future::join_all(search_futures).await; + information + .into_iter() + .filter_map(|i| i.ok().and_then(|s| s).map(Song::from)) + .collect() +} + +/// Executes youtube-dl asynchronously +/// An atomic U8 is used to control the number of parallel processes +/// to avoid using too much memory +async fn youtube_dl(args: &[&str]) -> BotResult { + lazy_static::lazy_static! { static ref THREAD_LOCK: Arc = Arc::new(AtomicU8::new(0)); } + log::trace!("Running youtube-dl with args {:?}", args); + + while THREAD_LOCK.load(Ordering::SeqCst) >= THREAD_LIMIT { + tokio::time::sleep(Duration::from_millis(100)).await; + } + THREAD_LOCK.fetch_add(1, Ordering::Relaxed); + + let output = run_command_async("youtube-dl", args).await.map_err(|e| { + THREAD_LOCK.fetch_sub(1, Ordering::Relaxed); + e + })?; + log::trace!("youtube-dl response is {}", output); + THREAD_LOCK.fetch_sub(1, Ordering::Relaxed); + + Ok(output) +} diff --git a/src/providers/qalc.rs b/src/providers/qalc.rs new file mode 100644 index 0000000..52225b2 --- /dev/null +++ b/src/providers/qalc.rs @@ -0,0 +1,8 @@ +use crate::utils::error::BotResult; +use crate::utils::run_command_async; + +/// Runs the qalc command with the given expression +pub async fn qalc(expression: &str) -> BotResult { + let result = run_command_async("qalc", &[expression]).await?; + Ok(result) +} diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 9467c6f..e6ef530 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -1,6 +1,10 @@ use std::collections::VecDeque; use rand::Rng; +use std::io; +use std::process::Stdio; +use tokio::io::AsyncReadExt; +use tokio::process::Command; pub(crate) mod context_data; pub(crate) mod error; @@ -16,3 +20,23 @@ pub fn shuffle_vec_deque(deque: &mut VecDeque) { deque.swap(i, rng.gen_range(0..i + 1)) } } + +/// Asynchronously runs a given command and returns the output +pub async fn run_command_async(command: &str, args: &[&str]) -> io::Result { + log::trace!("Running command '{}' with args {:?}", command, args); + let cmd = Command::new(command) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + let mut stderr = String::new(); + let mut output = String::new(); + cmd.stderr.unwrap().read_to_string(&mut stderr).await?; + if stderr.len() != 0 { + log::debug!("STDERR of command {}: {}", command, stderr); + } + cmd.stdout.unwrap().read_to_string(&mut output).await?; + log::trace!("Command output is {}", output); + + Ok(output) +}