diff --git a/mediarepo-daemon/Cargo.lock b/mediarepo-daemon/Cargo.lock index 065c2ad..3a90c45 100644 --- a/mediarepo-daemon/Cargo.lock +++ b/mediarepo-daemon/Cargo.lock @@ -541,6 +541,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "hashbrown" version = "0.11.2" @@ -701,6 +707,7 @@ name = "mediarepo" version = "0.1.0" dependencies = [ "env_logger", + "glob", "log", "mediarepo-core", "mediarepo-model", @@ -744,6 +751,8 @@ dependencies = [ "chrono", "mediarepo-core", "mediarepo-database", + "mime", + "mime_guess", "sea-orm", "serde", "tokio", @@ -768,6 +777,22 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.1.4" @@ -1747,6 +1772,15 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.6" diff --git a/mediarepo-daemon/Cargo.toml b/mediarepo-daemon/Cargo.toml index 62f1ab5..5c69d62 100644 --- a/mediarepo-daemon/Cargo.toml +++ b/mediarepo-daemon/Cargo.toml @@ -17,7 +17,8 @@ crate-type = ["lib"] [dependencies] toml = {version = "0.5.8", optional=true} structopt = {version="0.3.23", optional=true} -env_logger = "0.9.0" +env_logger = {version="0.9.0", optional=true} +glob = {version="0.3.0", optional=true} log = "0.4.14" [dependencies.mediarepo-core] @@ -37,5 +38,5 @@ features = ["macros", "rt-multi-thread", "io-std", "io-util"] [features] default = ["runtime"] -runtime = ["toml", "structopt", "mediarepo-model", "mediarepo-socket"] +runtime = ["toml", "structopt", "mediarepo-model", "mediarepo-socket", "env_logger", "glob"] library = ["mediarepo-socket"] \ No newline at end of file diff --git a/mediarepo-daemon/mediarepo-model/Cargo.lock b/mediarepo-daemon/mediarepo-model/Cargo.lock index 93996b7..a423148 100644 --- a/mediarepo-daemon/mediarepo-model/Cargo.lock +++ b/mediarepo-daemon/mediarepo-model/Cargo.lock @@ -666,6 +666,8 @@ dependencies = [ "chrono", "mediarepo-core", "mediarepo-database", + "mime", + "mime_guess", "sea-orm", "serde", "tokio", @@ -678,6 +680,22 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.1.3" @@ -1580,6 +1598,15 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.6" diff --git a/mediarepo-daemon/mediarepo-model/Cargo.toml b/mediarepo-daemon/mediarepo-model/Cargo.toml index 9747ce0..32f17e2 100644 --- a/mediarepo-daemon/mediarepo-model/Cargo.toml +++ b/mediarepo-daemon/mediarepo-model/Cargo.toml @@ -9,6 +9,8 @@ edition = "2018" chrono = "0.4.19" typemap_rev = "0.1.5" serde = "1.0.130" +mime_guess = "2.0.3" +mime = "0.3.16" [dependencies.mediarepo-core] path = "../mediarepo-core" diff --git a/mediarepo-daemon/mediarepo-model/src/file.rs b/mediarepo-daemon/mediarepo-model/src/file.rs index 0d07fcd..dfb30d5 100644 --- a/mediarepo-daemon/mediarepo-model/src/file.rs +++ b/mediarepo-daemon/mediarepo-model/src/file.rs @@ -1,6 +1,6 @@ use crate::file_type::FileType; use crate::storage::Storage; -use chrono::NaiveDateTime; +use chrono::{Local, NaiveDateTime}; use mediarepo_core::error::RepoResult; use mediarepo_database::entities::file; use mediarepo_database::entities::file::ActiveModel as ActiveFile; @@ -71,6 +71,37 @@ impl File { } } + /// Adds a file with its hash to the database + pub(crate) async fn add( + db: DatabaseConnection, + storage_id: i64, + hash: S, + file_type: FileType, + ) -> RepoResult { + let hash = hash::ActiveModel { + value: Set(hash.to_string()), + ..Default::default() + }; + let now = Local::now().naive_local(); + let hash: hash::ActiveModel = hash.insert(&db).await?; + let id: i64 = hash.id.unwrap(); + let file = file::ActiveModel { + hash_id: Set(id), + file_type: Set(file_type as u32), + storage_id: Set(storage_id), + import_time: Set(now.clone()), + creation_time: Set(now.clone()), + change_time: Set(now), + ..Default::default() + }; + let file: file::ActiveModel = file.insert(&db).await?.into(); + let file = Self::by_id(db, file.id.unwrap()) + .await? + .expect("Inserted file does not exist"); + + Ok(file) + } + /// Returns the unique identifier of the file pub fn id(&self) -> i64 { self.model.id diff --git a/mediarepo-daemon/mediarepo-model/src/file_type.rs b/mediarepo-daemon/mediarepo-model/src/file_type.rs index 1c34c24..50cec01 100644 --- a/mediarepo-daemon/mediarepo-model/src/file_type.rs +++ b/mediarepo-daemon/mediarepo-model/src/file_type.rs @@ -1,9 +1,27 @@ use serde::{Deserialize, Serialize}; +use std::path::PathBuf; #[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, PartialEq)] pub enum FileType { + Other = -1, Unknown = 0, Image = 1, Video = 2, Audio = 3, } + +impl From<&PathBuf> for FileType { + fn from(path: &PathBuf) -> Self { + let mime = mime_guess::from_path(path).first(); + if let Some(mime) = mime { + match mime.type_() { + mime::IMAGE => Self::Image, + mime::VIDEO => Self::Video, + mime::AUDIO => Self::Audio, + _ => Self::Other, + } + } else { + Self::Unknown + } + } +} diff --git a/mediarepo-daemon/mediarepo-model/src/repo.rs b/mediarepo-daemon/mediarepo-model/src/repo.rs index d356214..232f080 100644 --- a/mediarepo-daemon/mediarepo-model/src/repo.rs +++ b/mediarepo-daemon/mediarepo-model/src/repo.rs @@ -1,4 +1,5 @@ use crate::file::File; +use crate::file_type::FileType; use crate::storage::Storage; use mediarepo_core::error::{RepoError, RepoResult}; use mediarepo_database::get_database; @@ -56,6 +57,11 @@ impl Repo { File::by_hash(self.db.clone(), hash).await } + /// Returns a file by id + pub async fn file_by_id(&self, id: i64) -> RepoResult> { + File::by_id(self.db.clone(), id).await + } + /// Returns a list of all stored files pub async fn files(&self) -> RepoResult> { File::all(self.db.clone()).await @@ -63,17 +69,12 @@ impl Repo { /// Adds a file to the database by its readable path in the file system pub async fn add_file_by_path(&self, path: PathBuf) -> RepoResult { + let file_type = FileType::from(&path); let os_file = OpenOptions::new().read(true).open(&path).await?; let reader = BufReader::new(os_file); let storage = self.get_main_storage()?; - let hash = storage.add_file(reader).await?; - let file = self - .file_by_hash(hash) - .await? - .expect("Invalid database state."); - - Ok(file) + storage.add_file(reader, file_type).await } fn get_main_storage(&self) -> RepoResult<&Storage> { diff --git a/mediarepo-daemon/mediarepo-model/src/storage.rs b/mediarepo-daemon/mediarepo-model/src/storage.rs index 6b40b31..4c3a3bd 100644 --- a/mediarepo-daemon/mediarepo-model/src/storage.rs +++ b/mediarepo-daemon/mediarepo-model/src/storage.rs @@ -1,3 +1,5 @@ +use crate::file::File; +use crate::file_type::FileType; use mediarepo_core::error::RepoResult; use mediarepo_core::file_hash_store::FileHashStore; use mediarepo_database::entities::storage; @@ -132,8 +134,13 @@ impl Storage { } /// Adds a file to the store - pub async fn add_file(&self, reader: R) -> RepoResult { - self.store.add_file(reader, None).await + pub async fn add_file( + &self, + reader: R, + file_type: FileType, + ) -> RepoResult { + let hash = self.store.add_file(reader, None).await?; + File::add(self.db.clone(), self.id(), hash, file_type).await } /// Returns the buf reader to the given hash diff --git a/mediarepo-daemon/mediarepo-socket/Cargo.lock b/mediarepo-daemon/mediarepo-socket/Cargo.lock index cb74794..46771d4 100644 --- a/mediarepo-daemon/mediarepo-socket/Cargo.lock +++ b/mediarepo-daemon/mediarepo-socket/Cargo.lock @@ -667,6 +667,8 @@ dependencies = [ "chrono", "mediarepo-core", "mediarepo-database", + "mime", + "mime_guess", "sea-orm", "serde", "tokio", @@ -691,6 +693,22 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.1.4" @@ -1593,6 +1611,15 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.7" diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs index 3d4c29d..5688c7f 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs @@ -1,10 +1,12 @@ -use crate::types::requests::AddFileRequest; +use crate::types::requests::{AddFileRequest, ReadFileRequest}; use crate::types::responses::FileResponse; +use mediarepo_core::error::RepoError; use mediarepo_model::type_keys::RepoKey; use rmp_ipc::context::Context; use rmp_ipc::error::Result; use rmp_ipc::{Event, NamespaceBuilder}; use std::path::PathBuf; +use tokio::io::AsyncReadExt; pub const FILES_NAMESPACE: &str = "files"; @@ -12,6 +14,7 @@ pub fn build(builder: NamespaceBuilder) -> NamespaceBuilder { builder .on("all_files", |c, e| Box::pin(all_files(c, e))) .on("add_file", |c, e| Box::pin(add_file(c, e))) + .on("read_file", |c, e| Box::pin(read_file(c, e))) } /// Returns a list of all files @@ -49,3 +52,25 @@ async fn add_file(ctx: &Context, event: Event) -> Result<()> { Ok(()) } + +/// Reads the binary contents of a file +async fn read_file(ctx: &Context, event: Event) -> Result<()> { + let request = event.data::()?; + let mut reader = { + let data = ctx.data.read().await; + let repo = data.get::().unwrap(); + let file = match request { + ReadFileRequest::ID(id) => repo.file_by_id(id).await, + ReadFileRequest::Hash(hash) => repo.file_by_hash(hash).await, + }? + .ok_or_else(|| RepoError::from("File not found")); + file?.get_reader().await? + }; + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await?; + ctx.emitter + .emit_response_to(event.id(), FILES_NAMESPACE, "read_file", buf) + .await?; + + Ok(()) +} diff --git a/mediarepo-daemon/mediarepo-socket/src/types/requests.rs b/mediarepo-daemon/mediarepo-socket/src/types/requests.rs index d0e50f1..2cb01b8 100644 --- a/mediarepo-daemon/mediarepo-socket/src/types/requests.rs +++ b/mediarepo-daemon/mediarepo-socket/src/types/requests.rs @@ -4,3 +4,9 @@ use serde::{Deserialize, Serialize}; pub struct AddFileRequest { pub path: String, } + +#[derive(Serialize, Deserialize)] +pub enum ReadFileRequest { + ID(i64), + Hash(String), +} diff --git a/mediarepo-daemon/src/main.rs b/mediarepo-daemon/src/main.rs index b0dbcae..1202562 100644 --- a/mediarepo-daemon/src/main.rs +++ b/mediarepo-daemon/src/main.rs @@ -6,6 +6,7 @@ use crate::utils::{create_paths_for_repo, get_repo, load_settings}; use mediarepo_core::error::RepoResult; use mediarepo_core::settings::Settings; use mediarepo_core::type_keys::SettingsKey; +use mediarepo_model::repo::Repo; use mediarepo_model::type_keys::RepoKey; use mediarepo_socket::get_builder; use std::path::PathBuf; @@ -24,7 +25,7 @@ struct Opt { cmd: SubCommand, } -#[derive(Debug, StructOpt)] +#[derive(Clone, Debug, StructOpt)] enum SubCommand { /// Initializes an empty repository Init { @@ -34,6 +35,13 @@ enum SubCommand { force: bool, }, + /// Imports file from a folder (by glob pattern) into the repository + Import { + /// The path to the folder where the files are located + #[structopt()] + folder_path: String, + }, + /// Starts the event server for the selected repository Start, } @@ -42,19 +50,35 @@ enum SubCommand { async fn main() -> RepoResult<()> { build_logger(); let opt: Opt = Opt::from_args(); - match opt.cmd { + match opt.cmd.clone() { SubCommand::Init { force } => init(opt, force).await, SubCommand::Start => start_server(opt).await, + SubCommand::Import { folder_path } => import(opt, folder_path).await, }?; Ok(()) } -/// Starts the server -async fn start_server(opt: Opt) -> RepoResult<()> { +fn build_logger() { + env_logger::builder() + .filter_module("sqlx", log::LevelFilter::Warn) + .filter_module("tokio", log::LevelFilter::Info) + .filter_module("tracing", log::LevelFilter::Warn) + .init(); +} + +async fn init_repo(opt: &Opt) -> RepoResult<(Settings, Repo)> { let settings = load_settings(&opt.repo.join(SETTINGS_PATH)).await?; let mut repo = get_repo(&opt.repo.join(&settings.database_path).to_str().unwrap()).await?; - repo.set_main_storage(&settings.default_file_store).await?; + let main_storage_path = opt.repo.join(&settings.default_file_store); + repo.set_main_storage(main_storage_path.to_str().unwrap()) + .await?; + Ok((settings, repo)) +} + +/// Starts the server +async fn start_server(opt: Opt) -> RepoResult<()> { + let (settings, repo) = init_repo(&opt).await?; get_builder(&settings.listen_address) .insert::(settings) @@ -67,29 +91,47 @@ async fn start_server(opt: Opt) -> RepoResult<()> { /// Initializes an empty repository async fn init(opt: Opt, force: bool) -> RepoResult<()> { + log::info!("Initializing repository at {:?}", opt.repo); if force { + log::debug!("Removing old repository"); fs::remove_dir_all(&opt.repo).await?; } let settings = Settings::default(); + log::debug!("Creating paths"); create_paths_for_repo(&opt.repo, &settings).await?; let db_path = opt.repo.join(&settings.database_path); if db_path.exists() { panic!("Database already exists in location. Use --force with init to delete everything and start a new repository"); } + log::debug!("Creating repo"); let repo = get_repo(&db_path.to_str().unwrap()).await?; let storage_path = opt.repo.join(&settings.default_file_store); + log::debug!("Adding storage"); repo.add_storage(DEFAULT_STORAGE_NAME, storage_path.to_str().unwrap()) .await?; let settings_string = settings.to_toml_string()?; + log::debug!("Writing settings"); fs::write(opt.repo.join(SETTINGS_PATH), &settings_string.into_bytes()).await?; + log::info!("Repository initialized"); Ok(()) } -fn build_logger() { - env_logger::builder() - .filter_module("sqlx", log::LevelFilter::Warn) - .filter_module("tokio", log::LevelFilter::Info) - .filter_module("tracing", log::LevelFilter::Warn) - .init(); +/// Imports files from a source into the database +async fn import(opt: Opt, path: String) -> RepoResult<()> { + let (_s, repo) = init_repo(&opt).await?; + log::info!("Importing"); + + for entry in glob::glob(&path).unwrap() { + if let Ok(path) = entry { + if path.is_file() { + log::debug!("Importing {:?}", path); + if let Err(e) = repo.add_file_by_path(path).await { + log::error!("Failed to import: {:?}", e); + } + } + } + } + + Ok(()) }