Add read_file event and parsing of file types from mimes

Signed-off-by: trivernis <trivernis@protonmail.com>
pull/4/head
trivernis 3 years ago
parent 0d26370caa
commit d62f5b38bf

@ -541,6 +541,12 @@ dependencies = [
"wasi", "wasi",
] ]
[[package]]
name = "glob"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.11.2" version = "0.11.2"
@ -701,6 +707,7 @@ name = "mediarepo"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"env_logger", "env_logger",
"glob",
"log", "log",
"mediarepo-core", "mediarepo-core",
"mediarepo-model", "mediarepo-model",
@ -744,6 +751,8 @@ dependencies = [
"chrono", "chrono",
"mediarepo-core", "mediarepo-core",
"mediarepo-database", "mediarepo-database",
"mime",
"mime_guess",
"sea-orm", "sea-orm",
"serde", "serde",
"tokio", "tokio",
@ -768,6 +777,22 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "mime"
version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
[[package]]
name = "mime_guess"
version = "2.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212"
dependencies = [
"mime",
"unicase",
]
[[package]] [[package]]
name = "minimal-lexical" name = "minimal-lexical"
version = "0.1.4" version = "0.1.4"
@ -1747,6 +1772,15 @@ version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec"
[[package]]
name = "unicase"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
dependencies = [
"version_check",
]
[[package]] [[package]]
name = "unicode-bidi" name = "unicode-bidi"
version = "0.3.6" version = "0.3.6"

@ -17,7 +17,8 @@ crate-type = ["lib"]
[dependencies] [dependencies]
toml = {version = "0.5.8", optional=true} toml = {version = "0.5.8", optional=true}
structopt = {version="0.3.23", optional=true} structopt = {version="0.3.23", optional=true}
env_logger = "0.9.0" env_logger = {version="0.9.0", optional=true}
glob = {version="0.3.0", optional=true}
log = "0.4.14" log = "0.4.14"
[dependencies.mediarepo-core] [dependencies.mediarepo-core]
@ -37,5 +38,5 @@ features = ["macros", "rt-multi-thread", "io-std", "io-util"]
[features] [features]
default = ["runtime"] default = ["runtime"]
runtime = ["toml", "structopt", "mediarepo-model", "mediarepo-socket"] runtime = ["toml", "structopt", "mediarepo-model", "mediarepo-socket", "env_logger", "glob"]
library = ["mediarepo-socket"] library = ["mediarepo-socket"]

@ -666,6 +666,8 @@ dependencies = [
"chrono", "chrono",
"mediarepo-core", "mediarepo-core",
"mediarepo-database", "mediarepo-database",
"mime",
"mime_guess",
"sea-orm", "sea-orm",
"serde", "serde",
"tokio", "tokio",
@ -678,6 +680,22 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "mime"
version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
[[package]]
name = "mime_guess"
version = "2.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212"
dependencies = [
"mime",
"unicase",
]
[[package]] [[package]]
name = "minimal-lexical" name = "minimal-lexical"
version = "0.1.3" version = "0.1.3"
@ -1580,6 +1598,15 @@ version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec"
[[package]]
name = "unicase"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
dependencies = [
"version_check",
]
[[package]] [[package]]
name = "unicode-bidi" name = "unicode-bidi"
version = "0.3.6" version = "0.3.6"

@ -9,6 +9,8 @@ edition = "2018"
chrono = "0.4.19" chrono = "0.4.19"
typemap_rev = "0.1.5" typemap_rev = "0.1.5"
serde = "1.0.130" serde = "1.0.130"
mime_guess = "2.0.3"
mime = "0.3.16"
[dependencies.mediarepo-core] [dependencies.mediarepo-core]
path = "../mediarepo-core" path = "../mediarepo-core"

@ -1,6 +1,6 @@
use crate::file_type::FileType; use crate::file_type::FileType;
use crate::storage::Storage; use crate::storage::Storage;
use chrono::NaiveDateTime; use chrono::{Local, NaiveDateTime};
use mediarepo_core::error::RepoResult; use mediarepo_core::error::RepoResult;
use mediarepo_database::entities::file; use mediarepo_database::entities::file;
use mediarepo_database::entities::file::ActiveModel as ActiveFile; use mediarepo_database::entities::file::ActiveModel as ActiveFile;
@ -71,6 +71,37 @@ impl File {
} }
} }
/// Adds a file with its hash to the database
pub(crate) async fn add<S: ToString>(
db: DatabaseConnection,
storage_id: i64,
hash: S,
file_type: FileType,
) -> RepoResult<Self> {
let hash = hash::ActiveModel {
value: Set(hash.to_string()),
..Default::default()
};
let now = Local::now().naive_local();
let hash: hash::ActiveModel = hash.insert(&db).await?;
let id: i64 = hash.id.unwrap();
let file = file::ActiveModel {
hash_id: Set(id),
file_type: Set(file_type as u32),
storage_id: Set(storage_id),
import_time: Set(now.clone()),
creation_time: Set(now.clone()),
change_time: Set(now),
..Default::default()
};
let file: file::ActiveModel = file.insert(&db).await?.into();
let file = Self::by_id(db, file.id.unwrap())
.await?
.expect("Inserted file does not exist");
Ok(file)
}
/// Returns the unique identifier of the file /// Returns the unique identifier of the file
pub fn id(&self) -> i64 { pub fn id(&self) -> i64 {
self.model.id self.model.id

@ -1,9 +1,27 @@
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::path::PathBuf;
#[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, PartialEq)] #[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, PartialEq)]
pub enum FileType { pub enum FileType {
Other = -1,
Unknown = 0, Unknown = 0,
Image = 1, Image = 1,
Video = 2, Video = 2,
Audio = 3, Audio = 3,
} }
impl From<&PathBuf> for FileType {
fn from(path: &PathBuf) -> Self {
let mime = mime_guess::from_path(path).first();
if let Some(mime) = mime {
match mime.type_() {
mime::IMAGE => Self::Image,
mime::VIDEO => Self::Video,
mime::AUDIO => Self::Audio,
_ => Self::Other,
}
} else {
Self::Unknown
}
}
}

@ -1,4 +1,5 @@
use crate::file::File; use crate::file::File;
use crate::file_type::FileType;
use crate::storage::Storage; use crate::storage::Storage;
use mediarepo_core::error::{RepoError, RepoResult}; use mediarepo_core::error::{RepoError, RepoResult};
use mediarepo_database::get_database; use mediarepo_database::get_database;
@ -56,6 +57,11 @@ impl Repo {
File::by_hash(self.db.clone(), hash).await File::by_hash(self.db.clone(), hash).await
} }
/// Returns a file by id
pub async fn file_by_id(&self, id: i64) -> RepoResult<Option<File>> {
File::by_id(self.db.clone(), id).await
}
/// Returns a list of all stored files /// Returns a list of all stored files
pub async fn files(&self) -> RepoResult<Vec<File>> { pub async fn files(&self) -> RepoResult<Vec<File>> {
File::all(self.db.clone()).await File::all(self.db.clone()).await
@ -63,17 +69,12 @@ impl Repo {
/// Adds a file to the database by its readable path in the file system /// Adds a file to the database by its readable path in the file system
pub async fn add_file_by_path(&self, path: PathBuf) -> RepoResult<File> { pub async fn add_file_by_path(&self, path: PathBuf) -> RepoResult<File> {
let file_type = FileType::from(&path);
let os_file = OpenOptions::new().read(true).open(&path).await?; let os_file = OpenOptions::new().read(true).open(&path).await?;
let reader = BufReader::new(os_file); let reader = BufReader::new(os_file);
let storage = self.get_main_storage()?; let storage = self.get_main_storage()?;
let hash = storage.add_file(reader).await?; storage.add_file(reader, file_type).await
let file = self
.file_by_hash(hash)
.await?
.expect("Invalid database state.");
Ok(file)
} }
fn get_main_storage(&self) -> RepoResult<&Storage> { fn get_main_storage(&self) -> RepoResult<&Storage> {

@ -1,3 +1,5 @@
use crate::file::File;
use crate::file_type::FileType;
use mediarepo_core::error::RepoResult; use mediarepo_core::error::RepoResult;
use mediarepo_core::file_hash_store::FileHashStore; use mediarepo_core::file_hash_store::FileHashStore;
use mediarepo_database::entities::storage; use mediarepo_database::entities::storage;
@ -132,8 +134,13 @@ impl Storage {
} }
/// Adds a file to the store /// Adds a file to the store
pub async fn add_file<R: AsyncRead + Unpin>(&self, reader: R) -> RepoResult<String> { pub async fn add_file<R: AsyncRead + Unpin>(
self.store.add_file(reader, None).await &self,
reader: R,
file_type: FileType,
) -> RepoResult<File> {
let hash = self.store.add_file(reader, None).await?;
File::add(self.db.clone(), self.id(), hash, file_type).await
} }
/// Returns the buf reader to the given hash /// Returns the buf reader to the given hash

@ -667,6 +667,8 @@ dependencies = [
"chrono", "chrono",
"mediarepo-core", "mediarepo-core",
"mediarepo-database", "mediarepo-database",
"mime",
"mime_guess",
"sea-orm", "sea-orm",
"serde", "serde",
"tokio", "tokio",
@ -691,6 +693,22 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "mime"
version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
[[package]]
name = "mime_guess"
version = "2.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212"
dependencies = [
"mime",
"unicase",
]
[[package]] [[package]]
name = "minimal-lexical" name = "minimal-lexical"
version = "0.1.4" version = "0.1.4"
@ -1593,6 +1611,15 @@ version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec"
[[package]]
name = "unicase"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
dependencies = [
"version_check",
]
[[package]] [[package]]
name = "unicode-bidi" name = "unicode-bidi"
version = "0.3.7" version = "0.3.7"

@ -1,10 +1,12 @@
use crate::types::requests::AddFileRequest; use crate::types::requests::{AddFileRequest, ReadFileRequest};
use crate::types::responses::FileResponse; use crate::types::responses::FileResponse;
use mediarepo_core::error::RepoError;
use mediarepo_model::type_keys::RepoKey; use mediarepo_model::type_keys::RepoKey;
use rmp_ipc::context::Context; use rmp_ipc::context::Context;
use rmp_ipc::error::Result; use rmp_ipc::error::Result;
use rmp_ipc::{Event, NamespaceBuilder}; use rmp_ipc::{Event, NamespaceBuilder};
use std::path::PathBuf; use std::path::PathBuf;
use tokio::io::AsyncReadExt;
pub const FILES_NAMESPACE: &str = "files"; pub const FILES_NAMESPACE: &str = "files";
@ -12,6 +14,7 @@ pub fn build(builder: NamespaceBuilder) -> NamespaceBuilder {
builder builder
.on("all_files", |c, e| Box::pin(all_files(c, e))) .on("all_files", |c, e| Box::pin(all_files(c, e)))
.on("add_file", |c, e| Box::pin(add_file(c, e))) .on("add_file", |c, e| Box::pin(add_file(c, e)))
.on("read_file", |c, e| Box::pin(read_file(c, e)))
} }
/// Returns a list of all files /// Returns a list of all files
@ -49,3 +52,25 @@ async fn add_file(ctx: &Context, event: Event) -> Result<()> {
Ok(()) Ok(())
} }
/// Reads the binary contents of a file
async fn read_file(ctx: &Context, event: Event) -> Result<()> {
let request = event.data::<ReadFileRequest>()?;
let mut reader = {
let data = ctx.data.read().await;
let repo = data.get::<RepoKey>().unwrap();
let file = match request {
ReadFileRequest::ID(id) => repo.file_by_id(id).await,
ReadFileRequest::Hash(hash) => repo.file_by_hash(hash).await,
}?
.ok_or_else(|| RepoError::from("File not found"));
file?.get_reader().await?
};
let mut buf = Vec::new();
reader.read_to_end(&mut buf).await?;
ctx.emitter
.emit_response_to(event.id(), FILES_NAMESPACE, "read_file", buf)
.await?;
Ok(())
}

@ -4,3 +4,9 @@ use serde::{Deserialize, Serialize};
pub struct AddFileRequest { pub struct AddFileRequest {
pub path: String, pub path: String,
} }
#[derive(Serialize, Deserialize)]
pub enum ReadFileRequest {
ID(i64),
Hash(String),
}

@ -6,6 +6,7 @@ use crate::utils::{create_paths_for_repo, get_repo, load_settings};
use mediarepo_core::error::RepoResult; use mediarepo_core::error::RepoResult;
use mediarepo_core::settings::Settings; use mediarepo_core::settings::Settings;
use mediarepo_core::type_keys::SettingsKey; use mediarepo_core::type_keys::SettingsKey;
use mediarepo_model::repo::Repo;
use mediarepo_model::type_keys::RepoKey; use mediarepo_model::type_keys::RepoKey;
use mediarepo_socket::get_builder; use mediarepo_socket::get_builder;
use std::path::PathBuf; use std::path::PathBuf;
@ -24,7 +25,7 @@ struct Opt {
cmd: SubCommand, cmd: SubCommand,
} }
#[derive(Debug, StructOpt)] #[derive(Clone, Debug, StructOpt)]
enum SubCommand { enum SubCommand {
/// Initializes an empty repository /// Initializes an empty repository
Init { Init {
@ -34,6 +35,13 @@ enum SubCommand {
force: bool, force: bool,
}, },
/// Imports file from a folder (by glob pattern) into the repository
Import {
/// The path to the folder where the files are located
#[structopt()]
folder_path: String,
},
/// Starts the event server for the selected repository /// Starts the event server for the selected repository
Start, Start,
} }
@ -42,19 +50,35 @@ enum SubCommand {
async fn main() -> RepoResult<()> { async fn main() -> RepoResult<()> {
build_logger(); build_logger();
let opt: Opt = Opt::from_args(); let opt: Opt = Opt::from_args();
match opt.cmd { match opt.cmd.clone() {
SubCommand::Init { force } => init(opt, force).await, SubCommand::Init { force } => init(opt, force).await,
SubCommand::Start => start_server(opt).await, SubCommand::Start => start_server(opt).await,
SubCommand::Import { folder_path } => import(opt, folder_path).await,
}?; }?;
Ok(()) Ok(())
} }
/// Starts the server fn build_logger() {
async fn start_server(opt: Opt) -> RepoResult<()> { env_logger::builder()
.filter_module("sqlx", log::LevelFilter::Warn)
.filter_module("tokio", log::LevelFilter::Info)
.filter_module("tracing", log::LevelFilter::Warn)
.init();
}
async fn init_repo(opt: &Opt) -> RepoResult<(Settings, Repo)> {
let settings = load_settings(&opt.repo.join(SETTINGS_PATH)).await?; let settings = load_settings(&opt.repo.join(SETTINGS_PATH)).await?;
let mut repo = get_repo(&opt.repo.join(&settings.database_path).to_str().unwrap()).await?; let mut repo = get_repo(&opt.repo.join(&settings.database_path).to_str().unwrap()).await?;
repo.set_main_storage(&settings.default_file_store).await?; let main_storage_path = opt.repo.join(&settings.default_file_store);
repo.set_main_storage(main_storage_path.to_str().unwrap())
.await?;
Ok((settings, repo))
}
/// Starts the server
async fn start_server(opt: Opt) -> RepoResult<()> {
let (settings, repo) = init_repo(&opt).await?;
get_builder(&settings.listen_address) get_builder(&settings.listen_address)
.insert::<SettingsKey>(settings) .insert::<SettingsKey>(settings)
@ -67,29 +91,47 @@ async fn start_server(opt: Opt) -> RepoResult<()> {
/// Initializes an empty repository /// Initializes an empty repository
async fn init(opt: Opt, force: bool) -> RepoResult<()> { async fn init(opt: Opt, force: bool) -> RepoResult<()> {
log::info!("Initializing repository at {:?}", opt.repo);
if force { if force {
log::debug!("Removing old repository");
fs::remove_dir_all(&opt.repo).await?; fs::remove_dir_all(&opt.repo).await?;
} }
let settings = Settings::default(); let settings = Settings::default();
log::debug!("Creating paths");
create_paths_for_repo(&opt.repo, &settings).await?; create_paths_for_repo(&opt.repo, &settings).await?;
let db_path = opt.repo.join(&settings.database_path); let db_path = opt.repo.join(&settings.database_path);
if db_path.exists() { if db_path.exists() {
panic!("Database already exists in location. Use --force with init to delete everything and start a new repository"); panic!("Database already exists in location. Use --force with init to delete everything and start a new repository");
} }
log::debug!("Creating repo");
let repo = get_repo(&db_path.to_str().unwrap()).await?; let repo = get_repo(&db_path.to_str().unwrap()).await?;
let storage_path = opt.repo.join(&settings.default_file_store); let storage_path = opt.repo.join(&settings.default_file_store);
log::debug!("Adding storage");
repo.add_storage(DEFAULT_STORAGE_NAME, storage_path.to_str().unwrap()) repo.add_storage(DEFAULT_STORAGE_NAME, storage_path.to_str().unwrap())
.await?; .await?;
let settings_string = settings.to_toml_string()?; let settings_string = settings.to_toml_string()?;
log::debug!("Writing settings");
fs::write(opt.repo.join(SETTINGS_PATH), &settings_string.into_bytes()).await?; fs::write(opt.repo.join(SETTINGS_PATH), &settings_string.into_bytes()).await?;
log::info!("Repository initialized");
Ok(()) Ok(())
} }
fn build_logger() { /// Imports files from a source into the database
env_logger::builder() async fn import(opt: Opt, path: String) -> RepoResult<()> {
.filter_module("sqlx", log::LevelFilter::Warn) let (_s, repo) = init_repo(&opt).await?;
.filter_module("tokio", log::LevelFilter::Info) log::info!("Importing");
.filter_module("tracing", log::LevelFilter::Warn)
.init(); for entry in glob::glob(&path).unwrap() {
if let Ok(path) = entry {
if path.is_file() {
log::debug!("Importing {:?}", path);
if let Err(e) = repo.add_file_by_path(path).await {
log::error!("Failed to import: {:?}", e);
}
}
}
}
Ok(())
} }

Loading…
Cancel
Save