Add read_file event and parsing of file types from mimes

Signed-off-by: trivernis <trivernis@protonmail.com>
pull/4/head
trivernis 3 years ago
parent 0d26370caa
commit d62f5b38bf

@ -541,6 +541,12 @@ dependencies = [
"wasi",
]
[[package]]
name = "glob"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
[[package]]
name = "hashbrown"
version = "0.11.2"
@ -701,6 +707,7 @@ name = "mediarepo"
version = "0.1.0"
dependencies = [
"env_logger",
"glob",
"log",
"mediarepo-core",
"mediarepo-model",
@ -744,6 +751,8 @@ dependencies = [
"chrono",
"mediarepo-core",
"mediarepo-database",
"mime",
"mime_guess",
"sea-orm",
"serde",
"tokio",
@ -768,6 +777,22 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "mime"
version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
[[package]]
name = "mime_guess"
version = "2.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "minimal-lexical"
version = "0.1.4"
@ -1747,6 +1772,15 @@ version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec"
[[package]]
name = "unicase"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
dependencies = [
"version_check",
]
[[package]]
name = "unicode-bidi"
version = "0.3.6"

@ -17,7 +17,8 @@ crate-type = ["lib"]
[dependencies]
toml = {version = "0.5.8", optional=true}
structopt = {version="0.3.23", optional=true}
env_logger = "0.9.0"
env_logger = {version="0.9.0", optional=true}
glob = {version="0.3.0", optional=true}
log = "0.4.14"
[dependencies.mediarepo-core]
@ -37,5 +38,5 @@ features = ["macros", "rt-multi-thread", "io-std", "io-util"]
[features]
default = ["runtime"]
runtime = ["toml", "structopt", "mediarepo-model", "mediarepo-socket"]
runtime = ["toml", "structopt", "mediarepo-model", "mediarepo-socket", "env_logger", "glob"]
library = ["mediarepo-socket"]

@ -666,6 +666,8 @@ dependencies = [
"chrono",
"mediarepo-core",
"mediarepo-database",
"mime",
"mime_guess",
"sea-orm",
"serde",
"tokio",
@ -678,6 +680,22 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "mime"
version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
[[package]]
name = "mime_guess"
version = "2.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "minimal-lexical"
version = "0.1.3"
@ -1580,6 +1598,15 @@ version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec"
[[package]]
name = "unicase"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
dependencies = [
"version_check",
]
[[package]]
name = "unicode-bidi"
version = "0.3.6"

@ -9,6 +9,8 @@ edition = "2018"
chrono = "0.4.19"
typemap_rev = "0.1.5"
serde = "1.0.130"
mime_guess = "2.0.3"
mime = "0.3.16"
[dependencies.mediarepo-core]
path = "../mediarepo-core"

@ -1,6 +1,6 @@
use crate::file_type::FileType;
use crate::storage::Storage;
use chrono::NaiveDateTime;
use chrono::{Local, NaiveDateTime};
use mediarepo_core::error::RepoResult;
use mediarepo_database::entities::file;
use mediarepo_database::entities::file::ActiveModel as ActiveFile;
@ -71,6 +71,37 @@ impl File {
}
}
/// Adds a file with its hash to the database
pub(crate) async fn add<S: ToString>(
db: DatabaseConnection,
storage_id: i64,
hash: S,
file_type: FileType,
) -> RepoResult<Self> {
let hash = hash::ActiveModel {
value: Set(hash.to_string()),
..Default::default()
};
let now = Local::now().naive_local();
let hash: hash::ActiveModel = hash.insert(&db).await?;
let id: i64 = hash.id.unwrap();
let file = file::ActiveModel {
hash_id: Set(id),
file_type: Set(file_type as u32),
storage_id: Set(storage_id),
import_time: Set(now.clone()),
creation_time: Set(now.clone()),
change_time: Set(now),
..Default::default()
};
let file: file::ActiveModel = file.insert(&db).await?.into();
let file = Self::by_id(db, file.id.unwrap())
.await?
.expect("Inserted file does not exist");
Ok(file)
}
/// Returns the unique identifier of the file
pub fn id(&self) -> i64 {
self.model.id

@ -1,9 +1,27 @@
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
#[derive(Clone, Debug, Serialize, Deserialize, PartialOrd, PartialEq)]
pub enum FileType {
Other = -1,
Unknown = 0,
Image = 1,
Video = 2,
Audio = 3,
}
impl From<&PathBuf> for FileType {
fn from(path: &PathBuf) -> Self {
let mime = mime_guess::from_path(path).first();
if let Some(mime) = mime {
match mime.type_() {
mime::IMAGE => Self::Image,
mime::VIDEO => Self::Video,
mime::AUDIO => Self::Audio,
_ => Self::Other,
}
} else {
Self::Unknown
}
}
}

@ -1,4 +1,5 @@
use crate::file::File;
use crate::file_type::FileType;
use crate::storage::Storage;
use mediarepo_core::error::{RepoError, RepoResult};
use mediarepo_database::get_database;
@ -56,6 +57,11 @@ impl Repo {
File::by_hash(self.db.clone(), hash).await
}
/// Returns a file by id
pub async fn file_by_id(&self, id: i64) -> RepoResult<Option<File>> {
File::by_id(self.db.clone(), id).await
}
/// Returns a list of all stored files
pub async fn files(&self) -> RepoResult<Vec<File>> {
File::all(self.db.clone()).await
@ -63,17 +69,12 @@ impl Repo {
/// Adds a file to the database by its readable path in the file system
pub async fn add_file_by_path(&self, path: PathBuf) -> RepoResult<File> {
let file_type = FileType::from(&path);
let os_file = OpenOptions::new().read(true).open(&path).await?;
let reader = BufReader::new(os_file);
let storage = self.get_main_storage()?;
let hash = storage.add_file(reader).await?;
let file = self
.file_by_hash(hash)
.await?
.expect("Invalid database state.");
Ok(file)
storage.add_file(reader, file_type).await
}
fn get_main_storage(&self) -> RepoResult<&Storage> {

@ -1,3 +1,5 @@
use crate::file::File;
use crate::file_type::FileType;
use mediarepo_core::error::RepoResult;
use mediarepo_core::file_hash_store::FileHashStore;
use mediarepo_database::entities::storage;
@ -132,8 +134,13 @@ impl Storage {
}
/// Adds a file to the store
pub async fn add_file<R: AsyncRead + Unpin>(&self, reader: R) -> RepoResult<String> {
self.store.add_file(reader, None).await
pub async fn add_file<R: AsyncRead + Unpin>(
&self,
reader: R,
file_type: FileType,
) -> RepoResult<File> {
let hash = self.store.add_file(reader, None).await?;
File::add(self.db.clone(), self.id(), hash, file_type).await
}
/// Returns the buf reader to the given hash

@ -667,6 +667,8 @@ dependencies = [
"chrono",
"mediarepo-core",
"mediarepo-database",
"mime",
"mime_guess",
"sea-orm",
"serde",
"tokio",
@ -691,6 +693,22 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "mime"
version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
[[package]]
name = "mime_guess"
version = "2.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212"
dependencies = [
"mime",
"unicase",
]
[[package]]
name = "minimal-lexical"
version = "0.1.4"
@ -1593,6 +1611,15 @@ version = "1.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec"
[[package]]
name = "unicase"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
dependencies = [
"version_check",
]
[[package]]
name = "unicode-bidi"
version = "0.3.7"

@ -1,10 +1,12 @@
use crate::types::requests::AddFileRequest;
use crate::types::requests::{AddFileRequest, ReadFileRequest};
use crate::types::responses::FileResponse;
use mediarepo_core::error::RepoError;
use mediarepo_model::type_keys::RepoKey;
use rmp_ipc::context::Context;
use rmp_ipc::error::Result;
use rmp_ipc::{Event, NamespaceBuilder};
use std::path::PathBuf;
use tokio::io::AsyncReadExt;
pub const FILES_NAMESPACE: &str = "files";
@ -12,6 +14,7 @@ pub fn build(builder: NamespaceBuilder) -> NamespaceBuilder {
builder
.on("all_files", |c, e| Box::pin(all_files(c, e)))
.on("add_file", |c, e| Box::pin(add_file(c, e)))
.on("read_file", |c, e| Box::pin(read_file(c, e)))
}
/// Returns a list of all files
@ -49,3 +52,25 @@ async fn add_file(ctx: &Context, event: Event) -> Result<()> {
Ok(())
}
/// Reads the binary contents of a file
async fn read_file(ctx: &Context, event: Event) -> Result<()> {
let request = event.data::<ReadFileRequest>()?;
let mut reader = {
let data = ctx.data.read().await;
let repo = data.get::<RepoKey>().unwrap();
let file = match request {
ReadFileRequest::ID(id) => repo.file_by_id(id).await,
ReadFileRequest::Hash(hash) => repo.file_by_hash(hash).await,
}?
.ok_or_else(|| RepoError::from("File not found"));
file?.get_reader().await?
};
let mut buf = Vec::new();
reader.read_to_end(&mut buf).await?;
ctx.emitter
.emit_response_to(event.id(), FILES_NAMESPACE, "read_file", buf)
.await?;
Ok(())
}

@ -4,3 +4,9 @@ use serde::{Deserialize, Serialize};
pub struct AddFileRequest {
pub path: String,
}
#[derive(Serialize, Deserialize)]
pub enum ReadFileRequest {
ID(i64),
Hash(String),
}

@ -6,6 +6,7 @@ use crate::utils::{create_paths_for_repo, get_repo, load_settings};
use mediarepo_core::error::RepoResult;
use mediarepo_core::settings::Settings;
use mediarepo_core::type_keys::SettingsKey;
use mediarepo_model::repo::Repo;
use mediarepo_model::type_keys::RepoKey;
use mediarepo_socket::get_builder;
use std::path::PathBuf;
@ -24,7 +25,7 @@ struct Opt {
cmd: SubCommand,
}
#[derive(Debug, StructOpt)]
#[derive(Clone, Debug, StructOpt)]
enum SubCommand {
/// Initializes an empty repository
Init {
@ -34,6 +35,13 @@ enum SubCommand {
force: bool,
},
/// Imports file from a folder (by glob pattern) into the repository
Import {
/// The path to the folder where the files are located
#[structopt()]
folder_path: String,
},
/// Starts the event server for the selected repository
Start,
}
@ -42,19 +50,35 @@ enum SubCommand {
async fn main() -> RepoResult<()> {
build_logger();
let opt: Opt = Opt::from_args();
match opt.cmd {
match opt.cmd.clone() {
SubCommand::Init { force } => init(opt, force).await,
SubCommand::Start => start_server(opt).await,
SubCommand::Import { folder_path } => import(opt, folder_path).await,
}?;
Ok(())
}
/// Starts the server
async fn start_server(opt: Opt) -> RepoResult<()> {
fn build_logger() {
env_logger::builder()
.filter_module("sqlx", log::LevelFilter::Warn)
.filter_module("tokio", log::LevelFilter::Info)
.filter_module("tracing", log::LevelFilter::Warn)
.init();
}
async fn init_repo(opt: &Opt) -> RepoResult<(Settings, Repo)> {
let settings = load_settings(&opt.repo.join(SETTINGS_PATH)).await?;
let mut repo = get_repo(&opt.repo.join(&settings.database_path).to_str().unwrap()).await?;
repo.set_main_storage(&settings.default_file_store).await?;
let main_storage_path = opt.repo.join(&settings.default_file_store);
repo.set_main_storage(main_storage_path.to_str().unwrap())
.await?;
Ok((settings, repo))
}
/// Starts the server
async fn start_server(opt: Opt) -> RepoResult<()> {
let (settings, repo) = init_repo(&opt).await?;
get_builder(&settings.listen_address)
.insert::<SettingsKey>(settings)
@ -67,29 +91,47 @@ async fn start_server(opt: Opt) -> RepoResult<()> {
/// Initializes an empty repository
async fn init(opt: Opt, force: bool) -> RepoResult<()> {
log::info!("Initializing repository at {:?}", opt.repo);
if force {
log::debug!("Removing old repository");
fs::remove_dir_all(&opt.repo).await?;
}
let settings = Settings::default();
log::debug!("Creating paths");
create_paths_for_repo(&opt.repo, &settings).await?;
let db_path = opt.repo.join(&settings.database_path);
if db_path.exists() {
panic!("Database already exists in location. Use --force with init to delete everything and start a new repository");
}
log::debug!("Creating repo");
let repo = get_repo(&db_path.to_str().unwrap()).await?;
let storage_path = opt.repo.join(&settings.default_file_store);
log::debug!("Adding storage");
repo.add_storage(DEFAULT_STORAGE_NAME, storage_path.to_str().unwrap())
.await?;
let settings_string = settings.to_toml_string()?;
log::debug!("Writing settings");
fs::write(opt.repo.join(SETTINGS_PATH), &settings_string.into_bytes()).await?;
log::info!("Repository initialized");
Ok(())
}
fn build_logger() {
env_logger::builder()
.filter_module("sqlx", log::LevelFilter::Warn)
.filter_module("tokio", log::LevelFilter::Info)
.filter_module("tracing", log::LevelFilter::Warn)
.init();
/// Imports files from a source into the database
async fn import(opt: Opt, path: String) -> RepoResult<()> {
let (_s, repo) = init_repo(&opt).await?;
log::info!("Importing");
for entry in glob::glob(&path).unwrap() {
if let Ok(path) = entry {
if path.is_file() {
log::debug!("Importing {:?}", path);
if let Err(e) = repo.add_file_by_path(path).await {
log::error!("Failed to import: {:?}", e);
}
}
}
}
Ok(())
}

Loading…
Cancel
Save