From c3b744ed02a124a8cbe709fb1905e1d3326f8546 Mon Sep 17 00:00:00 2001 From: trivernis Date: Mon, 15 Nov 2021 21:14:23 +0100 Subject: [PATCH] Change implementation to store thumbnails independent from the database Signed-off-by: trivernis --- mediarepo-daemon/Cargo.lock | 5 +- mediarepo-daemon/mediarepo-core/Cargo.lock | 7 + mediarepo-daemon/mediarepo-core/Cargo.toml | 1 + .../src/{ => fs}/file_hash_store.rs | 0 mediarepo-daemon/mediarepo-core/src/fs/mod.rs | 2 + .../mediarepo-core/src/fs/thumbnail_store.rs | 77 ++++++++++ mediarepo-daemon/mediarepo-core/src/lib.rs | 12 +- .../mediarepo-core/src/settings.rs | 2 +- .../mediarepo-database/Cargo.lock | 7 + mediarepo-daemon/mediarepo-model/Cargo.lock | 7 + mediarepo-daemon/mediarepo-model/src/file.rs | 7 - mediarepo-daemon/mediarepo-model/src/hash.rs | 15 -- mediarepo-daemon/mediarepo-model/src/repo.rs | 73 ++++++---- .../mediarepo-model/src/storage.rs | 2 +- .../mediarepo-model/src/thumbnail.rs | 137 ++---------------- mediarepo-daemon/mediarepo-socket/Cargo.lock | 11 +- mediarepo-daemon/mediarepo-socket/Cargo.toml | 2 +- .../mediarepo-socket/src/from_model.rs | 10 +- .../mediarepo-socket/src/namespaces/files.rs | 51 ++----- .../mediarepo-socket/src/utils.rs | 13 ++ mediarepo-daemon/src/main.rs | 6 +- 21 files changed, 207 insertions(+), 240 deletions(-) rename mediarepo-daemon/mediarepo-core/src/{ => fs}/file_hash_store.rs (100%) create mode 100644 mediarepo-daemon/mediarepo-core/src/fs/mod.rs create mode 100644 mediarepo-daemon/mediarepo-core/src/fs/thumbnail_store.rs diff --git a/mediarepo-daemon/Cargo.lock b/mediarepo-daemon/Cargo.lock index 3ffed93..2a5650e 100644 --- a/mediarepo-daemon/Cargo.lock +++ b/mediarepo-daemon/Cargo.lock @@ -855,8 +855,8 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "mediarepo-api" -version = "0.7.0" -source = "git+https://github.com/Trivernis/mediarepo-api.git?rev=3275c328fa6ac362f71fc548301b7c7c70725ca8#3275c328fa6ac362f71fc548301b7c7c70725ca8" +version = "0.10.0" +source = "git+https://github.com/Trivernis/mediarepo-api.git?rev=3e8a415a4cf1c04e3e7ccefbcdd56b650ad40c85#3e8a415a4cf1c04e3e7ccefbcdd56b650ad40c85" dependencies = [ "chrono", "serde", @@ -871,6 +871,7 @@ version = "0.1.0" dependencies = [ "base64", "futures 0.3.17", + "glob", "itertools", "multibase", "multihash", diff --git a/mediarepo-daemon/mediarepo-core/Cargo.lock b/mediarepo-daemon/mediarepo-core/Cargo.lock index b09b88d..503932e 100644 --- a/mediarepo-daemon/mediarepo-core/Cargo.lock +++ b/mediarepo-daemon/mediarepo-core/Cargo.lock @@ -512,6 +512,12 @@ dependencies = [ "weezl", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "hashbrown" version = "0.11.2" @@ -702,6 +708,7 @@ version = "0.1.0" dependencies = [ "base64", "futures", + "glob", "itertools", "multibase", "multihash", diff --git a/mediarepo-daemon/mediarepo-core/Cargo.toml b/mediarepo-daemon/mediarepo-core/Cargo.toml index 9cf96e8..ab1bffc 100644 --- a/mediarepo-daemon/mediarepo-core/Cargo.toml +++ b/mediarepo-daemon/mediarepo-core/Cargo.toml @@ -17,6 +17,7 @@ typemap_rev = "0.1.5" futures = "0.3.17" thumbnailer = "0.1.0" itertools = "0.10.1" +glob = "0.3.0" [dependencies.sea-orm] version = "0.3.2" diff --git a/mediarepo-daemon/mediarepo-core/src/file_hash_store.rs b/mediarepo-daemon/mediarepo-core/src/fs/file_hash_store.rs similarity index 100% rename from mediarepo-daemon/mediarepo-core/src/file_hash_store.rs rename to mediarepo-daemon/mediarepo-core/src/fs/file_hash_store.rs diff --git a/mediarepo-daemon/mediarepo-core/src/fs/mod.rs b/mediarepo-daemon/mediarepo-core/src/fs/mod.rs new file mode 100644 index 0000000..00afe5a --- /dev/null +++ b/mediarepo-daemon/mediarepo-core/src/fs/mod.rs @@ -0,0 +1,2 @@ +pub mod file_hash_store; +pub mod thumbnail_store; diff --git a/mediarepo-daemon/mediarepo-core/src/fs/thumbnail_store.rs b/mediarepo-daemon/mediarepo-core/src/fs/thumbnail_store.rs new file mode 100644 index 0000000..d40f286 --- /dev/null +++ b/mediarepo-daemon/mediarepo-core/src/fs/thumbnail_store.rs @@ -0,0 +1,77 @@ +use std::io::Result; +use std::path::PathBuf; +use tokio::fs; +use tokio::fs::OpenOptions; +use tokio::io::{AsyncWriteExt, BufWriter}; + +#[derive(Clone, Debug)] +pub struct ThumbnailStore { + path: PathBuf, +} + +#[derive(Clone, Debug)] +pub struct Dimensions { + pub height: u32, + pub width: u32, +} + +impl ThumbnailStore { + pub fn new(path: PathBuf) -> Self { + Self { path } + } + + /// Adds a thumbnail to be stored for a parent id + /// if the thumbnail already exists it will be recreated without warning + pub async fn add_thumbnail( + &self, + parent_id: S, + size: Dimensions, + data: &[u8], + ) -> Result { + let parent_dir = self.path.join(parent_id.to_string()); + let entry_path = parent_dir.join(format!("{}-{}", size.height, size.width)); + + if !parent_dir.exists() { + fs::create_dir_all(parent_dir).await?; + } + + let file = OpenOptions::new() + .write(true) + .create(true) + .open(&entry_path) + .await?; + let mut writer = BufWriter::new(file); + writer.write_all(data).await?; + writer.flush().await?; + + Ok(entry_path) + } + + /// Returns all thumbnails for a parent id + pub async fn get_thumbnails( + &self, + parent_id: S, + ) -> Result> { + let mut entries = Vec::new(); + let parent_dir = self.path.join(parent_id.to_string()); + if !parent_dir.exists() { + return Ok(vec![]); + } + let mut dir = fs::read_dir(parent_dir).await?; + + while let Ok(Some(entry)) = dir.next_entry().await { + let file_name = entry.file_name(); + let name = file_name.to_string_lossy(); + + let (height, width) = name + .split_once("-") + .and_then(|(height, width)| { + Some((height.parse::().ok()?, width.parse::().ok()?)) + }) + .unwrap_or((255, 255)); + entries.push((Dimensions { height, width }, entry.path())) + } + + Ok(entries) + } +} diff --git a/mediarepo-daemon/mediarepo-core/src/lib.rs b/mediarepo-daemon/mediarepo-core/src/lib.rs index 3b94e53..31b9697 100644 --- a/mediarepo-daemon/mediarepo-core/src/lib.rs +++ b/mediarepo-daemon/mediarepo-core/src/lib.rs @@ -1,11 +1,11 @@ +pub use futures; +pub use itertools; +pub use rmp_ipc; +pub use thumbnailer; + pub mod context; pub mod error; -pub mod file_hash_store; +pub mod fs; pub mod settings; pub mod type_keys; pub mod utils; - -pub use futures; -pub use itertools; -pub use rmp_ipc; -pub use thumbnailer; diff --git a/mediarepo-daemon/mediarepo-core/src/settings.rs b/mediarepo-daemon/mediarepo-core/src/settings.rs index 423efcc..a6ee0bb 100644 --- a/mediarepo-daemon/mediarepo-core/src/settings.rs +++ b/mediarepo-daemon/mediarepo-core/src/settings.rs @@ -18,7 +18,7 @@ impl Default for Settings { port_range: (3400, 3500), database_path: "./db/repo.db".to_string(), default_file_store: "Main".to_string(), - thumbnail_store: "Thumbnails".to_string(), + thumbnail_store: "./thumbnails".to_string(), } } } diff --git a/mediarepo-daemon/mediarepo-database/Cargo.lock b/mediarepo-daemon/mediarepo-database/Cargo.lock index 0195bfc..7b86779 100644 --- a/mediarepo-daemon/mediarepo-database/Cargo.lock +++ b/mediarepo-daemon/mediarepo-database/Cargo.lock @@ -569,6 +569,12 @@ dependencies = [ "weezl", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "hashbrown" version = "0.11.2" @@ -770,6 +776,7 @@ version = "0.1.0" dependencies = [ "base64", "futures", + "glob", "itertools", "multibase", "multihash", diff --git a/mediarepo-daemon/mediarepo-model/Cargo.lock b/mediarepo-daemon/mediarepo-model/Cargo.lock index 965dbf1..899c871 100644 --- a/mediarepo-daemon/mediarepo-model/Cargo.lock +++ b/mediarepo-daemon/mediarepo-model/Cargo.lock @@ -569,6 +569,12 @@ dependencies = [ "weezl", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "hashbrown" version = "0.11.2" @@ -770,6 +776,7 @@ version = "0.1.0" dependencies = [ "base64", "futures", + "glob", "itertools", "multibase", "multihash", diff --git a/mediarepo-daemon/mediarepo-model/src/file.rs b/mediarepo-daemon/mediarepo-model/src/file.rs index a1cfbab..f87f5f8 100644 --- a/mediarepo-daemon/mediarepo-model/src/file.rs +++ b/mediarepo-daemon/mediarepo-model/src/file.rs @@ -20,7 +20,6 @@ use mediarepo_database::entities::tag; use crate::file_type::FileType; use crate::storage::Storage; use crate::tag::Tag; -use crate::thumbnail::Thumbnail; #[derive(Clone)] pub struct File { @@ -225,12 +224,6 @@ impl File { Ok(storage) } - /// Returns a list of thumbnails for the file - #[tracing::instrument(level = "debug", skip(self))] - pub async fn thumbnails(&self) -> RepoResult> { - Thumbnail::for_file_id(self.db.clone(), self.model.id).await - } - /// Returns the list of tags of the file #[tracing::instrument(level = "debug", skip(self))] pub async fn tags(&self) -> RepoResult> { diff --git a/mediarepo-daemon/mediarepo-model/src/hash.rs b/mediarepo-daemon/mediarepo-model/src/hash.rs index 79ce5d8..0d1930d 100644 --- a/mediarepo-daemon/mediarepo-model/src/hash.rs +++ b/mediarepo-daemon/mediarepo-model/src/hash.rs @@ -1,9 +1,7 @@ use crate::file::File; -use crate::thumbnail::Thumbnail; use mediarepo_core::error::RepoResult; use mediarepo_database::entities::file; use mediarepo_database::entities::hash; -use mediarepo_database::entities::thumbnail; use sea_orm::prelude::*; use sea_orm::{DatabaseConnection, Set}; use std::fmt::Debug; @@ -80,17 +78,4 @@ impl Hash { Ok(file) } - - /// Returns the the thumbnail associated with the hash - #[tracing::instrument(level = "debug", skip(self))] - pub async fn thumbnail(&self) -> RepoResult> { - let thumbnail = self - .model - .find_related(thumbnail::Entity) - .one(&self.db) - .await? - .map(|thumb_model| Thumbnail::new(self.db.clone(), thumb_model, self.model.clone())); - - Ok(thumbnail) - } } diff --git a/mediarepo-daemon/mediarepo-model/src/repo.rs b/mediarepo-daemon/mediarepo-model/src/repo.rs index 71be676..b90f6e2 100644 --- a/mediarepo-daemon/mediarepo-model/src/repo.rs +++ b/mediarepo-daemon/mediarepo-model/src/repo.rs @@ -6,6 +6,7 @@ use crate::tag::Tag; use crate::thumbnail::Thumbnail; use chrono::{Local, NaiveDateTime}; use mediarepo_core::error::{RepoError, RepoResult}; +use mediarepo_core::fs::thumbnail_store::{Dimensions, ThumbnailStore}; use mediarepo_core::itertools::Itertools; use mediarepo_core::thumbnailer::ThumbnailSize; use mediarepo_core::utils::parse_namespace_and_tag; @@ -24,7 +25,7 @@ use tokio::io::BufReader; pub struct Repo { db: DatabaseConnection, main_storage: Option, - thumbnail_storage: Option, + thumbnail_storage: Option, } impl Repo { @@ -65,15 +66,15 @@ impl Repo { /// Sets the main storage #[tracing::instrument(level = "debug", skip(self))] - pub async fn set_main_storage(&mut self, path: S) -> RepoResult<()> { - self.main_storage = Storage::by_name(self.db.clone(), path.to_string()).await?; + pub async fn set_main_storage(&mut self, name: S) -> RepoResult<()> { + self.main_storage = Storage::by_name(self.db.clone(), name.to_string()).await?; Ok(()) } /// Sets the default thumbnail storage #[tracing::instrument(level = "debug", skip(self))] - pub async fn set_thumbnail_storage(&mut self, path: S) -> RepoResult<()> { - self.thumbnail_storage = Storage::by_name(self.db.clone(), path.to_string()).await?; + pub async fn set_thumbnail_storage(&mut self, path: PathBuf) -> RepoResult<()> { + self.thumbnail_storage = Some(ThumbnailStore::new(path)); Ok(()) } @@ -189,29 +190,41 @@ impl Repo { .await } - /// Returns a thumbnail by its hash - #[tracing::instrument(level = "debug", skip(self))] - pub async fn thumbnail_by_hash + Debug>( - &self, - hash: S, - ) -> RepoResult> { - Thumbnail::by_hash(self.db.clone(), hash).await + /// Returns all thumbnails of a file + pub async fn get_file_thumbnails(&self, file_hash: String) -> RepoResult> { + let thumb_store = self.get_thumbnail_storage()?; + let thumbnails = thumb_store + .get_thumbnails(&file_hash) + .await? + .into_iter() + .map(|(size, path)| Thumbnail { + file_hash: file_hash.to_owned(), + path, + size, + mime_type: mime::IMAGE_PNG.to_string(), + }) + .collect_vec(); + + Ok(thumbnails) } /// Creates thumbnails of all sizes for a file #[tracing::instrument(level = "debug", skip(self, file))] - pub async fn create_thumbnails_for_file(&self, file: &File) -> RepoResult<()> { + pub async fn create_thumbnails_for_file(&self, file: &File) -> RepoResult> { let thumb_storage = self.get_thumbnail_storage()?; let size = ThumbnailSize::Medium; let (height, width) = size.dimensions(); let thumbs = file.create_thumbnail([size]).await?; + let mut created_thumbs = Vec::with_capacity(1); for thumb in thumbs { - self.store_single_thumbnail(file, thumb_storage, height, width, thumb) + let entry = self + .store_single_thumbnail(file.hash().to_owned(), thumb_storage, height, width, thumb) .await?; + created_thumbs.push(entry); } - Ok(()) + Ok(created_thumbs) } #[tracing::instrument(level = "debug", skip(self, file))] @@ -228,34 +241,34 @@ impl Repo { .pop() .ok_or_else(|| RepoError::from("Failed to create thumbnail"))?; let thumbnail = self - .store_single_thumbnail(file, thumb_storage, height, width, thumb) + .store_single_thumbnail(file.hash().to_owned(), thumb_storage, height, width, thumb) .await?; Ok(thumbnail) } + /// Stores a single thumbnail async fn store_single_thumbnail( &self, - file: &File, - thumb_storage: &Storage, + file_hash: String, + thumb_storage: &ThumbnailStore, height: u32, width: u32, thumb: mediarepo_core::thumbnailer::Thumbnail, ) -> RepoResult { let mut buf = Vec::new(); thumb.write_png(&mut buf)?; - let hash = thumb_storage.store_entry(Cursor::new(buf)).await?; + let size = Dimensions { height, width }; + let path = thumb_storage + .add_thumbnail(&file_hash, size.clone(), &buf) + .await?; - let thumbnail = Thumbnail::add( - self.db.clone(), - hash.id(), - file.id(), - thumb_storage.id(), - height as i32, - width as i32, - Some(mime::IMAGE_PNG.to_string()), - ) - .await?; + let thumbnail = Thumbnail { + file_hash, + path, + size, + mime_type: mime::IMAGE_PNG.to_string(), + }; Ok(thumbnail) } @@ -385,7 +398,7 @@ impl Repo { } #[tracing::instrument(level = "trace", skip(self))] - fn get_thumbnail_storage(&self) -> RepoResult<&Storage> { + fn get_thumbnail_storage(&self) -> RepoResult<&ThumbnailStore> { if let Some(storage) = &self.thumbnail_storage { Ok(storage) } else { diff --git a/mediarepo-daemon/mediarepo-model/src/storage.rs b/mediarepo-daemon/mediarepo-model/src/storage.rs index 8ce5f41..d6a9a8c 100644 --- a/mediarepo-daemon/mediarepo-model/src/storage.rs +++ b/mediarepo-daemon/mediarepo-model/src/storage.rs @@ -1,6 +1,6 @@ use crate::hash::Hash; use mediarepo_core::error::RepoResult; -use mediarepo_core::file_hash_store::FileHashStore; +use mediarepo_core::fs::file_hash_store::FileHashStore; use mediarepo_database::entities::storage; use mediarepo_database::entities::storage::ActiveModel as ActiveStorage; use mediarepo_database::entities::storage::Model as StorageModel; diff --git a/mediarepo-daemon/mediarepo-model/src/thumbnail.rs b/mediarepo-daemon/mediarepo-model/src/thumbnail.rs index 746be5c..6b2982b 100644 --- a/mediarepo-daemon/mediarepo-model/src/thumbnail.rs +++ b/mediarepo-daemon/mediarepo-model/src/thumbnail.rs @@ -1,140 +1,21 @@ -use crate::storage::Storage; use mediarepo_core::error::RepoResult; -use mediarepo_database::entities::hash; -use mediarepo_database::entities::thumbnail; -use sea_orm::prelude::*; -use sea_orm::{DatabaseConnection, Set}; -use std::fmt::Debug; -use tokio::fs::File; +use mediarepo_core::fs::thumbnail_store::Dimensions; +use std::path::PathBuf; +use tokio::fs::{File, OpenOptions}; use tokio::io::BufReader; pub struct Thumbnail { - db: DatabaseConnection, - model: thumbnail::Model, - hash: hash::Model, + pub file_hash: String, + pub path: PathBuf, + pub size: Dimensions, + pub mime_type: String, } impl Thumbnail { - #[tracing::instrument(level = "trace")] - pub(crate) fn new(db: DatabaseConnection, model: thumbnail::Model, hash: hash::Model) -> Self { - Self { db, model, hash } - } - - /// Returns the thumbnail by id - #[tracing::instrument(level = "debug", skip(db))] - pub async fn by_id(db: DatabaseConnection, id: i64) -> RepoResult> { - let model: Option<(thumbnail::Model, Option)> = - thumbnail::Entity::find_by_id(id) - .find_also_related(hash::Entity) - .one(&db) - .await?; - - if let Some((model, Some(hash))) = model { - Ok(Some(Self::new(db, model, hash))) - } else { - Ok(None) - } - } - - /// Returns a thumbnail by hash - #[tracing::instrument(level = "debug", skip(db))] - pub async fn by_hash + Debug>( - db: DatabaseConnection, - hash: S, - ) -> RepoResult> { - let result: Option<(hash::Model, Option)> = hash::Entity::find() - .filter(hash::Column::Value.eq(hash.as_ref())) - .find_also_related(thumbnail::Entity) - .one(&db) - .await?; - if let Some((hash, Some(model))) = result { - Ok(Some(Self::new(db, model, hash))) - } else { - Ok(None) - } - } - - /// Inserts a thumbnail into the database - #[tracing::instrument(level = "debug", skip(db))] - pub async fn add( - db: DatabaseConnection, - hash_id: i64, - file_id: i64, - storage_id: i64, - height: i32, - width: i32, - mime: Option, - ) -> RepoResult { - let active_model = thumbnail::ActiveModel { - storage_id: Set(storage_id), - hash_id: Set(hash_id), - file_id: Set(file_id), - height: Set(height), - width: Set(width), - mime: Set(mime), - ..Default::default() - }; - let active_model: thumbnail::ActiveModel = active_model.insert(&db).await?; - let thumbnail = Self::by_id(db, active_model.id.unwrap()) - .await? - .expect("Inserted thumbnail does not exist"); - - Ok(thumbnail) - } - - /// Returns all thumbnails for a given file - #[tracing::instrument(level = "debug", skip(db))] - pub async fn for_file_id(db: DatabaseConnection, file_id: i64) -> RepoResult> { - let thumb_models: Vec<(thumbnail::Model, Option)> = thumbnail::Entity::find() - .filter(thumbnail::Column::FileId.eq(file_id)) - .find_also_related(hash::Entity) - .all(&db) - .await?; - - Ok(thumb_models - .into_iter() - .filter_map(|(m, h)| Some(Self::new(db.clone(), m, h?))) - .collect()) - } - - pub fn id(&self) -> i64 { - self.model.id - } - - pub fn file_id(&self) -> i64 { - self.model.file_id - } - - pub fn hash(&self) -> &String { - &self.hash.value - } - - pub fn height(&self) -> i32 { - self.model.height - } - - pub fn width(&self) -> i32 { - self.model.width - } - - pub fn mime_type(&self) -> &Option { - &self.model.mime - } - - /// Returns the storage for the thumbnail - #[tracing::instrument(level = "debug", skip(self))] - pub async fn storage(&self) -> RepoResult { - let storage = Storage::by_id(self.db.clone(), self.model.storage_id) - .await? - .expect("The FK storage_id doesn't exist?!"); - - Ok(storage) - } - /// Returns the reader of the thumbnail file #[tracing::instrument(level = "debug", skip(self))] pub async fn get_reader(&self) -> RepoResult> { - let storage = self.storage().await?; - storage.get_file_reader(self.hash()).await + let file = OpenOptions::new().read(true).open(&self.path).await?; + Ok(BufReader::new(file)) } } diff --git a/mediarepo-daemon/mediarepo-socket/Cargo.lock b/mediarepo-daemon/mediarepo-socket/Cargo.lock index bc5dcde..64ec03c 100644 --- a/mediarepo-daemon/mediarepo-socket/Cargo.lock +++ b/mediarepo-daemon/mediarepo-socket/Cargo.lock @@ -599,6 +599,12 @@ dependencies = [ "weezl", ] +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "hashbrown" version = "0.11.2" @@ -796,8 +802,8 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "mediarepo-api" -version = "0.7.0" -source = "git+https://github.com/Trivernis/mediarepo-api.git?rev=3275c328fa6ac362f71fc548301b7c7c70725ca8#3275c328fa6ac362f71fc548301b7c7c70725ca8" +version = "0.10.0" +source = "git+https://github.com/Trivernis/mediarepo-api.git?rev=3e8a415a4cf1c04e3e7ccefbcdd56b650ad40c85#3e8a415a4cf1c04e3e7ccefbcdd56b650ad40c85" dependencies = [ "chrono", "serde", @@ -812,6 +818,7 @@ version = "0.1.0" dependencies = [ "base64", "futures 0.3.17", + "glob", "itertools", "multibase", "multihash", diff --git a/mediarepo-daemon/mediarepo-socket/Cargo.toml b/mediarepo-daemon/mediarepo-socket/Cargo.toml index beca3fd..a8b0d51 100644 --- a/mediarepo-daemon/mediarepo-socket/Cargo.toml +++ b/mediarepo-daemon/mediarepo-socket/Cargo.toml @@ -34,4 +34,4 @@ features = ["tokio-executor"] [dependencies.mediarepo-api] git = "https://github.com/Trivernis/mediarepo-api.git" -rev = "3275c328fa6ac362f71fc548301b7c7c70725ca8" \ No newline at end of file +rev = "3e8a415a4cf1c04e3e7ccefbcdd56b650ad40c85" \ No newline at end of file diff --git a/mediarepo-daemon/mediarepo-socket/src/from_model.rs b/mediarepo-daemon/mediarepo-socket/src/from_model.rs index b1e28db..fab629d 100644 --- a/mediarepo-daemon/mediarepo-socket/src/from_model.rs +++ b/mediarepo-daemon/mediarepo-socket/src/from_model.rs @@ -37,12 +37,10 @@ impl FromModel for TagResponse { impl FromModel for ThumbnailMetadataResponse { fn from_model(model: Thumbnail) -> Self { Self { - id: model.id(), - file_id: model.file_id(), - hash: model.hash().to_owned(), - height: model.height(), - width: model.width(), - mime_type: model.mime_type().to_owned(), + file_hash: model.file_hash, + height: model.size.height, + width: model.size.width, + mime_type: model.mime_type.to_owned(), } } } diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs index 773a7de..e286c55 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs @@ -1,5 +1,5 @@ use crate::from_model::FromModel; -use crate::utils::{file_by_identifier, get_repo_from_context}; +use crate::utils::{file_by_identifier, get_repo_from_context, hash_by_identifier}; use compare::Compare; use mediarepo_api::types::files::{ AddFileRequestHeader, FileMetadataResponse, FindFilesByTagsRequest, @@ -7,7 +7,7 @@ use mediarepo_api::types::files::{ SortKey, ThumbnailMetadataResponse, UpdateFileNameRequest, }; use mediarepo_api::types::identifier::FileIdentifier; -use mediarepo_core::error::RepoError; +use mediarepo_core::fs::thumbnail_store::Dimensions; use mediarepo_core::itertools::Itertools; use mediarepo_core::rmp_ipc::prelude::*; use mediarepo_core::thumbnailer::ThumbnailSize; @@ -33,7 +33,6 @@ impl NamespaceProvider for FilesNamespace { "add_file" => Self::add_file, "read_file" => Self::read_file, "get_thumbnails" => Self::thumbnails, - "read_thumbnail" => Self::read_thumbnail, "get_thumbnail_of_size" => Self::get_thumbnail_of_size, "update_file_name" => Self::update_file_name ); @@ -169,15 +168,15 @@ impl FilesNamespace { async fn thumbnails(ctx: &Context, event: Event) -> IPCResult<()> { let request = event.data::()?; let repo = get_repo_from_context(ctx).await; - let file = file_by_identifier(request.id, &repo).await?; - let mut thumbnails = file.thumbnails().await?; + let file_hash = hash_by_identifier(request.id.clone(), &repo).await?; + let mut thumbnails = repo.get_file_thumbnails(file_hash).await?; - if thumbnails.len() == 0 { + if thumbnails.is_empty() { tracing::debug!("No thumbnails for file found. Creating thumbnails..."); - repo.create_thumbnails_for_file(&file).await?; + let file = file_by_identifier(request.id, &repo).await?; + thumbnails = repo.create_thumbnails_for_file(&file).await?; tracing::debug!("Thumbnails for file created."); } - thumbnails = file.thumbnails().await?; let thumb_responses: Vec = thumbnails .into_iter() @@ -190,33 +189,6 @@ impl FilesNamespace { Ok(()) } - /// Reads a thumbnail for the given thumbnail hash - #[tracing::instrument(skip_all)] - async fn read_thumbnail( - ctx: &Context, - event: Event, - ) -> IPCResult<()> { - let hash = event.data::()?; - let repo = get_repo_from_context(ctx).await; - let thumbnail = repo - .thumbnail_by_hash(&hash) - .await? - .ok_or_else(|| RepoError::from("Thumbnail not found"))?; - let mut reader = thumbnail.get_reader().await?; - let mut buf = Vec::new(); - reader.read_to_end(&mut buf).await?; - ctx.emitter - .emit_response_to( - event.id(), - Self::name(), - "read_thumbnail", - BytePayload::new(buf), - ) - .await?; - - Ok(()) - } - /// Returns a thumbnail that is within the range of the requested sizes #[tracing::instrument(skip_all)] async fn get_thumbnail_of_size( @@ -225,14 +197,14 @@ impl FilesNamespace { ) -> IPCResult<()> { let request = event.data::()?; let repo = get_repo_from_context(ctx).await; - let file = file_by_identifier(request.id, &repo).await?; - let thumbnails = file.thumbnails().await?; + let file_hash = hash_by_identifier(request.id.clone(), &repo).await?; + let thumbnails = repo.get_file_thumbnails(file_hash).await?; let min_size = request.min_size; let max_size = request.max_size; let found_thumbnail = thumbnails.into_iter().find(|thumb| { - let height = thumb.height() as u32; - let width = thumb.width() as u32; + let Dimensions { height, width } = thumb.size; + height >= min_size.0 && height <= max_size.0 && width >= min_size.1 @@ -242,6 +214,7 @@ impl FilesNamespace { let thumbnail = if let Some(thumbnail) = found_thumbnail { thumbnail } else { + let file = file_by_identifier(request.id, &repo).await?; let middle_size = ((max_size.0 + min_size.0) / 2, (max_size.1 + min_size.1) / 2); let thumbnail = repo .create_file_thumbnail(&file, ThumbnailSize::Custom(middle_size)) diff --git a/mediarepo-daemon/mediarepo-socket/src/utils.rs b/mediarepo-daemon/mediarepo-socket/src/utils.rs index 6b2d759..23d051c 100644 --- a/mediarepo-daemon/mediarepo-socket/src/utils.rs +++ b/mediarepo-daemon/mediarepo-socket/src/utils.rs @@ -20,3 +20,16 @@ pub async fn file_by_identifier(identifier: FileIdentifier, repo: &Repo) -> Repo }?; file.ok_or_else(|| RepoError::from("Thumbnail not found")) } + +pub async fn hash_by_identifier(identifier: FileIdentifier, repo: &Repo) -> RepoResult { + match identifier { + FileIdentifier::ID(id) => { + let file = repo + .file_by_id(id) + .await? + .ok_or_else(|| "Thumbnail not found")?; + Ok(file.hash().to_owned()) + } + FileIdentifier::Hash(hash) => Ok(hash), + } +} diff --git a/mediarepo-daemon/src/main.rs b/mediarepo-daemon/src/main.rs index c6ac689..c39bffd 100644 --- a/mediarepo-daemon/src/main.rs +++ b/mediarepo-daemon/src/main.rs @@ -113,7 +113,7 @@ async fn init_repo(opt: &Opt) -> RepoResult<(Settings, Repo)> { let mut repo = get_repo(&opt.repo.join(&settings.database_path).to_str().unwrap()).await?; repo.set_main_storage(&settings.default_file_store).await?; - repo.set_thumbnail_storage(&settings.thumbnail_store) + repo.set_thumbnail_storage(opt.repo.join(&settings.thumbnail_store)) .await?; Ok((settings, repo)) } @@ -285,7 +285,9 @@ async fn add_tags_from_tags_file( #[tracing::instrument(skip(repo, file))] async fn create_file_thumbnails(repo: &Repo, file: File) -> RepoResult<()> { - if file.thumbnails().await?.len() == 0 { + let file_thumbnails = repo.get_file_thumbnails(file.hash().to_owned()).await?; + + if file_thumbnails.is_empty() { repo.create_thumbnails_for_file(&file).await?; } Ok(())