From ea14222e44bd780781b1375f1df66330adf85f97 Mon Sep 17 00:00:00 2001 From: trivernis Date: Wed, 19 Jan 2022 19:55:33 +0100 Subject: [PATCH 01/12] Add duplicate check on import Signed-off-by: trivernis --- mediarepo-daemon/Cargo.lock | 2 +- .../mediarepo-model/src/file/mod.rs | 17 ++++++------- .../src/namespaces/files/mod.rs | 24 ++++++++++++------- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/mediarepo-daemon/Cargo.lock b/mediarepo-daemon/Cargo.lock index fda4ce2..3112994 100644 --- a/mediarepo-daemon/Cargo.lock +++ b/mediarepo-daemon/Cargo.lock @@ -1219,7 +1219,7 @@ dependencies = [ [[package]] name = "mediarepo-daemon" -version = "0.13.0" +version = "0.13.1" dependencies = [ "console-subscriber", "glob", diff --git a/mediarepo-daemon/mediarepo-model/src/file/mod.rs b/mediarepo-daemon/mediarepo-model/src/file/mod.rs index dc7dfa8..0a38636 100644 --- a/mediarepo-daemon/mediarepo-model/src/file/mod.rs +++ b/mediarepo-daemon/mediarepo-model/src/file/mod.rs @@ -217,11 +217,7 @@ impl File { JoinType::LeftJoin, content_descriptor_tag::Relation::Tag.def().rev(), ) - .join( - JoinType::InnerJoin, - content_descriptor_tag::Relation::ContentDescriptorId.def(), - ) - .filter(content_descriptor::Column::Id.eq(self.content_descriptor.id)) + .filter(content_descriptor_tag::Column::CdId.eq(self.content_descriptor.id)) .all(&self.db) .await?; let tags = tags @@ -251,16 +247,21 @@ impl File { return Ok(()); } let cd_id = self.content_descriptor.id; + let own_tag_ids = self.tags().await?.into_iter().map(|t| t.id()).collect::>(); + let models: Vec = tag_ids .into_iter() + .filter(|tag_id|!own_tag_ids.contains(tag_id)) .map(|tag_id| content_descriptor_tag::ActiveModel { cd_id: Set(cd_id), tag_id: Set(tag_id), }) .collect(); - content_descriptor_tag::Entity::insert_many(models) - .exec(&self.db) - .await?; + if models.len() > 0 { + content_descriptor_tag::Entity::insert_many(models) + .exec(&self.db) + .await?; + } Ok(()) } diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs index f0d07a6..8d47a1e 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs @@ -18,6 +18,7 @@ use mediarepo_core::mediarepo_api::types::identifier::FileIdentifier; use mediarepo_core::thumbnailer::ThumbnailSize; use mediarepo_core::utils::parse_namespace_and_tag; use tokio::io::AsyncReadExt; +use mediarepo_core::content_descriptor::create_content_descriptor; pub struct FilesNamespace; @@ -135,15 +136,22 @@ impl FilesNamespace { .into_inner(); let AddFileRequestHeader { metadata, tags } = request; let repo = get_repo_from_context(ctx).await; + let bytes = bytes.into_inner(); + let cd = create_content_descriptor(&bytes); - let file = repo - .add_file( - metadata.mime_type, - bytes.into_inner(), - metadata.creation_time, - metadata.change_time, - ) - .await?; + let file = if let Some(file) = repo.file_by_cd(&cd).await? { + tracing::debug!("Inserted file already exists"); + file + } else { + repo + .add_file( + metadata.mime_type, + bytes, + metadata.creation_time, + metadata.change_time, + ) + .await? + }; file.metadata().await?.set_name(metadata.name).await?; let tags = repo From 1dedd8b327cd59b1743f79ad640ff78d462cd87d Mon Sep 17 00:00:00 2001 From: trivernis Date: Wed, 19 Jan 2022 20:00:41 +0100 Subject: [PATCH 02/12] Rename mediarepo-model to mediarepo-logic crate Signed-off-by: trivernis --- mediarepo-daemon/Cargo.lock | 6 +++--- mediarepo-daemon/Cargo.toml | 10 +++++----- .../{mediarepo-model => mediarepo-logic}/.gitignore | 0 .../{mediarepo-model => mediarepo-logic}/Cargo.toml | 2 +- .../src/content_descriptor.rs | 0 .../src/file/filter.rs | 0 .../src/file/mod.rs | 0 .../src/file_metadata.rs | 0 .../{mediarepo-model => mediarepo-logic}/src/lib.rs | 0 .../src/namespace.rs | 0 .../{mediarepo-model => mediarepo-logic}/src/repo.rs | 0 .../{mediarepo-model => mediarepo-logic}/src/tag.rs | 0 .../src/thumbnail.rs | 0 .../src/type_keys.rs | 0 mediarepo-daemon/mediarepo-socket/Cargo.toml | 4 ++-- mediarepo-daemon/mediarepo-socket/src/from_model.rs | 10 +++++----- mediarepo-daemon/mediarepo-socket/src/lib.rs | 4 ++-- .../mediarepo-socket/src/namespaces/files/searching.rs | 8 ++++---- .../mediarepo-socket/src/namespaces/files/sorting.rs | 6 +++--- mediarepo-daemon/mediarepo-socket/src/utils.rs | 6 +++--- mediarepo-daemon/src/main.rs | 2 +- mediarepo-daemon/src/utils.rs | 2 +- 22 files changed, 30 insertions(+), 30 deletions(-) rename mediarepo-daemon/{mediarepo-model => mediarepo-logic}/.gitignore (100%) rename mediarepo-daemon/{mediarepo-model => mediarepo-logic}/Cargo.toml (91%) rename mediarepo-daemon/{mediarepo-model => mediarepo-logic}/src/content_descriptor.rs (100%) rename mediarepo-daemon/{mediarepo-model => mediarepo-logic}/src/file/filter.rs (100%) rename mediarepo-daemon/{mediarepo-model => mediarepo-logic}/src/file/mod.rs (100%) rename mediarepo-daemon/{mediarepo-model => mediarepo-logic}/src/file_metadata.rs (100%) rename mediarepo-daemon/{mediarepo-model => mediarepo-logic}/src/lib.rs (100%) rename mediarepo-daemon/{mediarepo-model => mediarepo-logic}/src/namespace.rs (100%) rename mediarepo-daemon/{mediarepo-model => mediarepo-logic}/src/repo.rs (100%) rename mediarepo-daemon/{mediarepo-model => mediarepo-logic}/src/tag.rs (100%) rename mediarepo-daemon/{mediarepo-model => mediarepo-logic}/src/thumbnail.rs (100%) rename mediarepo-daemon/{mediarepo-model => mediarepo-logic}/src/type_keys.rs (100%) diff --git a/mediarepo-daemon/Cargo.lock b/mediarepo-daemon/Cargo.lock index 3112994..3643f40 100644 --- a/mediarepo-daemon/Cargo.lock +++ b/mediarepo-daemon/Cargo.lock @@ -1225,7 +1225,7 @@ dependencies = [ "glob", "log", "mediarepo-core", - "mediarepo-model", + "mediarepo-logic", "mediarepo-socket", "num-integer", "rolling-file", @@ -1251,7 +1251,7 @@ dependencies = [ ] [[package]] -name = "mediarepo-model" +name = "mediarepo-logic" version = "0.1.0" dependencies = [ "async-trait", @@ -1275,7 +1275,7 @@ dependencies = [ "compare", "mediarepo-core", "mediarepo-database", - "mediarepo-model", + "mediarepo-logic", "port_check", "rayon", "serde 1.0.132", diff --git a/mediarepo-daemon/Cargo.toml b/mediarepo-daemon/Cargo.toml index acca829..8a95499 100644 --- a/mediarepo-daemon/Cargo.toml +++ b/mediarepo-daemon/Cargo.toml @@ -1,6 +1,6 @@ [workspace] -members = ["mediarepo-core", "mediarepo-database", "mediarepo-model", "mediarepo-socket", "."] -default-members = ["mediarepo-core", "mediarepo-database", "mediarepo-model", "mediarepo-socket", "."] +members = ["mediarepo-core", "mediarepo-database", "mediarepo-logic", "mediarepo-socket", "."] +default-members = ["mediarepo-core", "mediarepo-database", "mediarepo-logic", "mediarepo-socket", "."] [package] name = "mediarepo-daemon" @@ -31,8 +31,8 @@ log = "^0.4.14" [dependencies.mediarepo-core] path = "./mediarepo-core" -[dependencies.mediarepo-model] -path = "./mediarepo-model" +[dependencies.mediarepo-logic] +path = "mediarepo-logic" [dependencies.mediarepo-socket] path = "./mediarepo-socket" @@ -47,4 +47,4 @@ features = ["env-filter", "ansi", "json"] [features] default = ["ffmpeg"] -ffmpeg = ["mediarepo-core/ffmpeg", "mediarepo-model/ffmpeg"] \ No newline at end of file +ffmpeg = ["mediarepo-core/ffmpeg", "mediarepo-logic/ffmpeg"] \ No newline at end of file diff --git a/mediarepo-daemon/mediarepo-model/.gitignore b/mediarepo-daemon/mediarepo-logic/.gitignore similarity index 100% rename from mediarepo-daemon/mediarepo-model/.gitignore rename to mediarepo-daemon/mediarepo-logic/.gitignore diff --git a/mediarepo-daemon/mediarepo-model/Cargo.toml b/mediarepo-daemon/mediarepo-logic/Cargo.toml similarity index 91% rename from mediarepo-daemon/mediarepo-model/Cargo.toml rename to mediarepo-daemon/mediarepo-logic/Cargo.toml index 0f14d81..75cab7b 100644 --- a/mediarepo-daemon/mediarepo-model/Cargo.toml +++ b/mediarepo-daemon/mediarepo-logic/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "mediarepo-model" +name = "mediarepo-logic" version = "0.1.0" edition = "2018" workspace = ".." diff --git a/mediarepo-daemon/mediarepo-model/src/content_descriptor.rs b/mediarepo-daemon/mediarepo-logic/src/content_descriptor.rs similarity index 100% rename from mediarepo-daemon/mediarepo-model/src/content_descriptor.rs rename to mediarepo-daemon/mediarepo-logic/src/content_descriptor.rs diff --git a/mediarepo-daemon/mediarepo-model/src/file/filter.rs b/mediarepo-daemon/mediarepo-logic/src/file/filter.rs similarity index 100% rename from mediarepo-daemon/mediarepo-model/src/file/filter.rs rename to mediarepo-daemon/mediarepo-logic/src/file/filter.rs diff --git a/mediarepo-daemon/mediarepo-model/src/file/mod.rs b/mediarepo-daemon/mediarepo-logic/src/file/mod.rs similarity index 100% rename from mediarepo-daemon/mediarepo-model/src/file/mod.rs rename to mediarepo-daemon/mediarepo-logic/src/file/mod.rs diff --git a/mediarepo-daemon/mediarepo-model/src/file_metadata.rs b/mediarepo-daemon/mediarepo-logic/src/file_metadata.rs similarity index 100% rename from mediarepo-daemon/mediarepo-model/src/file_metadata.rs rename to mediarepo-daemon/mediarepo-logic/src/file_metadata.rs diff --git a/mediarepo-daemon/mediarepo-model/src/lib.rs b/mediarepo-daemon/mediarepo-logic/src/lib.rs similarity index 100% rename from mediarepo-daemon/mediarepo-model/src/lib.rs rename to mediarepo-daemon/mediarepo-logic/src/lib.rs diff --git a/mediarepo-daemon/mediarepo-model/src/namespace.rs b/mediarepo-daemon/mediarepo-logic/src/namespace.rs similarity index 100% rename from mediarepo-daemon/mediarepo-model/src/namespace.rs rename to mediarepo-daemon/mediarepo-logic/src/namespace.rs diff --git a/mediarepo-daemon/mediarepo-model/src/repo.rs b/mediarepo-daemon/mediarepo-logic/src/repo.rs similarity index 100% rename from mediarepo-daemon/mediarepo-model/src/repo.rs rename to mediarepo-daemon/mediarepo-logic/src/repo.rs diff --git a/mediarepo-daemon/mediarepo-model/src/tag.rs b/mediarepo-daemon/mediarepo-logic/src/tag.rs similarity index 100% rename from mediarepo-daemon/mediarepo-model/src/tag.rs rename to mediarepo-daemon/mediarepo-logic/src/tag.rs diff --git a/mediarepo-daemon/mediarepo-model/src/thumbnail.rs b/mediarepo-daemon/mediarepo-logic/src/thumbnail.rs similarity index 100% rename from mediarepo-daemon/mediarepo-model/src/thumbnail.rs rename to mediarepo-daemon/mediarepo-logic/src/thumbnail.rs diff --git a/mediarepo-daemon/mediarepo-model/src/type_keys.rs b/mediarepo-daemon/mediarepo-logic/src/type_keys.rs similarity index 100% rename from mediarepo-daemon/mediarepo-model/src/type_keys.rs rename to mediarepo-daemon/mediarepo-logic/src/type_keys.rs diff --git a/mediarepo-daemon/mediarepo-socket/Cargo.toml b/mediarepo-daemon/mediarepo-socket/Cargo.toml index a804747..c1f51ee 100644 --- a/mediarepo-daemon/mediarepo-socket/Cargo.toml +++ b/mediarepo-daemon/mediarepo-socket/Cargo.toml @@ -19,8 +19,8 @@ path = "../mediarepo-core" [dependencies.mediarepo-database] path = "../mediarepo-database" -[dependencies.mediarepo-model] -path = "../mediarepo-model" +[dependencies.mediarepo-logic] +path = "../mediarepo-logic" [dependencies.tokio] version = "^1.15.0" diff --git a/mediarepo-daemon/mediarepo-socket/src/from_model.rs b/mediarepo-daemon/mediarepo-socket/src/from_model.rs index dc0f59f..abec320 100644 --- a/mediarepo-daemon/mediarepo-socket/src/from_model.rs +++ b/mediarepo-daemon/mediarepo-socket/src/from_model.rs @@ -2,11 +2,11 @@ use mediarepo_core::mediarepo_api::types::files::{ FileBasicDataResponse, FileMetadataResponse, FileStatus, ThumbnailMetadataResponse, }; use mediarepo_core::mediarepo_api::types::tags::{NamespaceResponse, TagResponse}; -use mediarepo_model::file::{File, FileStatus as FileStatusModel}; -use mediarepo_model::file_metadata::FileMetadata; -use mediarepo_model::namespace::Namespace; -use mediarepo_model::tag::Tag; -use mediarepo_model::thumbnail::Thumbnail; +use mediarepo_logic::file::{File, FileStatus as FileStatusModel}; +use mediarepo_logic::file_metadata::FileMetadata; +use mediarepo_logic::namespace::Namespace; +use mediarepo_logic::tag::Tag; +use mediarepo_logic::thumbnail::Thumbnail; pub trait FromModel { fn from_model(model: M) -> Self; diff --git a/mediarepo-daemon/mediarepo-socket/src/lib.rs b/mediarepo-daemon/mediarepo-socket/src/lib.rs index 50f6918..a2e40cb 100644 --- a/mediarepo-daemon/mediarepo-socket/src/lib.rs +++ b/mediarepo-daemon/mediarepo-socket/src/lib.rs @@ -4,8 +4,8 @@ use mediarepo_core::mediarepo_api::types::misc::InfoResponse; use mediarepo_core::settings::{PortSetting, Settings}; use mediarepo_core::tokio_graceful_shutdown::SubsystemHandle; use mediarepo_core::type_keys::{RepoPathKey, SettingsKey, SizeMetadataKey, SubsystemKey}; -use mediarepo_model::repo::Repo; -use mediarepo_model::type_keys::RepoKey; +use mediarepo_logic::repo::Repo; +use mediarepo_logic::type_keys::RepoKey; use std::net::SocketAddr; use std::path::PathBuf; use std::sync::Arc; diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs index e15c6df..3c04c8b 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs @@ -4,10 +4,10 @@ use mediarepo_core::mediarepo_api::types::files::FileStatus as ApiFileStatus; use mediarepo_core::mediarepo_api::types::filtering::{ FilterExpression, FilterQuery, PropertyQuery, TagQuery, ValueComparator, }; -use mediarepo_model::file::filter::NegatableComparator::{Is, IsNot}; -use mediarepo_model::file::filter::{FilterFileProperty, FilterProperty, OrderingComparator}; -use mediarepo_model::file::{File, FileStatus}; -use mediarepo_model::repo::Repo; +use mediarepo_logic::file::filter::NegatableComparator::{Is, IsNot}; +use mediarepo_logic::file::filter::{FilterFileProperty, FilterProperty, OrderingComparator}; +use mediarepo_logic::file::{File, FileStatus}; +use mediarepo_logic::repo::Repo; use std::collections::HashMap; #[tracing::instrument(level = "debug", skip(repo))] diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs index 1f36770..6d6fed8 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs @@ -5,9 +5,9 @@ use mediarepo_core::mediarepo_api::types::filtering::{SortDirection, SortKey}; use mediarepo_database::queries::tags::{ get_cids_with_namespaced_tags, get_content_descriptors_with_tag_count, }; -use mediarepo_model::file::File; -use mediarepo_model::file_metadata::FileMetadata; -use mediarepo_model::repo::Repo; +use mediarepo_logic::file::File; +use mediarepo_logic::file_metadata::FileMetadata; +use mediarepo_logic::repo::Repo; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::cmp::Ordering; use std::collections::HashMap; diff --git a/mediarepo-daemon/mediarepo-socket/src/utils.rs b/mediarepo-daemon/mediarepo-socket/src/utils.rs index 6c07756..568bf51 100644 --- a/mediarepo-daemon/mediarepo-socket/src/utils.rs +++ b/mediarepo-daemon/mediarepo-socket/src/utils.rs @@ -5,9 +5,9 @@ use mediarepo_core::mediarepo_api::types::identifier::FileIdentifier; use mediarepo_core::mediarepo_api::types::repo::SizeType; use mediarepo_core::type_keys::{RepoPathKey, SettingsKey}; use mediarepo_core::utils::get_folder_size; -use mediarepo_model::file::File; -use mediarepo_model::repo::Repo; -use mediarepo_model::type_keys::RepoKey; +use mediarepo_logic::file::File; +use mediarepo_logic::repo::Repo; +use mediarepo_logic::type_keys::RepoKey; use std::sync::Arc; use tokio::fs; diff --git a/mediarepo-daemon/src/main.rs b/mediarepo-daemon/src/main.rs index 3e60d09..4d615d4 100644 --- a/mediarepo-daemon/src/main.rs +++ b/mediarepo-daemon/src/main.rs @@ -9,7 +9,7 @@ use mediarepo_core::error::RepoResult; use mediarepo_core::fs::drop_file::DropFile; use mediarepo_core::settings::{PathSettings, Settings}; use mediarepo_core::tokio_graceful_shutdown::{SubsystemHandle, Toplevel}; -use mediarepo_model::repo::Repo; +use mediarepo_logic::repo::Repo; use mediarepo_socket::start_tcp_server; use std::env; use std::time::Duration; diff --git a/mediarepo-daemon/src/utils.rs b/mediarepo-daemon/src/utils.rs index 2abb7e1..cf9cf4b 100644 --- a/mediarepo-daemon/src/utils.rs +++ b/mediarepo-daemon/src/utils.rs @@ -1,7 +1,7 @@ use mediarepo_core::error::RepoResult; use mediarepo_core::settings::v1::SettingsV1; use mediarepo_core::settings::{PathSettings, Settings}; -use mediarepo_model::repo::Repo; +use mediarepo_logic::repo::Repo; use std::path::PathBuf; use tokio::fs; From a6da2b9e1e3ab30c9c33942db34e426780673ab9 Mon Sep 17 00:00:00 2001 From: trivernis Date: Wed, 19 Jan 2022 20:33:10 +0100 Subject: [PATCH 03/12] Add dto objects Signed-off-by: trivernis --- .../mediarepo-logic/src/dto/file.rs | 37 +++++++++++++++++ .../mediarepo-logic/src/dto/file_metadata.rs | 41 +++++++++++++++++++ .../mediarepo-logic/src/dto/mod.rs | 11 +++++ .../mediarepo-logic/src/dto/namespace.rs | 20 +++++++++ .../mediarepo-logic/src/dto/tag.rs | 30 ++++++++++++++ .../mediarepo-logic/src/dto/thumbnail.rs | 26 ++++++++++++ mediarepo-daemon/mediarepo-logic/src/lib.rs | 1 + 7 files changed, 166 insertions(+) create mode 100644 mediarepo-daemon/mediarepo-logic/src/dto/file.rs create mode 100644 mediarepo-daemon/mediarepo-logic/src/dto/file_metadata.rs create mode 100644 mediarepo-daemon/mediarepo-logic/src/dto/mod.rs create mode 100644 mediarepo-daemon/mediarepo-logic/src/dto/namespace.rs create mode 100644 mediarepo-daemon/mediarepo-logic/src/dto/tag.rs create mode 100644 mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/file.rs b/mediarepo-daemon/mediarepo-logic/src/dto/file.rs new file mode 100644 index 0000000..4832511 --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dto/file.rs @@ -0,0 +1,37 @@ +use mediarepo_database::entities::file; +use mediarepo_database::entities::file_metadata; +use mediarepo_database::entities::content_descriptor; +use crate::dto::FileMetadataDto; + +#[derive(Clone, Debug)] +pub struct FileDto { + model: file::Model, + content_descriptor: content_descriptor::Model, + metadata: Option, +} + +impl FileDto { + pub(crate) fn new(model: file::Model, content_descriptor: content_descriptor::Model, metadata: Option) -> Self { + Self { + model, + content_descriptor, + metadata: metadata.map(FileMetadataDto::new) + } + } + + pub fn id(&self) -> i64 { + self.model.id + } + + pub fn cd_id(&self) -> i64 { + self.model.cd_id + } + + pub fn cd(&self) -> &[u8] { + &self.content_descriptor.descriptor + } + + pub fn metadata(&self) -> Option<&FileMetadataDto> { + self.metadata.as_ref() + } +} \ No newline at end of file diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/file_metadata.rs b/mediarepo-daemon/mediarepo-logic/src/dto/file_metadata.rs new file mode 100644 index 0000000..651d209 --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dto/file_metadata.rs @@ -0,0 +1,41 @@ +use chrono::NaiveDateTime; +use mediarepo_database::entities::file_metadata; + +#[derive(Clone, Debug)] +pub struct FileMetadataDto { + model: file_metadata::Model, +} + +impl FileMetadataDto { + pub(crate) fn new(model: file_metadata::Model) -> Self { + Self {model} + } + + pub fn file_id(&self) -> i64 { + self.model.file_id + } + + pub fn name(&self) -> Option<&String> { + self.model.name.as_ref() + } + + pub fn comment(&self) -> Option<&String> { + self.model.comment.as_ref() + } + + pub fn size(&self) -> i64 { + self.model.size + } + + pub fn import_time(&self) -> NaiveDateTime { + self.model.import_time + } + + pub fn creation_time(&self) -> NaiveDateTime { + self.model.creation_time + } + + pub fn change_time(&self) -> NaiveDateTime { + self.model.change_time + } +} \ No newline at end of file diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dto/mod.rs new file mode 100644 index 0000000..e14493c --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dto/mod.rs @@ -0,0 +1,11 @@ +mod file; +mod file_metadata; +mod tag; +mod namespace; +mod thumbnail; + +pub use file::*; +pub use file_metadata::*; +pub use tag::*; +pub use namespace::*; +pub use thumbnail::*; \ No newline at end of file diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/namespace.rs b/mediarepo-daemon/mediarepo-logic/src/dto/namespace.rs new file mode 100644 index 0000000..2729ecc --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dto/namespace.rs @@ -0,0 +1,20 @@ +use mediarepo_database::entities::namespace; + +#[derive(Clone, Debug)] +pub struct NamespaceDto { + model: namespace::Model, +} + +impl NamespaceDto { + pub(crate) fn new(model: namespace::Model) -> Self { + Self {model} + } + + pub fn id(&self) -> i64 { + self.model.id + } + + pub fn name(&self) -> &String { + &self.model.name + } +} \ No newline at end of file diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/tag.rs b/mediarepo-daemon/mediarepo-logic/src/dto/tag.rs new file mode 100644 index 0000000..9a37e8a --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dto/tag.rs @@ -0,0 +1,30 @@ +pub use mediarepo_database::entities::tag; +pub use mediarepo_database::entities::namespace; +use crate::dto::NamespaceDto; + +#[derive(Clone, Debug)] +pub struct TagDto { + model: tag::Model, + namespace: Option, +} + +impl TagDto { + pub(crate) fn new(model: tag::Model, namespace_model: Option) -> Self { + Self { + model, + namespace: namespace_model.map(NamespaceDto::new) + } + } + + pub fn id(&self) -> i64 { + self.model.id + } + + pub fn name(&self) -> &String { + &self.model.name + } + + pub fn namespace(&self) -> Option<&NamespaceDto> { + self.namespace.as_ref() + } +} \ No newline at end of file diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs b/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs new file mode 100644 index 0000000..224d739 --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs @@ -0,0 +1,26 @@ +use mediarepo_core::fs::thumbnail_store::Dimensions; + +#[derive(Clone, Debug)] +pub struct ThumbnailDto { + parent_cd: String, + size: Dimensions, + mime_type: String, +} + +impl ThumbnailDto { + pub fn new(parent_cd: String, size: Dimensions, mime_type: String) -> Self { + Self {parent_cd, size, mime_type} + } + + pub fn parent_cd(&self) -> &String { + &self.parent_cd + } + + pub fn size(&self) -> &Dimensions { + &self.size + } + + pub fn mime_type(&self) -> &String { + &self.mime_type + } +} diff --git a/mediarepo-daemon/mediarepo-logic/src/lib.rs b/mediarepo-daemon/mediarepo-logic/src/lib.rs index 7706ef2..c10ebac 100644 --- a/mediarepo-daemon/mediarepo-logic/src/lib.rs +++ b/mediarepo-daemon/mediarepo-logic/src/lib.rs @@ -6,3 +6,4 @@ pub mod repo; pub mod tag; pub mod thumbnail; pub mod type_keys; +pub mod dto; From cb493b4651b2e98259bce0f49365843a7c46ab09 Mon Sep 17 00:00:00 2001 From: trivernis Date: Sat, 29 Jan 2022 16:26:36 +0100 Subject: [PATCH 04/12] Add file dao and tag dao Signed-off-by: trivernis --- mediarepo-daemon/Cargo.lock | 4 +- mediarepo-daemon/mediarepo-core/Cargo.toml | 2 +- .../mediarepo-logic/src/dao/file/add.rs | 71 +++++++++ .../mediarepo-logic/src/dao/file/delete.rs | 41 +++++ .../src/{file/filter.rs => dao/file/find.rs} | 27 +++- .../mediarepo-logic/src/dao/file/mod.rs | 149 ++++++++++++++++++ .../mediarepo-logic/src/dao/file/update.rs | 93 +++++++++++ .../mediarepo-logic/src/dao/mod.rs | 33 ++++ .../src/{repo.rs => dao/repo/mod.rs} | 39 ++--- .../mediarepo-logic/src/dao/tag/mappings.rs | 64 ++++++++ .../mediarepo-logic/src/dao/tag/mod.rs | 47 ++++++ .../mediarepo-logic/src/dto/file.rs | 83 +++++++++- .../mediarepo-logic/src/dto/file_metadata.rs | 13 +- .../mediarepo-logic/src/dto/thumbnail.rs | 20 ++- .../mediarepo-logic/src/file/mod.rs | 55 ++----- mediarepo-daemon/mediarepo-logic/src/lib.rs | 2 +- .../mediarepo-logic/src/type_keys.rs | 2 +- .../mediarepo-socket/src/from_model.rs | 50 +++++- mediarepo-daemon/mediarepo-socket/src/lib.rs | 2 +- .../src/namespaces/files/mod.rs | 99 ++++++++---- .../src/namespaces/files/searching.rs | 13 +- .../src/namespaces/files/sorting.rs | 14 +- .../mediarepo-socket/src/namespaces/tags.rs | 16 +- .../mediarepo-socket/src/utils.rs | 14 +- mediarepo-daemon/src/main.rs | 2 +- mediarepo-daemon/src/utils.rs | 2 +- 26 files changed, 813 insertions(+), 144 deletions(-) create mode 100644 mediarepo-daemon/mediarepo-logic/src/dao/file/add.rs create mode 100644 mediarepo-daemon/mediarepo-logic/src/dao/file/delete.rs rename mediarepo-daemon/mediarepo-logic/src/{file/filter.rs => dao/file/find.rs} (90%) create mode 100644 mediarepo-daemon/mediarepo-logic/src/dao/file/mod.rs create mode 100644 mediarepo-daemon/mediarepo-logic/src/dao/file/update.rs create mode 100644 mediarepo-daemon/mediarepo-logic/src/dao/mod.rs rename mediarepo-daemon/mediarepo-logic/src/{repo.rs => dao/repo/mod.rs} (93%) create mode 100644 mediarepo-daemon/mediarepo-logic/src/dao/tag/mappings.rs create mode 100644 mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs diff --git a/mediarepo-daemon/Cargo.lock b/mediarepo-daemon/Cargo.lock index 3643f40..756f8eb 100644 --- a/mediarepo-daemon/Cargo.lock +++ b/mediarepo-daemon/Cargo.lock @@ -2499,9 +2499,9 @@ dependencies = [ [[package]] name = "thumbnailer" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8981b60fe29b8213c829340b7d5bea9d0bfb145fc460450ea3db01cf54d8643a" +checksum = "6017341c89a0c406e38801119f67dd0b67d045ff0e50aa2cf8fc1de4a1b48c3b" dependencies = [ "ffmpeg-next", "image", diff --git a/mediarepo-daemon/mediarepo-core/Cargo.toml b/mediarepo-daemon/mediarepo-core/Cargo.toml index 7c2339a..71c1d40 100644 --- a/mediarepo-daemon/mediarepo-core/Cargo.toml +++ b/mediarepo-daemon/mediarepo-core/Cargo.toml @@ -22,7 +22,7 @@ data-encoding = "^2.3.2" tokio-graceful-shutdown = "^0.4.3" [dependencies.thumbnailer] -version = "^0.2.4" +version = "^0.2.5" default-features = false [dependencies.sea-orm] diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/file/add.rs b/mediarepo-daemon/mediarepo-logic/src/dao/file/add.rs new file mode 100644 index 0000000..c89fb64 --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dao/file/add.rs @@ -0,0 +1,71 @@ +use crate::dto::{AddFileDto, FileDto}; +use chrono::{Local, NaiveDateTime}; +use mediarepo_core::error::RepoResult; +use mediarepo_database::entities::{content_descriptor, file, file_metadata}; +use sea_orm::ActiveValue::Set; +use sea_orm::{ActiveModelTrait, ConnectionTrait, DatabaseTransaction}; +use std::io::Cursor; + +use crate::dao::file::FileDao; + +impl FileDao { + #[tracing::instrument(level = "debug", skip(self))] + pub async fn add(&self, add_dto: AddFileDto) -> RepoResult { + let trx = self.ctx.db.begin().await?; + let file_size = add_dto.content.len(); + let cd_bin = self + .ctx + .main_storage + .add_file(Cursor::new(add_dto.content), None) + .await?; + let cd_model = content_descriptor::ActiveModel { + descriptor: Set(cd_bin), + ..Default::default() + }; + let cd = cd_model.insert(&trx).await?; + + let model = file::ActiveModel { + cd_id: Set(cd.id), + mime_type: Set(add_dto.mime_type), + ..Default::default() + }; + let file: file::Model = model.insert(&trx).await?; + + let metadata = add_file_metadata( + &trx, + file.id, + file_size as i64, + add_dto.creation_time, + add_dto.change_time, + add_dto.name, + ) + .await?; + + trx.commit().await?; + + Ok(FileDto::new(file, cd, Some(metadata))) + } +} + +async fn add_file_metadata( + trx: &DatabaseTransaction, + file_id: i64, + size: i64, + creation_time: NaiveDateTime, + change_time: NaiveDateTime, + name: Option, +) -> RepoResult { + let metadata_model = file_metadata::ActiveModel { + file_id: Set(file_id), + size: Set(size), + import_time: Set(Local::now().naive_local()), + creation_time: Set(creation_time), + change_time: Set(change_time), + name: Set(name), + ..Default::default() + }; + + let metadata = metadata_model.insert(trx).await?; + + Ok(metadata) +} diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/file/delete.rs b/mediarepo-daemon/mediarepo-logic/src/dao/file/delete.rs new file mode 100644 index 0000000..337b7e7 --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dao/file/delete.rs @@ -0,0 +1,41 @@ +use crate::dao::file::{map_file_and_cd, FileDao}; +use crate::dto::FileDto; +use mediarepo_core::error::{RepoError, RepoResult}; +use mediarepo_database::entities::{ + content_descriptor, content_descriptor_tag, file, file_metadata, +}; +use sea_orm::prelude::*; +use sea_orm::ConnectionTrait; + +impl FileDao { + #[tracing::instrument(level = "debug", skip(self))] + pub async fn delete(&self, file: FileDto) -> RepoResult<()> { + let trx = self.ctx.db.begin().await?; + + file_metadata::Entity::delete_many() + .filter(file_metadata::Column::FileId.eq(file.id())) + .exec(&trx) + .await?; + file::Entity::delete_many() + .filter(file::Column::Id.eq(file.id())) + .exec(&trx) + .await?; + content_descriptor_tag::Entity::delete_many() + .filter(content_descriptor_tag::Column::CdId.eq(file.cd_id())) + .exec(&trx) + .await?; + content_descriptor::Entity::delete_many() + .filter(content_descriptor::Column::Id.eq(file.cd_id())) + .exec(&trx) + .await?; + + self.ctx + .thumbnail_storage + .delete_parent(&file.encoded_cd()) + .await?; + self.ctx.main_storage.delete_file(file.cd()).await?; + trx.commit().await?; + + Ok(()) + } +} diff --git a/mediarepo-daemon/mediarepo-logic/src/file/filter.rs b/mediarepo-daemon/mediarepo-logic/src/dao/file/find.rs similarity index 90% rename from mediarepo-daemon/mediarepo-logic/src/file/filter.rs rename to mediarepo-daemon/mediarepo-logic/src/dao/file/find.rs index c3a9313..2156063 100644 --- a/mediarepo-daemon/mediarepo-logic/src/file/filter.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/file/find.rs @@ -1,11 +1,14 @@ +use crate::dao::file::{map_cd_and_file, FileDao}; +use crate::dto::FileDto; use chrono::NaiveDateTime; +use mediarepo_core::error::RepoResult; use mediarepo_database::entities::content_descriptor; use mediarepo_database::entities::content_descriptor_tag; use mediarepo_database::entities::file; use mediarepo_database::entities::file_metadata; use sea_orm::sea_query::{Alias, Expr, Query, SimpleExpr}; -use sea_orm::ColumnTrait; use sea_orm::Condition; +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, QuerySelect}; macro_rules! apply_ordering_comparator { ($column:expr, $filter:expr) => { @@ -53,8 +56,28 @@ pub enum NegatableComparator { IsNot(T), } +impl FileDao { + /// Finds files by filters + #[tracing::instrument(level = "debug", skip(self))] + pub async fn find(&self, filters: Vec>) -> RepoResult> { + let main_condition = build_find_filter_conditions(filters); + + let files = content_descriptor::Entity::find() + .find_also_related(file::Entity) + .filter(main_condition) + .group_by(file::Column::Id) + .all(&self.ctx.db) + .await? + .into_iter() + .filter_map(map_cd_and_file) + .collect(); + + Ok(files) + } +} + #[tracing::instrument(level = "debug")] -pub fn build_find_filter_conditions(filters: Vec>) -> Condition { +fn build_find_filter_conditions(filters: Vec>) -> Condition { filters .into_iter() .fold(Condition::all(), |all_cond, mut expression| { diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/file/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/file/mod.rs new file mode 100644 index 0000000..3a601cb --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dao/file/mod.rs @@ -0,0 +1,149 @@ +use crate::dao::{DaoContext, DaoProvider}; +use crate::dto::{FileDto, FileMetadataDto, ThumbnailDto}; +use mediarepo_core::error::RepoResult; +use mediarepo_database::entities::{content_descriptor, file, file_metadata}; +use sea_orm::prelude::*; +use tokio::io::AsyncReadExt; + +pub mod add; +pub mod delete; +pub mod find; +pub mod update; + +pub struct FileDao { + ctx: DaoContext, +} + +impl DaoProvider for FileDao { + fn dao_ctx(&self) -> DaoContext { + self.ctx.clone() + } +} + +impl FileDao { + pub fn new(ctx: DaoContext) -> Self { + Self { ctx } + } + + #[tracing::instrument(level = "debug", skip(self))] + pub async fn all(&self) -> RepoResult> { + let files = file::Entity::find() + .find_also_related(content_descriptor::Entity) + .all(&self.ctx.db) + .await? + .into_iter() + .filter_map(map_file_and_cd) + .collect(); + + Ok(files) + } + + #[tracing::instrument(level = "debug", skip(self))] + #[inline] + pub async fn by_id(&self, id: i64) -> RepoResult> { + self.all_by_id(vec![id]).await.map(|f| f.into_iter().next()) + } + + #[tracing::instrument(level = "debug", skip(self))] + #[inline] + pub async fn by_cd(&self, cd: Vec) -> RepoResult> { + self.all_by_cd(vec![cd]).await.map(|f| f.into_iter().next()) + } + + #[tracing::instrument(level = "debug", skip(self))] + pub async fn all_by_cd(&self, cds: Vec>) -> RepoResult> { + if cds.is_empty() { + return Ok(vec![]); + } + + let files = file::Entity::find() + .find_also_related(content_descriptor::Entity) + .filter(content_descriptor::Column::Descriptor.is_in(cds)) + .all(&self.ctx.db) + .await? + .into_iter() + .filter_map(map_file_and_cd) + .collect(); + + Ok(files) + } + + #[tracing::instrument(level = "debug", skip(self))] + pub async fn all_by_id(&self, ids: Vec) -> RepoResult> { + if ids.is_empty() { + return Ok(vec![]); + } + + let files = file::Entity::find() + .find_also_related(content_descriptor::Entity) + .filter(file::Column::Id.is_in(ids)) + .all(&self.ctx.db) + .await? + .into_iter() + .filter_map(map_file_and_cd) + .collect(); + + Ok(files) + } + + pub async fn metadata(&self, file_id: i64) -> RepoResult> { + self.all_metadata(vec![file_id]) + .await + .map(|m| m.into_iter().next()) + } + + #[tracing::instrument(level = "debug", skip(self))] + pub async fn all_metadata(&self, file_ids: Vec) -> RepoResult> { + if file_ids.is_empty() { + return Ok(vec![]); + } + + let metadata = file_metadata::Entity::find() + .filter(file_metadata::Column::FileId.is_in(file_ids)) + .all(&self.ctx.db) + .await? + .into_iter() + .map(|m| FileMetadataDto::new(m)) + .collect(); + + Ok(metadata) + } + + /// Returns all thumbnails for a cd + #[tracing::instrument(level = "debug", skip(self))] + pub async fn thumbnails(&self, encoded_cd: String) -> RepoResult> { + let thumbnails = self + .ctx + .thumbnail_storage + .get_thumbnails(&encoded_cd) + .await? + .into_iter() + .map(|(size, path)| { + ThumbnailDto::new(path, encoded_cd.clone(), size, String::from("image/png")) + }) + .collect(); + + Ok(thumbnails) + } + + #[tracing::instrument(level = "debug", skip(self))] + pub async fn get_bytes(&self, cd: &[u8]) -> RepoResult> { + let mut buf = Vec::new(); + let mut reader = self.ctx.main_storage.get_file(cd).await?.1; + reader.read_to_end(&mut buf).await?; + + Ok(buf) + } +} + +fn map_file_and_cd( + (file, cd): (file::Model, Option), +) -> Option { + cd.map(|c| FileDto::new(file, c, None)) +} + +fn map_cd_and_file( + (cd, file): (content_descriptor::Model, Option), +) -> Option { + file.map(|f| FileDto::new(f, cd, None)) +} diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/file/update.rs b/mediarepo-daemon/mediarepo-logic/src/dao/file/update.rs new file mode 100644 index 0000000..69b4b72 --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dao/file/update.rs @@ -0,0 +1,93 @@ +use crate::dto::{FileDto, FileMetadataDto, ThumbnailDto, UpdateFileDto, UpdateFileMetadataDto}; +use mediarepo_core::error::{RepoError, RepoResult}; +use mediarepo_core::fs::thumbnail_store::Dimensions; +use mediarepo_core::thumbnailer; +use mediarepo_core::thumbnailer::{Thumbnail, ThumbnailSize}; +use mediarepo_database::entities::{content_descriptor, file, file_metadata}; +use sea_orm::prelude::*; +use sea_orm::ActiveValue::{Set, Unchanged}; +use sea_orm::{ConnectionTrait, NotSet}; +use std::fmt::Debug; +use std::io::Cursor; +use std::str::FromStr; + +use crate::dao::file::FileDao; +use crate::dao::opt_to_active_val; + +impl FileDao { + #[tracing::instrument(level = "debug", skip(self))] + pub async fn update(&self, update_dto: UpdateFileDto) -> RepoResult { + let trx = self.ctx.db.begin().await?; + let model = file::ActiveModel { + id: Set(update_dto.id), + cd_id: update_dto.cd_id.map(|v| Set(v)).unwrap_or(NotSet), + mime_type: update_dto.mime_type.map(|v| Set(v)).unwrap_or(NotSet), + status: update_dto.status.map(|v| Set(v as i32)).unwrap_or(NotSet), + }; + let file_model = model.update(&trx).await?; + let cd = file_model + .find_related(content_descriptor::Entity) + .one(&trx) + .await? + .ok_or_else(|| RepoError::from("Content descriptor not found"))?; + trx.commit().await?; + + Ok(FileDto::new(file_model, cd, None)) + } + + #[tracing::instrument(level = "debug", skip(self))] + pub async fn update_metadata( + &self, + update_dto: UpdateFileMetadataDto, + ) -> RepoResult { + let model = file_metadata::ActiveModel { + file_id: Unchanged(update_dto.file_id), + name: opt_to_active_val(update_dto.name), + comment: opt_to_active_val(update_dto.comment), + size: opt_to_active_val(update_dto.size), + change_time: opt_to_active_val(update_dto.change_time), + ..Default::default() + }; + let metadata = model.update(&self.ctx.db).await?; + + Ok(FileMetadataDto::new(metadata)) + } + + #[tracing::instrument(level = "debug", skip(self))] + pub async fn create_thumbnails + Debug>( + &self, + file: FileDto, + sizes: I, + ) -> RepoResult> { + let bytes = self.get_bytes(file.cd()).await?; + let mime_type = mime::Mime::from_str(file.mime_type()) + .unwrap_or_else(|_| mime::APPLICATION_OCTET_STREAM); + let thumbnails = + thumbnailer::create_thumbnails(Cursor::new(bytes), mime_type.clone(), sizes)?; + let mut dtos = Vec::new(); + + for thumbnail in thumbnails { + let mut buf = Vec::new(); + let size = thumbnail.size(); + let size = Dimensions { + height: size.1, + width: size.0, + }; + thumbnail.write_png(&mut buf)?; + + let path = self + .ctx + .thumbnail_storage + .add_thumbnail(file.encoded_cd(), size.clone(), &buf) + .await?; + dtos.push(ThumbnailDto::new( + path, + file.encoded_cd(), + size, + mime_type.to_string(), + )) + } + + Ok(dtos) + } +} diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/mod.rs new file mode 100644 index 0000000..268ecce --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dao/mod.rs @@ -0,0 +1,33 @@ +pub mod file; +pub mod repo; +pub mod tag; + +use crate::dao::file::FileDao; +use crate::dao::tag::TagDao; +use mediarepo_core::fs::file_hash_store::FileHashStore; +use mediarepo_core::fs::thumbnail_store::ThumbnailStore; +use sea_orm::{ActiveValue, DatabaseConnection}; + +#[derive(Clone)] +pub struct DaoContext { + pub db: DatabaseConnection, + pub main_storage: FileHashStore, + pub thumbnail_storage: ThumbnailStore, +} + +pub trait DaoProvider { + fn dao_ctx(&self) -> DaoContext; + + fn file(&self) -> FileDao { + FileDao::new(self.dao_ctx()) + } + + fn tag(&self) -> TagDao { + TagDao::new(self.dao_ctx()) + } +} + +fn opt_to_active_val>(opt: Option) -> ActiveValue { + opt.map(|v| ActiveValue::Set(v)) + .unwrap_or(ActiveValue::NotSet) +} diff --git a/mediarepo-daemon/mediarepo-logic/src/repo.rs b/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs similarity index 93% rename from mediarepo-daemon/mediarepo-logic/src/repo.rs rename to mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs index f81d37a..d0c1d5d 100644 --- a/mediarepo-daemon/mediarepo-logic/src/repo.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs @@ -1,5 +1,5 @@ use crate::content_descriptor::ContentDescriptor; -use crate::file::filter::FilterProperty; +use crate::dao::{DaoContext, DaoProvider}; use crate::file::File; use crate::file_metadata::FileMetadata; use crate::namespace::Namespace; @@ -32,6 +32,16 @@ pub struct Repo { thumbnail_storage: ThumbnailStore, } +impl DaoProvider for Repo { + fn dao_ctx(&self) -> DaoContext { + DaoContext { + db: self.db.clone(), + main_storage: self.main_storage.clone(), + thumbnail_storage: self.thumbnail_storage.clone(), + } + } +} + impl Repo { pub(crate) fn new( db: DatabaseConnection, @@ -61,33 +71,6 @@ impl Repo { &self.db } - /// Returns a file by its mapped hash - #[tracing::instrument(level = "debug", skip(self))] - pub async fn file_by_cd(&self, cd: &[u8]) -> RepoResult> { - File::by_cd(self.db.clone(), cd).await - } - - /// Returns a file by id - #[tracing::instrument(level = "debug", skip(self))] - pub async fn file_by_id(&self, id: i64) -> RepoResult> { - File::by_id(self.db.clone(), id).await - } - - /// Returns a list of all stored files - #[tracing::instrument(level = "debug", skip(self))] - pub async fn files(&self) -> RepoResult> { - File::all(self.db.clone()).await - } - - /// Finds all files by a list of tags - #[tracing::instrument(level = "debug", skip(self))] - pub async fn find_files_by_filters( - &self, - filters: Vec>, - ) -> RepoResult> { - File::find_by_filters(self.db.clone(), filters).await - } - /// Returns all file metadata entries for the given file ids #[tracing::instrument(level = "debug", skip(self))] pub async fn get_file_metadata_for_ids(&self, ids: Vec) -> RepoResult> { diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mappings.rs b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mappings.rs new file mode 100644 index 0000000..e76f6c9 --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mappings.rs @@ -0,0 +1,64 @@ +use crate::dao::tag::TagDao; +use mediarepo_core::error::RepoResult; +use mediarepo_database::entities::content_descriptor_tag; +use sea_orm::prelude::*; +use sea_orm::ActiveValue::Set; +use sea_orm::{ConnectionTrait, DatabaseTransaction}; + +impl TagDao { + #[tracing::instrument(level = "debug", skip(self))] + pub async fn upsert_mappings(&self, cd_ids: Vec, tag_ids: Vec) -> RepoResult<()> { + let trx = self.ctx.db.begin().await?; + + let existing_mappings = get_existing_mappings(&trx, &cd_ids, &tag_ids).await?; + + let active_models: Vec = cd_ids + .into_iter() + .flat_map(|cd_id: i64| { + tag_ids + .iter() + .filter(|tag_id| !existing_mappings.contains(&(cd_id, **tag_id))) + .map(move |tag_id| content_descriptor_tag::ActiveModel { + cd_id: Set(cd_id), + tag_id: Set(*tag_id), + }) + .collect::>() + }) + .collect(); + + content_descriptor_tag::Entity::insert_many(active_models) + .exec(&trx) + .await?; + + trx.commit().await?; + + Ok(()) + } + + #[tracing::instrument(level = "debug", skip(self))] + pub async fn remove_mappings(&self, cd_ids: Vec, tag_ids: Vec) -> RepoResult<()> { + content_descriptor_tag::Entity::delete_many() + .filter(content_descriptor_tag::Column::CdId.is_in(cd_ids)) + .filter(content_descriptor_tag::Column::TagId.is_in(tag_ids)) + .exec(&self.ctx.db) + .await?; + + Ok(()) + } +} + +async fn get_existing_mappings( + trx: &DatabaseTransaction, + cd_ids: &Vec, + tag_ids: &Vec, +) -> RepoResult> { + let existing_mappings: Vec<(i64, i64)> = content_descriptor_tag::Entity::find() + .filter(content_descriptor_tag::Column::CdId.is_in(cd_ids.clone())) + .filter(content_descriptor_tag::Column::TagId.is_in(tag_ids.clone())) + .all(trx) + .await? + .into_iter() + .map(|model: content_descriptor_tag::Model| (model.tag_id, model.cd_id)) + .collect(); + Ok(existing_mappings) +} diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs new file mode 100644 index 0000000..37429e3 --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs @@ -0,0 +1,47 @@ +pub mod mappings; + +use crate::dao::{DaoContext, DaoProvider}; +use crate::dto::TagDto; +use mediarepo_core::error::RepoResult; +use mediarepo_database::entities::{content_descriptor, content_descriptor_tag, namespace, tag}; +use sea_orm::prelude::*; +use sea_orm::QuerySelect; +use sea_orm::{DatabaseConnection, JoinType}; + +pub struct TagDao { + ctx: DaoContext, +} + +impl DaoProvider for TagDao { + fn dao_ctx(&self) -> DaoContext { + self.ctx.clone() + } +} + +impl TagDao { + pub fn new(ctx: DaoContext) -> Self { + Self { ctx } + } + + #[tracing::instrument(level = "debug", skip(self))] + pub async fn tags_for_cd(&self, cd_id: i64) -> RepoResult> { + let tags = tag::Entity::find() + .find_also_related(namespace::Entity) + .join( + JoinType::LeftJoin, + content_descriptor_tag::Relation::Tag.def().rev(), + ) + .join( + JoinType::InnerJoin, + content_descriptor_tag::Relation::ContentDescriptorId.def(), + ) + .filter(content_descriptor::Column::Id.eq(cd_id)) + .all(&self.ctx.db) + .await? + .into_iter() + .map(|(t, n)| TagDto::new(t, n)) + .collect(); + + Ok(tags) + } +} diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/file.rs b/mediarepo-daemon/mediarepo-logic/src/dto/file.rs index 4832511..b926201 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dto/file.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dto/file.rs @@ -1,7 +1,10 @@ +use crate::dto::FileMetadataDto; +use chrono::NaiveDateTime; +use mediarepo_core::content_descriptor::encode_content_descriptor; +use mediarepo_core::mediarepo_api::types::files::FileStatus as ApiFileStatus; +use mediarepo_database::entities::content_descriptor; use mediarepo_database::entities::file; use mediarepo_database::entities::file_metadata; -use mediarepo_database::entities::content_descriptor; -use crate::dto::FileMetadataDto; #[derive(Clone, Debug)] pub struct FileDto { @@ -11,11 +14,15 @@ pub struct FileDto { } impl FileDto { - pub(crate) fn new(model: file::Model, content_descriptor: content_descriptor::Model, metadata: Option) -> Self { + pub(crate) fn new( + model: file::Model, + content_descriptor: content_descriptor::Model, + metadata: Option, + ) -> Self { Self { model, content_descriptor, - metadata: metadata.map(FileMetadataDto::new) + metadata: metadata.map(FileMetadataDto::new), } } @@ -31,7 +38,73 @@ impl FileDto { &self.content_descriptor.descriptor } + pub fn encoded_cd(&self) -> String { + encode_content_descriptor(&self.content_descriptor.descriptor) + } + + pub fn status(&self) -> FileStatus { + match self.model.status { + 10 => FileStatus::Imported, + 20 => FileStatus::Archived, + 30 => FileStatus::Deleted, + _ => FileStatus::Imported, + } + } + + pub fn mime_type(&self) -> &String { + &self.model.mime_type + } + pub fn metadata(&self) -> Option<&FileMetadataDto> { self.metadata.as_ref() } -} \ No newline at end of file + + pub fn into_metadata(self) -> Option { + self.metadata + } +} + +#[derive(Clone, Debug)] +pub struct AddFileDto { + pub content: Vec, + pub mime_type: String, + pub creation_time: NaiveDateTime, + pub change_time: NaiveDateTime, + pub name: Option, +} + +#[derive(Clone, Debug)] +pub struct UpdateFileDto { + pub id: i64, + pub cd_id: Option, + pub mime_type: Option, + pub status: Option, +} + +impl Default for UpdateFileDto { + fn default() -> Self { + Self { + id: 0, + cd_id: None, + mime_type: None, + status: None, + } + } +} + +#[derive(Copy, Clone, Debug)] +pub enum FileStatus { + Imported = 10, + Archived = 20, + Deleted = 30, +} + +impl From for FileStatus { + fn from(s: ApiFileStatus) -> Self { + match s { + ApiFileStatus::Imported => Self::Imported, + ApiFileStatus::Archived => Self::Archived, + ApiFileStatus::Deleted => Self::Deleted, + } + } +} diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/file_metadata.rs b/mediarepo-daemon/mediarepo-logic/src/dto/file_metadata.rs index 651d209..e5cedc2 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dto/file_metadata.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dto/file_metadata.rs @@ -8,7 +8,7 @@ pub struct FileMetadataDto { impl FileMetadataDto { pub(crate) fn new(model: file_metadata::Model) -> Self { - Self {model} + Self { model } } pub fn file_id(&self) -> i64 { @@ -38,4 +38,13 @@ impl FileMetadataDto { pub fn change_time(&self) -> NaiveDateTime { self.model.change_time } -} \ No newline at end of file +} + +#[derive(Clone, Debug, Default)] +pub struct UpdateFileMetadataDto { + pub file_id: i64, + pub name: Option>, + pub comment: Option>, + pub size: Option, + pub change_time: Option, +} diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs b/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs index 224d739..eecbfe5 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs @@ -1,15 +1,25 @@ +use mediarepo_core::error::RepoResult; use mediarepo_core::fs::thumbnail_store::Dimensions; +use std::path::PathBuf; +use tokio::fs::{File, OpenOptions}; +use tokio::io::BufReader; #[derive(Clone, Debug)] pub struct ThumbnailDto { + path: PathBuf, parent_cd: String, size: Dimensions, mime_type: String, } impl ThumbnailDto { - pub fn new(parent_cd: String, size: Dimensions, mime_type: String) -> Self { - Self {parent_cd, size, mime_type} + pub fn new(path: PathBuf, parent_cd: String, size: Dimensions, mime_type: String) -> Self { + Self { + path, + parent_cd, + size, + mime_type, + } } pub fn parent_cd(&self) -> &String { @@ -23,4 +33,10 @@ impl ThumbnailDto { pub fn mime_type(&self) -> &String { &self.mime_type } + + #[tracing::instrument(level = "debug")] + pub async fn get_reader(&self) -> RepoResult> { + let file = OpenOptions::new().read(true).open(&self.path).await?; + Ok(BufReader::new(file)) + } } diff --git a/mediarepo-daemon/mediarepo-logic/src/file/mod.rs b/mediarepo-daemon/mediarepo-logic/src/file/mod.rs index 0a38636..f76baaa 100644 --- a/mediarepo-daemon/mediarepo-logic/src/file/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/file/mod.rs @@ -1,5 +1,3 @@ -pub mod filter; - use std::fmt::Debug; use std::io::Cursor; use std::str::FromStr; @@ -10,11 +8,12 @@ use sea_orm::{ConnectionTrait, DatabaseConnection, Set}; use sea_orm::{JoinType, QuerySelect}; use tokio::io::{AsyncReadExt, BufReader}; -use crate::file::filter::FilterProperty; +use crate::dao::file::find; +use crate::dao::file::find::FilterProperty; +use crate::dto::FileStatus; use crate::file_metadata::FileMetadata; use mediarepo_core::error::{RepoError, RepoResult}; use mediarepo_core::fs::file_hash_store::FileHashStore; -use mediarepo_core::mediarepo_api::types::files::FileStatus as ApiFileStatus; use mediarepo_core::thumbnailer::{self, Thumbnail as ThumbnailerThumb, ThumbnailSize}; use mediarepo_database::entities::content_descriptor; use mediarepo_database::entities::content_descriptor_tag; @@ -25,22 +24,6 @@ use mediarepo_database::entities::tag; use crate::tag::Tag; -pub enum FileStatus { - Imported = 10, - Archived = 20, - Deleted = 30, -} - -impl From for FileStatus { - fn from(s: ApiFileStatus) -> Self { - match s { - ApiFileStatus::Imported => Self::Imported, - ApiFileStatus::Archived => Self::Archived, - ApiFileStatus::Deleted => Self::Deleted, - } - } -} - #[derive(Clone)] pub struct File { db: DatabaseConnection, @@ -111,29 +94,6 @@ impl File { } } - /// Finds the file by tags - #[tracing::instrument(level = "debug", skip(db))] - pub(crate) async fn find_by_filters( - db: DatabaseConnection, - filters: Vec>, - ) -> RepoResult> { - let main_condition = filter::build_find_filter_conditions(filters); - - let results: Vec<(content_descriptor::Model, Option)> = - content_descriptor::Entity::find() - .find_also_related(file::Entity) - .filter(main_condition) - .group_by(file::Column::Id) - .all(&db) - .await?; - let files: Vec = results - .into_iter() - .filter_map(|(hash, tag)| Some(Self::new(db.clone(), tag?, hash))) - .collect(); - - Ok(files) - } - /// Adds a file with its hash to the database #[tracing::instrument(level = "debug", skip(db))] pub(crate) async fn add( @@ -247,11 +207,16 @@ impl File { return Ok(()); } let cd_id = self.content_descriptor.id; - let own_tag_ids = self.tags().await?.into_iter().map(|t| t.id()).collect::>(); + let own_tag_ids = self + .tags() + .await? + .into_iter() + .map(|t| t.id()) + .collect::>(); let models: Vec = tag_ids .into_iter() - .filter(|tag_id|!own_tag_ids.contains(tag_id)) + .filter(|tag_id| !own_tag_ids.contains(tag_id)) .map(|tag_id| content_descriptor_tag::ActiveModel { cd_id: Set(cd_id), tag_id: Set(tag_id), diff --git a/mediarepo-daemon/mediarepo-logic/src/lib.rs b/mediarepo-daemon/mediarepo-logic/src/lib.rs index c10ebac..817571d 100644 --- a/mediarepo-daemon/mediarepo-logic/src/lib.rs +++ b/mediarepo-daemon/mediarepo-logic/src/lib.rs @@ -2,8 +2,8 @@ pub mod content_descriptor; pub mod file; pub mod file_metadata; pub mod namespace; -pub mod repo; pub mod tag; pub mod thumbnail; pub mod type_keys; pub mod dto; +pub mod dao; diff --git a/mediarepo-daemon/mediarepo-logic/src/type_keys.rs b/mediarepo-daemon/mediarepo-logic/src/type_keys.rs index a604c7f..f70f645 100644 --- a/mediarepo-daemon/mediarepo-logic/src/type_keys.rs +++ b/mediarepo-daemon/mediarepo-logic/src/type_keys.rs @@ -1,4 +1,4 @@ -use crate::repo::Repo; +use crate::dao::repo::Repo; use std::sync::Arc; use typemap_rev::TypeMapKey; diff --git a/mediarepo-daemon/mediarepo-socket/src/from_model.rs b/mediarepo-daemon/mediarepo-socket/src/from_model.rs index abec320..bdccea0 100644 --- a/mediarepo-daemon/mediarepo-socket/src/from_model.rs +++ b/mediarepo-daemon/mediarepo-socket/src/from_model.rs @@ -2,7 +2,10 @@ use mediarepo_core::mediarepo_api::types::files::{ FileBasicDataResponse, FileMetadataResponse, FileStatus, ThumbnailMetadataResponse, }; use mediarepo_core::mediarepo_api::types::tags::{NamespaceResponse, TagResponse}; -use mediarepo_logic::file::{File, FileStatus as FileStatusModel}; +use mediarepo_logic::dto::{ + FileDto, FileMetadataDto, FileStatus as FileStatusModel, TagDto, ThumbnailDto, +}; +use mediarepo_logic::file::File; use mediarepo_logic::file_metadata::FileMetadata; use mediarepo_logic::namespace::Namespace; use mediarepo_logic::tag::Tag; @@ -25,6 +28,19 @@ impl FromModel for FileMetadataResponse { } } +impl FromModel for FileMetadataResponse { + fn from_model(model: FileMetadataDto) -> Self { + Self { + file_id: model.file_id(), + name: model.name().cloned(), + comment: model.comment().cloned(), + creation_time: model.creation_time().to_owned(), + change_time: model.change_time().to_owned(), + import_time: model.import_time().to_owned(), + } + } +} + impl FromModel for FileBasicDataResponse { fn from_model(file: File) -> Self { FileBasicDataResponse { @@ -36,6 +52,17 @@ impl FromModel for FileBasicDataResponse { } } +impl FromModel for FileBasicDataResponse { + fn from_model(model: FileDto) -> Self { + FileBasicDataResponse { + id: model.id(), + status: FileStatus::from_model(model.status()), + cd: model.encoded_cd(), + mime_type: model.mime_type().to_owned(), + } + } +} + impl FromModel for FileStatus { fn from_model(status: FileStatusModel) -> Self { match status { @@ -56,6 +83,16 @@ impl FromModel for TagResponse { } } +impl FromModel for TagResponse { + fn from_model(model: TagDto) -> Self { + Self { + id: model.id(), + namespace: model.namespace().map(|n| n.name().to_owned()), + name: model.name().to_owned(), + } + } +} + impl FromModel for ThumbnailMetadataResponse { fn from_model(model: Thumbnail) -> Self { Self { @@ -67,6 +104,17 @@ impl FromModel for ThumbnailMetadataResponse { } } +impl FromModel for ThumbnailMetadataResponse { + fn from_model(model: ThumbnailDto) -> Self { + Self { + file_hash: model.parent_cd().to_owned(), + height: model.size().height, + width: model.size().width, + mime_type: model.mime_type().to_owned(), + } + } +} + impl FromModel for NamespaceResponse { fn from_model(model: Namespace) -> Self { Self { diff --git a/mediarepo-daemon/mediarepo-socket/src/lib.rs b/mediarepo-daemon/mediarepo-socket/src/lib.rs index a2e40cb..70082ca 100644 --- a/mediarepo-daemon/mediarepo-socket/src/lib.rs +++ b/mediarepo-daemon/mediarepo-socket/src/lib.rs @@ -4,7 +4,7 @@ use mediarepo_core::mediarepo_api::types::misc::InfoResponse; use mediarepo_core::settings::{PortSetting, Settings}; use mediarepo_core::tokio_graceful_shutdown::SubsystemHandle; use mediarepo_core::type_keys::{RepoPathKey, SettingsKey, SizeMetadataKey, SubsystemKey}; -use mediarepo_logic::repo::Repo; +use mediarepo_logic::dao::repo::Repo; use mediarepo_logic::type_keys::RepoKey; use std::net::SocketAddr; use std::path::PathBuf; diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs index 8d47a1e..fdad59a 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs @@ -6,6 +6,8 @@ use crate::namespaces::files::searching::find_files_for_filters; use crate::namespaces::files::sorting::sort_files_by_properties; use crate::utils::{cd_by_identifier, file_by_identifier, get_repo_from_context}; use mediarepo_core::bromine::prelude::*; +use mediarepo_core::content_descriptor::{create_content_descriptor, encode_content_descriptor}; +use mediarepo_core::error::RepoError; use mediarepo_core::fs::thumbnail_store::Dimensions; use mediarepo_core::itertools::Itertools; use mediarepo_core::mediarepo_api::types::files::{ @@ -17,8 +19,9 @@ use mediarepo_core::mediarepo_api::types::filtering::FindFilesRequest; use mediarepo_core::mediarepo_api::types::identifier::FileIdentifier; use mediarepo_core::thumbnailer::ThumbnailSize; use mediarepo_core::utils::parse_namespace_and_tag; +use mediarepo_logic::dao::DaoProvider; +use mediarepo_logic::dto::{AddFileDto, UpdateFileDto, UpdateFileMetadataDto}; use tokio::io::AsyncReadExt; -use mediarepo_core::content_descriptor::create_content_descriptor; pub struct FilesNamespace; @@ -51,7 +54,7 @@ impl FilesNamespace { #[tracing::instrument(skip_all)] async fn all_files(ctx: &Context, _event: Event) -> IPCResult<()> { let repo = get_repo_from_context(ctx).await; - let files = repo.files().await?; + let files = repo.file().all().await?; let responses: Vec = files .into_iter() @@ -81,7 +84,17 @@ impl FilesNamespace { let id = event.payload::()?; let repo = get_repo_from_context(ctx).await; let file = file_by_identifier(id, &repo).await?; - let metadata = file.metadata().await?; + let file_id = file.id(); + + let metadata = if let Some(metadata) = file.into_metadata() { + metadata + } else { + repo.file() + .metadata(file_id) + .await? + .ok_or_else(|| RepoError::from("file metadata not found"))? + }; + ctx.emit_to( Self::name(), "get_file_metadata", @@ -139,26 +152,29 @@ impl FilesNamespace { let bytes = bytes.into_inner(); let cd = create_content_descriptor(&bytes); - let file = if let Some(file) = repo.file_by_cd(&cd).await? { + let file = if let Some(file) = repo.file().by_cd(cd).await? { tracing::debug!("Inserted file already exists"); file } else { - repo - .add_file( - metadata.mime_type, - bytes, - metadata.creation_time, - metadata.change_time, - ) - .await? + let add_dto = AddFileDto { + content: bytes, + mime_type: metadata + .mime_type + .unwrap_or(String::from("application/octet-stream")), + creation_time: metadata.creation_time, + change_time: metadata.change_time, + name: Some(metadata.name), + }; + repo.file().add(add_dto).await? }; - file.metadata().await?.set_name(metadata.name).await?; let tags = repo .add_all_tags(tags.into_iter().map(parse_namespace_and_tag).collect()) .await?; let tag_ids: Vec = tags.into_iter().map(|t| t.id()).unique().collect(); - file.add_tags(tag_ids).await?; + repo.tag() + .upsert_mappings(vec![file.cd_id()], tag_ids) + .await?; ctx.emit_to( Self::name(), @@ -175,7 +191,14 @@ impl FilesNamespace { let request = event.payload::()?; let repo = get_repo_from_context(ctx).await; let mut file = file_by_identifier(request.file_id, &repo).await?; - file.set_status(request.status.into()).await?; + file = repo + .file() + .update(UpdateFileDto { + id: file.id(), + status: Some(request.status.into()), + ..Default::default() + }) + .await?; ctx.emit_to( Self::name(), "update_file_status", @@ -192,7 +215,7 @@ impl FilesNamespace { let request = event.payload::()?; let repo = get_repo_from_context(ctx).await; let file = file_by_identifier(request.id, &repo).await?; - let bytes = repo.get_file_bytes(&file).await?; + let bytes = repo.file().get_bytes(file.cd()).await?; ctx.emit_to(Self::name(), "read_file", BytePayload::new(bytes)) .await?; @@ -206,7 +229,7 @@ impl FilesNamespace { let id = event.payload::()?; let repo = get_repo_from_context(ctx).await; let file = file_by_identifier(id, &repo).await?; - repo.delete_file(file).await?; + repo.file().delete(file).await?; ctx.emit_to(Self::name(), "delete_file", ()).await?; @@ -219,12 +242,18 @@ impl FilesNamespace { let request = event.payload::()?; let repo = get_repo_from_context(ctx).await; let file_cd = cd_by_identifier(request.id.clone(), &repo).await?; - let mut thumbnails = repo.get_file_thumbnails(&file_cd).await?; + let mut thumbnails = repo + .file() + .thumbnails(encode_content_descriptor(&file_cd)) + .await?; if thumbnails.is_empty() { tracing::debug!("No thumbnails for file found. Creating thumbnails..."); let file = file_by_identifier(request.id, &repo).await?; - thumbnails = repo.create_thumbnails_for_file(&file).await?; + thumbnails = repo + .file() + .create_thumbnails(file, vec![ThumbnailSize::Medium]) + .await?; tracing::debug!("Thumbnails for file created."); } @@ -244,17 +273,20 @@ impl FilesNamespace { let request = event.payload::()?; let repo = get_repo_from_context(ctx).await; let file_cd = cd_by_identifier(request.id.clone(), &repo).await?; - let thumbnails = repo.get_file_thumbnails(&file_cd).await?; let min_size = request.min_size; let max_size = request.max_size; + let thumbnails = repo + .file() + .thumbnails(encode_content_descriptor(&file_cd)) + .await?; let found_thumbnail = thumbnails.into_iter().find(|thumb| { - let Dimensions { height, width } = thumb.size; + let Dimensions { height, width } = thumb.size(); - height >= min_size.0 - && height <= max_size.0 - && width >= min_size.1 - && width <= max_size.1 + *height >= min_size.0 + && *height <= max_size.0 + && *width >= min_size.1 + && *width <= max_size.1 }); let thumbnail = if let Some(thumbnail) = found_thumbnail { @@ -263,10 +295,14 @@ impl FilesNamespace { let file = file_by_identifier(request.id, &repo).await?; let middle_size = ((max_size.0 + min_size.0) / 2, (max_size.1 + min_size.1) / 2); let thumbnail = repo - .create_file_thumbnail(&file, ThumbnailSize::Custom(middle_size)) + .file() + .create_thumbnails(file, vec![ThumbnailSize::Custom(middle_size)]) .await?; thumbnail + .into_iter() + .next() + .ok_or_else(|| RepoError::from("thumbnail could not be created"))? }; let mut buf = Vec::new(); thumbnail.get_reader().await?.read_to_end(&mut buf).await?; @@ -288,8 +324,15 @@ impl FilesNamespace { let repo = get_repo_from_context(ctx).await; let request = event.payload::()?; let file = file_by_identifier(request.file_id, &repo).await?; - let mut metadata = file.metadata().await?; - metadata.set_name(request.name).await?; + + let metadata = repo + .file() + .update_metadata(UpdateFileMetadataDto { + file_id: file.id(), + name: Some(Some(request.name)), + ..Default::default() + }) + .await?; ctx.emit_to( Self::name(), diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs index 3c04c8b..9ffa99e 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs @@ -4,22 +4,23 @@ use mediarepo_core::mediarepo_api::types::files::FileStatus as ApiFileStatus; use mediarepo_core::mediarepo_api::types::filtering::{ FilterExpression, FilterQuery, PropertyQuery, TagQuery, ValueComparator, }; -use mediarepo_logic::file::filter::NegatableComparator::{Is, IsNot}; -use mediarepo_logic::file::filter::{FilterFileProperty, FilterProperty, OrderingComparator}; -use mediarepo_logic::file::{File, FileStatus}; -use mediarepo_logic::repo::Repo; +use mediarepo_logic::dao::file::find::NegatableComparator::{Is, IsNot}; +use mediarepo_logic::dao::file::find::{FilterFileProperty, FilterProperty, OrderingComparator}; +use mediarepo_logic::dao::repo::Repo; +use mediarepo_logic::dao::DaoProvider; +use mediarepo_logic::dto::{FileDto, FileStatus}; use std::collections::HashMap; #[tracing::instrument(level = "debug", skip(repo))] pub async fn find_files_for_filters( repo: &Repo, expressions: Vec, -) -> RepoResult> { +) -> RepoResult> { let tag_names = get_tag_names_from_expressions(&expressions); let tag_id_map = repo.tag_names_to_ids(tag_names).await?; let filters = build_filters_from_expressions(expressions, &tag_id_map); - repo.find_files_by_filters(filters).await + repo.file().find(filters).await } #[tracing::instrument(level = "debug")] diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs index 6d6fed8..5a7972f 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs @@ -5,9 +5,11 @@ use mediarepo_core::mediarepo_api::types::filtering::{SortDirection, SortKey}; use mediarepo_database::queries::tags::{ get_cids_with_namespaced_tags, get_content_descriptors_with_tag_count, }; +use mediarepo_logic::dao::repo::Repo; +use mediarepo_logic::dao::DaoProvider; +use mediarepo_logic::dto::{FileDto, FileMetadataDto}; use mediarepo_logic::file::File; use mediarepo_logic::file_metadata::FileMetadata; -use mediarepo_logic::repo::Repo; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::cmp::Ordering; use std::collections::HashMap; @@ -28,7 +30,7 @@ pub struct FileSortContext { pub async fn sort_files_by_properties( repo: &Repo, sort_expression: Vec, - files: &mut Vec, + files: &mut Vec, ) -> RepoResult<()> { let contexts = build_sort_context(repo, files).await?; @@ -45,7 +47,7 @@ pub async fn sort_files_by_properties( async fn build_sort_context( repo: &Repo, - files: &Vec, + files: &Vec, ) -> RepoResult> { let hash_ids: Vec = files.par_iter().map(|f| f.cd_id()).collect(); let file_ids: Vec = files.par_iter().map(|f| f.id()).collect(); @@ -54,9 +56,9 @@ async fn build_sort_context( get_cids_with_namespaced_tags(repo.db(), hash_ids.clone()).await?; let mut cid_tag_counts = get_content_descriptors_with_tag_count(repo.db(), hash_ids).await?; - let files_metadata = repo.get_file_metadata_for_ids(file_ids).await?; + let files_metadata = repo.file().all_metadata(file_ids).await?; - let mut file_metadata_map: HashMap = + let mut file_metadata_map: HashMap = HashMap::from_iter(files_metadata.into_iter().map(|m| (m.file_id(), m))); let mut contexts = HashMap::new(); @@ -64,7 +66,7 @@ async fn build_sort_context( for file in files { if let Some(metadata) = file_metadata_map.remove(&file.id()) { let context = FileSortContext { - name: metadata.name().to_owned(), + name: metadata.name().cloned(), size: metadata.size() as u64, mime_type: file.mime_type().to_owned(), namespaces: cid_nsp diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs index cf2de6d..29ad8c8 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs @@ -6,6 +6,7 @@ use mediarepo_core::mediarepo_api::types::files::{GetFileTagsRequest, GetFilesTa use mediarepo_core::mediarepo_api::types::tags::{ ChangeFileTagsRequest, NamespaceResponse, TagResponse, }; +use mediarepo_logic::dao::DaoProvider; use rayon::iter::{IntoParallelIterator, ParallelIterator}; pub struct TagsNamespace; @@ -65,7 +66,7 @@ impl TagsNamespace { let repo = get_repo_from_context(ctx).await; let request = event.payload::()?; let file = file_by_identifier(request.id, &repo).await?; - let tags = file.tags().await?; + let tags = repo.tag().tags_for_cd(file.cd_id()).await?; let responses: Vec = tags.into_iter().map(TagResponse::from_model).collect(); ctx.emit_to(Self::name(), "tags_for_file", responses) @@ -126,14 +127,19 @@ impl TagsNamespace { let file = file_by_identifier(request.file_id, &repo).await?; if !request.added_tags.is_empty() { - file.add_tags(request.added_tags).await?; + repo.tag() + .upsert_mappings(vec![file.cd_id()], request.added_tags) + .await?; } if !request.removed_tags.is_empty() { - file.remove_tags(request.removed_tags).await?; + repo.tag() + .remove_mappings(vec![file.cd_id()], request.removed_tags) + .await?; } - let responses: Vec = file - .tags() + let responses: Vec = repo + .tag() + .tags_for_cd(file.cd_id()) .await? .into_iter() .map(TagResponse::from_model) diff --git a/mediarepo-daemon/mediarepo-socket/src/utils.rs b/mediarepo-daemon/mediarepo-socket/src/utils.rs index 568bf51..3769fb1 100644 --- a/mediarepo-daemon/mediarepo-socket/src/utils.rs +++ b/mediarepo-daemon/mediarepo-socket/src/utils.rs @@ -5,8 +5,9 @@ use mediarepo_core::mediarepo_api::types::identifier::FileIdentifier; use mediarepo_core::mediarepo_api::types::repo::SizeType; use mediarepo_core::type_keys::{RepoPathKey, SettingsKey}; use mediarepo_core::utils::get_folder_size; -use mediarepo_logic::file::File; -use mediarepo_logic::repo::Repo; +use mediarepo_logic::dao::repo::Repo; +use mediarepo_logic::dao::DaoProvider; +use mediarepo_logic::dto::FileDto; use mediarepo_logic::type_keys::RepoKey; use std::sync::Arc; use tokio::fs; @@ -17,10 +18,10 @@ pub async fn get_repo_from_context(ctx: &Context) -> Arc { Arc::clone(repo) } -pub async fn file_by_identifier(identifier: FileIdentifier, repo: &Repo) -> RepoResult { +pub async fn file_by_identifier(identifier: FileIdentifier, repo: &Repo) -> RepoResult { let file = match identifier { - FileIdentifier::ID(id) => repo.file_by_id(id).await, - FileIdentifier::CD(cd) => repo.file_by_cd(&decode_content_descriptor(cd)?).await, + FileIdentifier::ID(id) => repo.file().by_id(id).await, + FileIdentifier::CD(cd) => repo.file().by_cd(decode_content_descriptor(cd)?).await, }?; file.ok_or_else(|| RepoError::from("File not found")) } @@ -29,7 +30,8 @@ pub async fn cd_by_identifier(identifier: FileIdentifier, repo: &Repo) -> RepoRe match identifier { FileIdentifier::ID(id) => { let file = repo - .file_by_id(id) + .file() + .by_id(id) .await? .ok_or_else(|| "Thumbnail not found")?; Ok(file.cd().to_owned()) diff --git a/mediarepo-daemon/src/main.rs b/mediarepo-daemon/src/main.rs index 4d615d4..9de1330 100644 --- a/mediarepo-daemon/src/main.rs +++ b/mediarepo-daemon/src/main.rs @@ -9,7 +9,7 @@ use mediarepo_core::error::RepoResult; use mediarepo_core::fs::drop_file::DropFile; use mediarepo_core::settings::{PathSettings, Settings}; use mediarepo_core::tokio_graceful_shutdown::{SubsystemHandle, Toplevel}; -use mediarepo_logic::repo::Repo; +use mediarepo_logic::dao::repo::Repo; use mediarepo_socket::start_tcp_server; use std::env; use std::time::Duration; diff --git a/mediarepo-daemon/src/utils.rs b/mediarepo-daemon/src/utils.rs index cf9cf4b..8c96c9a 100644 --- a/mediarepo-daemon/src/utils.rs +++ b/mediarepo-daemon/src/utils.rs @@ -1,7 +1,7 @@ use mediarepo_core::error::RepoResult; use mediarepo_core::settings::v1::SettingsV1; use mediarepo_core::settings::{PathSettings, Settings}; -use mediarepo_logic::repo::Repo; +use mediarepo_logic::dao::repo::Repo; use std::path::PathBuf; use tokio::fs; From 745c6bf856107c4781cedd3fc4606fdaffaef7e8 Mon Sep 17 00:00:00 2001 From: trivernis Date: Sat, 29 Jan 2022 20:31:50 +0100 Subject: [PATCH 05/12] Delete old models Signed-off-by: trivernis --- .../mediarepo-logic/src/content_descriptor.rs | 101 ------ .../mediarepo-logic/src/dao/job/mod.rs | 59 ++++ .../mediarepo-logic/src/dao/mod.rs | 6 + .../mediarepo-logic/src/dao/repo/mod.rs | 186 +---------- .../mediarepo-logic/src/dto/thumbnail.rs | 9 + .../mediarepo-logic/src/file/mod.rs | 300 ------------------ mediarepo-daemon/mediarepo-logic/src/lib.rs | 7 +- .../mediarepo-logic/src/thumbnail.rs | 30 -- .../mediarepo-socket/src/from_model.rs | 24 -- .../src/namespaces/files/mod.rs | 2 +- .../src/namespaces/files/sorting.rs | 1 - .../mediarepo-socket/src/namespaces/jobs.rs | 5 +- 12 files changed, 84 insertions(+), 646 deletions(-) delete mode 100644 mediarepo-daemon/mediarepo-logic/src/content_descriptor.rs create mode 100644 mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs delete mode 100644 mediarepo-daemon/mediarepo-logic/src/file/mod.rs delete mode 100644 mediarepo-daemon/mediarepo-logic/src/thumbnail.rs diff --git a/mediarepo-daemon/mediarepo-logic/src/content_descriptor.rs b/mediarepo-daemon/mediarepo-logic/src/content_descriptor.rs deleted file mode 100644 index 2762278..0000000 --- a/mediarepo-daemon/mediarepo-logic/src/content_descriptor.rs +++ /dev/null @@ -1,101 +0,0 @@ -use crate::file::File; -use mediarepo_core::content_descriptor::convert_v1_descriptor_to_v2; -use mediarepo_core::error::RepoResult; -use mediarepo_database::entities::content_descriptor; -use mediarepo_database::entities::file; -use sea_orm::prelude::*; -use sea_orm::{DatabaseConnection, Set}; -use std::fmt::Debug; - -pub struct ContentDescriptor { - db: DatabaseConnection, - model: content_descriptor::Model, -} - -impl ContentDescriptor { - #[tracing::instrument(level = "trace")] - pub(crate) fn new(db: DatabaseConnection, model: content_descriptor::Model) -> Self { - Self { db, model } - } - - pub async fn all(db: DatabaseConnection) -> RepoResult> { - let descriptors = content_descriptor::Entity::find() - .all(&db) - .await? - .into_iter() - .map(|model| Self::new(db.clone(), model)) - .collect(); - - Ok(descriptors) - } - - /// Searches for the hash by id - #[tracing::instrument(level = "debug", skip(db))] - pub async fn by_id(db: DatabaseConnection, id: i64) -> RepoResult> { - let hash = content_descriptor::Entity::find_by_id(id) - .one(&db) - .await? - .map(|model| Self::new(db, model)); - - Ok(hash) - } - - /// Returns the hash by value - #[tracing::instrument(level = "debug", skip(db))] - pub async fn by_value + Debug>( - db: DatabaseConnection, - descriptor: D, - ) -> RepoResult> { - let cid = content_descriptor::Entity::find() - .filter(content_descriptor::Column::Descriptor.eq(descriptor.as_ref())) - .one(&db) - .await? - .map(|model| Self::new(db, model)); - - Ok(cid) - } - - /// Adds a new hash to the database - #[tracing::instrument(level = "debug", skip(db))] - pub async fn add(db: DatabaseConnection, descriptor: Vec) -> RepoResult { - let active_model = content_descriptor::ActiveModel { - descriptor: Set(descriptor), - ..Default::default() - }; - let model = active_model.insert(&db).await?; - - Ok(Self::new(db, model)) - } - - pub fn id(&self) -> i64 { - self.model.id - } - - pub fn descriptor(&self) -> &[u8] { - &self.model.descriptor[..] - } - - /// Returns the file associated with the hash - #[tracing::instrument(level = "debug", skip(self))] - pub async fn file(&self) -> RepoResult> { - let file = self - .model - .find_related(file::Entity) - .one(&self.db) - .await? - .map(|file_model| File::new(self.db.clone(), file_model, self.model.clone())); - - Ok(file) - } - - pub async fn convert_v1_to_v2(&mut self) -> RepoResult<()> { - let descriptor = convert_v1_descriptor_to_v2(&self.model.descriptor)?; - let active_model = content_descriptor::ActiveModel { - id: Set(self.id()), - descriptor: Set(descriptor), - }; - self.model = active_model.update(&self.db).await?; - - Ok(()) - } -} diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs new file mode 100644 index 0000000..e69f477 --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs @@ -0,0 +1,59 @@ +use crate::dao::{DaoContext, DaoProvider}; +use mediarepo_core::content_descriptor::{ + convert_v1_descriptor_to_v2, encode_content_descriptor, is_v1_content_descriptor, +}; +use mediarepo_core::error::RepoResult; +use mediarepo_database::entities::content_descriptor; +use sea_orm::prelude::*; +use sea_orm::ActiveValue::Set; +use sea_orm::ConnectionTrait; + +pub struct JobDao { + ctx: DaoContext, +} + +impl DaoProvider for JobDao { + fn dao_ctx(&self) -> DaoContext { + self.ctx.clone() + } +} + +impl JobDao { + pub fn new(ctx: DaoContext) -> JobDao { + Self { ctx } + } + + pub async fn migrate_content_descriptors(&self) -> RepoResult<()> { + let cds: Vec = + content_descriptor::Entity::find().all(&self.ctx.db).await?; + + tracing::info!("Converting content descriptors to v2 format..."); + let mut converted_count = 0; + + for mut cd in cds { + if is_v1_content_descriptor(&cd.descriptor) { + let trx = self.ctx.db.begin().await?; + let src_cd = cd.descriptor; + let dst_cd = convert_v1_descriptor_to_v2(&src_cd)?; + + let active_model = content_descriptor::ActiveModel { + id: Set(cd.id), + descriptor: Set(dst_cd.clone()), + }; + self.ctx.main_storage.rename_file(&src_cd, &dst_cd).await?; + self.ctx + .thumbnail_storage + .rename_parent( + encode_content_descriptor(&src_cd), + encode_content_descriptor(&dst_cd), + ) + .await?; + trx.commit().await?; + converted_count += 1; + } + } + tracing::info!("Converted {} descriptors", converted_count); + + Ok(()) + } +} diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/mod.rs index 268ecce..38101a4 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/mod.rs @@ -1,8 +1,10 @@ pub mod file; +pub mod job; pub mod repo; pub mod tag; use crate::dao::file::FileDao; +use crate::dao::job::JobDao; use crate::dao::tag::TagDao; use mediarepo_core::fs::file_hash_store::FileHashStore; use mediarepo_core::fs::thumbnail_store::ThumbnailStore; @@ -25,6 +27,10 @@ pub trait DaoProvider { fn tag(&self) -> TagDao { TagDao::new(self.dao_ctx()) } + + fn job(&self) -> JobDao { + JobDao::new(self.dao_ctx()) + } } fn opt_to_active_val>(opt: Option) -> ActiveValue { diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs index d0c1d5d..af954b9 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs @@ -1,18 +1,18 @@ -use crate::content_descriptor::ContentDescriptor; use crate::dao::{DaoContext, DaoProvider}; -use crate::file::File; use crate::file_metadata::FileMetadata; use crate::namespace::Namespace; use crate::tag::Tag; -use crate::thumbnail::Thumbnail; use chrono::{Local, NaiveDateTime}; -use mediarepo_core::content_descriptor::{encode_content_descriptor, is_v1_content_descriptor}; +use mediarepo_core::content_descriptor::{ + convert_v1_descriptor_to_v2, encode_content_descriptor, is_v1_content_descriptor, +}; use mediarepo_core::error::{RepoError, RepoResult}; use mediarepo_core::fs::file_hash_store::FileHashStore; use mediarepo_core::fs::thumbnail_store::{Dimensions, ThumbnailStore}; use mediarepo_core::itertools::Itertools; use mediarepo_core::thumbnailer::ThumbnailSize; use mediarepo_core::utils::parse_namespace_and_tag; +use mediarepo_database::entities::content_descriptor; use mediarepo_database::get_database; use mediarepo_database::queries::analysis::{get_all_counts, Counts}; use sea_orm::DatabaseConnection; @@ -77,158 +77,6 @@ impl Repo { FileMetadata::all_by_ids(self.db.clone(), ids).await } - /// Adds a file from bytes to the database - #[tracing::instrument(level = "debug", skip(self, content))] - pub async fn add_file( - &self, - mime_type: Option, - content: Vec, - creation_time: NaiveDateTime, - change_time: NaiveDateTime, - ) -> RepoResult { - let file_size = content.len(); - let reader = Cursor::new(content); - let cd_binary = self.main_storage.add_file(reader, None).await?; - let cd = ContentDescriptor::add(self.db.clone(), cd_binary).await?; - - let mime_type = mime_type - .and_then(|m| mime::Mime::from_str(&m).ok()) - .unwrap_or_else(|| mime::APPLICATION_OCTET_STREAM) - .to_string(); - - let file = File::add(self.db.clone(), cd.id(), mime_type).await?; - FileMetadata::add( - self.db.clone(), - file.id(), - file_size as i64, - creation_time, - change_time, - ) - .await?; - - Ok(file) - } - - /// Adds a file to the database by its readable path in the file system - #[tracing::instrument(level = "debug", skip(self))] - pub async fn add_file_by_path(&self, path: PathBuf) -> RepoResult { - let mime_type = mime_guess::from_path(&path).first().map(|m| m.to_string()); - - let mut os_file = OpenOptions::new().read(true).open(&path).await?; - let mut buf = Vec::new(); - os_file.read_to_end(&mut buf).await?; - - self.add_file( - mime_type, - buf, - Local::now().naive_local(), - Local::now().naive_local(), - ) - .await - } - - /// Deletes a file from the database and disk - #[tracing::instrument(level = "debug", skip(self, file))] - pub async fn delete_file(&self, file: File) -> RepoResult<()> { - let cd = file.cd().to_owned(); - let cd_string = file.encoded_cd(); - file.delete().await?; - self.main_storage.delete_file(&cd).await?; - self.thumbnail_storage.delete_parent(&cd_string).await?; - - Ok(()) - } - - /// Returns all thumbnails of a file - pub async fn get_file_thumbnails(&self, file_cd: &[u8]) -> RepoResult> { - let file_cd = encode_content_descriptor(file_cd); - let thumbnails = self - .thumbnail_storage - .get_thumbnails(&file_cd) - .await? - .into_iter() - .map(|(size, path)| Thumbnail { - file_hash: file_cd.to_owned(), - path, - size, - mime_type: mime::IMAGE_PNG.to_string(), - }) - .collect_vec(); - - Ok(thumbnails) - } - - pub async fn get_file_bytes(&self, file: &File) -> RepoResult> { - let mut buf = Vec::new(); - let mut reader = file.get_reader(&self.main_storage).await?; - reader.read_to_end(&mut buf).await?; - - Ok(buf) - } - - /// Creates thumbnails of all sizes for a file - #[tracing::instrument(level = "debug", skip(self, file))] - pub async fn create_thumbnails_for_file(&self, file: &File) -> RepoResult> { - let size = ThumbnailSize::Medium; - let (height, width) = size.dimensions(); - let thumbs = file.create_thumbnail(&self.main_storage, [size]).await?; - let mut created_thumbs = Vec::with_capacity(1); - - for thumb in thumbs { - let entry = self - .store_single_thumbnail(file.encoded_cd(), height, width, thumb) - .await?; - created_thumbs.push(entry); - } - - Ok(created_thumbs) - } - - #[tracing::instrument(level = "debug", skip(self, file))] - pub async fn create_file_thumbnail( - &self, - file: &File, - size: ThumbnailSize, - ) -> RepoResult { - let (height, width) = size.dimensions(); - let thumb = file - .create_thumbnail(&self.main_storage, [size]) - .await? - .pop() - .ok_or_else(|| RepoError::from("Failed to create thumbnail"))?; - let thumbnail = self - .store_single_thumbnail(file.encoded_cd(), height, width, thumb) - .await?; - - Ok(thumbnail) - } - - /// Stores a single thumbnail - async fn store_single_thumbnail( - &self, - file_hash: String, - height: u32, - width: u32, - thumb: mediarepo_core::thumbnailer::Thumbnail, - ) -> RepoResult { - let mut buf = Vec::new(); - thumb.write_png(&mut buf)?; - let size = Dimensions { height, width }; - let path = self - .thumbnail_storage - .add_thumbnail(&file_hash, size.clone(), &buf) - .await?; - - let thumbnail = Thumbnail { - file_hash, - path, - size, - mime_type: mime::IMAGE_PNG.to_string(), - }; - - Ok(thumbnail) - } - /// Returns all tags stored in the database #[tracing::instrument(level = "debug", skip(self))] pub async fn tags(&self) -> RepoResult> { @@ -386,30 +234,4 @@ impl Repo { pub async fn get_counts(&self) -> RepoResult { get_all_counts(&self.db).await } - - pub async fn migrate(&self) -> RepoResult<()> { - let cds = ContentDescriptor::all(self.db.clone()).await?; - - tracing::info!("Converting content descriptors to v2 format..."); - let mut converted_count = 0; - - for mut cd in cds { - if is_v1_content_descriptor(cd.descriptor()) { - let src_cd = cd.descriptor().to_owned(); - cd.convert_v1_to_v2().await?; - let dst_cd = cd.descriptor().to_owned(); - self.main_storage.rename_file(&src_cd, &dst_cd).await?; - self.thumbnail_storage - .rename_parent( - encode_content_descriptor(&src_cd), - encode_content_descriptor(&dst_cd), - ) - .await?; - converted_count += 1; - } - } - tracing::info!("Converted {} descriptors", converted_count); - - Ok(()) - } } diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs b/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs index eecbfe5..2ac6166 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs @@ -1,6 +1,7 @@ use mediarepo_core::error::RepoResult; use mediarepo_core::fs::thumbnail_store::Dimensions; use std::path::PathBuf; +use tokio::fs; use tokio::fs::{File, OpenOptions}; use tokio::io::BufReader; @@ -39,4 +40,12 @@ impl ThumbnailDto { let file = OpenOptions::new().read(true).open(&self.path).await?; Ok(BufReader::new(file)) } + + /// Deletes the thumbnail + #[tracing::instrument(level = "debug")] + pub async fn delete(self) -> RepoResult<()> { + fs::remove_file(&self.path).await?; + + Ok(()) + } } diff --git a/mediarepo-daemon/mediarepo-logic/src/file/mod.rs b/mediarepo-daemon/mediarepo-logic/src/file/mod.rs deleted file mode 100644 index f76baaa..0000000 --- a/mediarepo-daemon/mediarepo-logic/src/file/mod.rs +++ /dev/null @@ -1,300 +0,0 @@ -use std::fmt::Debug; -use std::io::Cursor; -use std::str::FromStr; - -use mediarepo_core::content_descriptor::encode_content_descriptor; -use sea_orm::prelude::*; -use sea_orm::{ConnectionTrait, DatabaseConnection, Set}; -use sea_orm::{JoinType, QuerySelect}; -use tokio::io::{AsyncReadExt, BufReader}; - -use crate::dao::file::find; -use crate::dao::file::find::FilterProperty; -use crate::dto::FileStatus; -use crate::file_metadata::FileMetadata; -use mediarepo_core::error::{RepoError, RepoResult}; -use mediarepo_core::fs::file_hash_store::FileHashStore; -use mediarepo_core::thumbnailer::{self, Thumbnail as ThumbnailerThumb, ThumbnailSize}; -use mediarepo_database::entities::content_descriptor; -use mediarepo_database::entities::content_descriptor_tag; -use mediarepo_database::entities::file; -use mediarepo_database::entities::file_metadata; -use mediarepo_database::entities::namespace; -use mediarepo_database::entities::tag; - -use crate::tag::Tag; - -#[derive(Clone)] -pub struct File { - db: DatabaseConnection, - model: file::Model, - content_descriptor: content_descriptor::Model, -} - -impl File { - #[tracing::instrument(level = "trace")] - pub(crate) fn new( - db: DatabaseConnection, - model: file::Model, - hash: content_descriptor::Model, - ) -> Self { - Self { - db, - model, - content_descriptor: hash, - } - } - - /// Returns a list of all known stored files - #[tracing::instrument(level = "debug", skip(db))] - pub async fn all(db: DatabaseConnection) -> RepoResult> { - let files: Vec<(file::Model, Option)> = file::Entity::find() - .find_also_related(content_descriptor::Entity) - .all(&db) - .await?; - let files = files - .into_iter() - .filter_map(|(f, h)| { - let h = h?; - Some(Self::new(db.clone(), f, h)) - }) - .collect(); - - Ok(files) - } - - /// Fetches the file by id - #[tracing::instrument(level = "debug", skip(db))] - pub async fn by_id(db: DatabaseConnection, id: i64) -> RepoResult> { - if let Some((model, Some(hash))) = file::Entity::find_by_id(id) - .find_also_related(content_descriptor::Entity) - .one(&db) - .await? - { - let file = File::new(db, model, hash); - Ok(Some(file)) - } else { - Ok(None) - } - } - - /// Finds the file by hash - #[tracing::instrument(level = "debug", skip(db))] - pub async fn by_cd(db: DatabaseConnection, cd: &[u8]) -> RepoResult> { - if let Some((hash, Some(model))) = content_descriptor::Entity::find() - .filter(content_descriptor::Column::Descriptor.eq(cd)) - .find_also_related(file::Entity) - .one(&db) - .await? - { - let file = File::new(db, model, hash); - Ok(Some(file)) - } else { - Ok(None) - } - } - - /// Adds a file with its hash to the database - #[tracing::instrument(level = "debug", skip(db))] - pub(crate) async fn add( - db: DatabaseConnection, - cd_id: i64, - mime_type: String, - ) -> RepoResult { - let file = file::ActiveModel { - cd_id: Set(cd_id), - mime_type: Set(mime_type), - ..Default::default() - }; - let file: file::ActiveModel = file.insert(&db).await?.into(); - let file = Self::by_id(db, file.id.unwrap()) - .await? - .expect("Inserted file does not exist"); - - Ok(file) - } - - /// Returns the unique identifier of the file - pub fn id(&self) -> i64 { - self.model.id - } - - /// Returns the hash of the file (content identifier) - pub fn cd(&self) -> &[u8] { - &self.content_descriptor.descriptor - } - - /// Returns the encoded content descriptor - pub fn encoded_cd(&self) -> String { - encode_content_descriptor(self.cd()) - } - - /// Returns the id of the civ (content identifier value) of the file - pub fn cd_id(&self) -> i64 { - self.content_descriptor.id - } - - /// Returns the mime type of the file - pub fn mime_type(&self) -> &String { - &self.model.mime_type - } - - /// Returns the status of the file - pub fn status(&self) -> FileStatus { - match self.model.status { - 10 => FileStatus::Imported, - 20 => FileStatus::Archived, - 30 => FileStatus::Deleted, - _ => FileStatus::Imported, - } - } - - pub async fn set_status(&mut self, status: FileStatus) -> RepoResult<()> { - let active_model = file::ActiveModel { - id: Set(self.model.id), - status: Set(status as i32), - ..Default::default() - }; - self.model = active_model.update(&self.db).await?; - - Ok(()) - } - - /// Returns the metadata associated with this file - /// A file MUST always have metadata associated - pub async fn metadata(&self) -> RepoResult { - FileMetadata::by_id(self.db.clone(), self.model.id) - .await - .and_then(|f| f.ok_or_else(|| RepoError::from("missing file metadata"))) - } - - /// Returns the list of tags of the file - #[tracing::instrument(level = "debug", skip(self))] - pub async fn tags(&self) -> RepoResult> { - let tags: Vec<(tag::Model, Option)> = tag::Entity::find() - .find_also_related(namespace::Entity) - .join( - JoinType::LeftJoin, - content_descriptor_tag::Relation::Tag.def().rev(), - ) - .filter(content_descriptor_tag::Column::CdId.eq(self.content_descriptor.id)) - .all(&self.db) - .await?; - let tags = tags - .into_iter() - .map(|(tag, namespace)| Tag::new(self.db.clone(), tag, namespace)) - .collect(); - - Ok(tags) - } - - /// Adds a single tag to the file - #[tracing::instrument(level = "debug", skip(self))] - pub async fn add_tag(&mut self, tag_id: i64) -> RepoResult<()> { - let cd_id = self.content_descriptor.id; - let active_model = content_descriptor_tag::ActiveModel { - cd_id: Set(cd_id), - tag_id: Set(tag_id), - }; - active_model.insert(&self.db).await?; - Ok(()) - } - - /// Adds multiple tags to the file at once - #[tracing::instrument(level = "debug", skip(self))] - pub async fn add_tags(&self, tag_ids: Vec) -> RepoResult<()> { - if tag_ids.is_empty() { - return Ok(()); - } - let cd_id = self.content_descriptor.id; - let own_tag_ids = self - .tags() - .await? - .into_iter() - .map(|t| t.id()) - .collect::>(); - - let models: Vec = tag_ids - .into_iter() - .filter(|tag_id| !own_tag_ids.contains(tag_id)) - .map(|tag_id| content_descriptor_tag::ActiveModel { - cd_id: Set(cd_id), - tag_id: Set(tag_id), - }) - .collect(); - if models.len() > 0 { - content_descriptor_tag::Entity::insert_many(models) - .exec(&self.db) - .await?; - } - - Ok(()) - } - - /// Removes multiple tags from the file - #[tracing::instrument(level = "debug", skip(self))] - pub async fn remove_tags(&self, tag_ids: Vec) -> RepoResult<()> { - let hash_id = self.content_descriptor.id; - content_descriptor_tag::Entity::delete_many() - .filter(content_descriptor_tag::Column::CdId.eq(hash_id)) - .filter(content_descriptor_tag::Column::TagId.is_in(tag_ids)) - .exec(&self.db) - .await?; - - Ok(()) - } - - /// Returns the reader for the file - #[tracing::instrument(level = "debug", skip(self))] - pub async fn get_reader( - &self, - storage: &FileHashStore, - ) -> RepoResult> { - storage - .get_file(&self.content_descriptor.descriptor) - .await - .map(|(_, f)| f) - } - - /// Creates a thumbnail for the file - #[tracing::instrument(level = "debug", skip(self))] - pub async fn create_thumbnail + Debug>( - &self, - storage: &FileHashStore, - sizes: I, - ) -> RepoResult> { - let mut buf = Vec::new(); - self.get_reader(storage) - .await? - .read_to_end(&mut buf) - .await?; - let mime_type = self.model.mime_type.clone(); - let mime_type = - mime::Mime::from_str(&mime_type).unwrap_or_else(|_| mime::APPLICATION_OCTET_STREAM); - let thumbs = thumbnailer::create_thumbnails(Cursor::new(buf), mime_type, sizes)?; - - Ok(thumbs) - } - - /// Deletes the file as well as the content descriptor, tag mappings and metadata about the file - #[tracing::instrument(level = "debug", skip(self))] - pub async fn delete(self) -> RepoResult<()> { - let trx = self.db.begin().await?; - file_metadata::Entity::delete_many() - .filter(file_metadata::Column::FileId.eq(self.model.id)) - .exec(&trx) - .await?; - self.model.delete(&trx).await?; - content_descriptor_tag::Entity::delete_many() - .filter(content_descriptor_tag::Column::CdId.eq(self.content_descriptor.id)) - .exec(&trx) - .await?; - content_descriptor::Entity::delete_many() - .filter(content_descriptor::Column::Id.eq(self.content_descriptor.id)) - .exec(&trx) - .await?; - trx.commit().await?; - - Ok(()) - } -} diff --git a/mediarepo-daemon/mediarepo-logic/src/lib.rs b/mediarepo-daemon/mediarepo-logic/src/lib.rs index 817571d..1d620f5 100644 --- a/mediarepo-daemon/mediarepo-logic/src/lib.rs +++ b/mediarepo-daemon/mediarepo-logic/src/lib.rs @@ -1,9 +1,6 @@ -pub mod content_descriptor; -pub mod file; +pub mod dao; +pub mod dto; pub mod file_metadata; pub mod namespace; pub mod tag; -pub mod thumbnail; pub mod type_keys; -pub mod dto; -pub mod dao; diff --git a/mediarepo-daemon/mediarepo-logic/src/thumbnail.rs b/mediarepo-daemon/mediarepo-logic/src/thumbnail.rs deleted file mode 100644 index be45be5..0000000 --- a/mediarepo-daemon/mediarepo-logic/src/thumbnail.rs +++ /dev/null @@ -1,30 +0,0 @@ -use mediarepo_core::error::RepoResult; -use mediarepo_core::fs::thumbnail_store::Dimensions; -use std::path::PathBuf; -use tokio::fs::{self, File, OpenOptions}; -use tokio::io::BufReader; - -#[derive(Debug)] -pub struct Thumbnail { - pub file_hash: String, - pub path: PathBuf, - pub size: Dimensions, - pub mime_type: String, -} - -impl Thumbnail { - /// Returns the reader of the thumbnail file - #[tracing::instrument(level = "debug")] - pub async fn get_reader(&self) -> RepoResult> { - let file = OpenOptions::new().read(true).open(&self.path).await?; - Ok(BufReader::new(file)) - } - - /// Deletes the thumbnail - #[tracing::instrument(level = "debug")] - pub async fn delete(self) -> RepoResult<()> { - fs::remove_file(&self.path).await?; - - Ok(()) - } -} diff --git a/mediarepo-daemon/mediarepo-socket/src/from_model.rs b/mediarepo-daemon/mediarepo-socket/src/from_model.rs index bdccea0..ec96095 100644 --- a/mediarepo-daemon/mediarepo-socket/src/from_model.rs +++ b/mediarepo-daemon/mediarepo-socket/src/from_model.rs @@ -5,11 +5,9 @@ use mediarepo_core::mediarepo_api::types::tags::{NamespaceResponse, TagResponse} use mediarepo_logic::dto::{ FileDto, FileMetadataDto, FileStatus as FileStatusModel, TagDto, ThumbnailDto, }; -use mediarepo_logic::file::File; use mediarepo_logic::file_metadata::FileMetadata; use mediarepo_logic::namespace::Namespace; use mediarepo_logic::tag::Tag; -use mediarepo_logic::thumbnail::Thumbnail; pub trait FromModel { fn from_model(model: M) -> Self; @@ -41,17 +39,6 @@ impl FromModel for FileMetadataResponse { } } -impl FromModel for FileBasicDataResponse { - fn from_model(file: File) -> Self { - FileBasicDataResponse { - id: file.id(), - status: FileStatus::from_model(file.status()), - cd: file.encoded_cd(), - mime_type: file.mime_type().to_owned(), - } - } -} - impl FromModel for FileBasicDataResponse { fn from_model(model: FileDto) -> Self { FileBasicDataResponse { @@ -93,17 +80,6 @@ impl FromModel for TagResponse { } } -impl FromModel for ThumbnailMetadataResponse { - fn from_model(model: Thumbnail) -> Self { - Self { - file_hash: model.file_hash, - height: model.size.height, - width: model.size.width, - mime_type: model.mime_type.to_owned(), - } - } -} - impl FromModel for ThumbnailMetadataResponse { fn from_model(model: ThumbnailDto) -> Self { Self { diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs index fdad59a..973daf4 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs @@ -350,7 +350,7 @@ impl FilesNamespace { let repo = get_repo_from_context(ctx).await; let id = event.payload::()?; let file = file_by_identifier(id, &repo).await?; - let thumbnails = repo.get_file_thumbnails(file.cd()).await?; + let thumbnails = repo.file().thumbnails(file.encoded_cd()).await?; for thumb in thumbnails { thumb.delete().await?; diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs index 5a7972f..e80bbe1 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs @@ -8,7 +8,6 @@ use mediarepo_database::queries::tags::{ use mediarepo_logic::dao::repo::Repo; use mediarepo_logic::dao::DaoProvider; use mediarepo_logic::dto::{FileDto, FileMetadataDto}; -use mediarepo_logic::file::File; use mediarepo_logic::file_metadata::FileMetadata; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::cmp::Ordering; diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs index 393fed8..83231bf 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs @@ -4,6 +4,7 @@ use mediarepo_core::error::RepoResult; use mediarepo_core::mediarepo_api::types::jobs::{JobType, RunJobRequest}; use mediarepo_core::mediarepo_api::types::repo::SizeType; use mediarepo_core::type_keys::SizeMetadataKey; +use mediarepo_logic::dao::DaoProvider; pub struct JobsNamespace; @@ -23,10 +24,10 @@ impl JobsNamespace { #[tracing::instrument(skip_all)] pub async fn run_job(ctx: &Context, event: Event) -> IPCResult<()> { let run_request = event.payload::()?; - let repo = get_repo_from_context(ctx).await; + let job_dao = get_repo_from_context(ctx).await.job(); match run_request.job_type { - JobType::MigrateContentDescriptors => repo.migrate().await?, + JobType::MigrateContentDescriptors => job_dao.migrate_content_descriptors().await?, JobType::CalculateSizes => calculate_all_sizes(ctx).await?, JobType::CheckIntegrity => {} } From acdad7ac7e1fb0410e21a453398720cfdfb1bc6d Mon Sep 17 00:00:00 2001 From: trivernis Date: Sat, 29 Jan 2022 20:40:15 +0100 Subject: [PATCH 06/12] Fix warnings Signed-off-by: trivernis --- .../mediarepo-core/src/content_descriptor.rs | 3 +- .../mediarepo-core/src/context.rs | 6 ++- mediarepo-daemon/mediarepo-core/src/error.rs | 3 +- .../mediarepo-core/src/fs/drop_file.rs | 1 + .../mediarepo-core/src/fs/file_hash_store.rs | 8 ++-- .../mediarepo-core/src/fs/thumbnail_store.rs | 6 ++- .../mediarepo-core/src/settings/mod.rs | 18 ++++---- .../mediarepo-core/src/settings/paths.rs | 3 +- .../mediarepo-core/src/settings/server.rs | 3 +- .../mediarepo-core/src/settings/v1.rs | 6 ++- .../mediarepo-core/src/type_keys.rs | 6 ++- mediarepo-daemon/mediarepo-core/src/utils.rs | 6 ++- .../mediarepo-database/src/lib.rs | 6 ++- .../mediarepo-database/src/queries/tags.rs | 10 +++-- .../mediarepo-logic/src/dao/file/add.rs | 10 +++-- .../mediarepo-logic/src/dao/file/delete.rs | 12 ++--- .../mediarepo-logic/src/dao/file/find.rs | 12 ++--- .../mediarepo-logic/src/dao/file/mod.rs | 10 +++-- .../mediarepo-logic/src/dao/file/update.rs | 18 ++++---- .../mediarepo-logic/src/dao/job/mod.rs | 14 +++--- .../mediarepo-logic/src/dao/mod.rs | 16 ++++--- .../mediarepo-logic/src/dao/repo/mod.rs | 45 ++++++++++--------- .../mediarepo-logic/src/dao/tag/mappings.rs | 10 +++-- .../mediarepo-logic/src/dao/tag/mod.rs | 14 +++--- .../mediarepo-logic/src/dto/file.rs | 4 +- .../mediarepo-logic/src/dto/file_metadata.rs | 1 + .../mediarepo-logic/src/dto/mod.rs | 11 ++--- .../mediarepo-logic/src/dto/tag.rs | 7 +-- .../mediarepo-logic/src/dto/thumbnail.rs | 6 ++- .../mediarepo-logic/src/namespace.rs | 10 +++-- mediarepo-daemon/mediarepo-logic/src/tag.rs | 11 ++--- .../mediarepo-logic/src/type_keys.rs | 4 +- mediarepo-daemon/mediarepo-socket/src/lib.rs | 12 ++--- .../src/namespaces/files/mod.rs | 16 ++++--- .../src/namespaces/files/searching.rs | 7 +-- .../src/namespaces/files/sorting.rs | 14 +++--- .../mediarepo-socket/src/namespaces/jobs.rs | 3 +- .../mediarepo-socket/src/namespaces/tags.rs | 8 ++-- .../mediarepo-socket/src/utils.rs | 8 ++-- mediarepo-daemon/src/logging.rs | 15 ++++--- mediarepo-daemon/src/main.rs | 6 +-- mediarepo-daemon/src/utils.rs | 8 ++-- 42 files changed, 233 insertions(+), 164 deletions(-) diff --git a/mediarepo-daemon/mediarepo-core/src/content_descriptor.rs b/mediarepo-daemon/mediarepo-core/src/content_descriptor.rs index 73c1ea3..e89f47e 100644 --- a/mediarepo-daemon/mediarepo-core/src/content_descriptor.rs +++ b/mediarepo-daemon/mediarepo-core/src/content_descriptor.rs @@ -1,6 +1,7 @@ -use crate::error::RepoResult; use multihash::{Code, MultihashDigest}; +use crate::error::RepoResult; + /// Creates a new content descriptor for the given file pub fn create_content_descriptor(bytes: &[u8]) -> Vec { Code::Sha2_256.digest(bytes).to_bytes() diff --git a/mediarepo-daemon/mediarepo-core/src/context.rs b/mediarepo-daemon/mediarepo-core/src/context.rs index cc3e52d..77b1e0f 100644 --- a/mediarepo-daemon/mediarepo-core/src/context.rs +++ b/mediarepo-daemon/mediarepo-core/src/context.rs @@ -1,8 +1,10 @@ -use crate::settings::Settings; -use sea_orm::DatabaseConnection; use std::sync::Arc; + +use sea_orm::DatabaseConnection; use tokio::sync::Mutex; +use crate::settings::Settings; + #[derive(Clone, Default)] pub struct Context { pub settings: Arc>, diff --git a/mediarepo-daemon/mediarepo-core/src/error.rs b/mediarepo-daemon/mediarepo-core/src/error.rs index b3e1d82..820e854 100644 --- a/mediarepo-daemon/mediarepo-core/src/error.rs +++ b/mediarepo-daemon/mediarepo-core/src/error.rs @@ -1,5 +1,6 @@ -use sea_orm::DbErr; use std::fmt::{Debug, Formatter}; + +use sea_orm::DbErr; use thiserror::Error; pub type RepoResult = Result; diff --git a/mediarepo-daemon/mediarepo-core/src/fs/drop_file.rs b/mediarepo-daemon/mediarepo-core/src/fs/drop_file.rs index 53fb266..ee6bdb3 100644 --- a/mediarepo-daemon/mediarepo-core/src/fs/drop_file.rs +++ b/mediarepo-daemon/mediarepo-core/src/fs/drop_file.rs @@ -1,5 +1,6 @@ use std::io::Result; use std::path::{Path, PathBuf}; + use tokio::fs::{File, OpenOptions}; /// A file that only exists while being owned. diff --git a/mediarepo-daemon/mediarepo-core/src/fs/file_hash_store.rs b/mediarepo-daemon/mediarepo-core/src/fs/file_hash_store.rs index b23fcf7..71034dc 100644 --- a/mediarepo-daemon/mediarepo-core/src/fs/file_hash_store.rs +++ b/mediarepo-daemon/mediarepo-core/src/fs/file_hash_store.rs @@ -1,11 +1,13 @@ -use crate::content_descriptor::{create_content_descriptor, encode_content_descriptor}; -use crate::error::RepoResult; -use crate::utils::get_folder_size; use std::path::PathBuf; + use tokio::fs; use tokio::fs::{File, OpenOptions}; use tokio::io::{AsyncRead, AsyncReadExt, BufReader}; +use crate::content_descriptor::{create_content_descriptor, encode_content_descriptor}; +use crate::error::RepoResult; +use crate::utils::get_folder_size; + #[derive(Clone, Debug)] pub struct FileHashStore { path: PathBuf, diff --git a/mediarepo-daemon/mediarepo-core/src/fs/thumbnail_store.rs b/mediarepo-daemon/mediarepo-core/src/fs/thumbnail_store.rs index 6a500ea..7b150c2 100644 --- a/mediarepo-daemon/mediarepo-core/src/fs/thumbnail_store.rs +++ b/mediarepo-daemon/mediarepo-core/src/fs/thumbnail_store.rs @@ -1,12 +1,14 @@ -use crate::error::RepoResult; -use crate::utils::get_folder_size; use std::fmt::Debug; use std::io::Result; use std::path::PathBuf; + use tokio::fs; use tokio::fs::OpenOptions; use tokio::io::{AsyncWriteExt, BufWriter}; +use crate::error::RepoResult; +use crate::utils::get_folder_size; + #[derive(Clone, Debug)] pub struct ThumbnailStore { path: PathBuf, diff --git a/mediarepo-daemon/mediarepo-core/src/settings/mod.rs b/mediarepo-daemon/mediarepo-core/src/settings/mod.rs index 329e32a..66a8960 100644 --- a/mediarepo-daemon/mediarepo-core/src/settings/mod.rs +++ b/mediarepo-daemon/mediarepo-core/src/settings/mod.rs @@ -1,19 +1,21 @@ -mod logging; -mod paths; -mod server; -pub mod v1; +use std::fs; +use std::path::PathBuf; -use crate::error::RepoResult; -use crate::settings::v1::SettingsV1; use config::{Config, FileFormat}; use serde::{Deserialize, Serialize}; -use std::fs; -use std::path::PathBuf; pub use logging::*; pub use paths::*; pub use server::*; +use crate::error::RepoResult; +use crate::settings::v1::SettingsV1; + +mod logging; +mod paths; +mod server; +pub mod v1; + #[derive(Clone, Debug, Deserialize, Serialize, Default)] pub struct Settings { pub server: ServerSettings, diff --git a/mediarepo-daemon/mediarepo-core/src/settings/paths.rs b/mediarepo-daemon/mediarepo-core/src/settings/paths.rs index 5e14e75..162e096 100644 --- a/mediarepo-daemon/mediarepo-core/src/settings/paths.rs +++ b/mediarepo-daemon/mediarepo-core/src/settings/paths.rs @@ -1,6 +1,7 @@ -use serde::{Deserialize, Serialize}; use std::path::PathBuf; +use serde::{Deserialize, Serialize}; + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct PathSettings { pub(crate) database_directory: String, diff --git a/mediarepo-daemon/mediarepo-core/src/settings/server.rs b/mediarepo-daemon/mediarepo-core/src/settings/server.rs index c722a11..32de932 100644 --- a/mediarepo-daemon/mediarepo-core/src/settings/server.rs +++ b/mediarepo-daemon/mediarepo-core/src/settings/server.rs @@ -1,6 +1,7 @@ -use serde::{Deserialize, Serialize}; use std::net::IpAddr; +use serde::{Deserialize, Serialize}; + #[derive(Clone, Debug, Deserialize, Serialize, Default)] pub struct ServerSettings { pub tcp: TcpServerSettings, diff --git a/mediarepo-daemon/mediarepo-core/src/settings/v1.rs b/mediarepo-daemon/mediarepo-core/src/settings/v1.rs index ccbb35e..f14cb68 100644 --- a/mediarepo-daemon/mediarepo-core/src/settings/v1.rs +++ b/mediarepo-daemon/mediarepo-core/src/settings/v1.rs @@ -1,7 +1,9 @@ -use crate::error::RepoResult; -use serde::{Deserialize, Serialize}; use std::net::IpAddr; +use serde::{Deserialize, Serialize}; + +use crate::error::RepoResult; + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct SettingsV1 { pub listen_address: IpAddr, diff --git a/mediarepo-daemon/mediarepo-core/src/type_keys.rs b/mediarepo-daemon/mediarepo-core/src/type_keys.rs index 5aa447d..8cebd2c 100644 --- a/mediarepo-daemon/mediarepo-core/src/type_keys.rs +++ b/mediarepo-daemon/mediarepo-core/src/type_keys.rs @@ -1,10 +1,12 @@ -use crate::settings::Settings; -use mediarepo_api::types::repo::SizeType; use std::collections::HashMap; use std::path::PathBuf; + +use mediarepo_api::types::repo::SizeType; use tokio_graceful_shutdown::SubsystemHandle; use typemap_rev::TypeMapKey; +use crate::settings::Settings; + pub struct SettingsKey; impl TypeMapKey for SettingsKey { diff --git a/mediarepo-daemon/mediarepo-core/src/utils.rs b/mediarepo-daemon/mediarepo-core/src/utils.rs index 609f7c2..4d901e8 100644 --- a/mediarepo-daemon/mediarepo-core/src/utils.rs +++ b/mediarepo-daemon/mediarepo-core/src/utils.rs @@ -1,9 +1,11 @@ -use crate::error::RepoResult; -use futures::future; use std::path::PathBuf; + +use futures::future; use tokio::fs::{self, OpenOptions}; use tokio::io::{AsyncBufReadExt, BufReader}; +use crate::error::RepoResult; + /// Parses a normalized tag into its two components of namespace and tag pub fn parse_namespace_and_tag(norm_tag: String) -> (Option, String) { norm_tag diff --git a/mediarepo-daemon/mediarepo-database/src/lib.rs b/mediarepo-daemon/mediarepo-database/src/lib.rs index ea5fb18..4e946a3 100644 --- a/mediarepo-daemon/mediarepo-database/src/lib.rs +++ b/mediarepo-daemon/mediarepo-database/src/lib.rs @@ -1,7 +1,9 @@ -use mediarepo_core::error::RepoDatabaseResult; +use std::time::Duration; + use sea_orm::{ConnectOptions, Database, DatabaseConnection}; use sqlx::migrate::MigrateDatabase; -use std::time::Duration; + +use mediarepo_core::error::RepoDatabaseResult; pub mod entities; pub mod queries; diff --git a/mediarepo-daemon/mediarepo-database/src/queries/tags.rs b/mediarepo-daemon/mediarepo-database/src/queries/tags.rs index d3726c5..c901031 100644 --- a/mediarepo-daemon/mediarepo-database/src/queries/tags.rs +++ b/mediarepo-daemon/mediarepo-database/src/queries/tags.rs @@ -1,11 +1,13 @@ -use mediarepo_core::error::RepoResult; -use sea_orm::DbBackend; -use sea_orm::FromQueryResult; -use sea_orm::{DatabaseConnection, Statement}; use std::collections::HashMap; use std::fmt::Display; use std::iter::FromIterator; +use sea_orm::{DatabaseConnection, Statement}; +use sea_orm::DbBackend; +use sea_orm::FromQueryResult; + +use mediarepo_core::error::RepoResult; + #[derive(Debug, FromQueryResult)] struct CIDNamespaceTag { cd_id: i64, diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/file/add.rs b/mediarepo-daemon/mediarepo-logic/src/dao/file/add.rs index c89fb64..6bb93c8 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/file/add.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/file/add.rs @@ -1,12 +1,14 @@ -use crate::dto::{AddFileDto, FileDto}; +use std::io::Cursor; + use chrono::{Local, NaiveDateTime}; -use mediarepo_core::error::RepoResult; -use mediarepo_database::entities::{content_descriptor, file, file_metadata}; use sea_orm::ActiveValue::Set; use sea_orm::{ActiveModelTrait, ConnectionTrait, DatabaseTransaction}; -use std::io::Cursor; + +use mediarepo_core::error::RepoResult; +use mediarepo_database::entities::{content_descriptor, file, file_metadata}; use crate::dao::file::FileDao; +use crate::dto::{AddFileDto, FileDto}; impl FileDao { #[tracing::instrument(level = "debug", skip(self))] diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/file/delete.rs b/mediarepo-daemon/mediarepo-logic/src/dao/file/delete.rs index 337b7e7..81c914a 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/file/delete.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/file/delete.rs @@ -1,11 +1,13 @@ -use crate::dao::file::{map_file_and_cd, FileDao}; -use crate::dto::FileDto; -use mediarepo_core::error::{RepoError, RepoResult}; +use sea_orm::ConnectionTrait; +use sea_orm::prelude::*; + +use mediarepo_core::error::{RepoResult}; use mediarepo_database::entities::{ content_descriptor, content_descriptor_tag, file, file_metadata, }; -use sea_orm::prelude::*; -use sea_orm::ConnectionTrait; + +use crate::dao::file::{FileDao}; +use crate::dto::FileDto; impl FileDao { #[tracing::instrument(level = "debug", skip(self))] diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/file/find.rs b/mediarepo-daemon/mediarepo-logic/src/dao/file/find.rs index 2156063..f96457d 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/file/find.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/file/find.rs @@ -1,14 +1,16 @@ -use crate::dao::file::{map_cd_and_file, FileDao}; -use crate::dto::FileDto; use chrono::NaiveDateTime; +use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, QuerySelect}; +use sea_orm::Condition; +use sea_orm::sea_query::{Alias, Expr, Query, SimpleExpr}; + use mediarepo_core::error::RepoResult; use mediarepo_database::entities::content_descriptor; use mediarepo_database::entities::content_descriptor_tag; use mediarepo_database::entities::file; use mediarepo_database::entities::file_metadata; -use sea_orm::sea_query::{Alias, Expr, Query, SimpleExpr}; -use sea_orm::Condition; -use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, QuerySelect}; + +use crate::dao::file::{FileDao, map_cd_and_file}; +use crate::dto::FileDto; macro_rules! apply_ordering_comparator { ($column:expr, $filter:expr) => { diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/file/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/file/mod.rs index 3a601cb..210cf50 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/file/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/file/mod.rs @@ -1,10 +1,12 @@ -use crate::dao::{DaoContext, DaoProvider}; -use crate::dto::{FileDto, FileMetadataDto, ThumbnailDto}; -use mediarepo_core::error::RepoResult; -use mediarepo_database::entities::{content_descriptor, file, file_metadata}; use sea_orm::prelude::*; use tokio::io::AsyncReadExt; +use mediarepo_core::error::RepoResult; +use mediarepo_database::entities::{content_descriptor, file, file_metadata}; + +use crate::dao::{DaoContext, DaoProvider}; +use crate::dto::{FileDto, FileMetadataDto, ThumbnailDto}; + pub mod add; pub mod delete; pub mod find; diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/file/update.rs b/mediarepo-daemon/mediarepo-logic/src/dao/file/update.rs index 69b4b72..c143964 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/file/update.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/file/update.rs @@ -1,18 +1,20 @@ -use crate::dto::{FileDto, FileMetadataDto, ThumbnailDto, UpdateFileDto, UpdateFileMetadataDto}; +use std::fmt::Debug; +use std::io::Cursor; +use std::str::FromStr; + +use sea_orm::prelude::*; +use sea_orm::ActiveValue::{Set, Unchanged}; +use sea_orm::{ConnectionTrait, NotSet}; + use mediarepo_core::error::{RepoError, RepoResult}; use mediarepo_core::fs::thumbnail_store::Dimensions; use mediarepo_core::thumbnailer; -use mediarepo_core::thumbnailer::{Thumbnail, ThumbnailSize}; +use mediarepo_core::thumbnailer::{ThumbnailSize}; use mediarepo_database::entities::{content_descriptor, file, file_metadata}; -use sea_orm::prelude::*; -use sea_orm::ActiveValue::{Set, Unchanged}; -use sea_orm::{ConnectionTrait, NotSet}; -use std::fmt::Debug; -use std::io::Cursor; -use std::str::FromStr; use crate::dao::file::FileDao; use crate::dao::opt_to_active_val; +use crate::dto::{FileDto, FileMetadataDto, ThumbnailDto, UpdateFileDto, UpdateFileMetadataDto}; impl FileDao { #[tracing::instrument(level = "debug", skip(self))] diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs index e69f477..1098fdb 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs @@ -1,12 +1,14 @@ -use crate::dao::{DaoContext, DaoProvider}; +use sea_orm::ActiveValue::Set; +use sea_orm::ConnectionTrait; +use sea_orm::prelude::*; + use mediarepo_core::content_descriptor::{ convert_v1_descriptor_to_v2, encode_content_descriptor, is_v1_content_descriptor, }; use mediarepo_core::error::RepoResult; use mediarepo_database::entities::content_descriptor; -use sea_orm::prelude::*; -use sea_orm::ActiveValue::Set; -use sea_orm::ConnectionTrait; + +use crate::dao::{DaoContext, DaoProvider}; pub struct JobDao { ctx: DaoContext, @@ -30,13 +32,13 @@ impl JobDao { tracing::info!("Converting content descriptors to v2 format..."); let mut converted_count = 0; - for mut cd in cds { + for cd in cds { if is_v1_content_descriptor(&cd.descriptor) { let trx = self.ctx.db.begin().await?; let src_cd = cd.descriptor; let dst_cd = convert_v1_descriptor_to_v2(&src_cd)?; - let active_model = content_descriptor::ActiveModel { + let _active_model = content_descriptor::ActiveModel { id: Set(cd.id), descriptor: Set(dst_cd.clone()), }; diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/mod.rs index 38101a4..f3cf620 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/mod.rs @@ -1,14 +1,16 @@ -pub mod file; -pub mod job; -pub mod repo; -pub mod tag; +use sea_orm::{ActiveValue, DatabaseConnection}; + +use mediarepo_core::fs::file_hash_store::FileHashStore; +use mediarepo_core::fs::thumbnail_store::ThumbnailStore; use crate::dao::file::FileDao; use crate::dao::job::JobDao; use crate::dao::tag::TagDao; -use mediarepo_core::fs::file_hash_store::FileHashStore; -use mediarepo_core::fs::thumbnail_store::ThumbnailStore; -use sea_orm::{ActiveValue, DatabaseConnection}; + +pub mod file; +pub mod job; +pub mod repo; +pub mod tag; #[derive(Clone)] pub struct DaoContext { diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs index af954b9..2954366 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs @@ -1,29 +1,30 @@ -use crate::dao::{DaoContext, DaoProvider}; -use crate::file_metadata::FileMetadata; -use crate::namespace::Namespace; -use crate::tag::Tag; -use chrono::{Local, NaiveDateTime}; -use mediarepo_core::content_descriptor::{ - convert_v1_descriptor_to_v2, encode_content_descriptor, is_v1_content_descriptor, -}; -use mediarepo_core::error::{RepoError, RepoResult}; -use mediarepo_core::fs::file_hash_store::FileHashStore; -use mediarepo_core::fs::thumbnail_store::{Dimensions, ThumbnailStore}; -use mediarepo_core::itertools::Itertools; -use mediarepo_core::thumbnailer::ThumbnailSize; -use mediarepo_core::utils::parse_namespace_and_tag; -use mediarepo_database::entities::content_descriptor; -use mediarepo_database::get_database; -use mediarepo_database::queries::analysis::{get_all_counts, Counts}; -use sea_orm::DatabaseConnection; use std::collections::{HashMap, HashSet}; use std::fmt::Debug; -use std::io::Cursor; + use std::iter::FromIterator; use std::path::PathBuf; -use std::str::FromStr; -use tokio::fs::OpenOptions; -use tokio::io::AsyncReadExt; + + + +use sea_orm::DatabaseConnection; + + + + +use mediarepo_core::error::{RepoResult}; +use mediarepo_core::fs::file_hash_store::FileHashStore; +use mediarepo_core::fs::thumbnail_store::{ThumbnailStore}; +use mediarepo_core::itertools::Itertools; + +use mediarepo_core::utils::parse_namespace_and_tag; + +use mediarepo_database::get_database; +use mediarepo_database::queries::analysis::{Counts, get_all_counts}; + +use crate::dao::{DaoContext, DaoProvider}; +use crate::file_metadata::FileMetadata; +use crate::namespace::Namespace; +use crate::tag::Tag; #[derive(Clone)] pub struct Repo { diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mappings.rs b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mappings.rs index e76f6c9..0e2f488 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mappings.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mappings.rs @@ -1,9 +1,11 @@ -use crate::dao::tag::TagDao; +use sea_orm::{ConnectionTrait, DatabaseTransaction}; +use sea_orm::ActiveValue::Set; +use sea_orm::prelude::*; + use mediarepo_core::error::RepoResult; use mediarepo_database::entities::content_descriptor_tag; -use sea_orm::prelude::*; -use sea_orm::ActiveValue::Set; -use sea_orm::{ConnectionTrait, DatabaseTransaction}; + +use crate::dao::tag::TagDao; impl TagDao { #[tracing::instrument(level = "debug", skip(self))] diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs index 37429e3..ac71383 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs @@ -1,12 +1,14 @@ -pub mod mappings; +use sea_orm::prelude::*; +use sea_orm::QuerySelect; +use sea_orm::{JoinType}; -use crate::dao::{DaoContext, DaoProvider}; -use crate::dto::TagDto; use mediarepo_core::error::RepoResult; use mediarepo_database::entities::{content_descriptor, content_descriptor_tag, namespace, tag}; -use sea_orm::prelude::*; -use sea_orm::QuerySelect; -use sea_orm::{DatabaseConnection, JoinType}; + +use crate::dao::{DaoContext, DaoProvider}; +use crate::dto::TagDto; + +pub mod mappings; pub struct TagDao { ctx: DaoContext, diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/file.rs b/mediarepo-daemon/mediarepo-logic/src/dto/file.rs index b926201..ced7cca 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dto/file.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dto/file.rs @@ -1,11 +1,13 @@ -use crate::dto::FileMetadataDto; use chrono::NaiveDateTime; + use mediarepo_core::content_descriptor::encode_content_descriptor; use mediarepo_core::mediarepo_api::types::files::FileStatus as ApiFileStatus; use mediarepo_database::entities::content_descriptor; use mediarepo_database::entities::file; use mediarepo_database::entities::file_metadata; +use crate::dto::FileMetadataDto; + #[derive(Clone, Debug)] pub struct FileDto { model: file::Model, diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/file_metadata.rs b/mediarepo-daemon/mediarepo-logic/src/dto/file_metadata.rs index e5cedc2..1bfb161 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dto/file_metadata.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dto/file_metadata.rs @@ -1,4 +1,5 @@ use chrono::NaiveDateTime; + use mediarepo_database::entities::file_metadata; #[derive(Clone, Debug)] diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dto/mod.rs index e14493c..0d7a962 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dto/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dto/mod.rs @@ -1,11 +1,12 @@ +pub use file::*; +pub use file_metadata::*; +pub use namespace::*; +pub use tag::*; +pub use thumbnail::*; + mod file; mod file_metadata; mod tag; mod namespace; mod thumbnail; -pub use file::*; -pub use file_metadata::*; -pub use tag::*; -pub use namespace::*; -pub use thumbnail::*; \ No newline at end of file diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/tag.rs b/mediarepo-daemon/mediarepo-logic/src/dto/tag.rs index 9a37e8a..df59a5f 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dto/tag.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dto/tag.rs @@ -1,5 +1,6 @@ -pub use mediarepo_database::entities::tag; pub use mediarepo_database::entities::namespace; +pub use mediarepo_database::entities::tag; + use crate::dto::NamespaceDto; #[derive(Clone, Debug)] @@ -12,7 +13,7 @@ impl TagDto { pub(crate) fn new(model: tag::Model, namespace_model: Option) -> Self { Self { model, - namespace: namespace_model.map(NamespaceDto::new) + namespace: namespace_model.map(NamespaceDto::new), } } @@ -27,4 +28,4 @@ impl TagDto { pub fn namespace(&self) -> Option<&NamespaceDto> { self.namespace.as_ref() } -} \ No newline at end of file +} diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs b/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs index 2ac6166..328302b 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dto/thumbnail.rs @@ -1,10 +1,12 @@ -use mediarepo_core::error::RepoResult; -use mediarepo_core::fs::thumbnail_store::Dimensions; use std::path::PathBuf; + use tokio::fs; use tokio::fs::{File, OpenOptions}; use tokio::io::BufReader; +use mediarepo_core::error::RepoResult; +use mediarepo_core::fs::thumbnail_store::Dimensions; + #[derive(Clone, Debug)] pub struct ThumbnailDto { path: PathBuf, diff --git a/mediarepo-daemon/mediarepo-logic/src/namespace.rs b/mediarepo-daemon/mediarepo-logic/src/namespace.rs index 1b67661..8f173c7 100644 --- a/mediarepo-daemon/mediarepo-logic/src/namespace.rs +++ b/mediarepo-daemon/mediarepo-logic/src/namespace.rs @@ -1,10 +1,12 @@ -use mediarepo_core::error::RepoResult; -use mediarepo_database::entities::namespace; -use sea_orm::prelude::*; +use std::fmt::Debug; + use sea_orm::{ Condition, ConnectionTrait, DatabaseBackend, DatabaseConnection, InsertResult, Set, Statement, }; -use std::fmt::Debug; +use sea_orm::prelude::*; + +use mediarepo_core::error::RepoResult; +use mediarepo_database::entities::namespace; #[derive(Clone)] pub struct Namespace { diff --git a/mediarepo-daemon/mediarepo-logic/src/tag.rs b/mediarepo-daemon/mediarepo-logic/src/tag.rs index 95b30d5..378c854 100644 --- a/mediarepo-daemon/mediarepo-logic/src/tag.rs +++ b/mediarepo-daemon/mediarepo-logic/src/tag.rs @@ -1,15 +1,16 @@ use std::fmt::Debug; +use sea_orm::{Condition, DatabaseBackend, DatabaseConnection, JoinType, Set, Statement}; +use sea_orm::{InsertResult, QuerySelect}; +use sea_orm::prelude::*; +use sea_orm::query::ConnectionTrait; +use sea_orm::sea_query::Expr; + use mediarepo_core::error::RepoResult; use mediarepo_database::entities::content_descriptor; use mediarepo_database::entities::content_descriptor_tag; use mediarepo_database::entities::namespace; use mediarepo_database::entities::tag; -use sea_orm::prelude::*; -use sea_orm::query::ConnectionTrait; -use sea_orm::sea_query::Expr; -use sea_orm::{Condition, DatabaseBackend, DatabaseConnection, JoinType, Set, Statement}; -use sea_orm::{InsertResult, QuerySelect}; use crate::namespace::Namespace; diff --git a/mediarepo-daemon/mediarepo-logic/src/type_keys.rs b/mediarepo-daemon/mediarepo-logic/src/type_keys.rs index f70f645..bd12d0c 100644 --- a/mediarepo-daemon/mediarepo-logic/src/type_keys.rs +++ b/mediarepo-daemon/mediarepo-logic/src/type_keys.rs @@ -1,7 +1,9 @@ -use crate::dao::repo::Repo; use std::sync::Arc; + use typemap_rev::TypeMapKey; +use crate::dao::repo::Repo; + pub struct RepoKey; impl TypeMapKey for RepoKey { diff --git a/mediarepo-daemon/mediarepo-socket/src/lib.rs b/mediarepo-daemon/mediarepo-socket/src/lib.rs index 70082ca..4979eda 100644 --- a/mediarepo-daemon/mediarepo-socket/src/lib.rs +++ b/mediarepo-daemon/mediarepo-socket/src/lib.rs @@ -1,3 +1,10 @@ +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; + +use tokio::net::TcpListener; +use tokio::task::JoinHandle; + use mediarepo_core::bromine::prelude::*; use mediarepo_core::error::{RepoError, RepoResult}; use mediarepo_core::mediarepo_api::types::misc::InfoResponse; @@ -6,11 +13,6 @@ use mediarepo_core::tokio_graceful_shutdown::SubsystemHandle; use mediarepo_core::type_keys::{RepoPathKey, SettingsKey, SizeMetadataKey, SubsystemKey}; use mediarepo_logic::dao::repo::Repo; use mediarepo_logic::type_keys::RepoKey; -use std::net::SocketAddr; -use std::path::PathBuf; -use std::sync::Arc; -use tokio::net::TcpListener; -use tokio::task::JoinHandle; mod from_model; mod namespaces; diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs index 973daf4..c379b58 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs @@ -1,10 +1,5 @@ -mod searching; -mod sorting; +use tokio::io::AsyncReadExt; -use crate::from_model::FromModel; -use crate::namespaces::files::searching::find_files_for_filters; -use crate::namespaces::files::sorting::sort_files_by_properties; -use crate::utils::{cd_by_identifier, file_by_identifier, get_repo_from_context}; use mediarepo_core::bromine::prelude::*; use mediarepo_core::content_descriptor::{create_content_descriptor, encode_content_descriptor}; use mediarepo_core::error::RepoError; @@ -21,7 +16,14 @@ use mediarepo_core::thumbnailer::ThumbnailSize; use mediarepo_core::utils::parse_namespace_and_tag; use mediarepo_logic::dao::DaoProvider; use mediarepo_logic::dto::{AddFileDto, UpdateFileDto, UpdateFileMetadataDto}; -use tokio::io::AsyncReadExt; + +use crate::from_model::FromModel; +use crate::namespaces::files::searching::find_files_for_filters; +use crate::namespaces::files::sorting::sort_files_by_properties; +use crate::utils::{cd_by_identifier, file_by_identifier, get_repo_from_context}; + +mod searching; +mod sorting; pub struct FilesNamespace; diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs index 9ffa99e..ab6d66b 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs @@ -1,15 +1,16 @@ +use std::collections::HashMap; + use mediarepo_core::content_descriptor::decode_content_descriptor; use mediarepo_core::error::RepoResult; use mediarepo_core::mediarepo_api::types::files::FileStatus as ApiFileStatus; use mediarepo_core::mediarepo_api::types::filtering::{ FilterExpression, FilterQuery, PropertyQuery, TagQuery, ValueComparator, }; -use mediarepo_logic::dao::file::find::NegatableComparator::{Is, IsNot}; +use mediarepo_logic::dao::DaoProvider; use mediarepo_logic::dao::file::find::{FilterFileProperty, FilterProperty, OrderingComparator}; +use mediarepo_logic::dao::file::find::NegatableComparator::{Is, IsNot}; use mediarepo_logic::dao::repo::Repo; -use mediarepo_logic::dao::DaoProvider; use mediarepo_logic::dto::{FileDto, FileStatus}; -use std::collections::HashMap; #[tracing::instrument(level = "debug", skip(repo))] pub async fn find_files_for_filters( diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs index e80bbe1..5f56c5d 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs @@ -1,18 +1,20 @@ +use std::cmp::Ordering; +use std::collections::HashMap; +use std::iter::FromIterator; + use chrono::NaiveDateTime; use compare::Compare; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + use mediarepo_core::error::RepoResult; use mediarepo_core::mediarepo_api::types::filtering::{SortDirection, SortKey}; use mediarepo_database::queries::tags::{ get_cids_with_namespaced_tags, get_content_descriptors_with_tag_count, }; -use mediarepo_logic::dao::repo::Repo; use mediarepo_logic::dao::DaoProvider; +use mediarepo_logic::dao::repo::Repo; use mediarepo_logic::dto::{FileDto, FileMetadataDto}; -use mediarepo_logic::file_metadata::FileMetadata; -use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; -use std::cmp::Ordering; -use std::collections::HashMap; -use std::iter::FromIterator; + pub struct FileSortContext { name: Option, diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs index 83231bf..208c5ce 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs @@ -1,4 +1,3 @@ -use crate::utils::{calculate_size, get_repo_from_context}; use mediarepo_core::bromine::prelude::*; use mediarepo_core::error::RepoResult; use mediarepo_core::mediarepo_api::types::jobs::{JobType, RunJobRequest}; @@ -6,6 +5,8 @@ use mediarepo_core::mediarepo_api::types::repo::SizeType; use mediarepo_core::type_keys::SizeMetadataKey; use mediarepo_logic::dao::DaoProvider; +use crate::utils::{calculate_size, get_repo_from_context}; + pub struct JobsNamespace; impl NamespaceProvider for JobsNamespace { diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs index 29ad8c8..269dcd4 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs @@ -1,5 +1,5 @@ -use crate::from_model::FromModel; -use crate::utils::{file_by_identifier, get_repo_from_context}; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; + use mediarepo_core::bromine::prelude::*; use mediarepo_core::content_descriptor::decode_content_descriptor; use mediarepo_core::mediarepo_api::types::files::{GetFileTagsRequest, GetFilesTagsRequest}; @@ -7,7 +7,9 @@ use mediarepo_core::mediarepo_api::types::tags::{ ChangeFileTagsRequest, NamespaceResponse, TagResponse, }; use mediarepo_logic::dao::DaoProvider; -use rayon::iter::{IntoParallelIterator, ParallelIterator}; + +use crate::from_model::FromModel; +use crate::utils::{file_by_identifier, get_repo_from_context}; pub struct TagsNamespace; diff --git a/mediarepo-daemon/mediarepo-socket/src/utils.rs b/mediarepo-daemon/mediarepo-socket/src/utils.rs index 3769fb1..5e48cdc 100644 --- a/mediarepo-daemon/mediarepo-socket/src/utils.rs +++ b/mediarepo-daemon/mediarepo-socket/src/utils.rs @@ -1,3 +1,7 @@ +use std::sync::Arc; + +use tokio::fs; + use mediarepo_core::bromine::ipc::context::Context; use mediarepo_core::content_descriptor::decode_content_descriptor; use mediarepo_core::error::{RepoError, RepoResult}; @@ -5,12 +9,10 @@ use mediarepo_core::mediarepo_api::types::identifier::FileIdentifier; use mediarepo_core::mediarepo_api::types::repo::SizeType; use mediarepo_core::type_keys::{RepoPathKey, SettingsKey}; use mediarepo_core::utils::get_folder_size; -use mediarepo_logic::dao::repo::Repo; use mediarepo_logic::dao::DaoProvider; +use mediarepo_logic::dao::repo::Repo; use mediarepo_logic::dto::FileDto; use mediarepo_logic::type_keys::RepoKey; -use std::sync::Arc; -use tokio::fs; pub async fn get_repo_from_context(ctx: &Context) -> Arc { let data = ctx.data.read().await; diff --git a/mediarepo-daemon/src/logging.rs b/mediarepo-daemon/src/logging.rs index 9e1692a..dfb5846 100644 --- a/mediarepo-daemon/src/logging.rs +++ b/mediarepo-daemon/src/logging.rs @@ -1,21 +1,22 @@ -use console_subscriber::ConsoleLayer; -use rolling_file::RollingConditionBasic; use std::fs; use std::path::PathBuf; -use mediarepo_core::settings::LoggingSettings; +use console_subscriber::ConsoleLayer; +use rolling_file::RollingConditionBasic; use tracing::Level; use tracing_appender::non_blocking::{NonBlocking, WorkerGuard}; use tracing_flame::FlameLayer; use tracing_log::LogTracer; -use tracing_subscriber::filter::{self, Targets}; -use tracing_subscriber::fmt::format::FmtSpan; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{ fmt::{self}, Layer, Registry, }; +use tracing_subscriber::filter::{self, Targets}; +use tracing_subscriber::fmt::format::FmtSpan; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; + +use mediarepo_core::settings::LoggingSettings; #[allow(dyn_drop)] pub type DropGuard = Box; diff --git a/mediarepo-daemon/src/main.rs b/mediarepo-daemon/src/main.rs index 9de1330..757dffe 100644 --- a/mediarepo-daemon/src/main.rs +++ b/mediarepo-daemon/src/main.rs @@ -1,7 +1,10 @@ +use std::env; use std::path::PathBuf; +use std::time::Duration; use structopt::StructOpt; use tokio::fs; +use tokio::io::AsyncWriteExt; use tokio::runtime; use tokio::runtime::Runtime; @@ -11,9 +14,6 @@ use mediarepo_core::settings::{PathSettings, Settings}; use mediarepo_core::tokio_graceful_shutdown::{SubsystemHandle, Toplevel}; use mediarepo_logic::dao::repo::Repo; use mediarepo_socket::start_tcp_server; -use std::env; -use std::time::Duration; -use tokio::io::AsyncWriteExt; use crate::utils::{create_paths_for_repo, get_repo, load_settings}; diff --git a/mediarepo-daemon/src/utils.rs b/mediarepo-daemon/src/utils.rs index 8c96c9a..130947a 100644 --- a/mediarepo-daemon/src/utils.rs +++ b/mediarepo-daemon/src/utils.rs @@ -1,9 +1,11 @@ +use std::path::PathBuf; + +use tokio::fs; + use mediarepo_core::error::RepoResult; -use mediarepo_core::settings::v1::SettingsV1; use mediarepo_core::settings::{PathSettings, Settings}; +use mediarepo_core::settings::v1::SettingsV1; use mediarepo_logic::dao::repo::Repo; -use std::path::PathBuf; -use tokio::fs; /// Loads the settings from a toml path pub fn load_settings(root_path: &PathBuf) -> RepoResult { From 744475dd1e533ececcfaa47ca426b99f3a181cf0 Mon Sep 17 00:00:00 2001 From: trivernis Date: Sat, 29 Jan 2022 20:53:57 +0100 Subject: [PATCH 07/12] Move tag and namespace retrieval to dao Signed-off-by: trivernis --- .../mediarepo-logic/src/dao/repo/mod.rs | 30 ++-------------- .../mediarepo-logic/src/dao/tag/mod.rs | 35 +++++++++++++++++-- .../mediarepo-socket/src/from_model.rs | 11 +++++- .../mediarepo-socket/src/namespaces/tags.rs | 6 ++-- 4 files changed, 49 insertions(+), 33 deletions(-) diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs index 2954366..e0713ab 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs @@ -4,25 +4,19 @@ use std::fmt::Debug; use std::iter::FromIterator; use std::path::PathBuf; - - use sea_orm::DatabaseConnection; - - - -use mediarepo_core::error::{RepoResult}; +use mediarepo_core::error::RepoResult; use mediarepo_core::fs::file_hash_store::FileHashStore; -use mediarepo_core::fs::thumbnail_store::{ThumbnailStore}; +use mediarepo_core::fs::thumbnail_store::ThumbnailStore; use mediarepo_core::itertools::Itertools; use mediarepo_core::utils::parse_namespace_and_tag; use mediarepo_database::get_database; -use mediarepo_database::queries::analysis::{Counts, get_all_counts}; +use mediarepo_database::queries::analysis::{get_all_counts, Counts}; use crate::dao::{DaoContext, DaoProvider}; -use crate::file_metadata::FileMetadata; use crate::namespace::Namespace; use crate::tag::Tag; @@ -72,24 +66,6 @@ impl Repo { &self.db } - /// Returns all file metadata entries for the given file ids - #[tracing::instrument(level = "debug", skip(self))] - pub async fn get_file_metadata_for_ids(&self, ids: Vec) -> RepoResult> { - FileMetadata::all_by_ids(self.db.clone(), ids).await - } - - /// Returns all tags stored in the database - #[tracing::instrument(level = "debug", skip(self))] - pub async fn tags(&self) -> RepoResult> { - Tag::all(self.db.clone()).await - } - - /// Returns all namespaces stored in the database - #[tracing::instrument(level = "debug", skip(self))] - pub async fn namespaces(&self) -> RepoResult> { - Namespace::all(self.db.clone()).await - } - /// Converts a list of tag names to tag ids #[tracing::instrument(level = "debug", skip(self))] pub async fn tag_names_to_ids(&self, tags: Vec) -> RepoResult> { diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs index ac71383..2f3dfa2 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs @@ -1,12 +1,12 @@ use sea_orm::prelude::*; +use sea_orm::JoinType; use sea_orm::QuerySelect; -use sea_orm::{JoinType}; use mediarepo_core::error::RepoResult; use mediarepo_database::entities::{content_descriptor, content_descriptor_tag, namespace, tag}; use crate::dao::{DaoContext, DaoProvider}; -use crate::dto::TagDto; +use crate::dto::{NamespaceDto, TagDto}; pub mod mappings; @@ -25,6 +25,31 @@ impl TagDao { Self { ctx } } + #[tracing::instrument(level = "debug", skip(self))] + pub async fn all(&self) -> RepoResult> { + let tags = tag::Entity::find() + .find_also_related(namespace::Entity) + .all(&self.ctx.db) + .await? + .into_iter() + .map(map_tag_dto) + .collect(); + + Ok(tags) + } + + #[tracing::instrument(level = "debug", skip(self))] + pub async fn all_namespaces(&self) -> RepoResult> { + let namespaces = namespace::Entity::find() + .all(&self.ctx.db) + .await? + .into_iter() + .map(NamespaceDto::new) + .collect(); + + Ok(namespaces) + } + #[tracing::instrument(level = "debug", skip(self))] pub async fn tags_for_cd(&self, cd_id: i64) -> RepoResult> { let tags = tag::Entity::find() @@ -41,9 +66,13 @@ impl TagDao { .all(&self.ctx.db) .await? .into_iter() - .map(|(t, n)| TagDto::new(t, n)) + .map(map_tag_dto) .collect(); Ok(tags) } } + +fn map_tag_dto(result: (tag::Model, Option)) -> TagDto { + TagDto::new(result.0, result.1) +} diff --git a/mediarepo-daemon/mediarepo-socket/src/from_model.rs b/mediarepo-daemon/mediarepo-socket/src/from_model.rs index ec96095..eaa6e7e 100644 --- a/mediarepo-daemon/mediarepo-socket/src/from_model.rs +++ b/mediarepo-daemon/mediarepo-socket/src/from_model.rs @@ -3,7 +3,7 @@ use mediarepo_core::mediarepo_api::types::files::{ }; use mediarepo_core::mediarepo_api::types::tags::{NamespaceResponse, TagResponse}; use mediarepo_logic::dto::{ - FileDto, FileMetadataDto, FileStatus as FileStatusModel, TagDto, ThumbnailDto, + FileDto, FileMetadataDto, FileStatus as FileStatusModel, NamespaceDto, TagDto, ThumbnailDto, }; use mediarepo_logic::file_metadata::FileMetadata; use mediarepo_logic::namespace::Namespace; @@ -99,3 +99,12 @@ impl FromModel for NamespaceResponse { } } } + +impl FromModel for NamespaceResponse { + fn from_model(model: NamespaceDto) -> Self { + Self { + id: model.id(), + name: model.name().to_owned(), + } + } +} diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs index 269dcd4..1bfde4c 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs @@ -36,7 +36,8 @@ impl TagsNamespace { async fn all_tags(ctx: &Context, _event: Event) -> IPCResult<()> { let repo = get_repo_from_context(ctx).await; let tags: Vec = repo - .tags() + .tag() + .all() .await? .into_iter() .map(TagResponse::from_model) @@ -51,7 +52,8 @@ impl TagsNamespace { async fn all_namespaces(ctx: &Context, _event: Event) -> IPCResult<()> { let repo = get_repo_from_context(ctx).await; let namespaces: Vec = repo - .namespaces() + .tag() + .all_namespaces() .await? .into_iter() .map(NamespaceResponse::from_model) From 31addcda87cbeb94ec1a673b12ce49cc9b15a980 Mon Sep 17 00:00:00 2001 From: trivernis Date: Sun, 30 Jan 2022 13:56:54 +0100 Subject: [PATCH 08/12] Move tag adding and retrieval by name to dao Signed-off-by: trivernis --- .../mediarepo-logic/src/dao/repo/mod.rs | 67 +--------- .../mediarepo-logic/src/dao/tag/add.rs | 120 ++++++++++++++++++ .../mediarepo-logic/src/dao/tag/by_name.rs | 63 +++++++++ .../mediarepo-logic/src/dao/tag/mod.rs | 27 ++++ .../mediarepo-logic/src/dto/tag.rs | 31 +++++ .../src/namespaces/files/mod.rs | 10 +- .../src/namespaces/files/searching.rs | 6 +- 7 files changed, 253 insertions(+), 71 deletions(-) create mode 100644 mediarepo-daemon/mediarepo-logic/src/dao/tag/add.rs create mode 100644 mediarepo-daemon/mediarepo-logic/src/dao/tag/by_name.rs diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs index e0713ab..a329f2a 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs @@ -16,6 +16,7 @@ use mediarepo_core::utils::parse_namespace_and_tag; use mediarepo_database::get_database; use mediarepo_database::queries::analysis::{get_all_counts, Counts}; +use crate::dao::tag::by_name::TagByNameQuery; use crate::dao::{DaoContext, DaoProvider}; use crate::namespace::Namespace; use crate::tag::Tag; @@ -66,78 +67,12 @@ impl Repo { &self.db } - /// Converts a list of tag names to tag ids - #[tracing::instrument(level = "debug", skip(self))] - pub async fn tag_names_to_ids(&self, tags: Vec) -> RepoResult> { - let parsed_tags = tags - .iter() - .map(|tag| parse_namespace_and_tag(tag.clone())) - .unique() - .collect(); - - let db_tags = self.tags_by_names(parsed_tags).await?; - let tag_map: HashMap = - HashMap::from_iter(db_tags.into_iter().map(|t| (t.normalized_name(), t.id()))); - - Ok(tag_map) - } - - /// Finds all tags by name - #[tracing::instrument(level = "debug", skip(self))] - pub async fn tags_by_names(&self, tags: Vec<(Option, String)>) -> RepoResult> { - Tag::all_by_name(self.db.clone(), tags).await - } - /// Finds all tags that are assigned to the given list of hashes #[tracing::instrument(level = "debug", skip_all)] pub async fn find_tags_for_file_identifiers(&self, cds: Vec>) -> RepoResult> { Tag::for_cd_list(self.db.clone(), cds).await } - /// Adds all tags that are not in the database to the database and returns the ones already existing as well - #[tracing::instrument(level = "debug", skip_all)] - pub async fn add_all_tags(&self, tags: Vec<(Option, String)>) -> RepoResult> { - let mut tags_to_add = tags.into_iter().unique().collect_vec(); - let mut namespaces_to_add = tags_to_add - .iter() - .filter_map(|(namespace, _)| namespace.clone()) - .unique() - .collect_vec(); - - let mut existing_namespaces = - Namespace::all_by_name(self.db.clone(), namespaces_to_add.clone()).await?; - { - let existing_namespaces_set = existing_namespaces - .iter() - .map(|n| n.name().clone()) - .collect::>(); - namespaces_to_add.retain(|namespace| !existing_namespaces_set.contains(namespace)); - } - existing_namespaces - .append(&mut Namespace::add_all(self.db.clone(), namespaces_to_add).await?); - - let mut existing_tags = self.tags_by_names(tags_to_add.clone()).await?; - { - let existing_tags_set = existing_tags - .iter() - .map(|t| (t.namespace().map(|n| n.name().clone()), t.name().clone())) - .collect::, String)>>(); - - tags_to_add.retain(|t| !existing_tags_set.contains(t)); - } - let namespace_map = existing_namespaces - .into_iter() - .map(|namespace| (namespace.name().clone(), namespace.id())) - .collect::>(); - let tags_to_add = tags_to_add - .into_iter() - .map(|(nsp, name)| (nsp.and_then(|n| namespace_map.get(&n)).map(|i| *i), name)) - .collect_vec(); - existing_tags.append(&mut Tag::add_all(self.db.clone(), tags_to_add).await?); - - Ok(existing_tags) - } - /// Adds or finds a tag #[tracing::instrument(level = "debug", skip(self))] pub async fn add_or_find_tag(&self, tag: S) -> RepoResult { diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/tag/add.rs b/mediarepo-daemon/mediarepo-logic/src/dao/tag/add.rs new file mode 100644 index 0000000..f52da09 --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dao/tag/add.rs @@ -0,0 +1,120 @@ +use crate::dao::tag::{map_tag_dto, TagDao}; +use crate::dto::{AddTagDto, NamespaceDto, TagDto}; +use mediarepo_core::error::RepoResult; +use mediarepo_database::entities::{namespace, tag}; +use sea_orm::prelude::*; +use sea_orm::ActiveValue::Set; +use sea_orm::{Condition, ConnectionTrait, DatabaseTransaction}; +use std::collections::HashMap; +use std::iter::FromIterator; + +impl TagDao { + #[tracing::instrument(level = "debug", skip(self))] + pub async fn add_all(&self, mut tags: Vec) -> RepoResult> { + let namespaces = tags.iter().filter_map(|t| t.namespace.clone()).collect(); + let trx = self.ctx.db.begin().await?; + let existing_tags = tags_by_name(&trx, tags.clone()).await?; + + if existing_tags.len() == tags.len() { + return Ok(existing_tags); + } + let existing_tag_map: HashMap = + HashMap::from_iter(existing_tags.into_iter().map(|t| (t.normalized_name(), t))); + + let namespace_map = add_or_get_all_namespaces(&trx, namespaces).await?; + tags.retain(|dto| !existing_tag_map.contains_key(&dto.normalized_name())); + let tag_models: Vec = tags + .iter() + .map(|t| tag::ActiveModel { + name: Set(t.name.to_owned()), + namespace_id: Set(t + .namespace + .as_ref() + .and_then(|n| namespace_map.get(n)) + .map(|n| n.id())), + ..Default::default() + }) + .collect(); + tag::Entity::insert_many(tag_models).exec(&trx).await?; + let mut tag_dtos = tags_by_name(&trx, tags).await?; + trx.commit().await?; + tag_dtos.append(&mut existing_tag_map.into_iter().map(|(_, dto)| dto).collect()); + + Ok(tag_dtos) + } +} + +async fn add_or_get_all_namespaces( + trx: &DatabaseTransaction, + mut namespaces: Vec, +) -> RepoResult> { + let existing_namespaces = namespaces_by_name(trx, namespaces.clone()).await?; + let mut namespace_map = HashMap::from_iter( + existing_namespaces + .into_iter() + .map(|nsp| (nsp.name().to_owned(), nsp)), + ); + if namespaces.len() == namespace_map.len() { + return Ok(namespace_map); + } + namespaces.retain(|nsp| !namespace_map.contains_key(nsp)); + let namespace_models: Vec = namespaces + .iter() + .map(|nsp| namespace::ActiveModel { + name: Set(nsp.to_owned()), + ..Default::default() + }) + .collect(); + namespace::Entity::insert_many(namespace_models) + .exec(trx) + .await?; + let additional_namespaces = namespaces_by_name(trx, namespaces.clone()).await?; + + for nsp in additional_namespaces { + namespace_map.insert(nsp.name().to_owned(), nsp); + } + + Ok(namespace_map) +} + +async fn namespaces_by_name( + trx: &DatabaseTransaction, + names: Vec, +) -> RepoResult> { + let namespaces: Vec = namespace::Entity::find() + .filter(namespace::Column::Name.is_in(names)) + .all(trx) + .await? + .into_iter() + .map(NamespaceDto::new) + .collect(); + + Ok(namespaces) +} + +async fn tags_by_name(trx: &DatabaseTransaction, tags: Vec) -> RepoResult> { + let condition = tags + .into_iter() + .map(build_tag_condition) + .fold(Condition::any(), Condition::add); + let tags = tag::Entity::find() + .find_also_related(namespace::Entity) + .filter(condition) + .all(trx) + .await? + .into_iter() + .map(map_tag_dto) + .collect(); + + Ok(tags) +} + +fn build_tag_condition(tag: AddTagDto) -> Condition { + if let Some(namespace) = tag.namespace { + Condition::all() + .add(tag::Column::Name.eq(tag.name)) + .add(namespace::Column::Name.eq(namespace)) + } else { + Condition::all().add(tag::Column::Name.eq(tag.name)) + } +} diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/tag/by_name.rs b/mediarepo-daemon/mediarepo-logic/src/dao/tag/by_name.rs new file mode 100644 index 0000000..de7e4c9 --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dao/tag/by_name.rs @@ -0,0 +1,63 @@ +use crate::dao::tag::{map_tag_dto, TagDao}; +use crate::dto::TagDto; +use mediarepo_core::error::RepoResult; +use mediarepo_database::entities::{namespace, tag}; +use sea_orm::prelude::*; +use sea_orm::sea_query::Expr; +use sea_orm::Condition; + +#[derive(Clone, Debug)] +pub struct TagByNameQuery { + pub namespace: Option, + pub name: String, +} + +impl TagDao { + /// Filters all tags by names + /// wildcards are supported + #[tracing::instrument(level = "debug", skip(self))] + pub async fn all_by_name(&self, names: Vec) -> RepoResult> { + let mut condition_count = 0; + let condition = names + .into_iter() + .filter_map(name_query_to_condition) + .inspect(|_| condition_count += 1) + .fold(Condition::any(), Condition::add); + if condition_count == 0 { + return Ok(vec![]); + } + + let tags = tag::Entity::find() + .find_also_related(namespace::Entity) + .filter(condition) + .all(&self.ctx.db) + .await? + .into_iter() + .map(map_tag_dto) + .collect(); + + Ok(tags) + } +} + +fn name_query_to_condition(query: TagByNameQuery) -> Option { + let TagByNameQuery { namespace, name } = query; + let mut condition = Condition::all(); + + if !name.ends_with('*') { + condition = condition.add(tag::Column::Name.eq(name)) + } else if name.len() > 1 { + condition = + condition.add(tag::Column::Name.like(&*format!("{}%", name.trim_end_matches("*")))) + } else if namespace.is_none() { + return None; + } + + condition = if let Some(namespace) = namespace { + condition.add(namespace::Column::Name.eq(namespace)) + } else { + condition.add(Expr::tbl(tag::Entity, tag::Column::NamespaceId).is_null()) + }; + + Some(condition) +} diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs index 2f3dfa2..6e08e01 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs @@ -1,13 +1,20 @@ use sea_orm::prelude::*; use sea_orm::JoinType; use sea_orm::QuerySelect; +use std::collections::HashMap; +use std::iter::FromIterator; use mediarepo_core::error::RepoResult; +use mediarepo_core::mediarepo_api::types::filtering::TagQuery; +use mediarepo_core::utils::parse_namespace_and_tag; use mediarepo_database::entities::{content_descriptor, content_descriptor_tag, namespace, tag}; +use crate::dao::tag::by_name::TagByNameQuery; use crate::dao::{DaoContext, DaoProvider}; use crate::dto::{NamespaceDto, TagDto}; +pub mod add; +pub mod by_name; pub mod mappings; pub struct TagDao { @@ -71,6 +78,26 @@ impl TagDao { Ok(tags) } + + /// Returns a map mapping tag names to ids + #[tracing::instrument(level = "debug", skip(self))] + pub async fn normalized_tags_to_ids( + &self, + names: Vec, + ) -> RepoResult> { + let queries = names + .into_iter() + .map(parse_namespace_and_tag) + .map(|(namespace, name)| TagByNameQuery { namespace, name }) + .collect(); + let tags = self.all_by_name(queries).await?; + let tag_map = HashMap::from_iter( + tags.into_iter() + .map(|tag| (tag.normalized_name(), tag.id())), + ); + + Ok(tag_map) + } } fn map_tag_dto(result: (tag::Model, Option)) -> TagDto { diff --git a/mediarepo-daemon/mediarepo-logic/src/dto/tag.rs b/mediarepo-daemon/mediarepo-logic/src/dto/tag.rs index df59a5f..d45dcd4 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dto/tag.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dto/tag.rs @@ -28,4 +28,35 @@ impl TagDto { pub fn namespace(&self) -> Option<&NamespaceDto> { self.namespace.as_ref() } + + /// Returns the normalized name of the tag (namespace:tag) + pub fn normalized_name(&self) -> String { + if let Some(namespace) = &self.namespace { + format!("{}:{}", namespace.name(), self.name()) + } else { + self.name().to_owned() + } + } +} + +#[derive(Clone, Debug)] +pub struct AddTagDto { + pub namespace: Option, + pub name: String, +} + +impl AddTagDto { + pub fn from_tuple(tuple: (Option, String)) -> Self { + let (namespace, name) = tuple; + Self { namespace, name } + } + + /// Returns the normalized name of the tag (namespace:tag) + pub fn normalized_name(&self) -> String { + if let Some(namespace) = &self.namespace { + format!("{}:{}", namespace, &self.name) + } else { + self.name.to_owned() + } + } } diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs index c379b58..644a6f0 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/mod.rs @@ -15,7 +15,7 @@ use mediarepo_core::mediarepo_api::types::identifier::FileIdentifier; use mediarepo_core::thumbnailer::ThumbnailSize; use mediarepo_core::utils::parse_namespace_and_tag; use mediarepo_logic::dao::DaoProvider; -use mediarepo_logic::dto::{AddFileDto, UpdateFileDto, UpdateFileMetadataDto}; +use mediarepo_logic::dto::{AddFileDto, AddTagDto, UpdateFileDto, UpdateFileMetadataDto}; use crate::from_model::FromModel; use crate::namespaces::files::searching::find_files_for_filters; @@ -171,7 +171,13 @@ impl FilesNamespace { }; let tags = repo - .add_all_tags(tags.into_iter().map(parse_namespace_and_tag).collect()) + .tag() + .add_all( + tags.into_iter() + .map(parse_namespace_and_tag) + .map(AddTagDto::from_tuple) + .collect(), + ) .await?; let tag_ids: Vec = tags.into_iter().map(|t| t.id()).unique().collect(); repo.tag() diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs index ab6d66b..fb78e4d 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files/searching.rs @@ -6,10 +6,10 @@ use mediarepo_core::mediarepo_api::types::files::FileStatus as ApiFileStatus; use mediarepo_core::mediarepo_api::types::filtering::{ FilterExpression, FilterQuery, PropertyQuery, TagQuery, ValueComparator, }; -use mediarepo_logic::dao::DaoProvider; -use mediarepo_logic::dao::file::find::{FilterFileProperty, FilterProperty, OrderingComparator}; use mediarepo_logic::dao::file::find::NegatableComparator::{Is, IsNot}; +use mediarepo_logic::dao::file::find::{FilterFileProperty, FilterProperty, OrderingComparator}; use mediarepo_logic::dao::repo::Repo; +use mediarepo_logic::dao::DaoProvider; use mediarepo_logic::dto::{FileDto, FileStatus}; #[tracing::instrument(level = "debug", skip(repo))] @@ -18,7 +18,7 @@ pub async fn find_files_for_filters( expressions: Vec, ) -> RepoResult> { let tag_names = get_tag_names_from_expressions(&expressions); - let tag_id_map = repo.tag_names_to_ids(tag_names).await?; + let tag_id_map = repo.tag().normalized_tags_to_ids(tag_names).await?; let filters = build_filters_from_expressions(expressions, &tag_id_map); repo.file().find(filters).await From e9641e604e82249f7526b9509a795f142d105cf4 Mon Sep 17 00:00:00 2001 From: trivernis Date: Sun, 30 Jan 2022 14:10:24 +0100 Subject: [PATCH 09/12] Improve order of execution for adding tags Signed-off-by: trivernis --- .../mediarepo-logic/src/dao/repo/mod.rs | 8 +++---- .../mediarepo-logic/src/dao/tag/add.rs | 21 +++++++++++++++++-- .../mediarepo-logic/src/dao/tag/mod.rs | 2 +- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs index a329f2a..89945af 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs @@ -1,7 +1,7 @@ -use std::collections::{HashMap, HashSet}; + use std::fmt::Debug; -use std::iter::FromIterator; + use std::path::PathBuf; use sea_orm::DatabaseConnection; @@ -9,14 +9,14 @@ use sea_orm::DatabaseConnection; use mediarepo_core::error::RepoResult; use mediarepo_core::fs::file_hash_store::FileHashStore; use mediarepo_core::fs::thumbnail_store::ThumbnailStore; -use mediarepo_core::itertools::Itertools; + use mediarepo_core::utils::parse_namespace_and_tag; use mediarepo_database::get_database; use mediarepo_database::queries::analysis::{get_all_counts, Counts}; -use crate::dao::tag::by_name::TagByNameQuery; + use crate::dao::{DaoContext, DaoProvider}; use crate::namespace::Namespace; use crate::tag::Tag; diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/tag/add.rs b/mediarepo-daemon/mediarepo-logic/src/dao/tag/add.rs index f52da09..602a7ea 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/tag/add.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/tag/add.rs @@ -21,8 +21,13 @@ impl TagDao { let existing_tag_map: HashMap = HashMap::from_iter(existing_tags.into_iter().map(|t| (t.normalized_name(), t))); - let namespace_map = add_or_get_all_namespaces(&trx, namespaces).await?; tags.retain(|dto| !existing_tag_map.contains_key(&dto.normalized_name())); + let namespace_map = add_or_get_all_namespaces(&trx, namespaces).await?; + + if tags.is_empty() { + return Ok(existing_tag_map.into_values().collect()); + } + let tag_models: Vec = tags .iter() .map(|t| tag::ActiveModel { @@ -38,7 +43,7 @@ impl TagDao { tag::Entity::insert_many(tag_models).exec(&trx).await?; let mut tag_dtos = tags_by_name(&trx, tags).await?; trx.commit().await?; - tag_dtos.append(&mut existing_tag_map.into_iter().map(|(_, dto)| dto).collect()); + tag_dtos.append(&mut existing_tag_map.into_values().collect()); Ok(tag_dtos) } @@ -48,6 +53,9 @@ async fn add_or_get_all_namespaces( trx: &DatabaseTransaction, mut namespaces: Vec, ) -> RepoResult> { + if namespaces.is_empty() { + return Ok(HashMap::with_capacity(0)); + } let existing_namespaces = namespaces_by_name(trx, namespaces.clone()).await?; let mut namespace_map = HashMap::from_iter( existing_namespaces @@ -58,6 +66,9 @@ async fn add_or_get_all_namespaces( return Ok(namespace_map); } namespaces.retain(|nsp| !namespace_map.contains_key(nsp)); + if namespaces.is_empty() { + return Ok(namespace_map); + } let namespace_models: Vec = namespaces .iter() .map(|nsp| namespace::ActiveModel { @@ -81,6 +92,9 @@ async fn namespaces_by_name( trx: &DatabaseTransaction, names: Vec, ) -> RepoResult> { + if names.is_empty() { + return Ok(vec![]); + } let namespaces: Vec = namespace::Entity::find() .filter(namespace::Column::Name.is_in(names)) .all(trx) @@ -93,6 +107,9 @@ async fn namespaces_by_name( } async fn tags_by_name(trx: &DatabaseTransaction, tags: Vec) -> RepoResult> { + if tags.is_empty() { + return Ok(vec![]); + } let condition = tags .into_iter() .map(build_tag_condition) diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs index 6e08e01..1403456 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use std::iter::FromIterator; use mediarepo_core::error::RepoResult; -use mediarepo_core::mediarepo_api::types::filtering::TagQuery; + use mediarepo_core::utils::parse_namespace_and_tag; use mediarepo_database::entities::{content_descriptor, content_descriptor_tag, namespace, tag}; From 0261d2fe3c073b9b5f4bfe0a053180810d46285c Mon Sep 17 00:00:00 2001 From: trivernis Date: Sun, 30 Jan 2022 14:25:38 +0100 Subject: [PATCH 10/12] Remove remaining models in favour of dao-dto architecture Signed-off-by: trivernis --- .../mediarepo-logic/src/dao/repo/mod.rs | 70 +----- .../mediarepo-logic/src/dao/tag/mod.rs | 22 ++ .../mediarepo-logic/src/file_metadata.rs | 124 ---------- mediarepo-daemon/mediarepo-logic/src/lib.rs | 3 - .../mediarepo-logic/src/namespace.rs | 143 ----------- mediarepo-daemon/mediarepo-logic/src/tag.rs | 227 ------------------ .../mediarepo-socket/src/from_model.rs | 35 --- .../mediarepo-socket/src/namespaces/tags.rs | 21 +- 8 files changed, 37 insertions(+), 608 deletions(-) delete mode 100644 mediarepo-daemon/mediarepo-logic/src/file_metadata.rs delete mode 100644 mediarepo-daemon/mediarepo-logic/src/namespace.rs delete mode 100644 mediarepo-daemon/mediarepo-logic/src/tag.rs diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs index 89945af..9157345 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/repo/mod.rs @@ -1,7 +1,5 @@ - use std::fmt::Debug; - use std::path::PathBuf; use sea_orm::DatabaseConnection; @@ -10,17 +8,10 @@ use mediarepo_core::error::RepoResult; use mediarepo_core::fs::file_hash_store::FileHashStore; use mediarepo_core::fs::thumbnail_store::ThumbnailStore; - -use mediarepo_core::utils::parse_namespace_and_tag; - +use crate::dao::{DaoContext, DaoProvider}; use mediarepo_database::get_database; use mediarepo_database::queries::analysis::{get_all_counts, Counts}; - -use crate::dao::{DaoContext, DaoProvider}; -use crate::namespace::Namespace; -use crate::tag::Tag; - #[derive(Clone)] pub struct Repo { db: DatabaseConnection, @@ -67,65 +58,6 @@ impl Repo { &self.db } - /// Finds all tags that are assigned to the given list of hashes - #[tracing::instrument(level = "debug", skip_all)] - pub async fn find_tags_for_file_identifiers(&self, cds: Vec>) -> RepoResult> { - Tag::for_cd_list(self.db.clone(), cds).await - } - - /// Adds or finds a tag - #[tracing::instrument(level = "debug", skip(self))] - pub async fn add_or_find_tag(&self, tag: S) -> RepoResult { - let (namespace, name) = parse_namespace_and_tag(tag.to_string()); - if let Some(namespace) = namespace { - self.add_or_find_namespaced_tag(name, namespace).await - } else { - self.add_or_find_unnamespaced_tag(name).await - } - } - - /// Adds or finds an unnamespaced tag - #[tracing::instrument(level = "debug", skip(self))] - pub async fn add_or_find_unnamespaced_tag(&self, name: String) -> RepoResult { - if let Some(tag) = Tag::by_name(self.db.clone(), &name, None).await? { - Ok(tag) - } else { - self.add_unnamespaced_tag(name).await - } - } - - /// Adds an unnamespaced tag - #[tracing::instrument(level = "debug", skip(self))] - pub async fn add_unnamespaced_tag(&self, name: String) -> RepoResult { - Tag::add(self.db.clone(), name, None).await - } - - /// Adds or finds a namespaced tag - #[tracing::instrument(level = "debug", skip(self))] - pub async fn add_or_find_namespaced_tag( - &self, - name: String, - namespace: String, - ) -> RepoResult { - if let Some(tag) = Tag::by_name(self.db.clone(), &name, Some(namespace.clone())).await? { - Ok(tag) - } else { - self.add_namespaced_tag(name, namespace).await - } - } - - /// Adds a namespaced tag - #[tracing::instrument(level = "debug", skip(self))] - pub async fn add_namespaced_tag(&self, name: String, namespace: String) -> RepoResult { - let namespace = - if let Some(namespace) = Namespace::by_name(self.db.clone(), &namespace).await? { - namespace - } else { - Namespace::add(self.db.clone(), namespace).await? - }; - Tag::add(self.db.clone(), name, Some(namespace.id())).await - } - /// Returns the size of the main storage #[inline] #[tracing::instrument(level = "debug", skip(self))] diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs index 1403456..bd4a6e9 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs @@ -57,6 +57,28 @@ impl TagDao { Ok(namespaces) } + #[tracing::instrument(level = "debug", skip(self))] + pub async fn all_for_cds(&self, cds: Vec>) -> RepoResult> { + let tags = tag::Entity::find() + .find_also_related(namespace::Entity) + .join( + JoinType::LeftJoin, + content_descriptor_tag::Relation::Tag.def().rev(), + ) + .join( + JoinType::InnerJoin, + content_descriptor_tag::Relation::ContentDescriptorId.def(), + ) + .filter(content_descriptor::Column::Descriptor.is_in(cds)) + .all(&self.ctx.db) + .await? + .into_iter() + .map(map_tag_dto) + .collect(); + + Ok(tags) + } + #[tracing::instrument(level = "debug", skip(self))] pub async fn tags_for_cd(&self, cd_id: i64) -> RepoResult> { let tags = tag::Entity::find() diff --git a/mediarepo-daemon/mediarepo-logic/src/file_metadata.rs b/mediarepo-daemon/mediarepo-logic/src/file_metadata.rs deleted file mode 100644 index 0af2fbb..0000000 --- a/mediarepo-daemon/mediarepo-logic/src/file_metadata.rs +++ /dev/null @@ -1,124 +0,0 @@ -use std::fmt::Debug; - -use chrono::{Local, NaiveDateTime}; -use sea_orm::prelude::*; -use sea_orm::{DatabaseConnection, Set}; - -use mediarepo_core::error::RepoResult; -use mediarepo_database::entities::file_metadata; - -#[derive(Clone)] -pub struct FileMetadata { - db: DatabaseConnection, - model: file_metadata::Model, -} - -impl FileMetadata { - #[tracing::instrument(level = "trace")] - pub(crate) fn new(db: DatabaseConnection, model: file_metadata::Model) -> Self { - Self { db, model } - } - - /// Fetches the file by id - #[tracing::instrument(level = "debug", skip(db))] - pub async fn by_id(db: DatabaseConnection, id: i64) -> RepoResult> { - let file_metadata = file_metadata::Entity::find_by_id(id) - .one(&db) - .await? - .map(|m| FileMetadata::new(db, m)); - - Ok(file_metadata) - } - - /// Fetches metadata for all given file ids - #[tracing::instrument(level = "debug", skip(db))] - pub async fn all_by_ids(db: DatabaseConnection, ids: Vec) -> RepoResult> { - let file_metadata = file_metadata::Entity::find() - .filter(file_metadata::Column::FileId.is_in(ids)) - .all(&db) - .await? - .into_iter() - .map(|m| FileMetadata::new(db.clone(), m)) - .collect(); - - Ok(file_metadata) - } - - /// Adds a file with its hash to the database - #[tracing::instrument(level = "debug", skip(db))] - pub(crate) async fn add( - db: DatabaseConnection, - file_id: i64, - size: i64, - creation_time: NaiveDateTime, - change_time: NaiveDateTime, - ) -> RepoResult { - let file = file_metadata::ActiveModel { - file_id: Set(file_id), - size: Set(size), - import_time: Set(Local::now().naive_local()), - creation_time: Set(creation_time), - change_time: Set(change_time), - ..Default::default() - }; - let model = file.insert(&db).await?; - - Ok(Self::new(db, model)) - } - - pub fn file_id(&self) -> i64 { - self.model.file_id - } - - pub fn size(&self) -> i64 { - self.model.size - } - - pub fn name(&self) -> &Option { - &self.model.name - } - - pub fn comment(&self) -> &Option { - &self.model.comment - } - - pub fn import_time(&self) -> &NaiveDateTime { - &self.model.import_time - } - - pub fn creation_time(&self) -> &NaiveDateTime { - &self.model.creation_time - } - - pub fn change_time(&self) -> &NaiveDateTime { - &self.model.change_time - } - - /// Changes the name of the file - #[tracing::instrument(level = "debug", skip(self))] - pub async fn set_name(&mut self, name: S) -> RepoResult<()> { - let mut active_model = self.get_active_model(); - active_model.name = Set(Some(name.to_string())); - self.model = active_model.update(&self.db).await?; - - Ok(()) - } - - /// Changes the comment of the file - #[tracing::instrument(level = "debug", skip(self))] - pub async fn set_comment(&mut self, comment: S) -> RepoResult<()> { - let mut active_file = self.get_active_model(); - active_file.comment = Set(Some(comment.to_string())); - self.model = active_file.update(&self.db).await?; - - Ok(()) - } - - /// Returns the active model of the file with only the id set - fn get_active_model(&self) -> file_metadata::ActiveModel { - file_metadata::ActiveModel { - file_id: Set(self.file_id()), - ..Default::default() - } - } -} diff --git a/mediarepo-daemon/mediarepo-logic/src/lib.rs b/mediarepo-daemon/mediarepo-logic/src/lib.rs index 1d620f5..054053a 100644 --- a/mediarepo-daemon/mediarepo-logic/src/lib.rs +++ b/mediarepo-daemon/mediarepo-logic/src/lib.rs @@ -1,6 +1,3 @@ pub mod dao; pub mod dto; -pub mod file_metadata; -pub mod namespace; -pub mod tag; pub mod type_keys; diff --git a/mediarepo-daemon/mediarepo-logic/src/namespace.rs b/mediarepo-daemon/mediarepo-logic/src/namespace.rs deleted file mode 100644 index 8f173c7..0000000 --- a/mediarepo-daemon/mediarepo-logic/src/namespace.rs +++ /dev/null @@ -1,143 +0,0 @@ -use std::fmt::Debug; - -use sea_orm::{ - Condition, ConnectionTrait, DatabaseBackend, DatabaseConnection, InsertResult, Set, Statement, -}; -use sea_orm::prelude::*; - -use mediarepo_core::error::RepoResult; -use mediarepo_database::entities::namespace; - -#[derive(Clone)] -pub struct Namespace { - #[allow(dead_code)] - db: DatabaseConnection, - model: namespace::Model, -} - -impl Namespace { - #[tracing::instrument(level = "trace")] - pub(crate) fn new(db: DatabaseConnection, model: namespace::Model) -> Self { - Self { db, model } - } - - /// Retrieves a list of all namespaces - #[tracing::instrument(level = "debug", skip(db))] - pub async fn all(db: DatabaseConnection) -> RepoResult> { - let namespaces = namespace::Entity::find() - .all(&db) - .await? - .into_iter() - .map(|model| Self::new(db.clone(), model)) - .collect(); - - Ok(namespaces) - } - - /// Retrieves the namespace by id - #[tracing::instrument(level = "debug", skip(db))] - pub async fn by_id(db: DatabaseConnection, id: i64) -> RepoResult> { - let namespace = namespace::Entity::find_by_id(id) - .one(&db) - .await? - .map(|model| Self::new(db, model)); - - Ok(namespace) - } - - /// Retrieves a namespace by its name - #[tracing::instrument(level = "debug", skip(db))] - pub async fn by_name + Debug>( - db: DatabaseConnection, - name: S, - ) -> RepoResult> { - let namespace = namespace::Entity::find() - .filter(namespace::Column::Name.eq(name.as_ref())) - .one(&db) - .await? - .map(|model| Self::new(db, model)); - - Ok(namespace) - } - - /// Returns all namespaces by name - #[tracing::instrument(level = "debug", skip(db))] - pub async fn all_by_name(db: DatabaseConnection, names: Vec) -> RepoResult> { - if names.is_empty() { - return Ok(Vec::with_capacity(0)); - } - let mut condition = Condition::any(); - for name in names { - condition = condition.add(namespace::Column::Name.eq(name)); - } - - let namespaces = namespace::Entity::find() - .filter(condition) - .all(&db) - .await? - .into_iter() - .map(|model| Self::new(db.clone(), model)) - .collect(); - - Ok(namespaces) - } - - /// Adds all namespaces to the database - #[tracing::instrument(level = "debug", skip(db))] - pub async fn add_all(db: DatabaseConnection, names: Vec) -> RepoResult> { - if names.is_empty() { - return Ok(vec![]); - } - let models: Vec = names - .into_iter() - .map(|name| namespace::ActiveModel { - name: Set(name), - ..Default::default() - }) - .collect(); - let txn = db.begin().await?; - let last_id = txn - .query_one(Statement::from_string( - DatabaseBackend::Sqlite, - r#"SELECT MAX(id) AS "max_id" FROM namespaces;"#.to_owned(), - )) - .await? - .and_then(|result| result.try_get("", "max_id").ok()) - .unwrap_or(-1); - let result: InsertResult = - namespace::Entity::insert_many(models).exec(&txn).await?; - - let namespaces = namespace::Entity::find() - .filter(namespace::Column::Id.between(last_id, result.last_insert_id + 1)) - .all(&txn) - .await? - .into_iter() - .map(|model| Self::new(db.clone(), model)) - .collect(); - txn.commit().await?; - - Ok(namespaces) - } - - /// Adds a namespace to the database - #[tracing::instrument(level = "debug", skip(db))] - pub async fn add(db: DatabaseConnection, name: S) -> RepoResult { - let active_model = namespace::ActiveModel { - name: Set(name.to_string()), - ..Default::default() - }; - let model = active_model.insert(&db).await?; - - Ok(Self::new(db, model)) - } - - /// The ID of the namespace - pub fn id(&self) -> i64 { - self.model.id - } - - /// The name of the namespace - pub fn name(&self) -> &String { - &self.model.name - } -} diff --git a/mediarepo-daemon/mediarepo-logic/src/tag.rs b/mediarepo-daemon/mediarepo-logic/src/tag.rs deleted file mode 100644 index 378c854..0000000 --- a/mediarepo-daemon/mediarepo-logic/src/tag.rs +++ /dev/null @@ -1,227 +0,0 @@ -use std::fmt::Debug; - -use sea_orm::{Condition, DatabaseBackend, DatabaseConnection, JoinType, Set, Statement}; -use sea_orm::{InsertResult, QuerySelect}; -use sea_orm::prelude::*; -use sea_orm::query::ConnectionTrait; -use sea_orm::sea_query::Expr; - -use mediarepo_core::error::RepoResult; -use mediarepo_database::entities::content_descriptor; -use mediarepo_database::entities::content_descriptor_tag; -use mediarepo_database::entities::namespace; -use mediarepo_database::entities::tag; - -use crate::namespace::Namespace; - -#[derive(Clone)] -pub struct Tag { - db: DatabaseConnection, - model: tag::Model, - namespace: Option, -} - -impl Tag { - #[tracing::instrument(level = "trace")] - pub(crate) fn new( - db: DatabaseConnection, - model: tag::Model, - namespace: Option, - ) -> Self { - Self { - db, - model, - namespace, - } - } - - /// Returns all tags stored in the database - #[tracing::instrument(level = "debug", skip(db))] - pub async fn all(db: DatabaseConnection) -> RepoResult> { - let tags: Vec = tag::Entity::find() - .left_join(namespace::Entity) - .select_also(namespace::Entity) - .all(&db) - .await? - .into_iter() - .map(|(tag, namespace)| Self::new(db.clone(), tag, namespace)) - .collect(); - - Ok(tags) - } - - /// Returns the tag by id - #[tracing::instrument(level = "debug", skip(db))] - pub async fn by_id(db: DatabaseConnection, id: i64) -> RepoResult> { - let tag = tag::Entity::find_by_id(id) - .find_also_related(namespace::Entity) - .one(&db) - .await? - .map(|(model, namespace)| Self::new(db, model, namespace)); - - Ok(tag) - } - - /// Returns one tag by name and namespace - #[tracing::instrument(level = "debug", skip(db))] - pub async fn by_name( - db: DatabaseConnection, - name: S1, - namespace: Option, - ) -> RepoResult> { - let mut entries = Self::all_by_name(db, vec![(namespace, name.to_string())]).await?; - - Ok(entries.pop()) - } - - /// Retrieves the namespaced tags by name and namespace - #[tracing::instrument(level = "debug", skip(db))] - pub async fn all_by_name( - db: DatabaseConnection, - namespaces_with_names: Vec<(Option, String)>, - ) -> RepoResult> { - if namespaces_with_names.is_empty() { - return Ok(vec![]); - } - let mut or_condition = Condition::any(); - - for (namespace, name) in namespaces_with_names { - let mut all_condition = Condition::all(); - if !name.ends_with('*') { - all_condition = all_condition.add(tag::Column::Name.eq(name)) - } else if name.len() > 1 { - all_condition = all_condition - .add(tag::Column::Name.like(&*format!("{}%", name.trim_end_matches("*")))) - } else if namespace.is_none() { - continue; // would result in an empty condition otherwise - } - - all_condition = if let Some(namespace) = namespace { - all_condition.add(namespace::Column::Name.eq(namespace)) - } else { - all_condition.add(Expr::tbl(tag::Entity, tag::Column::NamespaceId).is_null()) - }; - or_condition = or_condition.add(all_condition); - } - - let tags: Vec = tag::Entity::find() - .find_also_related(namespace::Entity) - .filter(or_condition) - .group_by(tag::Column::Id) - .all(&db) - .await? - .into_iter() - .map(|(t, n)| Self::new(db.clone(), t, n)) - .collect(); - - Ok(tags) - } - - /// Returns all tags that are assigned to any of the passed hashes - #[tracing::instrument(level = "debug", skip_all)] - pub async fn for_cd_list(db: DatabaseConnection, cds: Vec>) -> RepoResult> { - let tags: Vec = tag::Entity::find() - .find_also_related(namespace::Entity) - .join( - JoinType::LeftJoin, - content_descriptor_tag::Relation::Tag.def().rev(), - ) - .join( - JoinType::InnerJoin, - content_descriptor_tag::Relation::ContentDescriptorId.def(), - ) - .filter(content_descriptor::Column::Descriptor.is_in(cds)) - .group_by(tag::Column::Id) - .all(&db) - .await? - .into_iter() - .map(|(t, n)| Self::new(db.clone(), t, n)) - .collect(); - - Ok(tags) - } - - pub async fn add_all( - db: DatabaseConnection, - namespaces_with_names: Vec<(Option, String)>, - ) -> RepoResult> { - if namespaces_with_names.is_empty() { - return Ok(vec![]); - } - let models: Vec = namespaces_with_names - .into_iter() - .map(|(namespace_id, name)| tag::ActiveModel { - name: Set(name), - namespace_id: Set(namespace_id), - ..Default::default() - }) - .collect(); - let txn = db.begin().await?; - let last_id: i64 = txn - .query_one(Statement::from_string( - DatabaseBackend::Sqlite, - r#"SELECT MAX(id) as "max_id" FROM tags"#.to_owned(), - )) - .await? - .and_then(|res| res.try_get("", "max_id").ok()) - .unwrap_or(-1); - - let result: InsertResult = - tag::Entity::insert_many(models).exec(&txn).await?; - let tags: Vec = tag::Entity::find() - .find_also_related(namespace::Entity) - .filter(tag::Column::Id.between(last_id, result.last_insert_id + 1)) - .all(&txn) - .await? - .into_iter() - .map(|(t, n)| Self::new(db.clone(), t, n)) - .collect(); - txn.commit().await?; - - Ok(tags) - } - - /// Adds a new tag to the database - #[tracing::instrument(level = "debug", skip(db))] - pub async fn add( - db: DatabaseConnection, - name: S, - namespace_id: Option, - ) -> RepoResult { - let active_model = tag::ActiveModel { - name: Set(name.to_string()), - namespace_id: Set(namespace_id), - ..Default::default() - }; - let model: tag::Model = active_model.insert(&db).await?; - let namespace = model.find_related(namespace::Entity).one(&db).await?; - - Ok(Self::new(db, model, namespace)) - } - - /// The ID of the tag - pub fn id(&self) -> i64 { - self.model.id - } - - /// The name of the tag - pub fn name(&self) -> &String { - &self.model.name - } - - /// The namespace of the tag - pub fn namespace(&self) -> Option { - self.namespace - .clone() - .map(|n| Namespace::new(self.db.clone(), n)) - } - - /// Returns the normalized name of the tag (namespace:tag) - pub fn normalized_name(&self) -> String { - if let Some(namespace) = &self.namespace { - format!("{}:{}", namespace.name, self.model.name) - } else { - self.model.name.to_owned() - } - } -} diff --git a/mediarepo-daemon/mediarepo-socket/src/from_model.rs b/mediarepo-daemon/mediarepo-socket/src/from_model.rs index eaa6e7e..eeffd64 100644 --- a/mediarepo-daemon/mediarepo-socket/src/from_model.rs +++ b/mediarepo-daemon/mediarepo-socket/src/from_model.rs @@ -5,27 +5,11 @@ use mediarepo_core::mediarepo_api::types::tags::{NamespaceResponse, TagResponse} use mediarepo_logic::dto::{ FileDto, FileMetadataDto, FileStatus as FileStatusModel, NamespaceDto, TagDto, ThumbnailDto, }; -use mediarepo_logic::file_metadata::FileMetadata; -use mediarepo_logic::namespace::Namespace; -use mediarepo_logic::tag::Tag; pub trait FromModel { fn from_model(model: M) -> Self; } -impl FromModel for FileMetadataResponse { - fn from_model(metadata: FileMetadata) -> Self { - Self { - file_id: metadata.file_id(), - name: metadata.name().to_owned(), - comment: metadata.comment().to_owned(), - creation_time: metadata.creation_time().to_owned(), - change_time: metadata.change_time().to_owned(), - import_time: metadata.import_time().to_owned(), - } - } -} - impl FromModel for FileMetadataResponse { fn from_model(model: FileMetadataDto) -> Self { Self { @@ -60,16 +44,6 @@ impl FromModel for FileStatus { } } -impl FromModel for TagResponse { - fn from_model(model: Tag) -> Self { - Self { - id: model.id(), - namespace: model.namespace().map(|n| n.name().to_owned()), - name: model.name().to_owned(), - } - } -} - impl FromModel for TagResponse { fn from_model(model: TagDto) -> Self { Self { @@ -91,15 +65,6 @@ impl FromModel for ThumbnailMetadataResponse { } } -impl FromModel for NamespaceResponse { - fn from_model(model: Namespace) -> Self { - Self { - id: model.id(), - name: model.name().to_owned(), - } - } -} - impl FromModel for NamespaceResponse { fn from_model(model: NamespaceDto) -> Self { Self { diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs index 1bfde4c..bfd6895 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs @@ -6,7 +6,9 @@ use mediarepo_core::mediarepo_api::types::files::{GetFileTagsRequest, GetFilesTa use mediarepo_core::mediarepo_api::types::tags::{ ChangeFileTagsRequest, NamespaceResponse, TagResponse, }; +use mediarepo_core::utils::parse_namespace_and_tag; use mediarepo_logic::dao::DaoProvider; +use mediarepo_logic::dto::AddTagDto; use crate::from_model::FromModel; use crate::utils::{file_by_identifier, get_repo_from_context}; @@ -85,7 +87,8 @@ impl TagsNamespace { let repo = get_repo_from_context(ctx).await; let request = event.payload::()?; let tag_responses: Vec = repo - .find_tags_for_file_identifiers( + .tag() + .all_for_cds( request .cds .into_par_iter() @@ -102,17 +105,21 @@ impl TagsNamespace { Ok(()) } - /// Creates all tags given as input or returns the existing tag + /// Creates all tags given as input or returns the existing tags #[tracing::instrument(skip_all)] async fn create_tags(ctx: &Context, event: Event) -> IPCResult<()> { let repo = get_repo_from_context(ctx).await; let tags = event.payload::>()?; - let mut created_tags = Vec::new(); + let created_tags = repo + .tag() + .add_all( + tags.into_iter() + .map(parse_namespace_and_tag) + .map(AddTagDto::from_tuple) + .collect(), + ) + .await?; - for tag in tags { - let created_tag = repo.add_or_find_tag(tag).await?; - created_tags.push(created_tag); - } let responses: Vec = created_tags .into_iter() .map(TagResponse::from_model) From 272b0bf7dc4b71a4e23cd03ac651cf95b3df7d95 Mon Sep 17 00:00:00 2001 From: trivernis Date: Sun, 30 Jan 2022 14:38:50 +0100 Subject: [PATCH 11/12] Restructure job dao Signed-off-by: trivernis --- .../dao/job/migrate_content_descriptors.rs | 45 +++++++++++++++++++ .../mediarepo-logic/src/dao/job/mod.rs | 44 +----------------- 2 files changed, 46 insertions(+), 43 deletions(-) create mode 100644 mediarepo-daemon/mediarepo-logic/src/dao/job/migrate_content_descriptors.rs diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/job/migrate_content_descriptors.rs b/mediarepo-daemon/mediarepo-logic/src/dao/job/migrate_content_descriptors.rs new file mode 100644 index 0000000..a339cc8 --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dao/job/migrate_content_descriptors.rs @@ -0,0 +1,45 @@ +use crate::dao::job::JobDao; +use mediarepo_core::content_descriptor::{ + convert_v1_descriptor_to_v2, encode_content_descriptor, is_v1_content_descriptor, +}; +use mediarepo_core::error::RepoResult; +use mediarepo_database::entities::content_descriptor; +use sea_orm::prelude::*; +use sea_orm::ActiveValue::Set; +use sea_orm::ConnectionTrait; + +impl JobDao { + pub async fn migrate_content_descriptors(&self) -> RepoResult<()> { + let cds: Vec = + content_descriptor::Entity::find().all(&self.ctx.db).await?; + + tracing::info!("Converting content descriptors to v2 format..."); + let mut converted_count = 0; + + for cd in cds { + if is_v1_content_descriptor(&cd.descriptor) { + let trx = self.ctx.db.begin().await?; + let src_cd = cd.descriptor; + let dst_cd = convert_v1_descriptor_to_v2(&src_cd)?; + + let _active_model = content_descriptor::ActiveModel { + id: Set(cd.id), + descriptor: Set(dst_cd.clone()), + }; + self.ctx.main_storage.rename_file(&src_cd, &dst_cd).await?; + self.ctx + .thumbnail_storage + .rename_parent( + encode_content_descriptor(&src_cd), + encode_content_descriptor(&dst_cd), + ) + .await?; + trx.commit().await?; + converted_count += 1; + } + } + tracing::info!("Converted {} descriptors", converted_count); + + Ok(()) + } +} diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs index 1098fdb..1d6c510 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs @@ -1,12 +1,4 @@ -use sea_orm::ActiveValue::Set; -use sea_orm::ConnectionTrait; -use sea_orm::prelude::*; - -use mediarepo_core::content_descriptor::{ - convert_v1_descriptor_to_v2, encode_content_descriptor, is_v1_content_descriptor, -}; -use mediarepo_core::error::RepoResult; -use mediarepo_database::entities::content_descriptor; +pub mod migrate_content_descriptors; use crate::dao::{DaoContext, DaoProvider}; @@ -24,38 +16,4 @@ impl JobDao { pub fn new(ctx: DaoContext) -> JobDao { Self { ctx } } - - pub async fn migrate_content_descriptors(&self) -> RepoResult<()> { - let cds: Vec = - content_descriptor::Entity::find().all(&self.ctx.db).await?; - - tracing::info!("Converting content descriptors to v2 format..."); - let mut converted_count = 0; - - for cd in cds { - if is_v1_content_descriptor(&cd.descriptor) { - let trx = self.ctx.db.begin().await?; - let src_cd = cd.descriptor; - let dst_cd = convert_v1_descriptor_to_v2(&src_cd)?; - - let _active_model = content_descriptor::ActiveModel { - id: Set(cd.id), - descriptor: Set(dst_cd.clone()), - }; - self.ctx.main_storage.rename_file(&src_cd, &dst_cd).await?; - self.ctx - .thumbnail_storage - .rename_parent( - encode_content_descriptor(&src_cd), - encode_content_descriptor(&dst_cd), - ) - .await?; - trx.commit().await?; - converted_count += 1; - } - } - tracing::info!("Converted {} descriptors", converted_count); - - Ok(()) - } } From 37322b13a7225c5eff225d629b7b3ce3f68b3270 Mon Sep 17 00:00:00 2001 From: trivernis Date: Sun, 30 Jan 2022 15:03:00 +0100 Subject: [PATCH 12/12] Implement integrity check Signed-off-by: trivernis --- mediarepo-daemon/mediarepo-core/src/error.rs | 5 ++- .../dao/job/migrate_content_descriptors.rs | 1 + .../mediarepo-logic/src/dao/job/mod.rs | 1 + .../src/dao/job/sqlite_operations.rs | 44 +++++++++++++++++++ .../mediarepo-socket/src/namespaces/jobs.rs | 2 +- 5 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 mediarepo-daemon/mediarepo-logic/src/dao/job/sqlite_operations.rs diff --git a/mediarepo-daemon/mediarepo-core/src/error.rs b/mediarepo-daemon/mediarepo-core/src/error.rs index 820e854..7f5dfbc 100644 --- a/mediarepo-daemon/mediarepo-core/src/error.rs +++ b/mediarepo-daemon/mediarepo-core/src/error.rs @@ -38,8 +38,11 @@ pub enum RepoError { #[error("failed to decode data {0}")] Decode(#[from] data_encoding::DecodeError), - #[error("Failed to read repo.toml configuration file {0}")] + #[error("failed to read repo.toml configuration file {0}")] Config(#[from] config::ConfigError), + + #[error("the database file is corrupted {0}")] + Corrupted(String), } #[derive(Error, Debug)] diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/job/migrate_content_descriptors.rs b/mediarepo-daemon/mediarepo-logic/src/dao/job/migrate_content_descriptors.rs index a339cc8..6c395fe 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/job/migrate_content_descriptors.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/job/migrate_content_descriptors.rs @@ -9,6 +9,7 @@ use sea_orm::ActiveValue::Set; use sea_orm::ConnectionTrait; impl JobDao { + #[tracing::instrument(level = "debug", skip(self))] pub async fn migrate_content_descriptors(&self) -> RepoResult<()> { let cds: Vec = content_descriptor::Entity::find().all(&self.ctx.db).await?; diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs index 1d6c510..c47b28a 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/job/mod.rs @@ -1,4 +1,5 @@ pub mod migrate_content_descriptors; +pub mod sqlite_operations; use crate::dao::{DaoContext, DaoProvider}; diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/job/sqlite_operations.rs b/mediarepo-daemon/mediarepo-logic/src/dao/job/sqlite_operations.rs new file mode 100644 index 0000000..132bdf9 --- /dev/null +++ b/mediarepo-daemon/mediarepo-logic/src/dao/job/sqlite_operations.rs @@ -0,0 +1,44 @@ +use crate::dao::job::JobDao; +use mediarepo_core::error::RepoError::Corrupted; +use mediarepo_core::error::RepoResult; +use sea_orm::DatabaseBackend::Sqlite; +use sea_orm::{ConnectionTrait, FromQueryResult, Statement}; + +#[derive(Debug, FromQueryResult)] +struct IntegrityCheckResult { + integrity_check: String, +} + +impl JobDao { + #[tracing::instrument(level = "debug", skip(self))] + pub async fn check_integrity(&self) -> RepoResult<()> { + let check_result: Option = IntegrityCheckResult::find_by_statement( + Statement::from_string(Sqlite, String::from("PRAGMA integrity_check;")), + ) + .one(&self.ctx.db) + .await?; + tracing::debug!("check result = {:?}", check_result); + + check_result + .ok_or_else(|| Corrupted(String::from("no check result"))) + .and_then(map_check_result) + } + + #[tracing::instrument(level = "debug", skip(self))] + pub async fn vacuum(&self) -> RepoResult<()> { + self.ctx + .db + .execute(Statement::from_string(Sqlite, String::from("VACUUM;"))) + .await?; + + Ok(()) + } +} + +fn map_check_result(result: IntegrityCheckResult) -> RepoResult<()> { + if result.integrity_check == "ok" { + Ok(()) + } else { + Err(Corrupted(result.integrity_check)) + } +} diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs index 208c5ce..d345e42 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs @@ -30,7 +30,7 @@ impl JobsNamespace { match run_request.job_type { JobType::MigrateContentDescriptors => job_dao.migrate_content_descriptors().await?, JobType::CalculateSizes => calculate_all_sizes(ctx).await?, - JobType::CheckIntegrity => {} + JobType::CheckIntegrity => job_dao.check_integrity().await?, } ctx.emit_to(Self::name(), "run_job", ()).await?;