From 165151096d1530d3840bd3d771538cafe4136f08 Mon Sep 17 00:00:00 2001 From: trivernis Date: Tue, 16 Nov 2021 21:15:37 +0100 Subject: [PATCH] Implement missing sort keys Signed-off-by: trivernis --- .../20211116200116_remove-thumbnail-table.sql | 2 + .../20211116200356_add-file-size-column.sql | 3 + .../mediarepo-database/src/entities/file.rs | 10 +- .../mediarepo-database/src/entities/hash.rs | 6 - .../mediarepo-database/src/entities/mod.rs | 1 - .../src/entities/storage.rs | 8 -- .../src/entities/thumbnail.rs | 58 -------- .../mediarepo-database/src/queries/tags.rs | 65 +++++++-- mediarepo-daemon/mediarepo-model/src/file.rs | 19 +++ .../mediarepo-socket/src/namespaces/files.rs | 130 ++++++++++++------ 10 files changed, 169 insertions(+), 133 deletions(-) create mode 100644 mediarepo-daemon/mediarepo-database/migrations/20211116200116_remove-thumbnail-table.sql create mode 100644 mediarepo-daemon/mediarepo-database/migrations/20211116200356_add-file-size-column.sql delete mode 100644 mediarepo-daemon/mediarepo-database/src/entities/thumbnail.rs diff --git a/mediarepo-daemon/mediarepo-database/migrations/20211116200116_remove-thumbnail-table.sql b/mediarepo-daemon/mediarepo-database/migrations/20211116200116_remove-thumbnail-table.sql new file mode 100644 index 0000000..6df0dad --- /dev/null +++ b/mediarepo-daemon/mediarepo-database/migrations/20211116200116_remove-thumbnail-table.sql @@ -0,0 +1,2 @@ +-- Add migration script here +DROP TABLE thumbnails; \ No newline at end of file diff --git a/mediarepo-daemon/mediarepo-database/migrations/20211116200356_add-file-size-column.sql b/mediarepo-daemon/mediarepo-database/migrations/20211116200356_add-file-size-column.sql new file mode 100644 index 0000000..5d7932e --- /dev/null +++ b/mediarepo-daemon/mediarepo-database/migrations/20211116200356_add-file-size-column.sql @@ -0,0 +1,3 @@ +-- Add migration script here +ALTER TABLE files + ADD COLUMN size INTEGER; \ No newline at end of file diff --git a/mediarepo-daemon/mediarepo-database/src/entities/file.rs b/mediarepo-daemon/mediarepo-database/src/entities/file.rs index 521a8d4..5505def 100644 --- a/mediarepo-daemon/mediarepo-database/src/entities/file.rs +++ b/mediarepo-daemon/mediarepo-database/src/entities/file.rs @@ -10,6 +10,7 @@ pub struct Model { pub name: Option, pub comment: Option, pub mime_type: Option, + pub size: Option, pub storage_id: i64, pub hash_id: i64, pub import_time: NaiveDateTime, @@ -32,9 +33,6 @@ pub enum Relation { to = "super::storage::Column::Id" )] Storage, - - #[sea_orm(has_many = "super::thumbnail::Entity")] - Thumbnail, } impl Related for Entity { @@ -49,10 +47,4 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Thumbnail.def() - } -} - impl ActiveModelBehavior for ActiveModel {} diff --git a/mediarepo-daemon/mediarepo-database/src/entities/hash.rs b/mediarepo-daemon/mediarepo-database/src/entities/hash.rs index c7675e6..984b469 100644 --- a/mediarepo-daemon/mediarepo-database/src/entities/hash.rs +++ b/mediarepo-daemon/mediarepo-database/src/entities/hash.rs @@ -37,10 +37,4 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - super::thumbnail::Relation::Hash.def().rev() - } -} - impl ActiveModelBehavior for ActiveModel {} diff --git a/mediarepo-daemon/mediarepo-database/src/entities/mod.rs b/mediarepo-daemon/mediarepo-database/src/entities/mod.rs index 756c618..0131e8e 100644 --- a/mediarepo-daemon/mediarepo-database/src/entities/mod.rs +++ b/mediarepo-daemon/mediarepo-database/src/entities/mod.rs @@ -6,4 +6,3 @@ pub mod namespace; pub mod source; pub mod storage; pub mod tag; -pub mod thumbnail; diff --git a/mediarepo-daemon/mediarepo-database/src/entities/storage.rs b/mediarepo-daemon/mediarepo-database/src/entities/storage.rs index 32d3494..10b2ce8 100644 --- a/mediarepo-daemon/mediarepo-database/src/entities/storage.rs +++ b/mediarepo-daemon/mediarepo-database/src/entities/storage.rs @@ -13,8 +13,6 @@ pub struct Model { pub enum Relation { #[sea_orm(has_many = "super::file::Entity")] File, - #[sea_orm(has_many = "super::thumbnail::Entity")] - Thumbnail, } impl Related for Entity { @@ -23,10 +21,4 @@ impl Related for Entity { } } -impl Related for Entity { - fn to() -> RelationDef { - Relation::Thumbnail.def() - } -} - impl ActiveModelBehavior for ActiveModel {} diff --git a/mediarepo-daemon/mediarepo-database/src/entities/thumbnail.rs b/mediarepo-daemon/mediarepo-database/src/entities/thumbnail.rs deleted file mode 100644 index 06e3f98..0000000 --- a/mediarepo-daemon/mediarepo-database/src/entities/thumbnail.rs +++ /dev/null @@ -1,58 +0,0 @@ -use sea_orm::prelude::*; - -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] -#[sea_orm(table_name = "thumbnails")] -pub struct Model { - #[sea_orm(primary_key)] - pub id: i64, - pub file_id: i64, - pub storage_id: i64, - pub hash_id: i64, - pub height: i32, - pub width: i32, - pub mime: Option, -} - -#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] -pub enum Relation { - #[sea_orm( - belongs_to = "super::file::Entity", - from = "Column::FileId", - to = "super::file::Column::Id" - )] - File, - - #[sea_orm( - belongs_to = "super::hash::Entity", - from = "Column::HashId", - to = "super::hash::Column::Id" - )] - Hash, - - #[sea_orm( - belongs_to = "super::storage::Entity", - from = "Column::StorageId", - to = "super::storage::Column::Id" - )] - Storage, -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Hash.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::File.def() - } -} - -impl Related for Entity { - fn to() -> RelationDef { - Relation::Storage.def() - } -} - -impl ActiveModelBehavior for ActiveModel {} diff --git a/mediarepo-daemon/mediarepo-database/src/queries/tags.rs b/mediarepo-daemon/mediarepo-database/src/queries/tags.rs index 7b9bf90..5d4dc99 100644 --- a/mediarepo-daemon/mediarepo-database/src/queries/tags.rs +++ b/mediarepo-daemon/mediarepo-database/src/queries/tags.rs @@ -3,6 +3,7 @@ use sea_orm::DbBackend; use sea_orm::FromQueryResult; use sea_orm::{DatabaseConnection, Statement}; use std::collections::HashMap; +use std::fmt::Display; use std::iter::FromIterator; #[derive(Debug, FromQueryResult)] @@ -16,7 +17,7 @@ struct HashNamespaceTags { pub async fn get_hashes_with_namespaced_tags( db: &DatabaseConnection, hash_ids: Vec, -) -> RepoResult>> { +) -> RepoResult>>> { let hash_namespace_tags: Vec = HashNamespaceTags::find_by_statement(Statement::from_sql_and_values( DbBackend::Sqlite, @@ -26,28 +27,74 @@ pub async fn get_hashes_with_namespaced_tags( INNER JOIN tags t on htm.tag_id = t.id JOIN namespaces n on t.namespace_id = n.id WHERE t.namespace_id IS NOT NULL - AND htm.hash_id IN ({});"#, - hash_ids - .into_iter() - .fold(String::new(), |acc, val| format!("{}{},", acc, val)) - .trim_end_matches(",") + AND htm.hash_id IN ({}) ORDER BY t.namespace_id;"#, + vec_to_query_list(hash_ids) ) .as_str(), vec![], )) .all(db) .await?; - let mut hash_namespaces: HashMap> = HashMap::new(); + let mut hash_namespaces: HashMap>> = HashMap::new(); for hnt in hash_namespace_tags { if let Some(entry) = hash_namespaces.get_mut(&hnt.hash_id) { - entry.insert(hnt.namespace, hnt.tag); + if let Some(nsp_entry) = entry.get_mut(&hnt.namespace) { + nsp_entry.push(hnt.tag); + } else { + entry.insert(hnt.namespace, vec![hnt.tag]); + } } else { hash_namespaces.insert( hnt.hash_id, - HashMap::from_iter(vec![(hnt.namespace, hnt.tag)].into_iter()), + HashMap::from_iter(vec![(hnt.namespace, vec![hnt.tag])].into_iter()), ); } } Ok(hash_namespaces) } + +#[derive(Debug, FromQueryResult)] +struct HashTagCount { + hash_id: i64, + tag_count: i32, +} + +#[tracing::instrument(level = "debug", skip_all)] +pub async fn get_hashes_with_tag_count( + db: &DatabaseConnection, + hash_ids: Vec, +) -> RepoResult> { + let hash_tag_counts: Vec = + HashTagCount::find_by_statement(Statement::from_sql_and_values( + DbBackend::Sqlite, + format!( + r#" + SELECT htm.hash_id, COUNT(htm.tag_id) AS "tag_count" from hash_tag_mappings htm + WHERE htm.hash_id IN ({}) + GROUP BY hash_id + "#, + vec_to_query_list(hash_ids) + ) + .as_str(), + vec![], + )) + .all(db) + .await?; + + let mappings = hash_tag_counts + .into_iter() + .map(|HashTagCount { hash_id, tag_count }| (hash_id, tag_count as u32)) + .collect::>(); + + Ok(mappings) +} + +fn vec_to_query_list(input: Vec) -> String { + let mut entries = input + .into_iter() + .fold(String::new(), |acc, val| format!("{}{},", acc, val)); + entries.remove(entries.len() - 1); + + entries +} diff --git a/mediarepo-daemon/mediarepo-model/src/file.rs b/mediarepo-daemon/mediarepo-model/src/file.rs index f87f5f8..71a0528 100644 --- a/mediarepo-daemon/mediarepo-model/src/file.rs +++ b/mediarepo-daemon/mediarepo-model/src/file.rs @@ -327,6 +327,25 @@ impl File { storage.get_file_reader(&self.hash.value).await } + /// Retrieves the size of the file from its content + #[tracing::instrument(level = "debug", skip(self))] + pub async fn get_size(&self) -> RepoResult { + if let Some(size) = self.model.size { + Ok(size as u64) + } else { + let mut reader = self.get_reader().await?; + let size = { + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await + }?; + let mut model = self.get_active_model(); + model.size = Set(Some(size as i64)); + model.update(&self.db).await?; + + Ok(size as u64) + } + } + /// Creates a thumbnail for the file #[tracing::instrument(level = "debug", skip(self))] pub async fn create_thumbnail + Debug>( diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs index e286c55..820ace6 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs @@ -1,5 +1,6 @@ use crate::from_model::FromModel; use crate::utils::{file_by_identifier, get_repo_from_context, hash_by_identifier}; +use chrono::NaiveDateTime; use compare::Compare; use mediarepo_api::types::files::{ AddFileRequestHeader, FileMetadataResponse, FindFilesByTagsRequest, @@ -12,13 +13,24 @@ use mediarepo_core::itertools::Itertools; use mediarepo_core::rmp_ipc::prelude::*; use mediarepo_core::thumbnailer::ThumbnailSize; use mediarepo_core::utils::parse_namespace_and_tag; -use mediarepo_database::queries::tags::get_hashes_with_namespaced_tags; -use mediarepo_model::file::File; +use mediarepo_database::queries::tags::{ + get_hashes_with_namespaced_tags, get_hashes_with_tag_count, +}; use std::cmp::Ordering; use std::collections::HashMap; use tokio::io::AsyncReadExt; pub struct FilesNamespace; +pub struct FileSortContext { + name: Option, + size: u64, + mime_type: Option, + namespaces: HashMap>, + tag_count: u32, + import_time: NaiveDateTime, + create_time: NaiveDateTime, + change_time: NaiveDateTime, +} impl NamespaceProvider for FilesNamespace { fn name() -> &'static str { @@ -79,21 +91,36 @@ impl FilesNamespace { let repo = get_repo_from_context(ctx).await; let tags = req.tags.into_iter().map(|t| (t.name, t.negate)).collect(); let mut files = repo.find_files_by_tags(tags).await?; - let hash_ids = files.iter().map(|f| f.hash_id()).collect(); - - let hash_nsp: HashMap> = - get_hashes_with_namespaced_tags(repo.db(), hash_ids).await?; - + let hash_ids: Vec = files.iter().map(|f| f.hash_id()).collect(); + + let mut hash_nsp: HashMap>> = + get_hashes_with_namespaced_tags(repo.db(), hash_ids.clone()).await?; + let mut hash_tag_counts = get_hashes_with_tag_count(repo.db(), hash_ids).await?; + + let mut contexts = HashMap::new(); + + for file in &files { + let context = FileSortContext { + name: file.name().to_owned(), + size: file.get_size().await?, + mime_type: file.mime_type().to_owned(), + namespaces: hash_nsp + .remove(&file.hash_id()) + .unwrap_or(HashMap::with_capacity(0)), + tag_count: hash_tag_counts.remove(&file.hash_id()).unwrap_or(0), + import_time: file.import_time().to_owned(), + create_time: file.import_time().to_owned(), + change_time: file.change_time().to_owned(), + }; + contexts.insert(file.id(), context); + } let sort_expression = req.sort_expression; tracing::debug!("sort_expression = {:?}", sort_expression); - let empty_map = HashMap::with_capacity(0); files.sort_by(|a, b| { compare_files( - a, - hash_nsp.get(&a.hash_id()).unwrap_or(&empty_map), - b, - hash_nsp.get(&b.hash_id()).unwrap_or(&empty_map), + contexts.get(&a.hash_id()).unwrap(), + contexts.get(&b.hash_id()).unwrap(), &sort_expression, ) }); @@ -263,55 +290,56 @@ impl FilesNamespace { #[tracing::instrument(level = "trace", skip_all)] fn compare_files( - file_a: &File, - nsp_a: &HashMap, - file_b: &File, - nsp_b: &HashMap, + ctx_a: &FileSortContext, + ctx_b: &FileSortContext, expression: &Vec, ) -> Ordering { let cmp_date = compare::natural(); + let cmp_u64 = compare::natural(); + let cmp_u32 = compare::natural(); for sort_key in expression { let ordering = match sort_key { SortKey::Namespace(namespace) => { - let tag_a = nsp_a.get(&namespace.name); - let tag_b = nsp_b.get(&namespace.name); - - if let (Some(a), Some(b)) = ( - tag_a.and_then(|a| a.parse::().ok()), - tag_b.and_then(|b| b.parse::().ok()), - ) { - adjust_for_dir(compare_f32(a, b), &namespace.direction) + let list_a = ctx_a.namespaces.get(&namespace.name); + let list_b = ctx_b.namespaces.get(&namespace.name); + + let cmp_result = if let (Some(list_a), Some(list_b)) = (list_a, list_b) { + compare_tag_lists(list_a, list_b) + } else if list_a.is_some() { + Ordering::Greater + } else if list_b.is_some() { + Ordering::Less } else { - adjust_for_dir(compare_opts(tag_a, tag_b), &namespace.direction) - } + Ordering::Equal + }; + adjust_for_dir(cmp_result, &namespace.direction) } - SortKey::FileName(direction) => adjust_for_dir( - compare_opts(file_a.name().clone(), file_b.name().clone()), - direction, - ), - SortKey::FileSize(_direction) => { - Ordering::Equal // TODO: Retrieve file size + SortKey::FileName(direction) => { + adjust_for_dir(compare_opts(&ctx_a.name, &ctx_b.name), direction) + } + SortKey::FileSize(direction) => { + adjust_for_dir(cmp_u64.compare(&ctx_a.size, &ctx_b.size), direction) } SortKey::FileImportedTime(direction) => adjust_for_dir( - cmp_date.compare(file_a.import_time(), file_b.import_time()), + cmp_date.compare(&ctx_a.import_time, &ctx_b.import_time), direction, ), SortKey::FileCreatedTime(direction) => adjust_for_dir( - cmp_date.compare(file_a.creation_time(), file_b.creation_time()), + cmp_date.compare(&ctx_a.create_time, &ctx_b.create_time), direction, ), SortKey::FileChangeTime(direction) => adjust_for_dir( - cmp_date.compare(file_a.change_time(), file_b.change_time()), + cmp_date.compare(&ctx_a.change_time, &ctx_b.change_time), direction, ), - SortKey::FileType(direction) => adjust_for_dir( - compare_opts(file_a.mime_type().clone(), file_b.mime_type().clone()), + SortKey::FileType(direction) => { + adjust_for_dir(compare_opts(&ctx_a.mime_type, &ctx_b.mime_type), direction) + } + SortKey::NumTags(direction) => adjust_for_dir( + cmp_u32.compare(&ctx_a.tag_count, &ctx_b.tag_count), direction, ), - SortKey::NumTags(_) => { - Ordering::Equal // TODO: Count tags - } }; if !ordering.is_eq() { return ordering; @@ -321,9 +349,9 @@ fn compare_files( Ordering::Equal } -fn compare_opts(opt_a: Option, opt_b: Option) -> Ordering { +fn compare_opts(opt_a: &Option, opt_b: &Option) -> Ordering { let cmp = compare::natural(); - if let (Some(a), Some(b)) = (&opt_a, &opt_b) { + if let (Some(a), Some(b)) = (opt_a, opt_b) { cmp.compare(a, b) } else if opt_a.is_some() { Ordering::Greater @@ -351,3 +379,21 @@ fn adjust_for_dir(ordering: Ordering, direction: &SortDirection) -> Ordering { ordering } } + +fn compare_tag_lists(list_a: &Vec, list_b: &Vec) -> Ordering { + let first_diff = list_a + .into_iter() + .zip(list_b.into_iter()) + .find(|(a, b)| *a != *b); + if let Some(diff) = first_diff { + if let (Some(num_a), Some(num_b)) = (diff.0.parse::().ok(), diff.1.parse::().ok()) + { + compare_f32(num_a, num_b) + } else { + let cmp = compare::natural(); + cmp.compare(diff.0, diff.1) + } + } else { + Ordering::Equal + } +}