Implement missing sort keys

Signed-off-by: trivernis <trivernis@protonmail.com>

mediarepo-daemon/mediarepo-database/migrations/20211116200116_remove-thumbnail-table.sql

@@ -0,0 +1,2 @@
+-- Add migration script here
+DROP TABLE thumbnails;

mediarepo-daemon/mediarepo-database/migrations/20211116200356_add-file-size-column.sql

@@ -0,0 +1,3 @@
+-- Add migration script here
+ALTER TABLE files
+    ADD COLUMN size INTEGER;

mediarepo-daemon/mediarepo-database/src/entities/file.rs

@@ -10,6 +10,7 @@ pub struct Model {
     pub name: Option<String>,
     pub comment: Option<String>,
     pub mime_type: Option<String>,
+    pub size: Option<i64>,
     pub storage_id: i64,
     pub hash_id: i64,
     pub import_time: NaiveDateTime,
@@ -32,9 +33,6 @@ pub enum Relation {
         to = "super::storage::Column::Id"
     )]
     Storage,
-
-    #[sea_orm(has_many = "super::thumbnail::Entity")]
-    Thumbnail,
 }
 
 impl Related<super::hash::Entity> for Entity {
@@ -49,10 +47,4 @@ impl Related<super::storage::Entity> for Entity {
     }
 }
 
-impl Related<super::thumbnail::Entity> for Entity {
-    fn to() -> RelationDef {
-        Relation::Thumbnail.def()
-    }
-}
-
 impl ActiveModelBehavior for ActiveModel {}

mediarepo-daemon/mediarepo-database/src/entities/hash.rs

@@ -37,10 +37,4 @@ impl Related<super::source::Entity> for Entity {
     }
 }
 
-impl Related<super::thumbnail::Entity> for Entity {
-    fn to() -> RelationDef {
-        super::thumbnail::Relation::Hash.def().rev()
-    }
-}
-
 impl ActiveModelBehavior for ActiveModel {}

mediarepo-daemon/mediarepo-database/src/entities/mod.rs

@@ -6,4 +6,3 @@ pub mod namespace;
 pub mod source;
 pub mod storage;
 pub mod tag;
-pub mod thumbnail;

mediarepo-daemon/mediarepo-database/src/entities/storage.rs

@@ -13,8 +13,6 @@ pub struct Model {
 pub enum Relation {
     #[sea_orm(has_many = "super::file::Entity")]
     File,
-    #[sea_orm(has_many = "super::thumbnail::Entity")]
-    Thumbnail,
 }
 
 impl Related<super::file::Entity> for Entity {
@@ -23,10 +21,4 @@ impl Related<super::file::Entity> for Entity {
     }
 }
 
-impl Related<super::thumbnail::Entity> for Entity {
-    fn to() -> RelationDef {
-        Relation::Thumbnail.def()
-    }
-}
-
 impl ActiveModelBehavior for ActiveModel {}

mediarepo-daemon/mediarepo-database/src/entities/thumbnail.rs

@@ -1,58 +0,0 @@
-use sea_orm::prelude::*;
-
-#[derive(Clone, Debug, PartialEq, DeriveEntityModel)]
-#[sea_orm(table_name = "thumbnails")]
-pub struct Model {
-    #[sea_orm(primary_key)]
-    pub id: i64,
-    pub file_id: i64,
-    pub storage_id: i64,
-    pub hash_id: i64,
-    pub height: i32,
-    pub width: i32,
-    pub mime: Option<String>,
-}
-
-#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
-pub enum Relation {
-    #[sea_orm(
-        belongs_to = "super::file::Entity",
-        from = "Column::FileId",
-        to = "super::file::Column::Id"
-    )]
-    File,
-
-    #[sea_orm(
-        belongs_to = "super::hash::Entity",
-        from = "Column::HashId",
-        to = "super::hash::Column::Id"
-    )]
-    Hash,
-
-    #[sea_orm(
-        belongs_to = "super::storage::Entity",
-        from = "Column::StorageId",
-        to = "super::storage::Column::Id"
-    )]
-    Storage,
-}
-
-impl Related<super::hash::Entity> for Entity {
-    fn to() -> RelationDef {
-        Relation::Hash.def()
-    }
-}
-
-impl Related<super::file::Entity> for Entity {
-    fn to() -> RelationDef {
-        Relation::File.def()
-    }
-}
-
-impl Related<super::storage::Entity> for Entity {
-    fn to() -> RelationDef {
-        Relation::Storage.def()
-    }
-}
-
-impl ActiveModelBehavior for ActiveModel {}

mediarepo-daemon/mediarepo-database/src/queries/tags.rs

@@ -3,6 +3,7 @@ use sea_orm::DbBackend;
 use sea_orm::FromQueryResult;
 use sea_orm::{DatabaseConnection, Statement};
 use std::collections::HashMap;
+use std::fmt::Display;
 use std::iter::FromIterator;
 
 #[derive(Debug, FromQueryResult)]
@@ -16,7 +17,7 @@ struct HashNamespaceTags {
 pub async fn get_hashes_with_namespaced_tags(
     db: &DatabaseConnection,
     hash_ids: Vec<i64>,
-) -> RepoResult<HashMap<i64, HashMap<String, String>>> {
+) -> RepoResult<HashMap<i64, HashMap<String, Vec<String>>>> {
     let hash_namespace_tags: Vec<HashNamespaceTags> =
         HashNamespaceTags::find_by_statement(Statement::from_sql_and_values(
             DbBackend::Sqlite,
@@ -26,28 +27,74 @@ pub async fn get_hashes_with_namespaced_tags(
                      INNER JOIN tags t on htm.tag_id = t.id
                      JOIN namespaces n on t.namespace_id = n.id
             WHERE t.namespace_id IS NOT NULL
-              AND htm.hash_id IN ({});"#,
+              AND htm.hash_id IN ({}) ORDER BY t.namespace_id;"#,
-                hash_ids
+                vec_to_query_list(hash_ids)
-                    .into_iter()
-                    .fold(String::new(), |acc, val| format!("{}{},", acc, val))
-                    .trim_end_matches(",")
             )
             .as_str(),
             vec![],
         ))
         .all(db)
         .await?;
-    let mut hash_namespaces: HashMap<i64, HashMap<String, String>> = HashMap::new();
+    let mut hash_namespaces: HashMap<i64, HashMap<String, Vec<String>>> = HashMap::new();
     for hnt in hash_namespace_tags {
         if let Some(entry) = hash_namespaces.get_mut(&hnt.hash_id) {
-            entry.insert(hnt.namespace, hnt.tag);
+            if let Some(nsp_entry) = entry.get_mut(&hnt.namespace) {
+                nsp_entry.push(hnt.tag);
+            } else {
+                entry.insert(hnt.namespace, vec![hnt.tag]);
+            }
         } else {
             hash_namespaces.insert(
                 hnt.hash_id,
-                HashMap::from_iter(vec![(hnt.namespace, hnt.tag)].into_iter()),
+                HashMap::from_iter(vec![(hnt.namespace, vec![hnt.tag])].into_iter()),
             );
         }
     }
 
     Ok(hash_namespaces)
 }
+
+#[derive(Debug, FromQueryResult)]
+struct HashTagCount {
+    hash_id: i64,
+    tag_count: i32,
+}
+
+#[tracing::instrument(level = "debug", skip_all)]
+pub async fn get_hashes_with_tag_count(
+    db: &DatabaseConnection,
+    hash_ids: Vec<i64>,
+) -> RepoResult<HashMap<i64, u32>> {
+    let hash_tag_counts: Vec<HashTagCount> =
+        HashTagCount::find_by_statement(Statement::from_sql_and_values(
+            DbBackend::Sqlite,
+            format!(
+                r#"
+        SELECT htm.hash_id, COUNT(htm.tag_id) AS "tag_count" from hash_tag_mappings htm
+        WHERE htm.hash_id IN ({})
+        GROUP BY hash_id
+    "#,
+                vec_to_query_list(hash_ids)
+            )
+            .as_str(),
+            vec![],
+        ))
+        .all(db)
+        .await?;
+
+    let mappings = hash_tag_counts
+        .into_iter()
+        .map(|HashTagCount { hash_id, tag_count }| (hash_id, tag_count as u32))
+        .collect::<HashMap<i64, u32>>();
+
+    Ok(mappings)
+}
+
+fn vec_to_query_list<D: Display>(input: Vec<D>) -> String {
+    let mut entries = input
+        .into_iter()
+        .fold(String::new(), |acc, val| format!("{}{},", acc, val));
+    entries.remove(entries.len() - 1);
+
+    entries
+}

mediarepo-daemon/mediarepo-model/src/file.rs

@@ -327,6 +327,25 @@ impl File {
         storage.get_file_reader(&self.hash.value).await
     }
 
+    /// Retrieves the size of the file from its content
+    #[tracing::instrument(level = "debug", skip(self))]
+    pub async fn get_size(&self) -> RepoResult<u64> {
+        if let Some(size) = self.model.size {
+            Ok(size as u64)
+        } else {
+            let mut reader = self.get_reader().await?;
+            let size = {
+                let mut buf = Vec::new();
+                reader.read_to_end(&mut buf).await
+            }?;
+            let mut model = self.get_active_model();
+            model.size = Set(Some(size as i64));
+            model.update(&self.db).await?;
+
+            Ok(size as u64)
+        }
+    }
+
     /// Creates a thumbnail for the file
     #[tracing::instrument(level = "debug", skip(self))]
     pub async fn create_thumbnail<I: IntoIterator<Item = ThumbnailSize> + Debug>(

mediarepo-daemon/mediarepo-socket/src/namespaces/files.rs

@@ -1,5 +1,6 @@
 use crate::from_model::FromModel;
 use crate::utils::{file_by_identifier, get_repo_from_context, hash_by_identifier};
+use chrono::NaiveDateTime;
 use compare::Compare;
 use mediarepo_api::types::files::{
     AddFileRequestHeader, FileMetadataResponse, FindFilesByTagsRequest,
@@ -12,13 +13,24 @@ use mediarepo_core::itertools::Itertools;
 use mediarepo_core::rmp_ipc::prelude::*;
 use mediarepo_core::thumbnailer::ThumbnailSize;
 use mediarepo_core::utils::parse_namespace_and_tag;
-use mediarepo_database::queries::tags::get_hashes_with_namespaced_tags;
+use mediarepo_database::queries::tags::{
-use mediarepo_model::file::File;
+    get_hashes_with_namespaced_tags, get_hashes_with_tag_count,
+};
 use std::cmp::Ordering;
 use std::collections::HashMap;
 use tokio::io::AsyncReadExt;
 
 pub struct FilesNamespace;
+pub struct FileSortContext {
+    name: Option<String>,
+    size: u64,
+    mime_type: Option<String>,
+    namespaces: HashMap<String, Vec<String>>,
+    tag_count: u32,
+    import_time: NaiveDateTime,
+    create_time: NaiveDateTime,
+    change_time: NaiveDateTime,
+}
 
 impl NamespaceProvider for FilesNamespace {
     fn name() -> &'static str {
@@ -79,21 +91,36 @@ impl FilesNamespace {
         let repo = get_repo_from_context(ctx).await;
         let tags = req.tags.into_iter().map(|t| (t.name, t.negate)).collect();
         let mut files = repo.find_files_by_tags(tags).await?;
-        let hash_ids = files.iter().map(|f| f.hash_id()).collect();
+        let hash_ids: Vec<i64> = files.iter().map(|f| f.hash_id()).collect();
-
+
-        let hash_nsp: HashMap<i64, HashMap<String, String>> =
+        let mut hash_nsp: HashMap<i64, HashMap<String, Vec<String>>> =
-            get_hashes_with_namespaced_tags(repo.db(), hash_ids).await?;
+            get_hashes_with_namespaced_tags(repo.db(), hash_ids.clone()).await?;
-
+        let mut hash_tag_counts = get_hashes_with_tag_count(repo.db(), hash_ids).await?;
+
+        let mut contexts = HashMap::new();
+
+        for file in &files {
+            let context = FileSortContext {
+                name: file.name().to_owned(),
+                size: file.get_size().await?,
+                mime_type: file.mime_type().to_owned(),
+                namespaces: hash_nsp
+                    .remove(&file.hash_id())
+                    .unwrap_or(HashMap::with_capacity(0)),
+                tag_count: hash_tag_counts.remove(&file.hash_id()).unwrap_or(0),
+                import_time: file.import_time().to_owned(),
+                create_time: file.import_time().to_owned(),
+                change_time: file.change_time().to_owned(),
+            };
+            contexts.insert(file.id(), context);
+        }
         let sort_expression = req.sort_expression;
         tracing::debug!("sort_expression = {:?}", sort_expression);
-        let empty_map = HashMap::with_capacity(0);
 
         files.sort_by(|a, b| {
             compare_files(
-                a,
+                contexts.get(&a.hash_id()).unwrap(),
-                hash_nsp.get(&a.hash_id()).unwrap_or(&empty_map),
+                contexts.get(&b.hash_id()).unwrap(),
-                b,
-                hash_nsp.get(&b.hash_id()).unwrap_or(&empty_map),
                 &sort_expression,
             )
         });
@@ -263,55 +290,56 @@ impl FilesNamespace {
 
 #[tracing::instrument(level = "trace", skip_all)]
 fn compare_files(
-    file_a: &File,
+    ctx_a: &FileSortContext,
-    nsp_a: &HashMap<String, String>,
+    ctx_b: &FileSortContext,
-    file_b: &File,
-    nsp_b: &HashMap<String, String>,
     expression: &Vec<SortKey>,
 ) -> Ordering {
     let cmp_date = compare::natural();
+    let cmp_u64 = compare::natural();
+    let cmp_u32 = compare::natural();
 
     for sort_key in expression {
         let ordering = match sort_key {
             SortKey::Namespace(namespace) => {
-                let tag_a = nsp_a.get(&namespace.name);
+                let list_a = ctx_a.namespaces.get(&namespace.name);
-                let tag_b = nsp_b.get(&namespace.name);
+                let list_b = ctx_b.namespaces.get(&namespace.name);
-
+
-                if let (Some(a), Some(b)) = (
+                let cmp_result = if let (Some(list_a), Some(list_b)) = (list_a, list_b) {
-                    tag_a.and_then(|a| a.parse::<f32>().ok()),
+                    compare_tag_lists(list_a, list_b)
-                    tag_b.and_then(|b| b.parse::<f32>().ok()),
+                } else if list_a.is_some() {
-                ) {
+                    Ordering::Greater
-                    adjust_for_dir(compare_f32(a, b), &namespace.direction)
+                } else if list_b.is_some() {
+                    Ordering::Less
                 } else {
-                    adjust_for_dir(compare_opts(tag_a, tag_b), &namespace.direction)
+                    Ordering::Equal
-                }
+                };
+                adjust_for_dir(cmp_result, &namespace.direction)
             }
-            SortKey::FileName(direction) => adjust_for_dir(
+            SortKey::FileName(direction) => {
-                compare_opts(file_a.name().clone(), file_b.name().clone()),
+                adjust_for_dir(compare_opts(&ctx_a.name, &ctx_b.name), direction)
-                direction,
+            }
-            ),
+            SortKey::FileSize(direction) => {
-            SortKey::FileSize(_direction) => {
+                adjust_for_dir(cmp_u64.compare(&ctx_a.size, &ctx_b.size), direction)
-                Ordering::Equal // TODO: Retrieve file size
             }
             SortKey::FileImportedTime(direction) => adjust_for_dir(
-                cmp_date.compare(file_a.import_time(), file_b.import_time()),
+                cmp_date.compare(&ctx_a.import_time, &ctx_b.import_time),
                 direction,
             ),
             SortKey::FileCreatedTime(direction) => adjust_for_dir(
-                cmp_date.compare(file_a.creation_time(), file_b.creation_time()),
+                cmp_date.compare(&ctx_a.create_time, &ctx_b.create_time),
                 direction,
             ),
             SortKey::FileChangeTime(direction) => adjust_for_dir(
-                cmp_date.compare(file_a.change_time(), file_b.change_time()),
+                cmp_date.compare(&ctx_a.change_time, &ctx_b.change_time),
                 direction,
             ),
-            SortKey::FileType(direction) => adjust_for_dir(
+            SortKey::FileType(direction) => {
-                compare_opts(file_a.mime_type().clone(), file_b.mime_type().clone()),
+                adjust_for_dir(compare_opts(&ctx_a.mime_type, &ctx_b.mime_type), direction)
+            }
+            SortKey::NumTags(direction) => adjust_for_dir(
+                cmp_u32.compare(&ctx_a.tag_count, &ctx_b.tag_count),
                 direction,
             ),
-            SortKey::NumTags(_) => {
-                Ordering::Equal // TODO: Count tags
-            }
         };
         if !ordering.is_eq() {
             return ordering;
@@ -321,9 +349,9 @@ fn compare_files(
     Ordering::Equal
 }
 
-fn compare_opts<T: Ord + Sized>(opt_a: Option<T>, opt_b: Option<T>) -> Ordering {
+fn compare_opts<T: Ord + Sized>(opt_a: &Option<T>, opt_b: &Option<T>) -> Ordering {
     let cmp = compare::natural();
-    if let (Some(a), Some(b)) = (&opt_a, &opt_b) {
+    if let (Some(a), Some(b)) = (opt_a, opt_b) {
         cmp.compare(a, b)
     } else if opt_a.is_some() {
         Ordering::Greater
@@ -351,3 +379,21 @@ fn adjust_for_dir(ordering: Ordering, direction: &SortDirection) -> Ordering {
         ordering
     }
 }
+
+fn compare_tag_lists(list_a: &Vec<String>, list_b: &Vec<String>) -> Ordering {
+    let first_diff = list_a
+        .into_iter()
+        .zip(list_b.into_iter())
+        .find(|(a, b)| *a != *b);
+    if let Some(diff) = first_diff {
+        if let (Some(num_a), Some(num_b)) = (diff.0.parse::<f32>().ok(), diff.1.parse::<f32>().ok())
+        {
+            compare_f32(num_a, num_b)
+        } else {
+            let cmp = compare::natural();
+            cmp.compare(diff.0, diff.1)
+        }
+    } else {
+        Ordering::Equal
+    }
+}