Improve performance of sorting data fetching

Signed-off-by: trivernis <trivernis@protonmail.com>

mediarepo-daemon/mediarepo-database/src/queries/tags.rs

@@ -1,61 +1,12 @@
 use std::collections::HashMap;
 use std::fmt::Display;
-use std::iter::FromIterator;
 
-use sea_orm::{DatabaseConnection, Statement};
 use sea_orm::DbBackend;
 use sea_orm::FromQueryResult;
+use sea_orm::{DatabaseConnection, Statement};
 
 use mediarepo_core::error::RepoResult;
 
-#[derive(Debug, FromQueryResult)]
-struct CIDNamespaceTag {
-    cd_id: i64,
-    namespace: String,
-    tag: String,
-}
-
-#[tracing::instrument(level = "debug", skip_all)]
-pub async fn get_cids_with_namespaced_tags(
-    db: &DatabaseConnection,
-    hash_ids: Vec<i64>,
-) -> RepoResult<HashMap<i64, HashMap<String, Vec<String>>>> {
-    let hash_namespace_tags: Vec<CIDNamespaceTag> =
-        CIDNamespaceTag::find_by_statement(Statement::from_sql_and_values(
-            DbBackend::Sqlite,
-            format!(
-                r#"SELECT ctm.cd_id, n.name as namespace, t.name as tag
-            FROM cd_tag_mappings ctm
-                     INNER JOIN tags t on ctm.tag_id = t.id
-                     JOIN namespaces n on t.namespace_id = n.id
-            WHERE t.namespace_id IS NOT NULL
-              AND ctm.cd_id IN ({}) ORDER BY t.namespace_id;"#,
-                vec_to_query_list(hash_ids)
-            )
-            .as_str(),
-            vec![],
-        ))
-        .all(db)
-        .await?;
-    let mut cd_id_namespaces: HashMap<i64, HashMap<String, Vec<String>>> = HashMap::new();
-    for hnt in hash_namespace_tags {
-        if let Some(entry) = cd_id_namespaces.get_mut(&hnt.cd_id) {
-            if let Some(nsp_entry) = entry.get_mut(&hnt.namespace) {
-                nsp_entry.push(hnt.tag);
-            } else {
-                entry.insert(hnt.namespace, vec![hnt.tag]);
-            }
-        } else {
-            cd_id_namespaces.insert(
-                hnt.cd_id,
-                HashMap::from_iter(vec![(hnt.namespace, vec![hnt.tag])].into_iter()),
-            );
-        }
-    }
-
-    Ok(cd_id_namespaces)
-}
-
 #[derive(Debug, FromQueryResult)]
 struct CIDTagCount {
     cd_id: i64,

mediarepo-daemon/mediarepo-logic/src/dao/tag/cdids_with_namespaced_tags.rs

@@ -0,0 +1,56 @@
+use crate::dao::tag::TagDao;
+use mediarepo_core::error::RepoResult;
+use mediarepo_database::entities::{content_descriptor_tag, namespace, tag};
+use sea_orm::prelude::*;
+use sea_orm::JoinType;
+use sea_orm::{FromQueryResult, QuerySelect};
+use std::collections::HashMap;
+use std::iter::FromIterator;
+
+#[derive(Debug, FromQueryResult)]
+struct CDIDNamespaceTag {
+    cd_id: i64,
+    namespace: String,
+    tag: String,
+}
+
+impl TagDao {
+    #[tracing::instrument(level = "debug", skip(self, cdids))]
+    pub async fn cdids_with_namespaced_tags(
+        &self,
+        cdids: Vec<i64>,
+    ) -> RepoResult<HashMap<i64, HashMap<String, Vec<String>>>> {
+        let cd_namespace_tags: Vec<CDIDNamespaceTag> = content_descriptor_tag::Entity::find()
+            .select_only()
+            .column(content_descriptor_tag::Column::CdId)
+            .column_as(tag::Column::Name, "tag")
+            .column_as(namespace::Column::Name, "namespace")
+            .join(
+                JoinType::InnerJoin,
+                content_descriptor_tag::Relation::Tag.def(),
+            )
+            .join(JoinType::Join, namespace::Relation::Tag.def().rev())
+            .filter(content_descriptor_tag::Column::CdId.is_in(cdids))
+            .into_model::<CDIDNamespaceTag>()
+            .all(&self.ctx.db)
+            .await?;
+
+        let mut cd_id_namespaces: HashMap<i64, HashMap<String, Vec<String>>> = HashMap::new();
+        for cnt in cd_namespace_tags {
+            if let Some(entry) = cd_id_namespaces.get_mut(&cnt.cd_id) {
+                if let Some(nsp_entry) = entry.get_mut(&cnt.namespace) {
+                    nsp_entry.push(cnt.tag);
+                } else {
+                    entry.insert(cnt.namespace, vec![cnt.tag]);
+                }
+            } else {
+                cd_id_namespaces.insert(
+                    cnt.cd_id,
+                    HashMap::from_iter(vec![(cnt.namespace, vec![cnt.tag])].into_iter()),
+                );
+            }
+        }
+
+        Ok(cd_id_namespaces)
+    }
+}

mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs

@@ -16,6 +16,7 @@ use crate::dto::{NamespaceDto, TagDto};
 pub mod add;
 pub mod all_for_cds_map;
 pub mod by_name;
+pub mod cdids_with_namespaced_tags;
 pub mod mappings;
 
 pub struct TagDao {

mediarepo-daemon/mediarepo-socket/src/namespaces/files/sorting.rs

@@ -8,14 +8,11 @@ use rayon::iter::{IntoParallelRefIterator, ParallelIterator};
 
 use mediarepo_core::error::RepoResult;
 use mediarepo_core::mediarepo_api::types::filtering::{SortDirection, SortKey};
-use mediarepo_database::queries::tags::{
+use mediarepo_database::queries::tags::get_content_descriptors_with_tag_count;
-    get_cids_with_namespaced_tags, get_content_descriptors_with_tag_count,
-};
-use mediarepo_logic::dao::DaoProvider;
 use mediarepo_logic::dao::repo::Repo;
+use mediarepo_logic::dao::DaoProvider;
 use mediarepo_logic::dto::{FileDto, FileMetadataDto};
 
-
 pub struct FileSortContext {
     name: Option<String>,
     size: u64,
@@ -50,12 +47,14 @@ async fn build_sort_context(
     repo: &Repo,
     files: &Vec<FileDto>,
 ) -> RepoResult<HashMap<i64, FileSortContext>> {
-    let hash_ids: Vec<i64> = files.par_iter().map(|f| f.cd_id()).collect();
+    let cd_ids: Vec<i64> = files.par_iter().map(|f| f.cd_id()).collect();
     let file_ids: Vec<i64> = files.par_iter().map(|f| f.id()).collect();
 
-    let mut cid_nsp: HashMap<i64, HashMap<String, Vec<String>>> =
+    let mut cid_nsp: HashMap<i64, HashMap<String, Vec<String>>> = repo
-        get_cids_with_namespaced_tags(repo.db(), hash_ids.clone()).await?;
+        .tag()
-    let mut cid_tag_counts = get_content_descriptors_with_tag_count(repo.db(), hash_ids).await?;
+        .cdids_with_namespaced_tags(cd_ids.clone())
+        .await?;
+    let mut cid_tag_counts = get_content_descriptors_with_tag_count(repo.db(), cd_ids).await?;
 
     let files_metadata = repo.file().all_metadata(file_ids).await?;