Merge pull request #7 from Trivernis/feature/performance-improvements

Feature/performance improvements

mediarepo-api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mediarepo-api"
-version = "0.27.0"
+version = "0.28.0"
 edition = "2018"
 license = "gpl-3"
 

mediarepo-api/src/client_api/tag.rs

@@ -1,6 +1,7 @@
+use std::collections::HashMap;
 use crate::client_api::error::ApiResult;
 use crate::client_api::IPCApi;
-use crate::types::files::{GetFileTagsRequest, GetFilesTagsRequest};
+use crate::types::files::{GetFileTagsRequest, GetFilesTagsRequest, GetFileTagMapRequest};
 use crate::types::identifier::FileIdentifier;
 use crate::types::tags::{ChangeFileTagsRequest, NamespaceResponse, TagResponse};
 use async_trait::async_trait;
@@ -72,6 +73,12 @@ impl TagApi {
         .await
     }
 
+    /// Returns a map from files to assigned tags
+    #[tracing::instrument(level = "debug", skip_all)]
+    pub async fn get_file_tag_map(&self, cds: Vec<String>) -> ApiResult<HashMap<String, Vec<TagResponse>>> {
+        self.emit_and_get("file_tag_map", GetFileTagMapRequest{cds}, Some(Duration::from_secs(10))).await
+    }
+
     /// Creates a new tag and returns the created tag object
     #[tracing::instrument(level = "debug", skip(self))]
     pub async fn create_tags(&self, tags: Vec<String>) -> ApiResult<Vec<TagResponse>> {

mediarepo-api/src/tauri_plugin/commands/tag.rs

@@ -1,3 +1,4 @@
+use std::collections::HashMap;
 use crate::tauri_plugin::commands::ApiAccess;
 use crate::tauri_plugin::error::PluginResult;
 use crate::types::identifier::FileIdentifier;
@@ -41,6 +42,14 @@ pub async fn get_tags_for_files(
     Ok(tags)
 }
 
+#[tauri::command]
+pub async fn get_file_tag_map(cds: Vec<String>, api_state: ApiAccess<'_>) -> PluginResult<HashMap<String, Vec<TagResponse>>> {
+    let api = api_state.api().await?;
+    let mappings = api.tag.get_file_tag_map(cds).await?;
+
+    Ok(mappings)
+}
+
 #[tauri::command]
 pub async fn create_tags(
     api_state: ApiAccess<'_>,

mediarepo-api/src/tauri_plugin/mod.rs

@@ -69,7 +69,8 @@ impl<R: Runtime> MediarepoPlugin<R> {
                 get_file_metadata,
                 run_job,
                 update_file_status,
-                delete_file
+                delete_file,
+                get_file_tag_map
             ]),
         }
     }

mediarepo-api/src/types/files.rs

@@ -30,6 +30,11 @@ pub struct GetFilesTagsRequest {
     pub cds: Vec<String>,
 }
 
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct GetFileTagMapRequest {
+    pub cds: Vec<String>,
+}
+
 #[derive(Clone, Debug, Serialize, Deserialize)]
 pub struct FileBasicDataResponse {
     pub id: i64,

mediarepo-api/src/types/jobs.rs

@@ -11,4 +11,5 @@ pub enum JobType {
     MigrateContentDescriptors,
     CalculateSizes,
     CheckIntegrity,
+    Vacuum,
 }

mediarepo-daemon/Cargo.lock

@@ -1182,7 +1182,7 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
 
 [[package]]
 name = "mediarepo-api"
-version = "0.27.0"
+version = "0.28.0"
 dependencies = [
  "bromine",
  "chrono",

mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs

@@ -5,6 +5,7 @@ use std::collections::HashMap;
 use std::iter::FromIterator;
 
 use mediarepo_core::error::RepoResult;
+use mediarepo_core::itertools::Itertools;
 
 use mediarepo_core::utils::parse_namespace_and_tag;
 use mediarepo_database::entities::{content_descriptor, content_descriptor_tag, namespace, tag};
@@ -57,7 +58,7 @@ impl TagDao {
         Ok(namespaces)
     }
 
-    #[tracing::instrument(level = "debug", skip(self))]
+    #[tracing::instrument(level = "debug", skip(self, cds))]
     pub async fn all_for_cds(&self, cds: Vec<Vec<u8>>) -> RepoResult<Vec<TagDto>> {
         let tags = tag::Entity::find()
             .find_also_related(namespace::Entity)
@@ -80,6 +81,51 @@ impl TagDao {
         Ok(tags)
     }
 
+    #[tracing::instrument(level = "debug", skip(self, cds))]
+    pub async fn all_for_cds_map(
+        &self,
+        cds: Vec<Vec<u8>>,
+    ) -> RepoResult<HashMap<Vec<u8>, Vec<TagDto>>> {
+        let tag_cd_entries: Vec<(
+            content_descriptor_tag::Model,
+            Option<content_descriptor::Model>,
+        )> = content_descriptor_tag::Entity::find()
+            .find_also_related(content_descriptor::Entity)
+            .filter(content_descriptor::Column::Descriptor.is_in(cds))
+            .all(&self.ctx.db)
+            .await?;
+
+        let tag_ids: Vec<i64> = tag_cd_entries
+            .iter()
+            .map(|(t, _)| t.tag_id)
+            .unique()
+            .collect();
+
+        let tags: Vec<TagDto> = tag::Entity::find()
+            .find_also_related(namespace::Entity)
+            .filter(tag::Column::Id.is_in(tag_ids))
+            .all(&self.ctx.db)
+            .await?
+            .into_iter()
+            .map(map_tag_dto)
+            .collect();
+
+        let tag_id_map = tags
+            .into_iter()
+            .map(|t| (t.id(), t))
+            .collect::<HashMap<i64, TagDto>>();
+        let cd_tag_map = tag_cd_entries
+            .into_iter()
+            .filter_map(|(t, cd)| Some((cd?, tag_id_map.get(&t.tag_id)?.clone())))
+            .sorted_by_key(|(cd, _)| cd.id)
+            .group_by(|(cd, _)| cd.descriptor.to_owned())
+            .into_iter()
+            .map(|(key, group)| (key, group.map(|(_, t)| t).collect::<Vec<TagDto>>()))
+            .collect();
+
+        Ok(cd_tag_map)
+    }
+
     #[tracing::instrument(level = "debug", skip(self))]
     pub async fn tags_for_cd(&self, cd_id: i64) -> RepoResult<Vec<TagDto>> {
         let tags = tag::Entity::find()

mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs

@@ -31,6 +31,7 @@ impl JobsNamespace {
             JobType::MigrateContentDescriptors => job_dao.migrate_content_descriptors().await?,
             JobType::CalculateSizes => calculate_all_sizes(ctx).await?,
             JobType::CheckIntegrity => job_dao.check_integrity().await?,
+            JobType::Vacuum => job_dao.vacuum().await?,
         }
 
         ctx.emit_to(Self::name(), "run_job", ()).await?;

mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs

@@ -1,8 +1,11 @@
 use rayon::iter::{IntoParallelIterator, ParallelIterator};
+use std::collections::HashMap;
 
 use mediarepo_core::bromine::prelude::*;
-use mediarepo_core::content_descriptor::decode_content_descriptor;
+use mediarepo_core::content_descriptor::{decode_content_descriptor, encode_content_descriptor};
-use mediarepo_core::mediarepo_api::types::files::{GetFileTagsRequest, GetFilesTagsRequest};
+use mediarepo_core::mediarepo_api::types::files::{
+    GetFileTagMapRequest, GetFileTagsRequest, GetFilesTagsRequest,
+};
 use mediarepo_core::mediarepo_api::types::tags::{
     ChangeFileTagsRequest, NamespaceResponse, TagResponse,
 };
@@ -26,6 +29,7 @@ impl NamespaceProvider for TagsNamespace {
             "all_namespaces" => Self::all_namespaces,
             "tags_for_file" => Self::tags_for_file,
             "tags_for_files" => Self::tags_for_files,
+            "file_tag_map" => Self::tag_cd_map_for_files,
             "create_tags" => Self::create_tags,
             "change_file_tags" => Self::change_file_tags
         );
@@ -105,6 +109,38 @@ impl TagsNamespace {
         Ok(())
     }
 
+    /// Returns a map of content descriptors to assigned tags
+    #[tracing::instrument(skip_all)]
+    async fn tag_cd_map_for_files(ctx: &Context, event: Event) -> IPCResult<()> {
+        let request = event.payload::<GetFileTagMapRequest>()?;
+        let repo = get_repo_from_context(ctx).await;
+        let cds = request
+            .cds
+            .into_iter()
+            .filter_map(|c| decode_content_descriptor(c).ok())
+            .collect();
+
+        let mappings = repo
+            .tag()
+            .all_for_cds_map(cds)
+            .await?
+            .into_iter()
+            .map(|(cd, tags)| (encode_content_descriptor(&cd), tags))
+            .map(|(cd, tags)| {
+                (
+                    cd,
+                    tags.into_iter()
+                        .map(TagResponse::from_model)
+                        .collect::<Vec<TagResponse>>(),
+                )
+            })
+            .collect::<HashMap<String, Vec<TagResponse>>>();
+
+        ctx.emit_to(Self::name(), "file_tag_map", mappings).await?;
+
+        Ok(())
+    }
+
     /// Creates all tags given as input or returns the existing tags
     #[tracing::instrument(skip_all)]
     async fn create_tags(ctx: &Context, event: Event) -> IPCResult<()> {

mediarepo-ui/src-tauri/Cargo.lock

@@ -1499,7 +1499,7 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
 
 [[package]]
 name = "mediarepo-api"
-version = "0.27.0"
+version = "0.28.0"
 dependencies = [
  "async-trait",
  "bromine",

mediarepo-ui/src/api/Api.ts

@@ -13,6 +13,7 @@ import {
     DeleteThumbnailsRequest,
     FindFilesRequest,
     GetFileMetadataRequest,
+    GetFileTagMapRequest,
     GetSizeRequest,
     GetTagsForFilesRequest,
     InitRepositoryRequest,
@@ -28,7 +29,7 @@ import {
     UpdateFileStatusRequest
 } from "./api-types/requests";
 import {RepositoryData, RepositoryMetadata, SizeMetadata} from "./api-types/repo";
-import {NamespaceData, TagData} from "./api-types/tags";
+import {CdTagMappings, NamespaceData, TagData} from "./api-types/tags";
 import {ShortCache} from "./ShortCache";
 
 export class MediarepoApi {
@@ -146,6 +147,15 @@ export class MediarepoApi {
         );
     }
 
+    public static async getFileTagMap(request: GetFileTagMapRequest): Promise<CdTagMappings> {
+        return ShortCache.cached(
+            request,
+            () => this.invokePlugin(ApiFunction.GetFileTagMap, request),
+            1000,
+            "getFileTagMap"
+        );
+    }
+
     public static async createTags(request: CreateTagsRequest): Promise<TagData[]> {
         return this.invokePlugin(ApiFunction.CreateTags, request);
     }

mediarepo-ui/src/api/api-types/functions.ts

@@ -29,6 +29,7 @@ export enum ApiFunction {
     GetAllTags = "get_all_tags",
     GetAllNamespace = "get_all_namespaces",
     GetTagsForFiles = "get_tags_for_files",
+    GetFileTagMap = "get_file_tag_map",
     CreateTags = "create_tags",
     ChangeFileTags = "change_file_tags",
     // import

mediarepo-ui/src/api/api-types/job.ts

@@ -1,3 +1,4 @@
 export type JobType = "MigrateContentDescriptors"
     | "CalculateSizes"
-    | "CheckIntegrity";
+    | "CheckIntegrity"
+    | "Vacuum";

mediarepo-ui/src/api/api-types/requests.ts

@@ -73,6 +73,10 @@ export type GetTagsForFilesRequest = {
     cds: string[]
 };
 
+export type GetFileTagMapRequest = {
+    cds: string[]
+};
+
 export type CreateTagsRequest = {
     tags: string[]
 };

mediarepo-ui/src/api/api-types/tags.ts

@@ -8,3 +8,7 @@ export type NamespaceData = {
     id: number,
     name: string,
 };
+
+export type CdTagMappings = {
+    [key: string]: TagData[],
+};

mediarepo-ui/src/app/app.component.scss

@@ -14,4 +14,5 @@
 
 ::ng-deep .app-error {
     background-color: $warn;
+    color: $text;
 }

mediarepo-ui/src/app/app.component.ts

@@ -63,21 +63,21 @@ export class AppComponent implements OnInit {
         if (!environment.production) {
             switch (entry.getLevel()) {
                 case LogLevel.Trace:
-                    console.trace(entry.getMessage(), entry);
+                    console.trace(entry.getMessage());
                     break;
                 case LogLevel.Debug:
-                    console.debug(entry.getMessage(), entry);
+                    console.debug(entry.getMessage());
                     break;
                 case LogLevel.Info:
-                    console.info(entry.getMessage(), entry);
+                    console.info(entry.getMessage());
                     break;
                 case LogLevel.Warn:
-                    console.warn(entry.getMessage(), entry);
+                    console.warn(entry.getMessage());
                     break;
             }
         }
         if (entry.getLevel() == LogLevel.Error) {
-            console.error(entry.getMessage(), entry.getError(), entry);
+            console.error(entry.getMessage(), entry.getError());
         }
     }
 }

mediarepo-ui/src/app/components/core/repositories-tab/repositories-tab.component.ts

@@ -111,13 +111,15 @@ export class RepositoriesTabComponent implements OnInit, AfterViewInit {
     }
 
     private async runRepositoryStartupTasks(dialogContext: BusyDialogContext): Promise<void> {
+        dialogContext.message.next("Checking integrity...");
+        await this.jobService.runJob("CheckIntegrity");
+        dialogContext.message.next("Running a vacuum on the database...");
+        await this.jobService.runJob("Vacuum");
         dialogContext.message.next(
             "Migrating content descriptors to new format...");
         await this.jobService.runJob("MigrateContentDescriptors");
         dialogContext.message.next("Calculating repository sizes...");
         await this.jobService.runJob("CalculateSizes");
-        dialogContext.message.next("Checking integrity...");
-        await this.jobService.runJob("CheckIntegrity");
         dialogContext.message.next("Finished repository startup");
     }
 

mediarepo-ui/src/app/components/shared/sidebar/tag-edit/tag-edit.component.ts

@@ -36,7 +36,7 @@ export class TagEditComponent implements AfterViewInit, OnChanges {
     private fileTags: { [key: number]: Tag[] } = {};
 
     constructor(
-        private errorBroker: LoggingService,
+        private logger: LoggingService,
         private tagService: TagService,
     ) {
     }
@@ -148,16 +148,12 @@ export class TagEditComponent implements AfterViewInit, OnChanges {
 
     private async loadFileTags() {
         await this.wrapAsyncOperation(async () => {
-            const promises = [];
+            console.log("loading tags");
-            const loadFn = async (file: File) => {
+            const mappings = await this.tagService.getFileTagMappings(this.files.map(f => f.cd));
-                this.fileTags[file.id] = await this.tagService.getTagsForFiles(
+
-                    [file.cd]);
-            };
             for (const file of this.files) {
-                promises.push(loadFn(file));
+                this.fileTags[file.id] = mappings[file.cd];
             }
-
-            await Promise.all(promises);
             this.mapFileTagsToTagList();
         });
     }
@@ -176,11 +172,11 @@ export class TagEditComponent implements AfterViewInit, OnChanges {
     }
 
     private async wrapAsyncOperation<T>(cb: () => Promise<T>): Promise<T | undefined> {
-        if (!this.busyIndicator) {
+        if (!this.busyIndicator?.wrapAsyncOperation) {
             try {
                 return cb();
             } catch (err: any) {
-                this.errorBroker.error(err);
+                this.logger.error(err);
                 return undefined;
             }
         } else {

mediarepo-ui/src/app/services/tag/tag.service.ts

@@ -29,16 +29,31 @@ export class TagService {
     public async getTagsForFiles(cds: string[]): Promise<Tag[]> {
         let tags: Tag[] = [];
         if (cds.length > 0) {
-            tags = await MediarepoApi.getTagsForFiles({cds}).then(mapMany(mapNew(Tag)));
+            tags = await MediarepoApi.getTagsForFiles({ cds }).then(mapMany(mapNew(Tag)));
         }
         return tags;
     }
 
+    public async getFileTagMappings(cds: string[]): Promise<{ [key: string]: Tag[] }> {
+        if (cds.length > 0) {
+            return await MediarepoApi.getFileTagMap({ cds }).then((cdMappings) => {
+                let mappings: { [key: string]: Tag[] } = {};
+                console.log("TAG MAPPINGS", cdMappings);
+                for (const key in cdMappings) {
+                    mappings[key] = cdMappings[key].map(mapNew(Tag));
+                }
+                return mappings;
+            });
+        } else {
+            return {};
+        }
+    }
+
     public async createTags(tags: string[]): Promise<Tag[]> {
-        return MediarepoApi.createTags({tags}).then(mapMany(mapNew(Tag)));
+        return MediarepoApi.createTags({ tags }).then(mapMany(mapNew(Tag)));
     }
 
     public async changeFileTags(fileId: number, addedTags: number[], removedTags: number[]): Promise<Tag[]> {
-        return MediarepoApi.changeFileTags({id: fileId, addedTags, removedTags}).then(mapMany(mapNew(Tag)));
+        return MediarepoApi.changeFileTags({ id: fileId, addedTags, removedTags }).then(mapMany(mapNew(Tag)));
     }
 }