From 3c005c3a7ee0c4c546ca9c9fb8677ed0e5e2942f Mon Sep 17 00:00:00 2001 From: trivernis Date: Sun, 30 Jan 2022 19:16:30 +0100 Subject: [PATCH] Add single lookup for cd-tag-mappings for better performance Signed-off-by: trivernis --- mediarepo-api/Cargo.toml | 2 +- mediarepo-api/src/client_api/tag.rs | 9 +++- .../src/tauri_plugin/commands/tag.rs | 9 ++++ mediarepo-api/src/tauri_plugin/mod.rs | 3 +- mediarepo-api/src/types/files.rs | 5 ++ mediarepo-api/src/types/jobs.rs | 1 + .../mediarepo-logic/src/dao/tag/mod.rs | 48 ++++++++++++++++++- .../mediarepo-socket/src/namespaces/jobs.rs | 1 + .../mediarepo-socket/src/namespaces/tags.rs | 40 +++++++++++++++- mediarepo-ui/src/api/Api.ts | 12 ++++- mediarepo-ui/src/api/api-types/functions.ts | 1 + mediarepo-ui/src/api/api-types/requests.ts | 4 ++ mediarepo-ui/src/api/api-types/tags.ts | 4 ++ mediarepo-ui/src/app/app.component.ts | 10 ++-- .../sidebar/tag-edit/tag-edit.component.ts | 18 +++---- .../src/app/services/tag/tag.service.ts | 21 ++++++-- 16 files changed, 162 insertions(+), 26 deletions(-) diff --git a/mediarepo-api/Cargo.toml b/mediarepo-api/Cargo.toml index e350b59..5a8a80e 100644 --- a/mediarepo-api/Cargo.toml +++ b/mediarepo-api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mediarepo-api" -version = "0.27.0" +version = "0.28.0" edition = "2018" license = "gpl-3" diff --git a/mediarepo-api/src/client_api/tag.rs b/mediarepo-api/src/client_api/tag.rs index 45e5098..e4b7f79 100644 --- a/mediarepo-api/src/client_api/tag.rs +++ b/mediarepo-api/src/client_api/tag.rs @@ -1,6 +1,7 @@ +use std::collections::HashMap; use crate::client_api::error::ApiResult; use crate::client_api::IPCApi; -use crate::types::files::{GetFileTagsRequest, GetFilesTagsRequest}; +use crate::types::files::{GetFileTagsRequest, GetFilesTagsRequest, GetFileTagMapRequest}; use crate::types::identifier::FileIdentifier; use crate::types::tags::{ChangeFileTagsRequest, NamespaceResponse, TagResponse}; use async_trait::async_trait; @@ -72,6 +73,12 @@ impl TagApi { .await } + /// Returns a map from files to assigned tags + #[tracing::instrument(level = "debug", skip_all)] + pub async fn get_file_tag_map(&self, cds: Vec) -> ApiResult>> { + self.emit_and_get("file_tag_map", GetFileTagMapRequest{cds}, Some(Duration::from_secs(10))).await + } + /// Creates a new tag and returns the created tag object #[tracing::instrument(level = "debug", skip(self))] pub async fn create_tags(&self, tags: Vec) -> ApiResult> { diff --git a/mediarepo-api/src/tauri_plugin/commands/tag.rs b/mediarepo-api/src/tauri_plugin/commands/tag.rs index 809e5b0..3633282 100644 --- a/mediarepo-api/src/tauri_plugin/commands/tag.rs +++ b/mediarepo-api/src/tauri_plugin/commands/tag.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use crate::tauri_plugin::commands::ApiAccess; use crate::tauri_plugin::error::PluginResult; use crate::types::identifier::FileIdentifier; @@ -41,6 +42,14 @@ pub async fn get_tags_for_files( Ok(tags) } +#[tauri::command] +pub async fn get_file_tag_map(cds: Vec, api_state: ApiAccess<'_>) -> PluginResult>> { + let api = api_state.api().await?; + let mappings = api.tag.get_file_tag_map(cds).await?; + + Ok(mappings) +} + #[tauri::command] pub async fn create_tags( api_state: ApiAccess<'_>, diff --git a/mediarepo-api/src/tauri_plugin/mod.rs b/mediarepo-api/src/tauri_plugin/mod.rs index 5779dbe..37e11c6 100644 --- a/mediarepo-api/src/tauri_plugin/mod.rs +++ b/mediarepo-api/src/tauri_plugin/mod.rs @@ -69,7 +69,8 @@ impl MediarepoPlugin { get_file_metadata, run_job, update_file_status, - delete_file + delete_file, + get_file_tag_map ]), } } diff --git a/mediarepo-api/src/types/files.rs b/mediarepo-api/src/types/files.rs index bb8146e..ff9153e 100644 --- a/mediarepo-api/src/types/files.rs +++ b/mediarepo-api/src/types/files.rs @@ -30,6 +30,11 @@ pub struct GetFilesTagsRequest { pub cds: Vec, } +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct GetFileTagMapRequest { + pub cds: Vec, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub struct FileBasicDataResponse { pub id: i64, diff --git a/mediarepo-api/src/types/jobs.rs b/mediarepo-api/src/types/jobs.rs index b6260fd..e8b9d3b 100644 --- a/mediarepo-api/src/types/jobs.rs +++ b/mediarepo-api/src/types/jobs.rs @@ -11,4 +11,5 @@ pub enum JobType { MigrateContentDescriptors, CalculateSizes, CheckIntegrity, + Vacuum, } diff --git a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs index 9b8e142..0f44309 100644 --- a/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs +++ b/mediarepo-daemon/mediarepo-logic/src/dao/tag/mod.rs @@ -5,6 +5,7 @@ use std::collections::HashMap; use std::iter::FromIterator; use mediarepo_core::error::RepoResult; +use mediarepo_core::itertools::Itertools; use mediarepo_core::utils::parse_namespace_and_tag; use mediarepo_database::entities::{content_descriptor, content_descriptor_tag, namespace, tag}; @@ -57,7 +58,7 @@ impl TagDao { Ok(namespaces) } - #[tracing::instrument(level = "debug", skip(self))] + #[tracing::instrument(level = "debug", skip(self, cds))] pub async fn all_for_cds(&self, cds: Vec>) -> RepoResult> { let tags = tag::Entity::find() .find_also_related(namespace::Entity) @@ -80,6 +81,51 @@ impl TagDao { Ok(tags) } + #[tracing::instrument(level = "debug", skip(self, cds))] + pub async fn all_for_cds_map( + &self, + cds: Vec>, + ) -> RepoResult, Vec>> { + let tag_cd_entries: Vec<( + content_descriptor_tag::Model, + Option, + )> = content_descriptor_tag::Entity::find() + .find_also_related(content_descriptor::Entity) + .filter(content_descriptor::Column::Descriptor.is_in(cds)) + .all(&self.ctx.db) + .await?; + + let tag_ids: Vec = tag_cd_entries + .iter() + .map(|(t, _)| t.tag_id) + .unique() + .collect(); + + let tags: Vec = tag::Entity::find() + .find_also_related(namespace::Entity) + .filter(tag::Column::Id.is_in(tag_ids)) + .all(&self.ctx.db) + .await? + .into_iter() + .map(map_tag_dto) + .collect(); + + let tag_id_map = tags + .into_iter() + .map(|t| (t.id(), t)) + .collect::>(); + let cd_tag_map = tag_cd_entries + .into_iter() + .filter_map(|(t, cd)| Some((cd?, tag_id_map.get(&t.tag_id)?.clone()))) + .sorted_by_key(|(cd, _)| cd.id) + .group_by(|(cd, _)| cd.descriptor.to_owned()) + .into_iter() + .map(|(key, group)| (key, group.map(|(_, t)| t).collect::>())) + .collect(); + + Ok(cd_tag_map) + } + #[tracing::instrument(level = "debug", skip(self))] pub async fn tags_for_cd(&self, cd_id: i64) -> RepoResult> { let tags = tag::Entity::find() diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs index d345e42..52e1ef4 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/jobs.rs @@ -31,6 +31,7 @@ impl JobsNamespace { JobType::MigrateContentDescriptors => job_dao.migrate_content_descriptors().await?, JobType::CalculateSizes => calculate_all_sizes(ctx).await?, JobType::CheckIntegrity => job_dao.check_integrity().await?, + JobType::Vacuum => job_dao.vacuum().await?, } ctx.emit_to(Self::name(), "run_job", ()).await?; diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs index bfd6895..ef6543b 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs @@ -1,8 +1,11 @@ use rayon::iter::{IntoParallelIterator, ParallelIterator}; +use std::collections::HashMap; use mediarepo_core::bromine::prelude::*; -use mediarepo_core::content_descriptor::decode_content_descriptor; -use mediarepo_core::mediarepo_api::types::files::{GetFileTagsRequest, GetFilesTagsRequest}; +use mediarepo_core::content_descriptor::{decode_content_descriptor, encode_content_descriptor}; +use mediarepo_core::mediarepo_api::types::files::{ + GetFileTagMapRequest, GetFileTagsRequest, GetFilesTagsRequest, +}; use mediarepo_core::mediarepo_api::types::tags::{ ChangeFileTagsRequest, NamespaceResponse, TagResponse, }; @@ -26,6 +29,7 @@ impl NamespaceProvider for TagsNamespace { "all_namespaces" => Self::all_namespaces, "tags_for_file" => Self::tags_for_file, "tags_for_files" => Self::tags_for_files, + "file_tag_map" => Self::tag_cd_map_for_files, "create_tags" => Self::create_tags, "change_file_tags" => Self::change_file_tags ); @@ -105,6 +109,38 @@ impl TagsNamespace { Ok(()) } + /// Returns a map of content descriptors to assigned tags + #[tracing::instrument(skip_all)] + async fn tag_cd_map_for_files(ctx: &Context, event: Event) -> IPCResult<()> { + let request = event.payload::()?; + let repo = get_repo_from_context(ctx).await; + let cds = request + .cds + .into_iter() + .filter_map(|c| decode_content_descriptor(c).ok()) + .collect(); + + let mappings = repo + .tag() + .all_for_cds_map(cds) + .await? + .into_iter() + .map(|(cd, tags)| (encode_content_descriptor(&cd), tags)) + .map(|(cd, tags)| { + ( + cd, + tags.into_iter() + .map(TagResponse::from_model) + .collect::>(), + ) + }) + .collect::>>(); + + ctx.emit_to(Self::name(), "file_tag_map", mappings).await?; + + Ok(()) + } + /// Creates all tags given as input or returns the existing tags #[tracing::instrument(skip_all)] async fn create_tags(ctx: &Context, event: Event) -> IPCResult<()> { diff --git a/mediarepo-ui/src/api/Api.ts b/mediarepo-ui/src/api/Api.ts index 6faeec7..292cd8f 100644 --- a/mediarepo-ui/src/api/Api.ts +++ b/mediarepo-ui/src/api/Api.ts @@ -13,6 +13,7 @@ import { DeleteThumbnailsRequest, FindFilesRequest, GetFileMetadataRequest, + GetFileTagMapRequest, GetSizeRequest, GetTagsForFilesRequest, InitRepositoryRequest, @@ -28,7 +29,7 @@ import { UpdateFileStatusRequest } from "./api-types/requests"; import {RepositoryData, RepositoryMetadata, SizeMetadata} from "./api-types/repo"; -import {NamespaceData, TagData} from "./api-types/tags"; +import {CdTagMappings, NamespaceData, TagData} from "./api-types/tags"; import {ShortCache} from "./ShortCache"; export class MediarepoApi { @@ -146,6 +147,15 @@ export class MediarepoApi { ); } + public static async getFileTagMap(request: GetFileTagMapRequest): Promise { + return ShortCache.cached( + request, + () => this.invokePlugin(ApiFunction.GetFileTagMap, request), + 1000, + "getFileTagMap" + ); + } + public static async createTags(request: CreateTagsRequest): Promise { return this.invokePlugin(ApiFunction.CreateTags, request); } diff --git a/mediarepo-ui/src/api/api-types/functions.ts b/mediarepo-ui/src/api/api-types/functions.ts index 33a5140..5f61ed0 100644 --- a/mediarepo-ui/src/api/api-types/functions.ts +++ b/mediarepo-ui/src/api/api-types/functions.ts @@ -29,6 +29,7 @@ export enum ApiFunction { GetAllTags = "get_all_tags", GetAllNamespace = "get_all_namespaces", GetTagsForFiles = "get_tags_for_files", + GetFileTagMap = "get_file_tag_map", CreateTags = "create_tags", ChangeFileTags = "change_file_tags", // import diff --git a/mediarepo-ui/src/api/api-types/requests.ts b/mediarepo-ui/src/api/api-types/requests.ts index 5a261b5..c6c0d24 100644 --- a/mediarepo-ui/src/api/api-types/requests.ts +++ b/mediarepo-ui/src/api/api-types/requests.ts @@ -73,6 +73,10 @@ export type GetTagsForFilesRequest = { cds: string[] }; +export type GetFileTagMapRequest = { + cds: string[] +}; + export type CreateTagsRequest = { tags: string[] }; diff --git a/mediarepo-ui/src/api/api-types/tags.ts b/mediarepo-ui/src/api/api-types/tags.ts index 6b81aba..fcd8d8f 100644 --- a/mediarepo-ui/src/api/api-types/tags.ts +++ b/mediarepo-ui/src/api/api-types/tags.ts @@ -8,3 +8,7 @@ export type NamespaceData = { id: number, name: string, }; + +export type CdTagMappings = { + [key: string]: TagData[], +}; diff --git a/mediarepo-ui/src/app/app.component.ts b/mediarepo-ui/src/app/app.component.ts index af0801f..0665228 100644 --- a/mediarepo-ui/src/app/app.component.ts +++ b/mediarepo-ui/src/app/app.component.ts @@ -63,21 +63,21 @@ export class AppComponent implements OnInit { if (!environment.production) { switch (entry.getLevel()) { case LogLevel.Trace: - console.trace(entry.getMessage(), entry); + console.trace(entry.getMessage()); break; case LogLevel.Debug: - console.debug(entry.getMessage(), entry); + console.debug(entry.getMessage()); break; case LogLevel.Info: - console.info(entry.getMessage(), entry); + console.info(entry.getMessage()); break; case LogLevel.Warn: - console.warn(entry.getMessage(), entry); + console.warn(entry.getMessage()); break; } } if (entry.getLevel() == LogLevel.Error) { - console.error(entry.getMessage(), entry.getError(), entry); + console.error(entry.getMessage(), entry.getError()); } } } diff --git a/mediarepo-ui/src/app/components/shared/sidebar/tag-edit/tag-edit.component.ts b/mediarepo-ui/src/app/components/shared/sidebar/tag-edit/tag-edit.component.ts index 37c9538..06f839c 100644 --- a/mediarepo-ui/src/app/components/shared/sidebar/tag-edit/tag-edit.component.ts +++ b/mediarepo-ui/src/app/components/shared/sidebar/tag-edit/tag-edit.component.ts @@ -36,7 +36,7 @@ export class TagEditComponent implements AfterViewInit, OnChanges { private fileTags: { [key: number]: Tag[] } = {}; constructor( - private errorBroker: LoggingService, + private logger: LoggingService, private tagService: TagService, ) { } @@ -148,16 +148,12 @@ export class TagEditComponent implements AfterViewInit, OnChanges { private async loadFileTags() { await this.wrapAsyncOperation(async () => { - const promises = []; - const loadFn = async (file: File) => { - this.fileTags[file.id] = await this.tagService.getTagsForFiles( - [file.cd]); - }; + console.log("loading tags"); + const mappings = await this.tagService.getFileTagMappings(this.files.map(f => f.cd)); + for (const file of this.files) { - promises.push(loadFn(file)); + this.fileTags[file.id] = mappings[file.cd]; } - - await Promise.all(promises); this.mapFileTagsToTagList(); }); } @@ -176,11 +172,11 @@ export class TagEditComponent implements AfterViewInit, OnChanges { } private async wrapAsyncOperation(cb: () => Promise): Promise { - if (!this.busyIndicator) { + if (!this.busyIndicator?.wrapAsyncOperation) { try { return cb(); } catch (err: any) { - this.errorBroker.error(err); + this.logger.error(err); return undefined; } } else { diff --git a/mediarepo-ui/src/app/services/tag/tag.service.ts b/mediarepo-ui/src/app/services/tag/tag.service.ts index 6722383..e701bc7 100644 --- a/mediarepo-ui/src/app/services/tag/tag.service.ts +++ b/mediarepo-ui/src/app/services/tag/tag.service.ts @@ -29,16 +29,31 @@ export class TagService { public async getTagsForFiles(cds: string[]): Promise { let tags: Tag[] = []; if (cds.length > 0) { - tags = await MediarepoApi.getTagsForFiles({cds}).then(mapMany(mapNew(Tag))); + tags = await MediarepoApi.getTagsForFiles({ cds }).then(mapMany(mapNew(Tag))); } return tags; } + public async getFileTagMappings(cds: string[]): Promise<{ [key: string]: Tag[] }> { + if (cds.length > 0) { + return await MediarepoApi.getFileTagMap({ cds }).then((cdMappings) => { + let mappings: { [key: string]: Tag[] } = {}; + console.log("TAG MAPPINGS", cdMappings); + for (const key in cdMappings) { + mappings[key] = cdMappings[key].map(mapNew(Tag)); + } + return mappings; + }); + } else { + return {}; + } + } + public async createTags(tags: string[]): Promise { - return MediarepoApi.createTags({tags}).then(mapMany(mapNew(Tag))); + return MediarepoApi.createTags({ tags }).then(mapMany(mapNew(Tag))); } public async changeFileTags(fileId: number, addedTags: number[], removedTags: number[]): Promise { - return MediarepoApi.changeFileTags({id: fileId, addedTags, removedTags}).then(mapMany(mapNew(Tag))); + return MediarepoApi.changeFileTags({ id: fileId, addedTags, removedTags }).then(mapMany(mapNew(Tag))); } }