From dbe0c20f634bf4c2ab53c74f9075b2e28ff35d12 Mon Sep 17 00:00:00 2001 From: trivernis Date: Sun, 17 Oct 2021 17:38:43 +0200 Subject: [PATCH] Add tag import to file import Signed-off-by: trivernis --- mediarepo-daemon/mediarepo-core/src/utils.rs | 18 ++++++++ .../20211017141828_make-hashes-unique.sql | 8 ++++ ...211017145013_change-uniqueness-of-tags.sql | 33 ++++++++++++++ .../mediarepo-database/src/entities/tag.rs | 8 +++- mediarepo-daemon/mediarepo-model/src/file.rs | 41 ++++++++++++++---- mediarepo-daemon/mediarepo-model/src/repo.rs | 8 +++- mediarepo-daemon/mediarepo-model/src/tag.rs | 7 ++- .../mediarepo-socket/src/namespaces/tags.rs | 1 + mediarepo-daemon/src/main.rs | 43 ++++++++++++++++--- 9 files changed, 146 insertions(+), 21 deletions(-) create mode 100644 mediarepo-daemon/mediarepo-database/migrations/20211017141828_make-hashes-unique.sql create mode 100644 mediarepo-daemon/mediarepo-database/migrations/20211017145013_change-uniqueness-of-tags.sql diff --git a/mediarepo-daemon/mediarepo-core/src/utils.rs b/mediarepo-daemon/mediarepo-core/src/utils.rs index 830ddbd..400a312 100644 --- a/mediarepo-daemon/mediarepo-core/src/utils.rs +++ b/mediarepo-daemon/mediarepo-core/src/utils.rs @@ -1,3 +1,8 @@ +use crate::error::RepoResult; +use std::path::PathBuf; +use tokio::fs::OpenOptions; +use tokio::io::{AsyncBufReadExt, BufReader}; + /// Parses a normalized tag into its two components of namespace and tag pub fn parse_namespace_and_tag(norm_tag: String) -> (Option, String) { norm_tag @@ -5,3 +10,16 @@ pub fn parse_namespace_and_tag(norm_tag: String) -> (Option, String) { .map(|(n, t)| (Some(n.trim().to_string()), t.trim().to_string())) .unwrap_or((None, norm_tag.trim().to_string())) } + +/// Parses all tags from a file +pub async fn parse_tags_file(path: PathBuf) -> RepoResult, String)>> { + let file = OpenOptions::new().read(true).open(path).await?; + let mut lines = BufReader::new(file).lines(); + let mut tags = Vec::new(); + + while let Some(line) = lines.next_line().await? { + tags.push(parse_namespace_and_tag(line)); + } + + Ok(tags) +} diff --git a/mediarepo-daemon/mediarepo-database/migrations/20211017141828_make-hashes-unique.sql b/mediarepo-daemon/mediarepo-database/migrations/20211017141828_make-hashes-unique.sql new file mode 100644 index 0000000..a66073f --- /dev/null +++ b/mediarepo-daemon/mediarepo-database/migrations/20211017141828_make-hashes-unique.sql @@ -0,0 +1,8 @@ +-- Add migration script here +DELETE FROM thumbnails WHERE file_id NOT IN (SELECT MIN(files.id) FROM files GROUP BY hash_id); +DELETE FROM files WHERE ROWID NOT IN (SELECT MIN(ROWID) FROM files GROUP BY hash_id); +DELETE FROM thumbnails WHERE hash_id NOT IN (SELECT MIN(hashes.id) FROM hashes GROUP BY value); +DELETE FROM files WHERE hash_id NOT IN (SELECT MIN(hashes.id) FROM hashes GROUP BY value); +DELETE FROM hashes WHERE ROWID NOT IN (SELECT MIN(ROWID) FROM hashes GROUP BY value); +CREATE UNIQUE INDEX hash_value_index ON hashes (value); +CREATE UNIQUE INDEX file_hash_id ON files (hash_id); \ No newline at end of file diff --git a/mediarepo-daemon/mediarepo-database/migrations/20211017145013_change-uniqueness-of-tags.sql b/mediarepo-daemon/mediarepo-database/migrations/20211017145013_change-uniqueness-of-tags.sql new file mode 100644 index 0000000..2f278a8 --- /dev/null +++ b/mediarepo-daemon/mediarepo-database/migrations/20211017145013_change-uniqueness-of-tags.sql @@ -0,0 +1,33 @@ +-- Add migration script here +PRAGMA foreign_keys=off; + +ALTER TABLE tags RENAME TO _tags_old; +CREATE TABLE tags +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + namespace_id INTEGER, + name VARCHAR(128), + FOREIGN KEY (namespace_id) REFERENCES namespaces (id) +); +CREATE UNIQUE INDEX tag_namespace_name_index ON tags (namespace_id, name); + +INSERT INTO tags SELECT * FROM _tags_old; + +DROP TABLE _tags_old; + +ALTER TABLE hash_tag_mappings RENAME TO _hash_tag_mappings_old; +CREATE TABLE hash_tag_mappings +( + hash_id INTEGER NOT NULL, + tag_id INTEGER NOT NULL, + PRIMARY KEY (hash_id, tag_id), + FOREIGN KEY (hash_id) REFERENCES hashes (id), + FOREIGN KEY (tag_id) REFERENCES tags (id) +); +CREATE UNIQUE INDEX hash_tag_mappings_hash_tag ON hash_tag_mappings (hash_id, tag_id); + +INSERT INTO hash_tag_mappings SELECT * FROM _hash_tag_mappings_old; + +DROP TABLE _hash_tag_mappings_old; + +PRAGMA foreign_keys=on; diff --git a/mediarepo-daemon/mediarepo-database/src/entities/tag.rs b/mediarepo-daemon/mediarepo-database/src/entities/tag.rs index b890c4c..217e0e7 100644 --- a/mediarepo-daemon/mediarepo-database/src/entities/tag.rs +++ b/mediarepo-daemon/mediarepo-database/src/entities/tag.rs @@ -19,7 +19,7 @@ pub enum Relation { Namespace, } -impl Related for Entity { +impl Related for Entity { fn to() -> RelationDef { super::hash_tag::Relation::Hash.def() } @@ -29,4 +29,10 @@ impl Related for Entity { } } +impl Related for Entity { + fn to() -> RelationDef { + Relation::Namespace.def() + } +} + impl ActiveModelBehavior for ActiveModel {} diff --git a/mediarepo-daemon/mediarepo-model/src/file.rs b/mediarepo-daemon/mediarepo-model/src/file.rs index 41e74d7..041b740 100644 --- a/mediarepo-daemon/mediarepo-model/src/file.rs +++ b/mediarepo-daemon/mediarepo-model/src/file.rs @@ -8,23 +8,22 @@ use mediarepo_core::image_processing::{ create_thumbnail, get_image_bytes_png, read_image, ThumbnailSize, }; use mediarepo_database::entities::file; -use mediarepo_database::entities::file::ActiveModel as ActiveFile; -use mediarepo_database::entities::file::Model as FileModel; use mediarepo_database::entities::hash; -use mediarepo_database::entities::hash::Model as HashModel; +use mediarepo_database::entities::hash_tag; use mime::Mime; use sea_orm::prelude::*; use sea_orm::{DatabaseConnection, Set}; use tokio::io::BufReader; +#[derive(Clone)] pub struct File { db: DatabaseConnection, - model: FileModel, - hash: HashModel, + model: file::Model, + hash: hash::Model, } impl File { - pub(crate) fn new(db: DatabaseConnection, model: FileModel, hash: HashModel) -> Self { + pub(crate) fn new(db: DatabaseConnection, model: file::Model, hash: hash::Model) -> Self { Self { db, model, hash } } @@ -198,6 +197,32 @@ impl File { Ok(()) } + /// Adds a single tag to the file + pub async fn add_tag(&mut self, tag_id: i64) -> RepoResult<()> { + let hash_id = self.hash.id; + let active_model = hash_tag::ActiveModel { + hash_id: Set(hash_id), + tag_id: Set(tag_id), + }; + active_model.insert(&self.db).await?; + Ok(()) + } + + /// Adds multiple tags to the file at once + pub async fn add_tags(&self, tag_ids: Vec) -> RepoResult<()> { + let hash_id = self.hash.id; + let models: Vec = tag_ids + .into_iter() + .map(|tag_id| hash_tag::ActiveModel { + hash_id: Set(hash_id), + tag_id: Set(tag_id), + }) + .collect(); + hash_tag::Entity::insert_many(models).exec(&self.db).await?; + + Ok(()) + } + /// Returns the reader for the file pub async fn get_reader(&self) -> RepoResult> { let storage = self.storage().await?; @@ -233,8 +258,8 @@ impl File { } /// Returns the active model of the file with only the id set - fn get_active_model(&self) -> ActiveFile { - ActiveFile { + fn get_active_model(&self) -> file::ActiveModel { + file::ActiveModel { id: Set(self.id()), ..Default::default() } diff --git a/mediarepo-daemon/mediarepo-model/src/repo.rs b/mediarepo-daemon/mediarepo-model/src/repo.rs index 6e3862c..cf50415 100644 --- a/mediarepo-daemon/mediarepo-model/src/repo.rs +++ b/mediarepo-daemon/mediarepo-model/src/repo.rs @@ -152,7 +152,7 @@ impl Repo { } /// Adds or finds an unnamespaced tag - async fn add_or_find_unnamespaced_tag(&self, name: String) -> RepoResult { + pub async fn add_or_find_unnamespaced_tag(&self, name: String) -> RepoResult { if let Some(tag) = Tag::by_name(self.db.clone(), &name).await? { Ok(tag) } else { @@ -166,7 +166,11 @@ impl Repo { } /// Adds or finds a namespaced tag - async fn add_or_find_namespaced_tag(&self, name: String, namespace: String) -> RepoResult { + pub async fn add_or_find_namespaced_tag( + &self, + name: String, + namespace: String, + ) -> RepoResult { if let Some(tag) = Tag::by_name_and_namespace(self.db.clone(), &name, &namespace).await? { Ok(tag) } else { diff --git a/mediarepo-daemon/mediarepo-model/src/tag.rs b/mediarepo-daemon/mediarepo-model/src/tag.rs index 73a9481..9ab7756 100644 --- a/mediarepo-daemon/mediarepo-model/src/tag.rs +++ b/mediarepo-daemon/mediarepo-model/src/tag.rs @@ -3,8 +3,7 @@ use mediarepo_core::error::RepoResult; use mediarepo_database::entities::namespace; use mediarepo_database::entities::tag; use sea_orm::prelude::*; -use sea_orm::QuerySelect; -use sea_orm::{DatabaseConnection, JoinType, Set}; +use sea_orm::{DatabaseConnection, Set}; #[derive(Clone)] pub struct Tag { @@ -29,7 +28,8 @@ impl Tag { /// Returns all tags stored in the database pub async fn all(db: DatabaseConnection) -> RepoResult> { let tags: Vec = tag::Entity::find() - .find_also_related(namespace::Entity) + .inner_join(namespace::Entity) + .select_also(namespace::Entity) .all(&db) .await? .into_iter() @@ -73,7 +73,6 @@ impl Tag { ) -> RepoResult> { let tag = tag::Entity::find() .find_also_related(namespace::Entity) - .join(JoinType::InnerJoin, namespace::Relation::Tag.def()) .filter(namespace::Column::Name.eq(namespace.as_ref())) .filter(tag::Column::Name.eq(name.as_ref())) .one(&db) diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs index fdd69ed..42c2571 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs @@ -17,6 +17,7 @@ impl NamespaceProvider for TagsNamespace { } impl TagsNamespace { + /// Returns a list of all tags in the database async fn all_tags(ctx: &Context, event: Event) -> IPCResult<()> { let repo = get_repo_from_context(ctx).await; let tags: Vec = repo diff --git a/mediarepo-daemon/src/main.rs b/mediarepo-daemon/src/main.rs index e2966ef..da0aeb0 100644 --- a/mediarepo-daemon/src/main.rs +++ b/mediarepo-daemon/src/main.rs @@ -5,9 +5,10 @@ use crate::constants::{DEFAULT_STORAGE_NAME, SETTINGS_PATH, THUMBNAIL_STORAGE_NA use crate::utils::{create_paths_for_repo, get_repo, load_settings}; use log::LevelFilter; use mediarepo_core::error::RepoResult; -use mediarepo_core::futures::future; use mediarepo_core::settings::Settings; use mediarepo_core::type_keys::SettingsKey; +use mediarepo_core::utils::parse_tags_file; +use mediarepo_model::file::File; use mediarepo_model::repo::Repo; use mediarepo_model::type_keys::RepoKey; use mediarepo_socket::get_builder; @@ -163,14 +164,18 @@ async fn import(opt: Opt, path: String) -> RepoResult<()> { let (_s, repo) = init_repo(&opt).await?; log::info!("Importing"); - let promises = glob::glob(&path) + let paths: Vec = glob::glob(&path) .unwrap() .into_iter() .filter_map(|r| r.ok()) .filter(|e| e.is_file()) - .map(|path| import_single_image(path, &repo)); + .collect(); - future::join_all(promises).await; + for path in paths { + if let Err(e) = import_single_image(path, &repo).await { + log::error!("Import failed: {:?}", e); + } + } Ok(()) } @@ -178,9 +183,35 @@ async fn import(opt: Opt, path: String) -> RepoResult<()> { /// Creates thumbnails of all sizes async fn import_single_image(path: PathBuf, repo: &Repo) -> RepoResult<()> { log::info!("Importing file"); - let file = repo.add_file_by_path(path).await?; + let file = repo.add_file_by_path(path.clone()).await?; log::info!("Creating thumbnails"); - repo.create_thumbnails_for_file(file).await?; + repo.create_thumbnails_for_file(file.clone()).await?; + let tags_path = PathBuf::from(format!("{}{}", path.to_str().unwrap(), ".txt")); + add_tags_from_tags_file(tags_path, repo, file).await?; Ok(()) } + +async fn add_tags_from_tags_file(tags_path: PathBuf, repo: &Repo, file: File) -> RepoResult<()> { + log::info!("Adding tags"); + if tags_path.exists() { + let tags = parse_tags_file(tags_path).await?; + let mut tag_ids = Vec::new(); + + for (namespace, name) in tags { + let tag = if let Some(namespace) = namespace { + log::info!("Adding namespaced tag '{}:{}'", namespace, name); + repo.add_or_find_namespaced_tag(name, namespace).await? + } else { + log::info!("Adding unnamespaced tag '{}'", name); + repo.add_or_find_unnamespaced_tag(name).await? + }; + tag_ids.push(tag.id()); + } + log::info!("Mapping {} tags to the file", tag_ids.len()); + file.add_tags(tag_ids).await?; + } else { + log::info!("No tags file '{:?}' found", tags_path); + } + Ok(()) +}