Add tag import to file import

Signed-off-by: trivernis <trivernis@protonmail.com>

mediarepo-daemon/mediarepo-core/src/utils.rs

@@ -1,3 +1,8 @@
+use crate::error::RepoResult;
+use std::path::PathBuf;
+use tokio::fs::OpenOptions;
+use tokio::io::{AsyncBufReadExt, BufReader};
+
 /// Parses a normalized tag into its two components of namespace and tag
 pub fn parse_namespace_and_tag(norm_tag: String) -> (Option<String>, String) {
     norm_tag
@@ -5,3 +10,16 @@ pub fn parse_namespace_and_tag(norm_tag: String) -> (Option<String>, String) {
         .map(|(n, t)| (Some(n.trim().to_string()), t.trim().to_string()))
         .unwrap_or((None, norm_tag.trim().to_string()))
 }
+
+/// Parses all tags from a file
+pub async fn parse_tags_file(path: PathBuf) -> RepoResult<Vec<(Option<String>, String)>> {
+    let file = OpenOptions::new().read(true).open(path).await?;
+    let mut lines = BufReader::new(file).lines();
+    let mut tags = Vec::new();
+
+    while let Some(line) = lines.next_line().await? {
+        tags.push(parse_namespace_and_tag(line));
+    }
+
+    Ok(tags)
+}

mediarepo-daemon/mediarepo-database/migrations/20211017141828_make-hashes-unique.sql

@@ -0,0 +1,8 @@
+-- Add migration script here
+DELETE FROM thumbnails WHERE file_id NOT IN (SELECT MIN(files.id) FROM files GROUP BY hash_id);
+DELETE FROM files WHERE ROWID NOT IN (SELECT MIN(ROWID) FROM files GROUP BY hash_id);
+DELETE FROM thumbnails WHERE hash_id NOT IN (SELECT MIN(hashes.id) FROM hashes GROUP BY value);
+DELETE FROM files WHERE hash_id NOT IN (SELECT MIN(hashes.id) FROM hashes GROUP BY value);
+DELETE FROM hashes WHERE ROWID NOT IN (SELECT MIN(ROWID) FROM hashes GROUP BY value);
+CREATE UNIQUE INDEX hash_value_index ON hashes (value);
+CREATE UNIQUE INDEX file_hash_id ON files (hash_id);

mediarepo-daemon/mediarepo-database/migrations/20211017145013_change-uniqueness-of-tags.sql

@@ -0,0 +1,33 @@
+-- Add migration script here
+PRAGMA foreign_keys=off;
+
+ALTER TABLE tags RENAME TO _tags_old;
+CREATE TABLE tags
+(
+    id           INTEGER PRIMARY KEY AUTOINCREMENT,
+    namespace_id INTEGER,
+    name         VARCHAR(128),
+    FOREIGN KEY (namespace_id) REFERENCES namespaces (id)
+);
+CREATE UNIQUE INDEX tag_namespace_name_index ON tags (namespace_id, name);
+
+INSERT INTO tags SELECT * FROM _tags_old;
+
+DROP TABLE _tags_old;
+
+ALTER TABLE hash_tag_mappings RENAME TO _hash_tag_mappings_old;
+CREATE TABLE hash_tag_mappings
+(
+    hash_id INTEGER NOT NULL,
+    tag_id  INTEGER NOT NULL,
+    PRIMARY KEY (hash_id, tag_id),
+    FOREIGN KEY (hash_id) REFERENCES hashes (id),
+    FOREIGN KEY (tag_id) REFERENCES tags (id)
+);
+CREATE UNIQUE INDEX hash_tag_mappings_hash_tag ON hash_tag_mappings (hash_id, tag_id);
+
+INSERT INTO hash_tag_mappings SELECT * FROM _hash_tag_mappings_old;
+
+DROP TABLE _hash_tag_mappings_old;
+
+PRAGMA foreign_keys=on;

mediarepo-daemon/mediarepo-database/src/entities/tag.rs

@@ -19,7 +19,7 @@ pub enum Relation {
     Namespace,
 }
 
-impl Related<super::namespace::Entity> for Entity {
+impl Related<super::hash::Entity> for Entity {
     fn to() -> RelationDef {
         super::hash_tag::Relation::Hash.def()
     }
@@ -29,4 +29,10 @@ impl Related<super::namespace::Entity> for Entity {
     }
 }
 
+impl Related<super::namespace::Entity> for Entity {
+    fn to() -> RelationDef {
+        Relation::Namespace.def()
+    }
+}
+
 impl ActiveModelBehavior for ActiveModel {}

mediarepo-daemon/mediarepo-model/src/file.rs

@@ -8,23 +8,22 @@ use mediarepo_core::image_processing::{
     create_thumbnail, get_image_bytes_png, read_image, ThumbnailSize,
 };
 use mediarepo_database::entities::file;
-use mediarepo_database::entities::file::ActiveModel as ActiveFile;
-use mediarepo_database::entities::file::Model as FileModel;
 use mediarepo_database::entities::hash;
-use mediarepo_database::entities::hash::Model as HashModel;
+use mediarepo_database::entities::hash_tag;
 use mime::Mime;
 use sea_orm::prelude::*;
 use sea_orm::{DatabaseConnection, Set};
 use tokio::io::BufReader;
 
+#[derive(Clone)]
 pub struct File {
     db: DatabaseConnection,
-    model: FileModel,
+    model: file::Model,
-    hash: HashModel,
+    hash: hash::Model,
 }
 
 impl File {
-    pub(crate) fn new(db: DatabaseConnection, model: FileModel, hash: HashModel) -> Self {
+    pub(crate) fn new(db: DatabaseConnection, model: file::Model, hash: hash::Model) -> Self {
         Self { db, model, hash }
     }
 
@@ -198,6 +197,32 @@ impl File {
         Ok(())
     }
 
+    /// Adds a single tag to the file
+    pub async fn add_tag(&mut self, tag_id: i64) -> RepoResult<()> {
+        let hash_id = self.hash.id;
+        let active_model = hash_tag::ActiveModel {
+            hash_id: Set(hash_id),
+            tag_id: Set(tag_id),
+        };
+        active_model.insert(&self.db).await?;
+        Ok(())
+    }
+
+    /// Adds multiple tags to the file at once
+    pub async fn add_tags(&self, tag_ids: Vec<i64>) -> RepoResult<()> {
+        let hash_id = self.hash.id;
+        let models: Vec<hash_tag::ActiveModel> = tag_ids
+            .into_iter()
+            .map(|tag_id| hash_tag::ActiveModel {
+                hash_id: Set(hash_id),
+                tag_id: Set(tag_id),
+            })
+            .collect();
+        hash_tag::Entity::insert_many(models).exec(&self.db).await?;
+
+        Ok(())
+    }
+
     /// Returns the reader for the file
     pub async fn get_reader(&self) -> RepoResult<BufReader<tokio::fs::File>> {
         let storage = self.storage().await?;
@@ -233,8 +258,8 @@ impl File {
     }
 
     /// Returns the active model of the file with only the id set
-    fn get_active_model(&self) -> ActiveFile {
+    fn get_active_model(&self) -> file::ActiveModel {
-        ActiveFile {
+        file::ActiveModel {
             id: Set(self.id()),
             ..Default::default()
         }

mediarepo-daemon/mediarepo-model/src/repo.rs

@@ -152,7 +152,7 @@ impl Repo {
     }
 
     /// Adds or finds an unnamespaced tag
-    async fn add_or_find_unnamespaced_tag(&self, name: String) -> RepoResult<Tag> {
+    pub async fn add_or_find_unnamespaced_tag(&self, name: String) -> RepoResult<Tag> {
         if let Some(tag) = Tag::by_name(self.db.clone(), &name).await? {
             Ok(tag)
         } else {
@@ -166,7 +166,11 @@ impl Repo {
     }
 
     /// Adds or finds a namespaced tag
-    async fn add_or_find_namespaced_tag(&self, name: String, namespace: String) -> RepoResult<Tag> {
+    pub async fn add_or_find_namespaced_tag(
+        &self,
+        name: String,
+        namespace: String,
+    ) -> RepoResult<Tag> {
         if let Some(tag) = Tag::by_name_and_namespace(self.db.clone(), &name, &namespace).await? {
             Ok(tag)
         } else {

mediarepo-daemon/mediarepo-model/src/tag.rs

@@ -3,8 +3,7 @@ use mediarepo_core::error::RepoResult;
 use mediarepo_database::entities::namespace;
 use mediarepo_database::entities::tag;
 use sea_orm::prelude::*;
-use sea_orm::QuerySelect;
+use sea_orm::{DatabaseConnection, Set};
-use sea_orm::{DatabaseConnection, JoinType, Set};
 
 #[derive(Clone)]
 pub struct Tag {
@@ -29,7 +28,8 @@ impl Tag {
     /// Returns all tags stored in the database
     pub async fn all(db: DatabaseConnection) -> RepoResult<Vec<Self>> {
         let tags: Vec<Self> = tag::Entity::find()
-            .find_also_related(namespace::Entity)
+            .inner_join(namespace::Entity)
+            .select_also(namespace::Entity)
             .all(&db)
             .await?
             .into_iter()
@@ -73,7 +73,6 @@ impl Tag {
     ) -> RepoResult<Option<Self>> {
         let tag = tag::Entity::find()
             .find_also_related(namespace::Entity)
-            .join(JoinType::InnerJoin, namespace::Relation::Tag.def())
             .filter(namespace::Column::Name.eq(namespace.as_ref()))
             .filter(tag::Column::Name.eq(name.as_ref()))
             .one(&db)

mediarepo-daemon/mediarepo-socket/src/namespaces/tags.rs

@@ -17,6 +17,7 @@ impl NamespaceProvider for TagsNamespace {
 }
 
 impl TagsNamespace {
+    /// Returns a list of all tags in the database
     async fn all_tags(ctx: &Context, event: Event) -> IPCResult<()> {
         let repo = get_repo_from_context(ctx).await;
         let tags: Vec<TagResponse> = repo

mediarepo-daemon/src/main.rs

@@ -5,9 +5,10 @@ use crate::constants::{DEFAULT_STORAGE_NAME, SETTINGS_PATH, THUMBNAIL_STORAGE_NA
 use crate::utils::{create_paths_for_repo, get_repo, load_settings};
 use log::LevelFilter;
 use mediarepo_core::error::RepoResult;
-use mediarepo_core::futures::future;
 use mediarepo_core::settings::Settings;
 use mediarepo_core::type_keys::SettingsKey;
+use mediarepo_core::utils::parse_tags_file;
+use mediarepo_model::file::File;
 use mediarepo_model::repo::Repo;
 use mediarepo_model::type_keys::RepoKey;
 use mediarepo_socket::get_builder;
@@ -163,14 +164,18 @@ async fn import(opt: Opt, path: String) -> RepoResult<()> {
     let (_s, repo) = init_repo(&opt).await?;
     log::info!("Importing");
 
-    let promises = glob::glob(&path)
+    let paths: Vec<PathBuf> = glob::glob(&path)
         .unwrap()
         .into_iter()
         .filter_map(|r| r.ok())
         .filter(|e| e.is_file())
-        .map(|path| import_single_image(path, &repo));
+        .collect();
 
-    future::join_all(promises).await;
+    for path in paths {
+        if let Err(e) = import_single_image(path, &repo).await {
+            log::error!("Import failed: {:?}", e);
+        }
+    }
 
     Ok(())
 }
@@ -178,9 +183,35 @@ async fn import(opt: Opt, path: String) -> RepoResult<()> {
 /// Creates thumbnails of all sizes
 async fn import_single_image(path: PathBuf, repo: &Repo) -> RepoResult<()> {
     log::info!("Importing file");
-    let file = repo.add_file_by_path(path).await?;
+    let file = repo.add_file_by_path(path.clone()).await?;
     log::info!("Creating thumbnails");
-    repo.create_thumbnails_for_file(file).await?;
+    repo.create_thumbnails_for_file(file.clone()).await?;
+    let tags_path = PathBuf::from(format!("{}{}", path.to_str().unwrap(), ".txt"));
+    add_tags_from_tags_file(tags_path, repo, file).await?;
 
     Ok(())
 }
+
+async fn add_tags_from_tags_file(tags_path: PathBuf, repo: &Repo, file: File) -> RepoResult<()> {
+    log::info!("Adding tags");
+    if tags_path.exists() {
+        let tags = parse_tags_file(tags_path).await?;
+        let mut tag_ids = Vec::new();
+
+        for (namespace, name) in tags {
+            let tag = if let Some(namespace) = namespace {
+                log::info!("Adding namespaced tag '{}:{}'", namespace, name);
+                repo.add_or_find_namespaced_tag(name, namespace).await?
+            } else {
+                log::info!("Adding unnamespaced tag '{}'", name);
+                repo.add_or_find_unnamespaced_tag(name).await?
+            };
+            tag_ids.push(tag.id());
+        }
+        log::info!("Mapping {} tags to the file", tag_ids.len());
+        file.add_tags(tag_ids).await?;
+    } else {
+        log::info!("No tags file '{:?}' found", tags_path);
+    }
+    Ok(())
+}