Implement missing sort keys

Signed-off-by: trivernis <trivernis@protonmail.com>
pull/4/head
trivernis 3 years ago
parent c3b744ed02
commit 165151096d

@ -0,0 +1,3 @@
-- Add migration script here
ALTER TABLE files
ADD COLUMN size INTEGER;

@ -10,6 +10,7 @@ pub struct Model {
pub name: Option<String>, pub name: Option<String>,
pub comment: Option<String>, pub comment: Option<String>,
pub mime_type: Option<String>, pub mime_type: Option<String>,
pub size: Option<i64>,
pub storage_id: i64, pub storage_id: i64,
pub hash_id: i64, pub hash_id: i64,
pub import_time: NaiveDateTime, pub import_time: NaiveDateTime,
@ -32,9 +33,6 @@ pub enum Relation {
to = "super::storage::Column::Id" to = "super::storage::Column::Id"
)] )]
Storage, Storage,
#[sea_orm(has_many = "super::thumbnail::Entity")]
Thumbnail,
} }
impl Related<super::hash::Entity> for Entity { impl Related<super::hash::Entity> for Entity {
@ -49,10 +47,4 @@ impl Related<super::storage::Entity> for Entity {
} }
} }
impl Related<super::thumbnail::Entity> for Entity {
fn to() -> RelationDef {
Relation::Thumbnail.def()
}
}
impl ActiveModelBehavior for ActiveModel {} impl ActiveModelBehavior for ActiveModel {}

@ -37,10 +37,4 @@ impl Related<super::source::Entity> for Entity {
} }
} }
impl Related<super::thumbnail::Entity> for Entity {
fn to() -> RelationDef {
super::thumbnail::Relation::Hash.def().rev()
}
}
impl ActiveModelBehavior for ActiveModel {} impl ActiveModelBehavior for ActiveModel {}

@ -6,4 +6,3 @@ pub mod namespace;
pub mod source; pub mod source;
pub mod storage; pub mod storage;
pub mod tag; pub mod tag;
pub mod thumbnail;

@ -13,8 +13,6 @@ pub struct Model {
pub enum Relation { pub enum Relation {
#[sea_orm(has_many = "super::file::Entity")] #[sea_orm(has_many = "super::file::Entity")]
File, File,
#[sea_orm(has_many = "super::thumbnail::Entity")]
Thumbnail,
} }
impl Related<super::file::Entity> for Entity { impl Related<super::file::Entity> for Entity {
@ -23,10 +21,4 @@ impl Related<super::file::Entity> for Entity {
} }
} }
impl Related<super::thumbnail::Entity> for Entity {
fn to() -> RelationDef {
Relation::Thumbnail.def()
}
}
impl ActiveModelBehavior for ActiveModel {} impl ActiveModelBehavior for ActiveModel {}

@ -1,58 +0,0 @@
use sea_orm::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel)]
#[sea_orm(table_name = "thumbnails")]
pub struct Model {
#[sea_orm(primary_key)]
pub id: i64,
pub file_id: i64,
pub storage_id: i64,
pub hash_id: i64,
pub height: i32,
pub width: i32,
pub mime: Option<String>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {
#[sea_orm(
belongs_to = "super::file::Entity",
from = "Column::FileId",
to = "super::file::Column::Id"
)]
File,
#[sea_orm(
belongs_to = "super::hash::Entity",
from = "Column::HashId",
to = "super::hash::Column::Id"
)]
Hash,
#[sea_orm(
belongs_to = "super::storage::Entity",
from = "Column::StorageId",
to = "super::storage::Column::Id"
)]
Storage,
}
impl Related<super::hash::Entity> for Entity {
fn to() -> RelationDef {
Relation::Hash.def()
}
}
impl Related<super::file::Entity> for Entity {
fn to() -> RelationDef {
Relation::File.def()
}
}
impl Related<super::storage::Entity> for Entity {
fn to() -> RelationDef {
Relation::Storage.def()
}
}
impl ActiveModelBehavior for ActiveModel {}

@ -3,6 +3,7 @@ use sea_orm::DbBackend;
use sea_orm::FromQueryResult; use sea_orm::FromQueryResult;
use sea_orm::{DatabaseConnection, Statement}; use sea_orm::{DatabaseConnection, Statement};
use std::collections::HashMap; use std::collections::HashMap;
use std::fmt::Display;
use std::iter::FromIterator; use std::iter::FromIterator;
#[derive(Debug, FromQueryResult)] #[derive(Debug, FromQueryResult)]
@ -16,7 +17,7 @@ struct HashNamespaceTags {
pub async fn get_hashes_with_namespaced_tags( pub async fn get_hashes_with_namespaced_tags(
db: &DatabaseConnection, db: &DatabaseConnection,
hash_ids: Vec<i64>, hash_ids: Vec<i64>,
) -> RepoResult<HashMap<i64, HashMap<String, String>>> { ) -> RepoResult<HashMap<i64, HashMap<String, Vec<String>>>> {
let hash_namespace_tags: Vec<HashNamespaceTags> = let hash_namespace_tags: Vec<HashNamespaceTags> =
HashNamespaceTags::find_by_statement(Statement::from_sql_and_values( HashNamespaceTags::find_by_statement(Statement::from_sql_and_values(
DbBackend::Sqlite, DbBackend::Sqlite,
@ -26,28 +27,74 @@ pub async fn get_hashes_with_namespaced_tags(
INNER JOIN tags t on htm.tag_id = t.id INNER JOIN tags t on htm.tag_id = t.id
JOIN namespaces n on t.namespace_id = n.id JOIN namespaces n on t.namespace_id = n.id
WHERE t.namespace_id IS NOT NULL WHERE t.namespace_id IS NOT NULL
AND htm.hash_id IN ({});"#, AND htm.hash_id IN ({}) ORDER BY t.namespace_id;"#,
hash_ids vec_to_query_list(hash_ids)
.into_iter()
.fold(String::new(), |acc, val| format!("{}{},", acc, val))
.trim_end_matches(",")
) )
.as_str(), .as_str(),
vec![], vec![],
)) ))
.all(db) .all(db)
.await?; .await?;
let mut hash_namespaces: HashMap<i64, HashMap<String, String>> = HashMap::new(); let mut hash_namespaces: HashMap<i64, HashMap<String, Vec<String>>> = HashMap::new();
for hnt in hash_namespace_tags { for hnt in hash_namespace_tags {
if let Some(entry) = hash_namespaces.get_mut(&hnt.hash_id) { if let Some(entry) = hash_namespaces.get_mut(&hnt.hash_id) {
entry.insert(hnt.namespace, hnt.tag); if let Some(nsp_entry) = entry.get_mut(&hnt.namespace) {
nsp_entry.push(hnt.tag);
} else {
entry.insert(hnt.namespace, vec![hnt.tag]);
}
} else { } else {
hash_namespaces.insert( hash_namespaces.insert(
hnt.hash_id, hnt.hash_id,
HashMap::from_iter(vec![(hnt.namespace, hnt.tag)].into_iter()), HashMap::from_iter(vec![(hnt.namespace, vec![hnt.tag])].into_iter()),
); );
} }
} }
Ok(hash_namespaces) Ok(hash_namespaces)
} }
#[derive(Debug, FromQueryResult)]
struct HashTagCount {
hash_id: i64,
tag_count: i32,
}
#[tracing::instrument(level = "debug", skip_all)]
pub async fn get_hashes_with_tag_count(
db: &DatabaseConnection,
hash_ids: Vec<i64>,
) -> RepoResult<HashMap<i64, u32>> {
let hash_tag_counts: Vec<HashTagCount> =
HashTagCount::find_by_statement(Statement::from_sql_and_values(
DbBackend::Sqlite,
format!(
r#"
SELECT htm.hash_id, COUNT(htm.tag_id) AS "tag_count" from hash_tag_mappings htm
WHERE htm.hash_id IN ({})
GROUP BY hash_id
"#,
vec_to_query_list(hash_ids)
)
.as_str(),
vec![],
))
.all(db)
.await?;
let mappings = hash_tag_counts
.into_iter()
.map(|HashTagCount { hash_id, tag_count }| (hash_id, tag_count as u32))
.collect::<HashMap<i64, u32>>();
Ok(mappings)
}
fn vec_to_query_list<D: Display>(input: Vec<D>) -> String {
let mut entries = input
.into_iter()
.fold(String::new(), |acc, val| format!("{}{},", acc, val));
entries.remove(entries.len() - 1);
entries
}

@ -327,6 +327,25 @@ impl File {
storage.get_file_reader(&self.hash.value).await storage.get_file_reader(&self.hash.value).await
} }
/// Retrieves the size of the file from its content
#[tracing::instrument(level = "debug", skip(self))]
pub async fn get_size(&self) -> RepoResult<u64> {
if let Some(size) = self.model.size {
Ok(size as u64)
} else {
let mut reader = self.get_reader().await?;
let size = {
let mut buf = Vec::new();
reader.read_to_end(&mut buf).await
}?;
let mut model = self.get_active_model();
model.size = Set(Some(size as i64));
model.update(&self.db).await?;
Ok(size as u64)
}
}
/// Creates a thumbnail for the file /// Creates a thumbnail for the file
#[tracing::instrument(level = "debug", skip(self))] #[tracing::instrument(level = "debug", skip(self))]
pub async fn create_thumbnail<I: IntoIterator<Item = ThumbnailSize> + Debug>( pub async fn create_thumbnail<I: IntoIterator<Item = ThumbnailSize> + Debug>(

@ -1,5 +1,6 @@
use crate::from_model::FromModel; use crate::from_model::FromModel;
use crate::utils::{file_by_identifier, get_repo_from_context, hash_by_identifier}; use crate::utils::{file_by_identifier, get_repo_from_context, hash_by_identifier};
use chrono::NaiveDateTime;
use compare::Compare; use compare::Compare;
use mediarepo_api::types::files::{ use mediarepo_api::types::files::{
AddFileRequestHeader, FileMetadataResponse, FindFilesByTagsRequest, AddFileRequestHeader, FileMetadataResponse, FindFilesByTagsRequest,
@ -12,13 +13,24 @@ use mediarepo_core::itertools::Itertools;
use mediarepo_core::rmp_ipc::prelude::*; use mediarepo_core::rmp_ipc::prelude::*;
use mediarepo_core::thumbnailer::ThumbnailSize; use mediarepo_core::thumbnailer::ThumbnailSize;
use mediarepo_core::utils::parse_namespace_and_tag; use mediarepo_core::utils::parse_namespace_and_tag;
use mediarepo_database::queries::tags::get_hashes_with_namespaced_tags; use mediarepo_database::queries::tags::{
use mediarepo_model::file::File; get_hashes_with_namespaced_tags, get_hashes_with_tag_count,
};
use std::cmp::Ordering; use std::cmp::Ordering;
use std::collections::HashMap; use std::collections::HashMap;
use tokio::io::AsyncReadExt; use tokio::io::AsyncReadExt;
pub struct FilesNamespace; pub struct FilesNamespace;
pub struct FileSortContext {
name: Option<String>,
size: u64,
mime_type: Option<String>,
namespaces: HashMap<String, Vec<String>>,
tag_count: u32,
import_time: NaiveDateTime,
create_time: NaiveDateTime,
change_time: NaiveDateTime,
}
impl NamespaceProvider for FilesNamespace { impl NamespaceProvider for FilesNamespace {
fn name() -> &'static str { fn name() -> &'static str {
@ -79,21 +91,36 @@ impl FilesNamespace {
let repo = get_repo_from_context(ctx).await; let repo = get_repo_from_context(ctx).await;
let tags = req.tags.into_iter().map(|t| (t.name, t.negate)).collect(); let tags = req.tags.into_iter().map(|t| (t.name, t.negate)).collect();
let mut files = repo.find_files_by_tags(tags).await?; let mut files = repo.find_files_by_tags(tags).await?;
let hash_ids = files.iter().map(|f| f.hash_id()).collect(); let hash_ids: Vec<i64> = files.iter().map(|f| f.hash_id()).collect();
let hash_nsp: HashMap<i64, HashMap<String, String>> = let mut hash_nsp: HashMap<i64, HashMap<String, Vec<String>>> =
get_hashes_with_namespaced_tags(repo.db(), hash_ids).await?; get_hashes_with_namespaced_tags(repo.db(), hash_ids.clone()).await?;
let mut hash_tag_counts = get_hashes_with_tag_count(repo.db(), hash_ids).await?;
let mut contexts = HashMap::new();
for file in &files {
let context = FileSortContext {
name: file.name().to_owned(),
size: file.get_size().await?,
mime_type: file.mime_type().to_owned(),
namespaces: hash_nsp
.remove(&file.hash_id())
.unwrap_or(HashMap::with_capacity(0)),
tag_count: hash_tag_counts.remove(&file.hash_id()).unwrap_or(0),
import_time: file.import_time().to_owned(),
create_time: file.import_time().to_owned(),
change_time: file.change_time().to_owned(),
};
contexts.insert(file.id(), context);
}
let sort_expression = req.sort_expression; let sort_expression = req.sort_expression;
tracing::debug!("sort_expression = {:?}", sort_expression); tracing::debug!("sort_expression = {:?}", sort_expression);
let empty_map = HashMap::with_capacity(0);
files.sort_by(|a, b| { files.sort_by(|a, b| {
compare_files( compare_files(
a, contexts.get(&a.hash_id()).unwrap(),
hash_nsp.get(&a.hash_id()).unwrap_or(&empty_map), contexts.get(&b.hash_id()).unwrap(),
b,
hash_nsp.get(&b.hash_id()).unwrap_or(&empty_map),
&sort_expression, &sort_expression,
) )
}); });
@ -263,55 +290,56 @@ impl FilesNamespace {
#[tracing::instrument(level = "trace", skip_all)] #[tracing::instrument(level = "trace", skip_all)]
fn compare_files( fn compare_files(
file_a: &File, ctx_a: &FileSortContext,
nsp_a: &HashMap<String, String>, ctx_b: &FileSortContext,
file_b: &File,
nsp_b: &HashMap<String, String>,
expression: &Vec<SortKey>, expression: &Vec<SortKey>,
) -> Ordering { ) -> Ordering {
let cmp_date = compare::natural(); let cmp_date = compare::natural();
let cmp_u64 = compare::natural();
let cmp_u32 = compare::natural();
for sort_key in expression { for sort_key in expression {
let ordering = match sort_key { let ordering = match sort_key {
SortKey::Namespace(namespace) => { SortKey::Namespace(namespace) => {
let tag_a = nsp_a.get(&namespace.name); let list_a = ctx_a.namespaces.get(&namespace.name);
let tag_b = nsp_b.get(&namespace.name); let list_b = ctx_b.namespaces.get(&namespace.name);
if let (Some(a), Some(b)) = ( let cmp_result = if let (Some(list_a), Some(list_b)) = (list_a, list_b) {
tag_a.and_then(|a| a.parse::<f32>().ok()), compare_tag_lists(list_a, list_b)
tag_b.and_then(|b| b.parse::<f32>().ok()), } else if list_a.is_some() {
) { Ordering::Greater
adjust_for_dir(compare_f32(a, b), &namespace.direction) } else if list_b.is_some() {
Ordering::Less
} else { } else {
adjust_for_dir(compare_opts(tag_a, tag_b), &namespace.direction) Ordering::Equal
} };
adjust_for_dir(cmp_result, &namespace.direction)
} }
SortKey::FileName(direction) => adjust_for_dir( SortKey::FileName(direction) => {
compare_opts(file_a.name().clone(), file_b.name().clone()), adjust_for_dir(compare_opts(&ctx_a.name, &ctx_b.name), direction)
direction, }
), SortKey::FileSize(direction) => {
SortKey::FileSize(_direction) => { adjust_for_dir(cmp_u64.compare(&ctx_a.size, &ctx_b.size), direction)
Ordering::Equal // TODO: Retrieve file size
} }
SortKey::FileImportedTime(direction) => adjust_for_dir( SortKey::FileImportedTime(direction) => adjust_for_dir(
cmp_date.compare(file_a.import_time(), file_b.import_time()), cmp_date.compare(&ctx_a.import_time, &ctx_b.import_time),
direction, direction,
), ),
SortKey::FileCreatedTime(direction) => adjust_for_dir( SortKey::FileCreatedTime(direction) => adjust_for_dir(
cmp_date.compare(file_a.creation_time(), file_b.creation_time()), cmp_date.compare(&ctx_a.create_time, &ctx_b.create_time),
direction, direction,
), ),
SortKey::FileChangeTime(direction) => adjust_for_dir( SortKey::FileChangeTime(direction) => adjust_for_dir(
cmp_date.compare(file_a.change_time(), file_b.change_time()), cmp_date.compare(&ctx_a.change_time, &ctx_b.change_time),
direction, direction,
), ),
SortKey::FileType(direction) => adjust_for_dir( SortKey::FileType(direction) => {
compare_opts(file_a.mime_type().clone(), file_b.mime_type().clone()), adjust_for_dir(compare_opts(&ctx_a.mime_type, &ctx_b.mime_type), direction)
}
SortKey::NumTags(direction) => adjust_for_dir(
cmp_u32.compare(&ctx_a.tag_count, &ctx_b.tag_count),
direction, direction,
), ),
SortKey::NumTags(_) => {
Ordering::Equal // TODO: Count tags
}
}; };
if !ordering.is_eq() { if !ordering.is_eq() {
return ordering; return ordering;
@ -321,9 +349,9 @@ fn compare_files(
Ordering::Equal Ordering::Equal
} }
fn compare_opts<T: Ord + Sized>(opt_a: Option<T>, opt_b: Option<T>) -> Ordering { fn compare_opts<T: Ord + Sized>(opt_a: &Option<T>, opt_b: &Option<T>) -> Ordering {
let cmp = compare::natural(); let cmp = compare::natural();
if let (Some(a), Some(b)) = (&opt_a, &opt_b) { if let (Some(a), Some(b)) = (opt_a, opt_b) {
cmp.compare(a, b) cmp.compare(a, b)
} else if opt_a.is_some() { } else if opt_a.is_some() {
Ordering::Greater Ordering::Greater
@ -351,3 +379,21 @@ fn adjust_for_dir(ordering: Ordering, direction: &SortDirection) -> Ordering {
ordering ordering
} }
} }
fn compare_tag_lists(list_a: &Vec<String>, list_b: &Vec<String>) -> Ordering {
let first_diff = list_a
.into_iter()
.zip(list_b.into_iter())
.find(|(a, b)| *a != *b);
if let Some(diff) = first_diff {
if let (Some(num_a), Some(num_b)) = (diff.0.parse::<f32>().ok(), diff.1.parse::<f32>().ok())
{
compare_f32(num_a, num_b)
} else {
let cmp = compare::natural();
cmp.compare(diff.0, diff.1)
}
} else {
Ordering::Equal
}
}

Loading…
Cancel
Save