From 77fcd2e4f5c3cbcaeeac85808648806cfd9ff3d4 Mon Sep 17 00:00:00 2001 From: Trivernis Date: Mon, 27 Dec 2021 19:15:46 +0100 Subject: [PATCH] Implement api to retrieve repository metadata Signed-off-by: Trivernis --- mediarepo-daemon/Cargo.lock | 5 +- mediarepo-daemon/mediarepo-core/Cargo.toml | 3 +- .../mediarepo-core/src/fs/file_hash_store.rs | 7 +++ .../mediarepo-core/src/fs/thumbnail_store.rs | 8 +++ mediarepo-daemon/mediarepo-core/src/utils.rs | 40 +++++++++++++- .../src/queries/analysis.rs | 36 +++++++++++++ .../mediarepo-database/src/queries/mod.rs | 1 + mediarepo-daemon/mediarepo-model/src/repo.rs | 24 +++++++++ .../mediarepo-model/src/storage.rs | 7 +++ .../mediarepo-socket/src/namespaces/repo.rs | 52 +++++++++++++++++-- 10 files changed, 176 insertions(+), 7 deletions(-) create mode 100644 mediarepo-daemon/mediarepo-database/src/queries/analysis.rs diff --git a/mediarepo-daemon/Cargo.lock b/mediarepo-daemon/Cargo.lock index de5d560..fa19ab7 100644 --- a/mediarepo-daemon/Cargo.lock +++ b/mediarepo-daemon/Cargo.lock @@ -1136,8 +1136,8 @@ checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "mediarepo-api" -version = "0.17.0" -source = "git+https://github.com/Trivernis/mediarepo-api.git?rev=72e4d902aff409feab8db10546b81a452ec6f75f#72e4d902aff409feab8db10546b81a452ec6f75f" +version = "0.18.0" +source = "git+https://github.com/Trivernis/mediarepo-api.git?rev=cb4510c57fb25bd39f2e994e928359599e7cbfaa#cb4510c57fb25bd39f2e994e928359599e7cbfaa" dependencies = [ "bromine", "chrono", @@ -1165,6 +1165,7 @@ dependencies = [ "thumbnailer", "tokio", "toml", + "tracing", "typemap_rev", ] diff --git a/mediarepo-daemon/mediarepo-core/Cargo.toml b/mediarepo-daemon/mediarepo-core/Cargo.toml index 9a5ed66..4940757 100644 --- a/mediarepo-daemon/mediarepo-core/Cargo.toml +++ b/mediarepo-daemon/mediarepo-core/Cargo.toml @@ -17,6 +17,7 @@ typemap_rev = "^0.1.5" futures = "^0.3.19" itertools = "^0.10.3" glob = "^0.3.0" +tracing = "0.1.29" [dependencies.thumbnailer] version = "^0.2.4" @@ -37,7 +38,7 @@ features = ["fs", "io-util", "io-std"] [dependencies.mediarepo-api] git = "https://github.com/Trivernis/mediarepo-api.git" -rev = "72e4d902aff409feab8db10546b81a452ec6f75f" +rev = "cb4510c57fb25bd39f2e994e928359599e7cbfaa" features = ["bromine"] [features] diff --git a/mediarepo-daemon/mediarepo-core/src/fs/file_hash_store.rs b/mediarepo-daemon/mediarepo-core/src/fs/file_hash_store.rs index 682d187..6884cac 100644 --- a/mediarepo-daemon/mediarepo-core/src/fs/file_hash_store.rs +++ b/mediarepo-daemon/mediarepo-core/src/fs/file_hash_store.rs @@ -1,4 +1,5 @@ use crate::error::RepoResult; +use crate::utils::get_folder_size; use multibase::Base; use multihash::{Code, MultihashDigest}; use std::path::PathBuf; @@ -62,6 +63,12 @@ impl FileHashStore { Ok((extension, reader)) } + /// Scans the size of the folder + #[inline] + pub async fn get_size(&self) -> RepoResult { + get_folder_size(self.path.to_owned()).await + } + fn hash_to_file_path(&self, hash: &str) -> PathBuf { let mut path = self.hash_to_folder_path(hash); path.push(hash); diff --git a/mediarepo-daemon/mediarepo-core/src/fs/thumbnail_store.rs b/mediarepo-daemon/mediarepo-core/src/fs/thumbnail_store.rs index d40f286..92a0563 100644 --- a/mediarepo-daemon/mediarepo-core/src/fs/thumbnail_store.rs +++ b/mediarepo-daemon/mediarepo-core/src/fs/thumbnail_store.rs @@ -1,3 +1,5 @@ +use crate::error::RepoResult; +use crate::utils::get_folder_size; use std::io::Result; use std::path::PathBuf; use tokio::fs; @@ -74,4 +76,10 @@ impl ThumbnailStore { Ok(entries) } + + /// Returns the size of the folder + #[tracing::instrument(level = "debug")] + pub async fn get_size(&self) -> RepoResult { + get_folder_size(self.path.to_owned()).await + } } diff --git a/mediarepo-daemon/mediarepo-core/src/utils.rs b/mediarepo-daemon/mediarepo-core/src/utils.rs index 400a312..6217037 100644 --- a/mediarepo-daemon/mediarepo-core/src/utils.rs +++ b/mediarepo-daemon/mediarepo-core/src/utils.rs @@ -1,6 +1,7 @@ use crate::error::RepoResult; +use futures::future; use std::path::PathBuf; -use tokio::fs::OpenOptions; +use tokio::fs::{self, OpenOptions}; use tokio::io::{AsyncBufReadExt, BufReader}; /// Parses a normalized tag into its two components of namespace and tag @@ -23,3 +24,40 @@ pub async fn parse_tags_file(path: PathBuf) -> RepoResult, S Ok(tags) } + +/// Iteratively scans the size of a folder +#[tracing::instrument(level = "debug")] +pub async fn get_folder_size(path: PathBuf) -> RepoResult { + let mut unchecked_dirs = vec![path]; + let mut all_files = Vec::new(); + + while !unchecked_dirs.is_empty() { + let dir = unchecked_dirs.remove(0); + let mut read_dir = fs::read_dir(dir).await?; + + while let Some(entry) = read_dir.next_entry().await? { + let file_type = entry.file_type().await?; + + if file_type.is_file() { + all_files.push(entry.path()); + } else if file_type.is_dir() { + unchecked_dirs.push(entry.path()) + } + } + } + let futures = all_files.into_iter().map(|f| read_file_size(f)); + let results = future::join_all(futures).await; + + let size = results + .into_iter() + .filter_map(|r| r.ok()) + .fold(0u64, |acc, val| acc + val); + + Ok(size) +} + +async fn read_file_size(path: PathBuf) -> RepoResult { + let metadata = fs::metadata(path).await?; + + Ok(metadata.len()) +} diff --git a/mediarepo-daemon/mediarepo-database/src/queries/analysis.rs b/mediarepo-daemon/mediarepo-database/src/queries/analysis.rs new file mode 100644 index 0000000..8ea114f --- /dev/null +++ b/mediarepo-daemon/mediarepo-database/src/queries/analysis.rs @@ -0,0 +1,36 @@ +use sea_orm::DbBackend; +use sea_orm::FromQueryResult; +use sea_orm::{DatabaseConnection, Statement}; + +use mediarepo_core::error::{RepoError, RepoResult}; + +#[derive(Debug, FromQueryResult)] +pub struct Counts { + pub file_count: i64, + pub hash_count: i64, + pub tag_count: i64, + pub namespace_count: i64, + pub source_count: i64, + pub mapping_count: i64, +} + +pub async fn get_all_counts(db: &DatabaseConnection) -> RepoResult { + let counts = Counts::find_by_statement(Statement::from_sql_and_values( + DbBackend::Sqlite, + r#" + SELECT * + FROM (SELECT COUNT(*) AS file_count FROM files), + (SELECT COUNT(*) AS hash_count FROM hashes), + (SELECT COUNT(*) AS tag_count FROM tags), + (SELECT COUNT(*) AS namespace_count FROM namespaces), + (SELECT COUNT(*) AS source_count FROM sources), + (SELECT COUNT(*) AS mapping_count FROM hash_tag_mappings) + "#, + vec![], + )) + .one(db) + .await? + .ok_or(RepoError::from("could not retrieve metadata from database"))?; + + Ok(counts) +} diff --git a/mediarepo-daemon/mediarepo-database/src/queries/mod.rs b/mediarepo-daemon/mediarepo-database/src/queries/mod.rs index cb4849d..6826b20 100644 --- a/mediarepo-daemon/mediarepo-database/src/queries/mod.rs +++ b/mediarepo-daemon/mediarepo-database/src/queries/mod.rs @@ -1 +1,2 @@ +pub mod analysis; pub mod tags; diff --git a/mediarepo-daemon/mediarepo-model/src/repo.rs b/mediarepo-daemon/mediarepo-model/src/repo.rs index e291cc9..fc12221 100644 --- a/mediarepo-daemon/mediarepo-model/src/repo.rs +++ b/mediarepo-daemon/mediarepo-model/src/repo.rs @@ -11,6 +11,7 @@ use mediarepo_core::itertools::Itertools; use mediarepo_core::thumbnailer::ThumbnailSize; use mediarepo_core::utils::parse_namespace_and_tag; use mediarepo_database::get_database; +use mediarepo_database::queries::analysis::{get_all_counts, Counts}; use sea_orm::DatabaseConnection; use std::collections::{HashMap, HashSet}; use std::fmt::Debug; @@ -388,6 +389,29 @@ impl Repo { Tag::add(self.db.clone(), name, Some(namespace.id())).await } + /// Returns the size of the main storage + #[inline] + #[tracing::instrument(level = "debug", skip(self))] + pub async fn get_main_store_size(&self) -> RepoResult { + let main_storage = self.get_main_storage()?; + main_storage.get_size().await + } + + /// Returns the size of the thumbnail storage + #[inline] + #[tracing::instrument(level = "debug", skip(self))] + pub async fn get_thumb_store_size(&self) -> RepoResult { + let thumb_storage = self.get_thumbnail_storage()?; + thumb_storage.get_size().await + } + + /// Returns all entity counts + #[inline] + #[tracing::instrument(level = "debug", skip(self))] + pub async fn get_counts(&self) -> RepoResult { + get_all_counts(&self.db).await + } + #[tracing::instrument(level = "trace", skip(self))] fn get_main_storage(&self) -> RepoResult<&Storage> { if let Some(storage) = &self.main_storage { diff --git a/mediarepo-daemon/mediarepo-model/src/storage.rs b/mediarepo-daemon/mediarepo-model/src/storage.rs index a95c861..8749de2 100644 --- a/mediarepo-daemon/mediarepo-model/src/storage.rs +++ b/mediarepo-daemon/mediarepo-model/src/storage.rs @@ -181,6 +181,13 @@ impl Storage { Ok(reader) } + /// Returns the size of the storage + #[inline] + #[tracing::instrument(level = "debug", skip(self))] + pub async fn get_size(&self) -> RepoResult { + self.store.get_size().await + } + /// Returns the active model with only the ID filled so saves always perform an update fn get_active_model(&self) -> ActiveStorage { ActiveStorage { diff --git a/mediarepo-daemon/mediarepo-socket/src/namespaces/repo.rs b/mediarepo-daemon/mediarepo-socket/src/namespaces/repo.rs index a784598..ce7a300 100644 --- a/mediarepo-daemon/mediarepo-socket/src/namespaces/repo.rs +++ b/mediarepo-daemon/mediarepo-socket/src/namespaces/repo.rs @@ -1,9 +1,14 @@ -use mediarepo_core::bromine::prelude::*; -use mediarepo_core::mediarepo_api::types::repo::FrontendState; -use mediarepo_core::type_keys::{RepoPathKey, SettingsKey}; use std::path::PathBuf; + use tokio::fs; +use mediarepo_core::bromine::prelude::*; +use mediarepo_core::mediarepo_api::types::repo::{FrontendState, RepositoryMetadata}; +use mediarepo_core::type_keys::{RepoPathKey, SettingsKey}; +use mediarepo_core::utils::get_folder_size; + +use crate::utils::get_repo_from_context; + pub struct RepoNamespace; impl NamespaceProvider for RepoNamespace { @@ -13,6 +18,7 @@ impl NamespaceProvider for RepoNamespace { fn register(handler: &mut EventHandler) { events!(handler, + "repository_metadata" => Self::get_metadata, "frontend_state" => Self::frontend_state, "set_frontend_state" => Self::set_frontend_state ); @@ -20,6 +26,46 @@ impl NamespaceProvider for RepoNamespace { } impl RepoNamespace { + #[tracing::instrument(skip_all)] + async fn get_metadata(ctx: &Context, _: Event) -> IPCResult<()> { + let repo = get_repo_from_context(ctx).await; + let counts = repo.get_counts().await?; + let file_size = repo.get_main_store_size().await?; + let thumbnail_size = repo.get_thumb_store_size().await?; + + let (repo_path, settings) = { + let data = ctx.data.read().await; + ( + data.get::().unwrap().clone(), + data.get::().unwrap().clone(), + ) + }; + let db_path = repo_path.join(settings.database_path); + let total_size = get_folder_size(repo_path).await?; + let database_metadata = fs::metadata(db_path).await?; + let database_size = database_metadata.len(); + + let metadata = RepositoryMetadata { + version: env!("CARGO_PKG_VERSION").to_string(), + file_count: counts.file_count as u64, + tag_count: counts.tag_count as u64, + namespace_count: counts.namespace_count as u64, + mapping_count: counts.mapping_count as u64, + hash_count: counts.hash_count as u64, + total_size, + file_size, + database_size, + thumbnail_size, + }; + + tracing::debug!("metadata = {:?}", metadata); + + ctx.emit_to(Self::name(), "repository_metadata", metadata) + .await?; + + Ok(()) + } + #[tracing::instrument(skip_all)] async fn frontend_state(ctx: &Context, _: Event) -> IPCResult<()> { let path = get_frontend_state_path(ctx).await?;