GitHub
commit
16cb977e45
@ -0,0 +1,42 @@
|
||||
ARG BASE_IMAGE=docker.io/alpine:latest
|
||||
|
||||
FROM ${BASE_IMAGE} AS base
|
||||
RUN apk update
|
||||
RUN apk add --no-cache \
|
||||
build-base \
|
||||
openssl3-dev \
|
||||
gtk+3.0-dev \
|
||||
libappindicator-dev \
|
||||
patchelf \
|
||||
librsvg-dev \
|
||||
curl \
|
||||
wget \
|
||||
clang \
|
||||
nodejs \
|
||||
npm \
|
||||
libsoup-dev \
|
||||
webkit2gtk-dev \
|
||||
file \
|
||||
python3 \
|
||||
bash \
|
||||
protoc
|
||||
RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
|
||||
ENV PATH="/root/.cargo/bin:${PATH}"
|
||||
RUN rm -rf /var/lib/{cache,log}/ /var/cache
|
||||
|
||||
FROM base AS sources
|
||||
WORKDIR /usr/src
|
||||
COPY mediarepo-api ./mediarepo-api
|
||||
COPY mediarepo-daemon ./mediarepo-daemon
|
||||
COPY mediarepo-ui ./mediarepo-ui
|
||||
COPY scripts ./scripts
|
||||
RUN python3 scripts/clean.py
|
||||
RUN python3 scripts/check.py --install
|
||||
|
||||
FROM sources AS build_daemon
|
||||
WORKDIR /usr/src
|
||||
RUN python3 scripts/build.py daemon --verbose
|
||||
|
||||
FROM sources AS build_ui
|
||||
WORKDIR /usr/src
|
||||
RUN python3 scripts/build.py ui --verbose --bundles deb
|
||||
@ -1,37 +0,0 @@
|
||||
ARG DEBIAN_RELEASE=bullseye
|
||||
|
||||
FROM bitnami/minideb:${DEBIAN_RELEASE} AS builder
|
||||
|
||||
WORKDIR /usr/src
|
||||
COPY mediarepo-api ./mediarepo-api
|
||||
COPY mediarepo-daemon ./mediarepo-daemon
|
||||
COPY mediarepo-ui ./mediarepo-ui
|
||||
COPY scripts ./scripts
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y \
|
||||
build-essential \
|
||||
libssl-dev \
|
||||
libgtk-3-dev \
|
||||
libappindicator3-0.1-cil-dev \
|
||||
patchelf \
|
||||
librsvg2-dev \
|
||||
curl \
|
||||
wget \
|
||||
pkg-config \
|
||||
clang \
|
||||
nodejs \
|
||||
npm \
|
||||
libsoup2.4-dev \
|
||||
libwebkit2gtk-4.0-dev \
|
||||
file \
|
||||
python
|
||||
|
||||
RUN apt remove cmdtest -y
|
||||
|
||||
RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
|
||||
ENV PATH="/root/.cargo/bin:${PATH}"
|
||||
|
||||
RUN python3 scripts/clean.py
|
||||
RUN python3 scripts/check.py --install
|
||||
RUN python3 scripts/build.py all --verbose
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,14 +1,17 @@
|
||||
pub use bincode;
|
||||
pub use futures;
|
||||
pub use itertools;
|
||||
pub use mediarepo_api;
|
||||
pub use mediarepo_api::bromine;
|
||||
pub use thumbnailer;
|
||||
pub use tokio_graceful_shutdown;
|
||||
pub use trait_bound_typemap;
|
||||
|
||||
pub mod content_descriptor;
|
||||
pub mod context;
|
||||
pub mod error;
|
||||
pub mod fs;
|
||||
pub mod settings;
|
||||
pub mod tracing_layer_list;
|
||||
pub mod type_keys;
|
||||
pub mod utils;
|
||||
|
||||
@ -0,0 +1,118 @@
|
||||
use std::slice::{Iter, IterMut};
|
||||
use tracing::level_filters::LevelFilter;
|
||||
use tracing::span::{Attributes, Record};
|
||||
use tracing::subscriber::Interest;
|
||||
use tracing::{Event, Id, Metadata, Subscriber};
|
||||
use tracing_subscriber::Layer;
|
||||
|
||||
pub struct DynLayerList<S>(Vec<Box<dyn Layer<S> + Send + Sync + 'static>>);
|
||||
|
||||
impl<S> DynLayerList<S> {
|
||||
pub fn new() -> Self {
|
||||
Self(Vec::new())
|
||||
}
|
||||
|
||||
pub fn iter(&self) -> Iter<'_, Box<dyn Layer<S> + Send + Sync>> {
|
||||
self.0.iter()
|
||||
}
|
||||
|
||||
pub fn iter_mut(&mut self) -> IterMut<'_, Box<dyn Layer<S> + Send + Sync>> {
|
||||
self.0.iter_mut()
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> DynLayerList<S>
|
||||
where
|
||||
S: Subscriber,
|
||||
{
|
||||
pub fn add<L: Layer<S> + Send + Sync>(&mut self, layer: L) {
|
||||
self.0.push(Box::new(layer));
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for DynLayerList<S>
|
||||
where
|
||||
S: Subscriber,
|
||||
{
|
||||
fn on_layer(&mut self, subscriber: &mut S) {
|
||||
self.iter_mut().for_each(|l| l.on_layer(subscriber));
|
||||
}
|
||||
|
||||
fn register_callsite(&self, metadata: &'static Metadata<'static>) -> Interest {
|
||||
// Return highest level of interest.
|
||||
let mut interest = Interest::never();
|
||||
for layer in &self.0 {
|
||||
let new_interest = layer.register_callsite(metadata);
|
||||
if (interest.is_sometimes() && new_interest.is_always())
|
||||
|| (interest.is_never() && !new_interest.is_never())
|
||||
{
|
||||
interest = new_interest;
|
||||
}
|
||||
}
|
||||
interest
|
||||
}
|
||||
|
||||
fn enabled(
|
||||
&self,
|
||||
metadata: &Metadata<'_>,
|
||||
ctx: tracing_subscriber::layer::Context<'_, S>,
|
||||
) -> bool {
|
||||
self.iter().any(|l| l.enabled(metadata, ctx.clone()))
|
||||
}
|
||||
|
||||
fn on_new_span(
|
||||
&self,
|
||||
attrs: &Attributes<'_>,
|
||||
id: &Id,
|
||||
ctx: tracing_subscriber::layer::Context<'_, S>,
|
||||
) {
|
||||
self.iter()
|
||||
.for_each(|l| l.on_new_span(attrs, id, ctx.clone()));
|
||||
}
|
||||
|
||||
fn max_level_hint(&self) -> Option<LevelFilter> {
|
||||
self.iter().filter_map(|l| l.max_level_hint()).max()
|
||||
}
|
||||
|
||||
fn on_record(
|
||||
&self,
|
||||
span: &Id,
|
||||
values: &Record<'_>,
|
||||
ctx: tracing_subscriber::layer::Context<'_, S>,
|
||||
) {
|
||||
self.iter()
|
||||
.for_each(|l| l.on_record(span, values, ctx.clone()));
|
||||
}
|
||||
|
||||
fn on_follows_from(
|
||||
&self,
|
||||
span: &Id,
|
||||
follows: &Id,
|
||||
ctx: tracing_subscriber::layer::Context<'_, S>,
|
||||
) {
|
||||
self.iter()
|
||||
.for_each(|l| l.on_follows_from(span, follows, ctx.clone()));
|
||||
}
|
||||
|
||||
fn on_event(&self, event: &Event<'_>, ctx: tracing_subscriber::layer::Context<'_, S>) {
|
||||
self.iter().for_each(|l| l.on_event(event, ctx.clone()));
|
||||
}
|
||||
|
||||
fn on_enter(&self, id: &Id, ctx: tracing_subscriber::layer::Context<'_, S>) {
|
||||
self.iter().for_each(|l| l.on_enter(id, ctx.clone()));
|
||||
}
|
||||
|
||||
fn on_exit(&self, id: &Id, ctx: tracing_subscriber::layer::Context<'_, S>) {
|
||||
self.iter().for_each(|l| l.on_exit(id, ctx.clone()));
|
||||
}
|
||||
|
||||
fn on_close(&self, id: Id, ctx: tracing_subscriber::layer::Context<'_, S>) {
|
||||
self.iter()
|
||||
.for_each(|l| l.on_close(id.clone(), ctx.clone()));
|
||||
}
|
||||
|
||||
fn on_id_change(&self, old: &Id, new: &Id, ctx: tracing_subscriber::layer::Context<'_, S>) {
|
||||
self.iter()
|
||||
.for_each(|l| l.on_id_change(old, new, ctx.clone()));
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,5 @@
|
||||
CREATE TABLE job_states (
|
||||
job_type INTEGER NOT NULL,
|
||||
value BLOB,
|
||||
PRIMARY KEY (job_type)
|
||||
);
|
||||
@ -0,0 +1,45 @@
|
||||
use sea_orm::prelude::*;
|
||||
use sea_orm::TryFromU64;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, DeriveEntityModel)]
|
||||
#[sea_orm(table_name = "job_states")]
|
||||
pub struct Model {
|
||||
#[sea_orm(primary_key)]
|
||||
pub job_type: JobType,
|
||||
pub value: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, EnumIter, DeriveActiveEnum)]
|
||||
#[sea_orm(rs_type = "u32", db_type = "Integer")]
|
||||
pub enum JobType {
|
||||
#[sea_orm(num_value = 10)]
|
||||
MigrateCDs,
|
||||
#[sea_orm(num_value = 20)]
|
||||
CalculateSizes,
|
||||
#[sea_orm(num_value = 30)]
|
||||
GenerateThumbs,
|
||||
#[sea_orm(num_value = 40)]
|
||||
CheckIntegrity,
|
||||
#[sea_orm(num_value = 50)]
|
||||
Vacuum,
|
||||
}
|
||||
|
||||
impl TryFromU64 for JobType {
|
||||
fn try_from_u64(n: u64) -> Result<Self, DbErr> {
|
||||
let value = match n {
|
||||
10 => Self::MigrateCDs,
|
||||
20 => Self::CalculateSizes,
|
||||
30 => Self::GenerateThumbs,
|
||||
40 => Self::CheckIntegrity,
|
||||
50 => Self::Vacuum,
|
||||
_ => return Err(DbErr::Custom(String::from("Invalid job type"))),
|
||||
};
|
||||
|
||||
Ok(value)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
|
||||
pub enum Relation {}
|
||||
|
||||
impl ActiveModelBehavior for ActiveModel {}
|
||||
@ -0,0 +1,58 @@
|
||||
use crate::dao::job::JobDao;
|
||||
use crate::dto::{JobStateDto, UpsertJobStateDto};
|
||||
use mediarepo_core::error::RepoResult;
|
||||
use mediarepo_database::entities::job_state;
|
||||
use mediarepo_database::entities::job_state::JobType;
|
||||
use sea_orm::prelude::*;
|
||||
use sea_orm::ActiveValue::Set;
|
||||
use sea_orm::{Condition, TransactionTrait};
|
||||
|
||||
impl JobDao {
|
||||
/// Returns all job states for a given job id
|
||||
pub async fn state_for_job_type(&self, job_type: JobType) -> RepoResult<Option<JobStateDto>> {
|
||||
let state = job_state::Entity::find()
|
||||
.filter(job_state::Column::JobType.eq(job_type))
|
||||
.one(&self.ctx.db)
|
||||
.await?
|
||||
.map(JobStateDto::new);
|
||||
|
||||
Ok(state)
|
||||
}
|
||||
|
||||
pub async fn upsert_state(&self, state: UpsertJobStateDto) -> RepoResult<()> {
|
||||
self.upsert_multiple_states(vec![state]).await
|
||||
}
|
||||
|
||||
pub async fn upsert_multiple_states(&self, states: Vec<UpsertJobStateDto>) -> RepoResult<()> {
|
||||
let trx = self.ctx.db.begin().await?;
|
||||
|
||||
job_state::Entity::delete_many()
|
||||
.filter(build_state_filters(&states))
|
||||
.exec(&trx)
|
||||
.await?;
|
||||
job_state::Entity::insert_many(build_active_state_models(states))
|
||||
.exec(&trx)
|
||||
.await?;
|
||||
|
||||
trx.commit().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn build_state_filters(states: &Vec<UpsertJobStateDto>) -> Condition {
|
||||
states
|
||||
.iter()
|
||||
.map(|s| Condition::all().add(job_state::Column::JobType.eq(s.job_type)))
|
||||
.fold(Condition::any(), |acc, cond| acc.add(cond))
|
||||
}
|
||||
|
||||
fn build_active_state_models(states: Vec<UpsertJobStateDto>) -> Vec<job_state::ActiveModel> {
|
||||
states
|
||||
.into_iter()
|
||||
.map(|s| job_state::ActiveModel {
|
||||
job_type: Set(s.job_type),
|
||||
value: Set(s.value),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
use mediarepo_database::entities::job_state;
|
||||
use mediarepo_database::entities::job_state::JobType;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct JobStateDto {
|
||||
model: job_state::Model,
|
||||
}
|
||||
|
||||
impl JobStateDto {
|
||||
pub(crate) fn new(model: job_state::Model) -> Self {
|
||||
Self { model }
|
||||
}
|
||||
|
||||
pub fn job_type(&self) -> JobType {
|
||||
self.model.job_type
|
||||
}
|
||||
|
||||
pub fn value(&self) -> &[u8] {
|
||||
&self.model.value
|
||||
}
|
||||
|
||||
pub fn into_value(self) -> Vec<u8> {
|
||||
self.model.value
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct UpsertJobStateDto {
|
||||
pub job_type: JobType,
|
||||
pub value: Vec<u8>,
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
[package]
|
||||
name = "mediarepo-worker"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
async-trait = "0.1.53"
|
||||
tracing = "0.1.32"
|
||||
|
||||
[dependencies.mediarepo-core]
|
||||
path = "../mediarepo-core"
|
||||
|
||||
[dependencies.mediarepo-logic]
|
||||
path = "../mediarepo-logic"
|
||||
|
||||
[dependencies.mediarepo-database]
|
||||
path = "../mediarepo-database"
|
||||
|
||||
[dependencies.tokio]
|
||||
version = "1.17.0"
|
||||
features = ["macros"]
|
||||
|
||||
[dependencies.chrono]
|
||||
version = "0.4.19"
|
||||
features = ["serde"]
|
||||
|
||||
[dependencies.serde]
|
||||
version = "1.0.136"
|
||||
features = ["derive"]
|
||||
@ -0,0 +1,101 @@
|
||||
use mediarepo_core::error::{RepoError, RepoResult};
|
||||
use std::ops::{Deref, DerefMut};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::broadcast::{Receiver, Sender};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
pub struct JobHandle<T: Send + Sync, R: Send + Sync> {
|
||||
status: Arc<RwLock<T>>,
|
||||
state: Arc<RwLock<JobState>>,
|
||||
result_receiver: CloneableReceiver<Arc<RwLock<Option<RepoResult<R>>>>>,
|
||||
}
|
||||
|
||||
impl<T: Send + Sync, R: Send + Sync> Clone for JobHandle<T, R> {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
status: self.status.clone(),
|
||||
state: self.state.clone(),
|
||||
result_receiver: self.result_receiver.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Send + Sync, R: Send + Sync> JobHandle<T, R> {
|
||||
pub fn new(
|
||||
status: Arc<RwLock<T>>,
|
||||
state: Arc<RwLock<JobState>>,
|
||||
result_receiver: CloneableReceiver<Arc<RwLock<Option<RepoResult<R>>>>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
status,
|
||||
state,
|
||||
result_receiver,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn state(&self) -> JobState {
|
||||
*self.state.read().await
|
||||
}
|
||||
|
||||
pub fn status(&self) -> &Arc<RwLock<T>> {
|
||||
&self.status
|
||||
}
|
||||
|
||||
pub async fn result(&mut self) -> Arc<RwLock<Option<RepoResult<R>>>> {
|
||||
match self.result_receiver.recv().await {
|
||||
Ok(v) => v,
|
||||
Err(e) => Arc::new(RwLock::new(Some(Err(RepoError::from(&*e.to_string()))))),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn take_result(&mut self) -> Option<RepoResult<R>> {
|
||||
let shared_result = self.result().await;
|
||||
let mut result = shared_result.write().await;
|
||||
result.take()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Ord, PartialOrd, Eq, PartialEq)]
|
||||
pub enum JobState {
|
||||
Queued,
|
||||
Scheduled,
|
||||
Running,
|
||||
Finished,
|
||||
}
|
||||
|
||||
pub struct CloneableReceiver<T: Clone> {
|
||||
receiver: Receiver<T>,
|
||||
sender: Sender<T>,
|
||||
}
|
||||
|
||||
impl<T: Clone> CloneableReceiver<T> {
|
||||
pub fn new(sender: Sender<T>) -> Self {
|
||||
Self {
|
||||
receiver: sender.subscribe(),
|
||||
sender,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Clone> Clone for CloneableReceiver<T> {
|
||||
fn clone(&self) -> Self {
|
||||
Self {
|
||||
sender: self.sender.clone(),
|
||||
receiver: self.sender.subscribe(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Clone> Deref for CloneableReceiver<T> {
|
||||
type Target = Receiver<T>;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.receiver
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Clone> DerefMut for CloneableReceiver<T> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.receiver
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,139 @@
|
||||
use crate::handle::{CloneableReceiver, JobHandle, JobState};
|
||||
use crate::jobs::{Job, JobTypeKey};
|
||||
use mediarepo_core::error::RepoError;
|
||||
use mediarepo_core::tokio_graceful_shutdown::SubsystemHandle;
|
||||
use mediarepo_core::trait_bound_typemap::{SendSyncTypeMap, TypeMap, TypeMapKey};
|
||||
use mediarepo_logic::dao::repo::Repo;
|
||||
use mediarepo_logic::dao::DaoProvider;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::broadcast::channel;
|
||||
use tokio::sync::RwLock;
|
||||
use tokio::time::Instant;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct JobDispatcher {
|
||||
subsystem: SubsystemHandle,
|
||||
job_handle_map: Arc<RwLock<SendSyncTypeMap>>,
|
||||
repo: Arc<Repo>,
|
||||
}
|
||||
|
||||
impl JobDispatcher {
|
||||
pub fn new(subsystem: SubsystemHandle, repo: Repo) -> Self {
|
||||
Self {
|
||||
job_handle_map: Arc::new(RwLock::new(SendSyncTypeMap::new())),
|
||||
subsystem,
|
||||
repo: Arc::new(repo),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn dispatch<T: 'static + Job>(&self, job: T) -> JobHandle<T::JobStatus, T::Result> {
|
||||
self._dispatch(job, None).await
|
||||
}
|
||||
|
||||
pub async fn dispatch_periodically<T: 'static + Job>(
|
||||
&self,
|
||||
job: T,
|
||||
interval: Duration,
|
||||
) -> JobHandle<T::JobStatus, T::Result> {
|
||||
self._dispatch(job, Some(interval)).await
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
async fn _dispatch<T: 'static + Job>(
|
||||
&self,
|
||||
job: T,
|
||||
interval: Option<Duration>,
|
||||
) -> JobHandle<T::JobStatus, T::Result> {
|
||||
let status = job.status();
|
||||
let state = Arc::new(RwLock::new(JobState::Queued));
|
||||
let (sender, mut receiver) = channel(1);
|
||||
self.subsystem
|
||||
.start::<RepoError, _, _>("channel-consumer", move |subsystem| async move {
|
||||
tokio::select! {
|
||||
_ = receiver.recv() => (),
|
||||
_ = subsystem.on_shutdown_requested() => (),
|
||||
}
|
||||
|
||||
Ok(())
|
||||
});
|
||||
let receiver = CloneableReceiver::new(sender.clone());
|
||||
let handle = JobHandle::new(status.clone(), state.clone(), receiver);
|
||||
self.add_handle::<T>(handle.clone()).await;
|
||||
|
||||
let repo = self.repo.clone();
|
||||
|
||||
self.subsystem
|
||||
.start::<RepoError, _, _>("worker-job", move |subsystem| async move {
|
||||
loop {
|
||||
let start = Instant::now();
|
||||
let job_2 = job.clone();
|
||||
{
|
||||
let mut state = state.write().await;
|
||||
*state = JobState::Running;
|
||||
}
|
||||
if let Err(e) = job.load_state(repo.job()).await {
|
||||
tracing::error!("failed to load the jobs state: {}", e);
|
||||
}
|
||||
let result = tokio::select! {
|
||||
_ = subsystem.on_shutdown_requested() => {
|
||||
job_2.save_state(repo.job()).await
|
||||
}
|
||||
r = job.run(repo.clone()) => {
|
||||
match r {
|
||||
Err(e) => Err(e),
|
||||
Ok(v) => {
|
||||
let _ = sender.send(Arc::new(RwLock::new(Some(Ok(v)))));
|
||||
job.save_state(repo.job()).await
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
if let Err(e) = result {
|
||||
tracing::error!("job failed with error: {}", e);
|
||||
let _ = sender.send(Arc::new(RwLock::new(Some(Err(e)))));
|
||||
}
|
||||
if let Some(interval) = interval {
|
||||
{
|
||||
let mut state = state.write().await;
|
||||
*state = JobState::Scheduled;
|
||||
}
|
||||
let sleep_duration = interval - start.elapsed();
|
||||
tokio::select! {
|
||||
_ = tokio::time::sleep(sleep_duration) => {},
|
||||
_ = subsystem.on_shutdown_requested() => {break}
|
||||
}
|
||||
} else {
|
||||
let mut state = state.write().await;
|
||||
*state = JobState::Finished;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
});
|
||||
|
||||
handle
|
||||
}
|
||||
|
||||
#[inline]
|
||||
async fn add_handle<T: 'static + Job>(&self, handle: JobHandle<T::JobStatus, T::Result>) {
|
||||
let mut status_map = self.job_handle_map.write().await;
|
||||
status_map.insert::<JobTypeKey<T>>(handle);
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub async fn get_handle<T: 'static + Job>(&self) -> Option<JobHandle<T::JobStatus, T::Result>> {
|
||||
let map = self.job_handle_map.read().await;
|
||||
map.get::<JobTypeKey<T>>().cloned()
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DispatcherKey;
|
||||
|
||||
impl TypeMapKey for DispatcherKey {
|
||||
type Value = JobDispatcher;
|
||||
}
|
||||
|
||||
unsafe impl Send for JobDispatcher {}
|
||||
unsafe impl Sync for JobDispatcher {}
|
||||
@ -0,0 +1,91 @@
|
||||
use crate::jobs::Job;
|
||||
use crate::status_utils::SimpleProgress;
|
||||
use async_trait::async_trait;
|
||||
use mediarepo_core::error::{RepoError, RepoResult};
|
||||
use mediarepo_core::mediarepo_api::types::repo::SizeType;
|
||||
use mediarepo_core::settings::Settings;
|
||||
use mediarepo_core::utils::get_folder_size;
|
||||
use mediarepo_logic::dao::repo::Repo;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tokio::fs;
|
||||
use tokio::sync::broadcast::{self, Sender};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
pub struct CalculateSizesState {
|
||||
pub progress: SimpleProgress,
|
||||
pub sizes_channel: Sender<(SizeType, u64)>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct CalculateSizesJob {
|
||||
repo_path: PathBuf,
|
||||
settings: Settings,
|
||||
state: Arc<RwLock<CalculateSizesState>>,
|
||||
}
|
||||
|
||||
impl CalculateSizesJob {
|
||||
pub fn new(repo_path: PathBuf, settings: Settings) -> Self {
|
||||
let (tx, _) = broadcast::channel(4);
|
||||
Self {
|
||||
repo_path,
|
||||
settings,
|
||||
state: Arc::new(RwLock::new(CalculateSizesState {
|
||||
sizes_channel: tx,
|
||||
progress: SimpleProgress::new(4),
|
||||
})),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Job for CalculateSizesJob {
|
||||
type JobStatus = CalculateSizesState;
|
||||
type Result = ();
|
||||
|
||||
fn status(&self) -> Arc<RwLock<Self::JobStatus>> {
|
||||
self.state.clone()
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
async fn run(&self, repo: Arc<Repo>) -> RepoResult<()> {
|
||||
let size_types = vec![
|
||||
SizeType::Total,
|
||||
SizeType::FileFolder,
|
||||
SizeType::ThumbFolder,
|
||||
SizeType::DatabaseFile,
|
||||
];
|
||||
for size_type in size_types {
|
||||
let size = calculate_size(&size_type, &repo, &self.repo_path, &self.settings).await?;
|
||||
let mut state = self.state.write().await;
|
||||
state
|
||||
.sizes_channel
|
||||
.send((size_type, size))
|
||||
.map_err(|_| RepoError::from("failed to broadcast new size"))?;
|
||||
state.progress.tick();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn calculate_size(
|
||||
size_type: &SizeType,
|
||||
repo: &Repo,
|
||||
repo_path: &PathBuf,
|
||||
settings: &Settings,
|
||||
) -> RepoResult<u64> {
|
||||
let size = match &size_type {
|
||||
SizeType::Total => get_folder_size(repo_path.clone()).await?,
|
||||
SizeType::FileFolder => repo.get_main_store_size().await?,
|
||||
SizeType::ThumbFolder => repo.get_thumb_store_size().await?,
|
||||
SizeType::DatabaseFile => {
|
||||
let db_path = settings.paths.db_file_path(repo_path);
|
||||
|
||||
let database_metadata = fs::metadata(db_path).await?;
|
||||
database_metadata.len()
|
||||
}
|
||||
};
|
||||
|
||||
Ok(size)
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
use crate::jobs::Job;
|
||||
use crate::status_utils::SimpleProgress;
|
||||
use async_trait::async_trait;
|
||||
use mediarepo_core::error::RepoResult;
|
||||
use mediarepo_logic::dao::repo::Repo;
|
||||
use mediarepo_logic::dao::DaoProvider;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct CheckIntegrityJob {
|
||||
progress: Arc<RwLock<SimpleProgress>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Job for CheckIntegrityJob {
|
||||
type JobStatus = SimpleProgress;
|
||||
type Result = ();
|
||||
|
||||
fn status(&self) -> Arc<RwLock<Self::JobStatus>> {
|
||||
self.progress.clone()
|
||||
}
|
||||
|
||||
async fn run(&self, repo: Arc<Repo>) -> RepoResult<Self::Result> {
|
||||
repo.job().check_integrity().await?;
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_total(100);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,109 @@
|
||||
use crate::jobs::{deserialize_state, serialize_state, Job};
|
||||
use crate::status_utils::SimpleProgress;
|
||||
use async_trait::async_trait;
|
||||
use mediarepo_core::error::RepoResult;
|
||||
use mediarepo_core::thumbnailer::ThumbnailSize;
|
||||
use mediarepo_database::entities::job_state::JobType;
|
||||
use mediarepo_logic::dao::job::JobDao;
|
||||
use mediarepo_logic::dao::repo::Repo;
|
||||
use mediarepo_logic::dao::DaoProvider;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::mem;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct GenerateMissingThumbsJob {
|
||||
state: Arc<RwLock<SimpleProgress>>,
|
||||
inner_state: Arc<RwLock<GenerateThumbsState>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Job for GenerateMissingThumbsJob {
|
||||
type JobStatus = SimpleProgress;
|
||||
type Result = ();
|
||||
|
||||
fn status(&self) -> Arc<RwLock<Self::JobStatus>> {
|
||||
self.state.clone()
|
||||
}
|
||||
|
||||
async fn load_state(&self, job_dao: JobDao) -> RepoResult<()> {
|
||||
if let Some(state) = job_dao.state_for_job_type(JobType::GenerateThumbs).await? {
|
||||
let mut inner_state = self.inner_state.write().await;
|
||||
let state = deserialize_state::<GenerateThumbsState>(state)?;
|
||||
let _ = mem::replace(&mut *inner_state, state);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run(&self, repo: Arc<Repo>) -> RepoResult<()> {
|
||||
if !self.needs_generation(&repo).await? {
|
||||
return Ok(());
|
||||
}
|
||||
let file_dao = repo.file();
|
||||
let all_files = file_dao.all().await?;
|
||||
{
|
||||
let mut progress = self.state.write().await;
|
||||
progress.set_total(all_files.len() as u64);
|
||||
}
|
||||
|
||||
for file in all_files {
|
||||
if file_dao.thumbnails(file.encoded_cd()).await?.is_empty() {
|
||||
let _ = file_dao
|
||||
.create_thumbnails(&file, vec![ThumbnailSize::Medium])
|
||||
.await;
|
||||
}
|
||||
{
|
||||
let mut progress = self.state.write().await;
|
||||
progress.tick();
|
||||
}
|
||||
}
|
||||
|
||||
self.refresh_state(&repo).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn save_state(&self, job_dao: JobDao) -> RepoResult<()> {
|
||||
let state = self.inner_state.read().await;
|
||||
let state = serialize_state(JobType::GenerateThumbs, &*state)?;
|
||||
job_dao.upsert_state(state).await
|
||||
}
|
||||
}
|
||||
|
||||
impl GenerateMissingThumbsJob {
|
||||
async fn needs_generation(&self, repo: &Repo) -> RepoResult<bool> {
|
||||
let repo_counts = repo.get_counts().await?;
|
||||
let file_count = repo_counts.file_count as u64;
|
||||
let state = self.inner_state.read().await;
|
||||
|
||||
Ok(state.file_count != file_count
|
||||
|| state.last_run.elapsed().unwrap() > Duration::from_secs(60 * 60))
|
||||
}
|
||||
|
||||
async fn refresh_state(&self, repo: &Repo) -> RepoResult<()> {
|
||||
let repo_counts = repo.get_counts().await?;
|
||||
let mut state = self.inner_state.write().await;
|
||||
state.last_run = SystemTime::now();
|
||||
state.file_count = repo_counts.file_count as u64;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct GenerateThumbsState {
|
||||
file_count: u64,
|
||||
last_run: SystemTime,
|
||||
}
|
||||
|
||||
impl Default for GenerateThumbsState {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
file_count: 0,
|
||||
last_run: SystemTime::now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,65 @@
|
||||
use crate::jobs::{deserialize_state, serialize_state, Job};
|
||||
use crate::status_utils::SimpleProgress;
|
||||
use async_trait::async_trait;
|
||||
use mediarepo_core::error::RepoResult;
|
||||
use mediarepo_database::entities::job_state::JobType;
|
||||
use mediarepo_logic::dao::job::JobDao;
|
||||
use mediarepo_logic::dao::repo::Repo;
|
||||
use mediarepo_logic::dao::DaoProvider;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct MigrateCDsJob {
|
||||
progress: Arc<RwLock<SimpleProgress>>,
|
||||
migrated: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Job for MigrateCDsJob {
|
||||
type JobStatus = SimpleProgress;
|
||||
type Result = ();
|
||||
|
||||
fn status(&self) -> Arc<tokio::sync::RwLock<Self::JobStatus>> {
|
||||
self.progress.clone()
|
||||
}
|
||||
|
||||
async fn load_state(&self, job_dao: JobDao) -> RepoResult<()> {
|
||||
if let Some(state) = job_dao.state_for_job_type(JobType::MigrateCDs).await? {
|
||||
let state = deserialize_state::<MigrationStatus>(state)?;
|
||||
self.migrated.store(state.migrated, Ordering::SeqCst);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run(&self, repo: Arc<Repo>) -> RepoResult<Self::Result> {
|
||||
if self.migrated.load(Ordering::SeqCst) {
|
||||
return Ok(());
|
||||
}
|
||||
let job_dao = repo.job();
|
||||
|
||||
job_dao.migrate_content_descriptors().await?;
|
||||
self.migrated.store(true, Ordering::Relaxed);
|
||||
{
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.set_total(100);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn save_state(&self, job_dao: JobDao) -> RepoResult<()> {
|
||||
if self.migrated.load(Ordering::Relaxed) {
|
||||
let state = serialize_state(JobType::MigrateCDs, &MigrationStatus { migrated: true })?;
|
||||
job_dao.upsert_state(state).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct MigrationStatus {
|
||||
pub migrated: bool,
|
||||
}
|
||||
@ -0,0 +1,71 @@
|
||||
mod calculate_sizes;
|
||||
mod check_integrity;
|
||||
mod generate_missing_thumbnails;
|
||||
mod migrate_content_descriptors;
|
||||
mod vacuum;
|
||||
|
||||
pub use calculate_sizes::*;
|
||||
pub use check_integrity::*;
|
||||
pub use generate_missing_thumbnails::*;
|
||||
pub use migrate_content_descriptors::*;
|
||||
use std::marker::PhantomData;
|
||||
use std::sync::Arc;
|
||||
pub use vacuum::*;
|
||||
|
||||
use crate::handle::JobHandle;
|
||||
use async_trait::async_trait;
|
||||
use mediarepo_core::bincode;
|
||||
use mediarepo_core::error::{RepoError, RepoResult};
|
||||
use mediarepo_core::trait_bound_typemap::TypeMapKey;
|
||||
use mediarepo_database::entities::job_state::JobType;
|
||||
use mediarepo_logic::dao::job::JobDao;
|
||||
use mediarepo_logic::dao::repo::Repo;
|
||||
use mediarepo_logic::dto::{JobStateDto, UpsertJobStateDto};
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::Serialize;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
type EmptyStatus = Arc<RwLock<()>>;
|
||||
|
||||
#[async_trait]
|
||||
pub trait Job: Clone + Send + Sync {
|
||||
type JobStatus: Send + Sync;
|
||||
type Result: Send + Sync;
|
||||
|
||||
fn status(&self) -> Arc<RwLock<Self::JobStatus>>;
|
||||
|
||||
async fn load_state(&self, _job_dao: JobDao) -> RepoResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run(&self, repo: Arc<Repo>) -> RepoResult<Self::Result>;
|
||||
|
||||
async fn save_state(&self, _job_dao: JobDao) -> RepoResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub struct JobTypeKey<T: Job>(PhantomData<T>);
|
||||
|
||||
impl<T: 'static> TypeMapKey for JobTypeKey<T>
|
||||
where
|
||||
T: Job,
|
||||
{
|
||||
type Value = JobHandle<T::JobStatus, T::Result>;
|
||||
}
|
||||
|
||||
pub fn deserialize_state<T: DeserializeOwned>(dto: JobStateDto) -> RepoResult<T> {
|
||||
bincode::deserialize(dto.value()).map_err(RepoError::from)
|
||||
}
|
||||
|
||||
pub fn serialize_state<T: Serialize>(
|
||||
job_type: JobType,
|
||||
state: &T,
|
||||
) -> RepoResult<UpsertJobStateDto> {
|
||||
let dto = UpsertJobStateDto {
|
||||
value: bincode::serialize(state)?,
|
||||
job_type,
|
||||
};
|
||||
|
||||
Ok(dto)
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
use crate::jobs::{EmptyStatus, Job};
|
||||
use async_trait::async_trait;
|
||||
use mediarepo_core::error::RepoResult;
|
||||
use mediarepo_logic::dao::repo::Repo;
|
||||
use mediarepo_logic::dao::DaoProvider;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct VacuumJob;
|
||||
|
||||
#[async_trait]
|
||||
impl Job for VacuumJob {
|
||||
type JobStatus = ();
|
||||
type Result = ();
|
||||
|
||||
fn status(&self) -> Arc<RwLock<Self::JobStatus>> {
|
||||
EmptyStatus::default()
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
async fn run(&self, repo: Arc<Repo>) -> RepoResult<()> {
|
||||
repo.job().vacuum().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
use crate::job_dispatcher::JobDispatcher;
|
||||
use crate::jobs::{CheckIntegrityJob, MigrateCDsJob, VacuumJob};
|
||||
use mediarepo_core::error::RepoError;
|
||||
use mediarepo_core::tokio_graceful_shutdown::Toplevel;
|
||||
use mediarepo_logic::dao::repo::Repo;
|
||||
use std::time::Duration;
|
||||
use tokio::sync::oneshot::channel;
|
||||
|
||||
pub mod handle;
|
||||
pub mod job_dispatcher;
|
||||
pub mod jobs;
|
||||
pub mod status_utils;
|
||||
|
||||
pub async fn start(top_level: Toplevel, repo: Repo) -> (Toplevel, JobDispatcher) {
|
||||
let (tx, rx) = channel();
|
||||
|
||||
let top_level =
|
||||
top_level.start::<RepoError, _, _>("mediarepo-worker", |subsystem| async move {
|
||||
let dispatcher = JobDispatcher::new(subsystem, repo);
|
||||
tx.send(dispatcher.clone())
|
||||
.map_err(|_| RepoError::from("failed to send dispatcher"))?;
|
||||
dispatcher
|
||||
.dispatch_periodically(VacuumJob::default(), Duration::from_secs(60 * 30))
|
||||
.await;
|
||||
dispatcher
|
||||
.dispatch_periodically(
|
||||
CheckIntegrityJob::default(),
|
||||
Duration::from_secs(60 * 60 * 24),
|
||||
)
|
||||
.await;
|
||||
dispatcher.dispatch(MigrateCDsJob::default()).await;
|
||||
|
||||
Ok(())
|
||||
});
|
||||
let receiver = rx
|
||||
.await
|
||||
.expect("failed to create background job dispatcher");
|
||||
|
||||
(top_level, receiver)
|
||||
}
|
||||
@ -0,0 +1,39 @@
|
||||
pub struct SimpleProgress {
|
||||
pub current: u64,
|
||||
pub total: u64,
|
||||
}
|
||||
|
||||
impl Default for SimpleProgress {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
total: 100,
|
||||
current: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SimpleProgress {
|
||||
pub fn new(total: u64) -> Self {
|
||||
Self { total, current: 0 }
|
||||
}
|
||||
|
||||
/// Sets the total count
|
||||
pub fn set_total(&mut self, total: u64) {
|
||||
self.total = total;
|
||||
}
|
||||
|
||||
/// Increments the current progress by 1
|
||||
pub fn tick(&mut self) {
|
||||
self.current += 1;
|
||||
}
|
||||
|
||||
/// Sets the current progress to a defined value
|
||||
pub fn set_current(&mut self, current: u64) {
|
||||
self.current = current;
|
||||
}
|
||||
|
||||
/// Returns the total progress in percent
|
||||
pub fn percent(&self) -> f64 {
|
||||
(self.current as f64) / (self.total as f64)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue