From 381a1fb50750594d295b5fda132f94c15b25f0eb Mon Sep 17 00:00:00 2001 From: trivernis Date: Fri, 15 Oct 2021 21:02:11 +0200 Subject: [PATCH] Add automatic thumbnail generation on import Signed-off-by: trivernis --- mediarepo-daemon/Cargo.lock | 217 +++++++++++++++ mediarepo-daemon/mediarepo-core/Cargo.lock | 246 ++++++++++++++++++ mediarepo-daemon/mediarepo-core/Cargo.toml | 2 + mediarepo-daemon/mediarepo-core/src/error.rs | 3 + .../mediarepo-core/src/image_processing.rs | 58 +++++ mediarepo-daemon/mediarepo-core/src/lib.rs | 4 + .../mediarepo-core/src/settings.rs | 2 + .../mediarepo-database/Cargo.lock | 1 + mediarepo-daemon/mediarepo-model/Cargo.lock | 217 +++++++++++++++ mediarepo-daemon/mediarepo-model/src/file.rs | 38 ++- mediarepo-daemon/mediarepo-model/src/hash.rs | 14 + mediarepo-daemon/mediarepo-model/src/repo.rs | 55 +++- .../mediarepo-model/src/storage.rs | 18 +- .../mediarepo-model/src/thumbnail.rs | 8 + mediarepo-daemon/mediarepo-socket/Cargo.lock | 217 +++++++++++++++ mediarepo-daemon/src/constants.rs | 1 + mediarepo-daemon/src/main.rs | 66 +++-- 17 files changed, 1130 insertions(+), 37 deletions(-) diff --git a/mediarepo-daemon/Cargo.lock b/mediarepo-daemon/Cargo.lock index 6945710..9bec627 100644 --- a/mediarepo-daemon/Cargo.lock +++ b/mediarepo-daemon/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "ahash" version = "0.7.4" @@ -195,6 +207,12 @@ version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9df67f7bf9ef8498769f994239c45613ef0c5899415fb58e9add412d2c1a538" +[[package]] +name = "bytemuck" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72957246c41db82b8ef88a5486143830adeb8227ef9837740bdec67724cf2c5b" + [[package]] name = "byteorder" version = "1.4.3" @@ -248,6 +266,12 @@ dependencies = [ "vec_map", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -300,6 +324,15 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.1" @@ -310,6 +343,30 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-queue" version = "0.3.2" @@ -356,6 +413,16 @@ dependencies = [ "syn", ] +[[package]] +name = "deflate" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73770f8e1fe7d64df17ca66ad28994a0a623ea497fa69486e14984e715c5d174" +dependencies = [ + "adler32", + "byteorder", +] + [[package]] name = "digest" version = "0.9.0" @@ -541,6 +608,16 @@ dependencies = [ "wasi", ] +[[package]] +name = "gif" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a7187e78088aead22ceedeee99779455b23fc231fe13ec443f99bb71694e5b" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "glob" version = "0.3.0" @@ -606,6 +683,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.23.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24ffcb7e7244a9bf19d35bf2883b9c080c4ced3c07a9895572178cdb8f13f6a1" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "gif", + "jpeg-decoder", + "num-iter", + "num-rational", + "num-traits", + "png", + "scoped_threadpool", + "tiff", +] + [[package]] name = "indexmap" version = "1.7.0" @@ -640,6 +736,15 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "jpeg-decoder" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2" +dependencies = [ + "rayon", +] + [[package]] name = "js-sys" version = "0.3.55" @@ -722,6 +827,8 @@ name = "mediarepo-core" version = "0.1.0" dependencies = [ "base64", + "futures", + "image", "multibase", "multihash", "rmp-ipc", @@ -777,6 +884,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memoffset" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -799,6 +915,25 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c64630dcdd71f1a64c435f54885086a0de5d6a12d104d69b165fb7d5286d677" +[[package]] +name = "miniz_oxide" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" +dependencies = [ + "adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.7.13" @@ -922,6 +1057,28 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.14" @@ -1035,6 +1192,18 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb" +[[package]] +name = "png" +version = "0.16.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3287920cb847dee3de33d301c463fba14dda99db24214ddf93f83d3021f4c6" +dependencies = [ + "bitflags", + "crc32fast", + "deflate", + "miniz_oxide 0.3.7", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1171,6 +1340,31 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.10" @@ -1269,6 +1463,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "scoped_threadpool" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" + [[package]] name = "scopeguard" version = "1.1.0" @@ -1676,6 +1876,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tiff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437" +dependencies = [ + "jpeg-decoder", + "miniz_oxide 0.4.4", + "weezl", +] + [[package]] name = "time" version = "0.1.44" @@ -1936,6 +2147,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b77fdfd5a253be4ab714e4ffa3c49caf146b4de743e97510c0656cf90f1e8e" + [[package]] name = "whoami" version = "1.1.5" diff --git a/mediarepo-daemon/mediarepo-core/Cargo.lock b/mediarepo-daemon/mediarepo-core/Cargo.lock index 7e3bb88..3891482 100644 --- a/mediarepo-daemon/mediarepo-core/Cargo.lock +++ b/mediarepo-daemon/mediarepo-core/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "ahash" version = "0.7.4" @@ -153,6 +165,12 @@ version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9df67f7bf9ef8498769f994239c45613ef0c5899415fb58e9add412d2c1a538" +[[package]] +name = "bytemuck" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72957246c41db82b8ef88a5486143830adeb8227ef9837740bdec67724cf2c5b" + [[package]] name = "byteorder" version = "1.4.3" @@ -177,6 +195,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -207,6 +231,15 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.1" @@ -217,6 +250,30 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-queue" version = "0.3.2" @@ -263,6 +320,16 @@ dependencies = [ "syn", ] +[[package]] +name = "deflate" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73770f8e1fe7d64df17ca66ad28994a0a623ea497fa69486e14984e715c5d174" +dependencies = [ + "adler32", + "byteorder", +] + [[package]] name = "digest" version = "0.9.0" @@ -420,6 +487,16 @@ dependencies = [ "wasi", ] +[[package]] +name = "gif" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a7187e78088aead22ceedeee99779455b23fc231fe13ec443f99bb71694e5b" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "hashbrown" version = "0.11.2" @@ -447,6 +524,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + [[package]] name = "hex" version = "0.4.3" @@ -464,6 +550,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.23.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24ffcb7e7244a9bf19d35bf2883b9c080c4ced3c07a9895572178cdb8f13f6a1" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "gif", + "jpeg-decoder", + "num-iter", + "num-rational", + "num-traits", + "png", + "scoped_threadpool", + "tiff", +] + [[package]] name = "indexmap" version = "1.7.0" @@ -498,6 +603,15 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "jpeg-decoder" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2" +dependencies = [ + "rayon", +] + [[package]] name = "js-sys" version = "0.3.55" @@ -554,6 +668,8 @@ name = "mediarepo-core" version = "0.1.0" dependencies = [ "base64", + "futures", + "image", "multibase", "multihash", "rmp-ipc", @@ -572,12 +688,40 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memoffset" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +dependencies = [ + "autocfg", +] + [[package]] name = "minimal-lexical" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c835948974f68e0bd58636fc6c5b1fbff7b297e3046f11b3b3c18bbac012c6d" +[[package]] +name = "miniz_oxide" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" +dependencies = [ + "adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.7.13" @@ -662,6 +806,38 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.14" @@ -671,6 +847,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "once_cell" version = "1.8.0" @@ -726,6 +912,18 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "png" +version = "0.16.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3287920cb847dee3de33d301c463fba14dda99db24214ddf93f83d3021f4c6" +dependencies = [ + "bitflags", + "crc32fast", + "deflate", + "miniz_oxide 0.3.7", +] + [[package]] name = "proc-macro-crate" version = "1.1.0" @@ -790,6 +988,31 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.10" @@ -835,6 +1058,12 @@ dependencies = [ "serde", ] +[[package]] +name = "scoped_threadpool" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" + [[package]] name = "scopeguard" version = "1.1.0" @@ -1095,6 +1324,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tiff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437" +dependencies = [ + "jpeg-decoder", + "miniz_oxide 0.4.4", + "weezl", +] + [[package]] name = "tinyvec" version = "1.5.0" @@ -1273,6 +1513,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b77fdfd5a253be4ab714e4ffa3c49caf146b4de743e97510c0656cf90f1e8e" + [[package]] name = "whoami" version = "1.1.5" diff --git a/mediarepo-daemon/mediarepo-core/Cargo.toml b/mediarepo-daemon/mediarepo-core/Cargo.toml index db4e526..9f8cb8c 100644 --- a/mediarepo-daemon/mediarepo-core/Cargo.toml +++ b/mediarepo-daemon/mediarepo-core/Cargo.toml @@ -14,6 +14,8 @@ toml = "0.5.8" serde = "1.0.130" rmp-ipc = "0.4.0" typemap_rev = "0.1.5" +image = "0.23.14" +futures = "0.3.17" [dependencies.sea-orm] version = "0.2.4" diff --git a/mediarepo-daemon/mediarepo-core/src/error.rs b/mediarepo-daemon/mediarepo-core/src/error.rs index 2509a90..96f396e 100644 --- a/mediarepo-daemon/mediarepo-core/src/error.rs +++ b/mediarepo-daemon/mediarepo-core/src/error.rs @@ -28,6 +28,9 @@ pub enum RepoError { #[error(transparent)] Raw(StringError), + + #[error(transparent)] + Image(#[from] image::error::ImageError), } #[derive(Error, Debug)] diff --git a/mediarepo-daemon/mediarepo-core/src/image_processing.rs b/mediarepo-daemon/mediarepo-core/src/image_processing.rs index e69de29..5d7222d 100644 --- a/mediarepo-daemon/mediarepo-core/src/image_processing.rs +++ b/mediarepo-daemon/mediarepo-core/src/image_processing.rs @@ -0,0 +1,58 @@ +use crate::error::RepoResult; +use image::imageops::FilterType; +use image::io::Reader as ImageReader; +use image::{DynamicImage, ImageOutputFormat}; +use std::io::Cursor; +use std::path::PathBuf; +use tokio::fs::OpenOptions; +use tokio::io::{AsyncRead, AsyncReadExt, BufReader}; + +/// Represents the different sizes of a thumbnail +#[derive(Clone, Copy)] +pub enum ThumbnailSize { + Small, + Medium, + Large, +} + +impl ThumbnailSize { + pub fn dimensions(&self) -> (u32, u32) { + match self { + ThumbnailSize::Small => (128, 128), + ThumbnailSize::Medium => (256, 256), + ThumbnailSize::Large => (512, 512), + } + } +} + +/// Reads an image from a path +pub async fn read_image_from_path(path: &PathBuf) -> RepoResult { + let file = OpenOptions::new().read(true).open(path).await?; + let mut reader = BufReader::new(file); + read_image(&mut reader).await +} + +/// Reads an image from a reader +pub async fn read_image(reader: &mut R) -> RepoResult { + let mut buf = Vec::new(); + reader.read_to_end(&mut buf).await?; + let image = ImageReader::new(Cursor::new(buf)) + .with_guessed_format()? + .decode()?; + + Ok(image) +} + +/// Returns the bytes of an image in the png format +pub fn get_image_bytes_png(image: DynamicImage) -> RepoResult> { + let mut buf = Vec::new(); + image.write_to(&mut buf, ImageOutputFormat::Png)?; + + Ok(buf) +} + +/// Creates a thumbnail with the defined thumbnail size +pub fn create_thumbnail(image: DynamicImage, size: ThumbnailSize) -> DynamicImage { + let (height, width) = size.dimensions(); + image.resize(height, width, FilterType::Nearest) +} diff --git a/mediarepo-daemon/mediarepo-core/src/lib.rs b/mediarepo-daemon/mediarepo-core/src/lib.rs index d065439..0e29cc4 100644 --- a/mediarepo-daemon/mediarepo-core/src/lib.rs +++ b/mediarepo-daemon/mediarepo-core/src/lib.rs @@ -1,5 +1,9 @@ pub mod context; pub mod error; pub mod file_hash_store; +pub mod image_processing; pub mod settings; pub mod type_keys; + +pub use futures; +pub use image; diff --git a/mediarepo-daemon/mediarepo-core/src/settings.rs b/mediarepo-daemon/mediarepo-core/src/settings.rs index 45467fc..c212b2a 100644 --- a/mediarepo-daemon/mediarepo-core/src/settings.rs +++ b/mediarepo-daemon/mediarepo-core/src/settings.rs @@ -6,6 +6,7 @@ pub struct Settings { pub listen_address: String, pub database_path: String, pub default_file_store: String, + pub thumbnail_store: String, } impl Default for Settings { @@ -14,6 +15,7 @@ impl Default for Settings { listen_address: "127.0.0.1:3425".to_string(), database_path: "./db/repo.db".to_string(), default_file_store: "./files".to_string(), + thumbnail_store: "./thumb".to_string(), } } } diff --git a/mediarepo-daemon/mediarepo-database/Cargo.lock b/mediarepo-daemon/mediarepo-database/Cargo.lock index cbd083e..d0390d2 100644 --- a/mediarepo-daemon/mediarepo-database/Cargo.lock +++ b/mediarepo-daemon/mediarepo-database/Cargo.lock @@ -742,6 +742,7 @@ name = "mediarepo-core" version = "0.1.0" dependencies = [ "base64", + "futures", "image", "multibase", "multihash", diff --git a/mediarepo-daemon/mediarepo-model/Cargo.lock b/mediarepo-daemon/mediarepo-model/Cargo.lock index bb94e4d..29c5112 100644 --- a/mediarepo-daemon/mediarepo-model/Cargo.lock +++ b/mediarepo-daemon/mediarepo-model/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "ahash" version = "0.7.4" @@ -166,6 +178,12 @@ version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9df67f7bf9ef8498769f994239c45613ef0c5899415fb58e9add412d2c1a538" +[[package]] +name = "bytemuck" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72957246c41db82b8ef88a5486143830adeb8227ef9837740bdec67724cf2c5b" + [[package]] name = "byteorder" version = "1.4.3" @@ -203,6 +221,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -255,6 +279,15 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.1" @@ -265,6 +298,30 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-queue" version = "0.3.2" @@ -311,6 +368,16 @@ dependencies = [ "syn", ] +[[package]] +name = "deflate" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73770f8e1fe7d64df17ca66ad28994a0a623ea497fa69486e14984e715c5d174" +dependencies = [ + "adler32", + "byteorder", +] + [[package]] name = "digest" version = "0.9.0" @@ -483,6 +550,16 @@ dependencies = [ "wasi", ] +[[package]] +name = "gif" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a7187e78088aead22ceedeee99779455b23fc231fe13ec443f99bb71694e5b" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "hashbrown" version = "0.11.2" @@ -536,6 +613,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.23.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24ffcb7e7244a9bf19d35bf2883b9c080c4ced3c07a9895572178cdb8f13f6a1" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "gif", + "jpeg-decoder", + "num-iter", + "num-rational", + "num-traits", + "png", + "scoped_threadpool", + "tiff", +] + [[package]] name = "indexmap" version = "1.7.0" @@ -570,6 +666,15 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "jpeg-decoder" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2" +dependencies = [ + "rayon", +] + [[package]] name = "js-sys" version = "0.3.55" @@ -637,6 +742,8 @@ name = "mediarepo-core" version = "0.1.0" dependencies = [ "base64", + "futures", + "image", "multibase", "multihash", "rmp-ipc", @@ -680,6 +787,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memoffset" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -702,6 +818,25 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c835948974f68e0bd58636fc6c5b1fbff7b297e3046f11b3b3c18bbac012c6d" +[[package]] +name = "miniz_oxide" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" +dependencies = [ + "adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.7.13" @@ -825,6 +960,28 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.14" @@ -938,6 +1095,18 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb" +[[package]] +name = "png" +version = "0.16.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3287920cb847dee3de33d301c463fba14dda99db24214ddf93f83d3021f4c6" +dependencies = [ + "bitflags", + "crc32fast", + "deflate", + "miniz_oxide 0.3.7", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1074,6 +1243,31 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.10" @@ -1155,6 +1349,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "scoped_threadpool" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" + [[package]] name = "scopeguard" version = "1.1.0" @@ -1514,6 +1714,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tiff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437" +dependencies = [ + "jpeg-decoder", + "miniz_oxide 0.4.4", + "weezl", +] + [[package]] name = "time" version = "0.1.44" @@ -1750,6 +1961,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b77fdfd5a253be4ab714e4ffa3c49caf146b4de743e97510c0656cf90f1e8e" + [[package]] name = "whoami" version = "1.1.5" diff --git a/mediarepo-daemon/mediarepo-model/src/file.rs b/mediarepo-daemon/mediarepo-model/src/file.rs index 81e06c9..42844f2 100644 --- a/mediarepo-daemon/mediarepo-model/src/file.rs +++ b/mediarepo-daemon/mediarepo-model/src/file.rs @@ -2,12 +2,16 @@ use crate::file_type::FileType; use crate::storage::Storage; use crate::thumbnail::Thumbnail; use chrono::{Local, NaiveDateTime}; -use mediarepo_core::error::RepoResult; +use mediarepo_core::error::{RepoError, RepoResult}; +use mediarepo_core::image_processing::{ + create_thumbnail, get_image_bytes_png, read_image, ThumbnailSize, +}; use mediarepo_database::entities::file; use mediarepo_database::entities::file::ActiveModel as ActiveFile; use mediarepo_database::entities::file::Model as FileModel; use mediarepo_database::entities::hash; use mediarepo_database::entities::hash::Model as HashModel; +use mime::Mime; use sea_orm::prelude::*; use sea_orm::{DatabaseConnection, Set}; use tokio::io::BufReader; @@ -73,22 +77,16 @@ impl File { } /// Adds a file with its hash to the database - pub(crate) async fn add( + pub(crate) async fn add( db: DatabaseConnection, storage_id: i64, - hash: S, + hash_id: i64, file_type: FileType, mime_type: Option, ) -> RepoResult { - let hash = hash::ActiveModel { - value: Set(hash.to_string()), - ..Default::default() - }; let now = Local::now().naive_local(); - let hash: hash::ActiveModel = hash.insert(&db).await?; - let id: i64 = hash.id.unwrap(); let file = file::ActiveModel { - hash_id: Set(id), + hash_id: Set(hash_id), file_type: Set(file_type as u32), mime_type: Set(mime_type), storage_id: Set(storage_id), @@ -206,6 +204,26 @@ impl File { storage.get_file_reader(&self.hash.value).await } + /// Creates a thumbnail for the file + pub async fn create_thumbnail(&self, size: ThumbnailSize) -> RepoResult<(Vec, Mime)> { + match self.file_type() { + FileType::Image => self.create_image_thumbnail(size).await, + _ => Err(RepoError::from( + "Unsupported file type for thumbnail generation", + )), + } + } + + /// Creates a thumbnail for an image + async fn create_image_thumbnail(&self, size: ThumbnailSize) -> RepoResult<(Vec, Mime)> { + let mut reader = self.get_reader().await?; + let image = read_image(&mut reader).await?; + let thumb_image = create_thumbnail(image, size); + let bytes = get_image_bytes_png(thumb_image)?; + + Ok((bytes, mime::IMAGE_PNG)) + } + /// Returns the active model of the file with only the id set fn get_active_model(&self) -> ActiveFile { ActiveFile { diff --git a/mediarepo-daemon/mediarepo-model/src/hash.rs b/mediarepo-daemon/mediarepo-model/src/hash.rs index 536a590..a33e59d 100644 --- a/mediarepo-daemon/mediarepo-model/src/hash.rs +++ b/mediarepo-daemon/mediarepo-model/src/hash.rs @@ -27,6 +27,20 @@ impl Hash { Ok(hash) } + /// Returns the hash by value + pub async fn by_value>( + db: DatabaseConnection, + value: S, + ) -> RepoResult> { + let hash = hash::Entity::find() + .filter(hash::Column::Value.eq(value.as_ref())) + .one(&db) + .await? + .map(|model| Self::new(db, model)); + + Ok(hash) + } + /// Adds a new hash to the database pub async fn add(db: DatabaseConnection, value: String) -> RepoResult { let active_model = hash::ActiveModel { diff --git a/mediarepo-daemon/mediarepo-model/src/repo.rs b/mediarepo-daemon/mediarepo-model/src/repo.rs index a3b957e..fab2fcc 100644 --- a/mediarepo-daemon/mediarepo-model/src/repo.rs +++ b/mediarepo-daemon/mediarepo-model/src/repo.rs @@ -1,9 +1,12 @@ use crate::file::File; use crate::file_type::FileType; use crate::storage::Storage; +use crate::thumbnail::Thumbnail; use mediarepo_core::error::{RepoError, RepoResult}; +use mediarepo_core::image_processing::ThumbnailSize; use mediarepo_database::get_database; use sea_orm::DatabaseConnection; +use std::io::Cursor; use std::path::PathBuf; use tokio::fs::OpenOptions; use tokio::io::BufReader; @@ -11,6 +14,7 @@ use tokio::io::BufReader; pub struct Repo { db: DatabaseConnection, main_storage: Option, + thumbnail_storage: Option, } impl Repo { @@ -18,6 +22,7 @@ impl Repo { Self { db, main_storage: None, + thumbnail_storage: None, } } @@ -43,6 +48,12 @@ impl Repo { Ok(()) } + /// Sets the default thumbnail storage + pub async fn set_thumbnail_storage(&mut self, path: S) -> RepoResult<()> { + self.thumbnail_storage = Storage::by_path(self.db.clone(), path).await?; + Ok(()) + } + /// Adds a storage to the repository pub async fn add_storage( &self, @@ -80,7 +91,41 @@ impl Repo { let reader = BufReader::new(os_file); let storage = self.get_main_storage()?; - storage.add_file(reader, file_type, mime_type).await + let hash = storage.store_entry(reader).await?; + File::add( + self.db.clone(), + storage.id(), + hash.id(), + file_type, + mime_type, + ) + .await + } + + /// Creates thumbnails of all sizes for a file + pub async fn create_thumbnails_for_file(&self, file: File) -> RepoResult<()> { + let thumb_storage = self.get_thumbnail_storage()?; + for size in [ + ThumbnailSize::Small, + ThumbnailSize::Medium, + ThumbnailSize::Large, + ] { + let (bytes, mime) = file.create_thumbnail(size).await?; + let hash = thumb_storage.store_entry(Cursor::new(bytes)).await?; + let (height, width) = size.dimensions(); + Thumbnail::add( + self.db.clone(), + hash.id(), + file.id(), + thumb_storage.id(), + height as i32, + width as i32, + Some(mime.to_string()), + ) + .await?; + } + + Ok(()) } fn get_main_storage(&self) -> RepoResult<&Storage> { @@ -90,4 +135,12 @@ impl Repo { Err(RepoError::from("No main storage configured.")) } } + + fn get_thumbnail_storage(&self) -> RepoResult<&Storage> { + if let Some(storage) = &self.thumbnail_storage { + Ok(storage) + } else { + Err(RepoError::from("No thumbnail storage configured.")) + } + } } diff --git a/mediarepo-daemon/mediarepo-model/src/storage.rs b/mediarepo-daemon/mediarepo-model/src/storage.rs index e08e5e8..4afc89a 100644 --- a/mediarepo-daemon/mediarepo-model/src/storage.rs +++ b/mediarepo-daemon/mediarepo-model/src/storage.rs @@ -1,5 +1,4 @@ -use crate::file::File; -use crate::file_type::FileType; +use crate::hash::Hash; use mediarepo_core::error::RepoResult; use mediarepo_core::file_hash_store::FileHashStore; use mediarepo_database::entities::storage; @@ -133,15 +132,14 @@ impl Storage { path.exists() } - /// Adds a file to the store - pub async fn add_file( - &self, - reader: R, - file_type: FileType, - mime_type: Option, - ) -> RepoResult { + /// Adds a thumbnail + pub async fn store_entry(&self, reader: R) -> RepoResult { let hash = self.store.add_file(reader, None).await?; - File::add(self.db.clone(), self.id(), hash, file_type, mime_type).await + if let Some(hash) = Hash::by_value(self.db.clone(), &hash).await? { + Ok(hash) + } else { + Hash::add(self.db.clone(), hash).await + } } /// Returns the buf reader to the given hash diff --git a/mediarepo-daemon/mediarepo-model/src/thumbnail.rs b/mediarepo-daemon/mediarepo-model/src/thumbnail.rs index 0e90a81..8f632e7 100644 --- a/mediarepo-daemon/mediarepo-model/src/thumbnail.rs +++ b/mediarepo-daemon/mediarepo-model/src/thumbnail.rs @@ -36,14 +36,18 @@ impl Thumbnail { db: DatabaseConnection, hash_id: i64, file_id: i64, + storage_id: i64, height: i32, width: i32, + mime: Option, ) -> RepoResult { let active_model = thumbnail::ActiveModel { + storage_id: Set(storage_id), hash_id: Set(hash_id), file_id: Set(file_id), height: Set(height), width: Set(width), + mime: Set(mime), ..Default::default() }; let active_model: thumbnail::ActiveModel = active_model.insert(&db).await?; @@ -84,6 +88,10 @@ impl Thumbnail { self.model.width } + pub fn mime_type(&self) -> &Option { + &self.model.mime + } + /// Returns the storage for the thumbnail pub async fn storage(&self) -> RepoResult { let storage = Storage::by_id(self.db.clone(), self.model.storage_id) diff --git a/mediarepo-daemon/mediarepo-socket/Cargo.lock b/mediarepo-daemon/mediarepo-socket/Cargo.lock index c1f96cb..64c2dc4 100644 --- a/mediarepo-daemon/mediarepo-socket/Cargo.lock +++ b/mediarepo-daemon/mediarepo-socket/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "ahash" version = "0.7.4" @@ -166,6 +178,12 @@ version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9df67f7bf9ef8498769f994239c45613ef0c5899415fb58e9add412d2c1a538" +[[package]] +name = "bytemuck" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72957246c41db82b8ef88a5486143830adeb8227ef9837740bdec67724cf2c5b" + [[package]] name = "byteorder" version = "1.4.3" @@ -204,6 +222,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -256,6 +280,15 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-channel" version = "0.5.1" @@ -266,6 +299,30 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-queue" version = "0.3.2" @@ -312,6 +369,16 @@ dependencies = [ "syn", ] +[[package]] +name = "deflate" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73770f8e1fe7d64df17ca66ad28994a0a623ea497fa69486e14984e715c5d174" +dependencies = [ + "adler32", + "byteorder", +] + [[package]] name = "digest" version = "0.9.0" @@ -484,6 +551,16 @@ dependencies = [ "wasi", ] +[[package]] +name = "gif" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a7187e78088aead22ceedeee99779455b23fc231fe13ec443f99bb71694e5b" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "hashbrown" version = "0.11.2" @@ -537,6 +614,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.23.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24ffcb7e7244a9bf19d35bf2883b9c080c4ced3c07a9895572178cdb8f13f6a1" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "gif", + "jpeg-decoder", + "num-iter", + "num-rational", + "num-traits", + "png", + "scoped_threadpool", + "tiff", +] + [[package]] name = "indexmap" version = "1.7.0" @@ -571,6 +667,15 @@ version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" +[[package]] +name = "jpeg-decoder" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2" +dependencies = [ + "rayon", +] + [[package]] name = "js-sys" version = "0.3.55" @@ -638,6 +743,8 @@ name = "mediarepo-core" version = "0.1.0" dependencies = [ "base64", + "futures", + "image", "multibase", "multihash", "rmp-ipc", @@ -693,6 +800,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memoffset" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -715,6 +831,25 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c64630dcdd71f1a64c435f54885086a0de5d6a12d104d69b165fb7d5286d677" +[[package]] +name = "miniz_oxide" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" +dependencies = [ + "adler32", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.7.13" @@ -838,6 +973,28 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.14" @@ -951,6 +1108,18 @@ version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb" +[[package]] +name = "png" +version = "0.16.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3287920cb847dee3de33d301c463fba14dda99db24214ddf93f83d3021f4c6" +dependencies = [ + "bitflags", + "crc32fast", + "deflate", + "miniz_oxide 0.3.7", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1087,6 +1256,31 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.10" @@ -1168,6 +1362,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "scoped_threadpool" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" + [[package]] name = "scopeguard" version = "1.1.0" @@ -1527,6 +1727,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tiff" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437" +dependencies = [ + "jpeg-decoder", + "miniz_oxide 0.4.4", + "weezl", +] + [[package]] name = "time" version = "0.1.44" @@ -1763,6 +1974,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "weezl" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b77fdfd5a253be4ab714e4ffa3c49caf146b4de743e97510c0656cf90f1e8e" + [[package]] name = "whoami" version = "1.1.5" diff --git a/mediarepo-daemon/src/constants.rs b/mediarepo-daemon/src/constants.rs index 4d39e62..a42a0f8 100644 --- a/mediarepo-daemon/src/constants.rs +++ b/mediarepo-daemon/src/constants.rs @@ -1,2 +1,3 @@ pub static SETTINGS_PATH: &str = "./repo.toml"; pub static DEFAULT_STORAGE_NAME: &str = "Main"; +pub static THUMBNAIL_STORAGE_NAME: &str = "Thumbnails"; diff --git a/mediarepo-daemon/src/main.rs b/mediarepo-daemon/src/main.rs index 1202562..e9cd34c 100644 --- a/mediarepo-daemon/src/main.rs +++ b/mediarepo-daemon/src/main.rs @@ -1,9 +1,10 @@ mod constants; mod utils; -use crate::constants::{DEFAULT_STORAGE_NAME, SETTINGS_PATH}; +use crate::constants::{DEFAULT_STORAGE_NAME, SETTINGS_PATH, THUMBNAIL_STORAGE_NAME}; use crate::utils::{create_paths_for_repo, get_repo, load_settings}; use mediarepo_core::error::RepoResult; +use mediarepo_core::futures::future; use mediarepo_core::settings::Settings; use mediarepo_core::type_keys::SettingsKey; use mediarepo_model::repo::Repo; @@ -12,6 +13,8 @@ use mediarepo_socket::get_builder; use std::path::PathBuf; use structopt::StructOpt; use tokio::fs; +use tokio::runtime; +use tokio::runtime::Runtime; #[derive(Debug, StructOpt)] #[structopt(name = "mediarepo", about = "A multimedia repository")] @@ -46,19 +49,36 @@ enum SubCommand { Start, } -#[tokio::main] -async fn main() -> RepoResult<()> { +fn main() -> RepoResult<()> { build_logger(); let opt: Opt = Opt::from_args(); + match opt.cmd.clone() { - SubCommand::Init { force } => init(opt, force).await, - SubCommand::Start => start_server(opt).await, - SubCommand::Import { folder_path } => import(opt, folder_path).await, + SubCommand::Init { force } => get_single_thread_runtime().block_on(init(opt, force)), + SubCommand::Start => get_multi_thread_runtime().block_on(start_server(opt)), + SubCommand::Import { folder_path } => { + get_single_thread_runtime().block_on(import(opt, folder_path)) + } }?; Ok(()) } +fn get_single_thread_runtime() -> Runtime { + runtime::Builder::new_current_thread() + .enable_all() + .max_blocking_threads(1) + .build() + .unwrap() +} + +fn get_multi_thread_runtime() -> Runtime { + runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap() +} + fn build_logger() { env_logger::builder() .filter_module("sqlx", log::LevelFilter::Warn) @@ -71,8 +91,11 @@ async fn init_repo(opt: &Opt) -> RepoResult<(Settings, Repo)> { let settings = load_settings(&opt.repo.join(SETTINGS_PATH)).await?; let mut repo = get_repo(&opt.repo.join(&settings.database_path).to_str().unwrap()).await?; let main_storage_path = opt.repo.join(&settings.default_file_store); + let thumb_storage_path = opt.repo.join(&settings.thumbnail_store); repo.set_main_storage(main_storage_path.to_str().unwrap()) .await?; + repo.set_thumbnail_storage(thumb_storage_path.to_str().unwrap()) + .await?; Ok((settings, repo)) } @@ -109,6 +132,9 @@ async fn init(opt: Opt, force: bool) -> RepoResult<()> { log::debug!("Adding storage"); repo.add_storage(DEFAULT_STORAGE_NAME, storage_path.to_str().unwrap()) .await?; + let thumb_storage_path = opt.repo.join(&settings.thumbnail_store); + repo.add_storage(THUMBNAIL_STORAGE_NAME, thumb_storage_path.to_str().unwrap()) + .await?; let settings_string = settings.to_toml_string()?; log::debug!("Writing settings"); fs::write(opt.repo.join(SETTINGS_PATH), &settings_string.into_bytes()).await?; @@ -122,16 +148,24 @@ async fn import(opt: Opt, path: String) -> RepoResult<()> { let (_s, repo) = init_repo(&opt).await?; log::info!("Importing"); - for entry in glob::glob(&path).unwrap() { - if let Ok(path) = entry { - if path.is_file() { - log::debug!("Importing {:?}", path); - if let Err(e) = repo.add_file_by_path(path).await { - log::error!("Failed to import: {:?}", e); - } - } - } - } + let promises = glob::glob(&path) + .unwrap() + .into_iter() + .filter_map(|r| r.ok()) + .filter(|e| e.is_file()) + .map(|path| import_single_image(path, &repo)); + + future::join_all(promises).await; + + Ok(()) +} + +/// Creates thumbnails of all sizes +async fn import_single_image(path: PathBuf, repo: &Repo) -> RepoResult<()> { + log::info!("Importing file"); + let file = repo.add_file_by_path(path).await?; + log::info!("Creating thumbnails"); + repo.create_thumbnails_for_file(file).await?; Ok(()) }