From b33bbd1c14c0252dc14331db3581d05c5bf29882 Mon Sep 17 00:00:00 2001 From: Trivernis Date: Wed, 22 Jan 2020 17:32:47 +0100 Subject: [PATCH] Add max file size and video processing - Add processing for videos to UploadRoute --- config/default.toml | 3 ++ package.json | 5 +++ src/routes/UploadRoute.ts | 42 +++++++++++++++++--- yarn.lock | 82 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 126 insertions(+), 6 deletions(-) diff --git a/config/default.toml b/config/default.toml index 5b76c38..62bec3b 100644 --- a/config/default.toml +++ b/config/default.toml @@ -64,6 +64,9 @@ maxQueryComplexity = 5000 # sets the image format for the site. Values: png or webp imageFormat = "png" +# the max file size for uploading in bytes +maxFileSize = 5_242_880 + # Configuration for the api rate limit [api.rateLimit] diff --git a/package.json b/package.json index ffd79fe..575f954 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "@types/express-graphql": "^0.8.0", "@types/express-session": "^1.15.14", "@types/express-socket.io-session": "^1.3.2", + "@types/fluent-ffmpeg": "^2.1.13", "@types/fs-extra": "^8.0.0", "@types/graphql-query-complexity": "^0.2.1", "@types/http-status": "^0.2.30", @@ -45,6 +46,7 @@ "@types/sharp": "^0.23.1", "@types/socket.io": "^2.1.2", "@types/socket.io-redis": "^1.0.25", + "@types/stream-buffers": "^3.0.3", "@types/uuid": "^3.4.6", "@types/validator": "^10.11.3", "chai": "^4.2.0", @@ -59,6 +61,7 @@ "typescript": "^3.7.2" }, "dependencies": { + "@ffmpeg-installer/ffmpeg": "^1.0.20", "@types/body-parser": "^1.17.1", "body-parser": "^1.19.0", "compression": "^1.7.4", @@ -72,6 +75,7 @@ "express-limiter": "^1.6.1", "express-session": "^1.16.2", "express-socket.io-session": "^1.3.5", + "fluent-ffmpeg": "^2.1.2", "fs-extra": "^8.1.0", "graphql": "^14.4.2", "graphql-import": "^0.7.1", @@ -91,6 +95,7 @@ "socket.io": "^2.2.0", "socket.io-redis": "^5.2.0", "sqlite3": "^4.1.0", + "stream-buffers": "^3.0.2", "toml": "^3.0.0", "uuid": "^3.3.3", "winston": "^3.2.1", diff --git a/src/routes/UploadRoute.ts b/src/routes/UploadRoute.ts index 6f8b10a..66d2c1c 100644 --- a/src/routes/UploadRoute.ts +++ b/src/routes/UploadRoute.ts @@ -4,14 +4,17 @@ import * as crypto from "crypto"; import {Router} from "express"; import * as fileUpload from "express-fileupload"; import {UploadedFile} from "express-fileupload"; +import * as ffmpeg from "fluent-ffmpeg"; import * as fsx from "fs-extra"; import * as status from "http-status"; import * as path from "path"; import * as sharp from "sharp"; +import {ReadableStreamBuffer} from "stream-buffers"; import globals from "../lib/globals"; import {Group, User} from "../lib/models"; import Route from "../lib/Route"; +const ffmpegPath = require("@ffmpeg-installer/ffmpeg").path; const dataDirName = "data"; interface IUploadConfirmation { @@ -56,6 +59,7 @@ export class UploadRoute extends Route { super(); this.router = Router(); this.dataDir = path.join(this.publicPath, dataDirName); + ffmpeg.setFfmpegPath(ffmpegPath); } /** @@ -63,7 +67,9 @@ export class UploadRoute extends Route { */ public async init() { await fsx.ensureDir(this.dataDir); - this.router.use(fileUpload()); + this.router.use(fileUpload({ + limits: config.get("api.maxFileSize"), + })); this.router.use(bodyParser()); // Uploads a file to the data directory and returns the filename this.router.use(async (req, res) => { @@ -114,7 +120,7 @@ export class UploadRoute extends Route { if (user) { fileName = await this.processAndStoreImage(profilePic.data); if (user.profilePicture) { - await this.deleteWebImage(user.profilePicture); + await this.deleteWebFile(user.profilePicture); } user.profilePicture = fileName; await user.save(); @@ -158,7 +164,7 @@ export class UploadRoute extends Route { if (isAdmin) { fileName = await this.processAndStoreImage(groupPicture.data); if (group.picture) { - await this.deleteWebImage(group.picture); + await this.deleteWebFile(group.picture); } group.picture = fileName; await group.save(); @@ -199,7 +205,7 @@ export class UploadRoute extends Route { fit, }) .normalise(); - if (config.get("api.imageFormat") === "webp") { + if (config.get("api.imageFormat") === "webp") { image = await image.webp({ reductionEffort: 6, smartSubsample: true, @@ -214,11 +220,37 @@ export class UploadRoute extends Route { return `/${dataDirName}/${fileBasename}`; } + /** + * Converts a video into a smaller format and .mp4 and returns the web path + * @param data + * @param width + */ + private async processAndStoreVideo(data: Buffer, width: number = 720): Promise { + return new Promise(async (resolve) => { + const fileBasename = UploadRoute.getFileName() + ".mp4"; + await fsx.ensureDir(this.dataDir); + const filePath = path.join(this.dataDir, fileBasename); + const videoFileStream = new ReadableStreamBuffer({ + chunkSize: 2048, + frequency: 10, + }); + videoFileStream.put(data); + const video = ffmpeg(videoFileStream); + video + .on("end", () => { + resolve(`/${dataDirName}/${fileBasename}`); + }) + .size(`${width}x?`) + .toFormat("libx264") + .output(filePath); + }); + } + /** * Deletes an image for a provided web path. * @param webPath */ - private async deleteWebImage(webPath: string) { + private async deleteWebFile(webPath: string) { const realPath = path.join(this.dataDir, path.basename(webPath)); if (await fsx.pathExists(realPath)) { await fsx.unlink(realPath); diff --git a/yarn.lock b/yarn.lock index 8c6b810..52703c3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -18,6 +18,54 @@ esutils "^2.0.2" js-tokens "^4.0.0" +"@ffmpeg-installer/darwin-x64@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-x64/-/darwin-x64-4.1.0.tgz#48e1706c690e628148482bfb64acb67472089aaa" + integrity sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw== + +"@ffmpeg-installer/ffmpeg@^1.0.20": + version "1.0.20" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/ffmpeg/-/ffmpeg-1.0.20.tgz#d3c9c2bbcd76149468fb0886c2b3fe9e4795490b" + integrity sha512-wbgd//6OdwbFXYgV68ZyKrIcozEQpUKlvV66XHaqO2h3sFbX0jYLzx62Q0v8UcFWN21LoxT98NU2P+K0OWsKNA== + optionalDependencies: + "@ffmpeg-installer/darwin-x64" "4.1.0" + "@ffmpeg-installer/linux-arm" "4.1.3" + "@ffmpeg-installer/linux-arm64" "4.1.4" + "@ffmpeg-installer/linux-ia32" "4.1.0" + "@ffmpeg-installer/linux-x64" "4.1.0" + "@ffmpeg-installer/win32-ia32" "4.1.0" + "@ffmpeg-installer/win32-x64" "4.1.0" + +"@ffmpeg-installer/linux-arm64@4.1.4": + version "4.1.4" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm64/-/linux-arm64-4.1.4.tgz#7219f3f901bb67f7926cb060b56b6974a6cad29f" + integrity sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg== + +"@ffmpeg-installer/linux-arm@4.1.3": + version "4.1.3" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm/-/linux-arm-4.1.3.tgz#c554f105ed5f10475ec25d7bec94926ce18db4c1" + integrity sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg== + +"@ffmpeg-installer/linux-ia32@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-ia32/-/linux-ia32-4.1.0.tgz#adad70b0d0d9d8d813983d6e683c5a338a75e442" + integrity sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ== + +"@ffmpeg-installer/linux-x64@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-x64/-/linux-x64-4.1.0.tgz#b4a5d89c4e12e6d9306dbcdc573df716ec1c4323" + integrity sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A== + +"@ffmpeg-installer/win32-ia32@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-ia32/-/win32-ia32-4.1.0.tgz#6eac4fb691b64c02e7a116c1e2d167f3e9b40638" + integrity sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw== + +"@ffmpeg-installer/win32-x64@4.1.0": + version "4.1.0" + resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-x64/-/win32-x64-4.1.0.tgz#17e8699b5798d4c60e36e2d6326a8ebe5e95a2c5" + integrity sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg== + "@types/babel-types@*", "@types/babel-types@^7.0.0": version "7.0.7" resolved "https://registry.yarnpkg.com/@types/babel-types/-/babel-types-7.0.7.tgz#667eb1640e8039436028055737d2b9986ee336e3" @@ -145,6 +193,13 @@ "@types/express-serve-static-core" "*" "@types/serve-static" "*" +"@types/fluent-ffmpeg@^2.1.13": + version "2.1.13" + resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.13.tgz#bfffbcf298b0980924e9ba9aa471aba234626afb" + integrity sha512-hg87ZQb9WVcNGQHNhrYwWJM0ARNYbQbLGh1c6CfPl55/I+BH5UTpFJAr5aZWYGbl8BFVY82oF5iG4I+Ra3btiQ== + dependencies: + "@types/node" "*" + "@types/fs-extra@^8.0.0": version "8.0.1" resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-8.0.1.tgz#a2378d6e7e8afea1564e44aafa2e207dadf77686" @@ -265,6 +320,13 @@ dependencies: "@types/node" "*" +"@types/stream-buffers@^3.0.3": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@types/stream-buffers/-/stream-buffers-3.0.3.tgz#34e565bf64e3e4bdeee23fd4aa58d4636014a02b" + integrity sha512-NeFeX7YfFZDYsCfbuaOmFQ0OjSmHreKBpp7MQ4alWQBHeh2USLsj7qyMyn9t82kjqIX516CR/5SRHnARduRtbQ== + dependencies: + "@types/node" "*" + "@types/uuid@^3.4.6": version "3.4.6" resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-3.4.6.tgz#d2c4c48eb85a757bf2927f75f939942d521e3016" @@ -601,6 +663,11 @@ async-settle@^1.0.0: dependencies: async-done "^1.2.2" +async@>=0.2.9: + version "3.1.0" + resolved "https://registry.yarnpkg.com/async/-/async-3.1.0.tgz#42b3b12ae1b74927b5217d8c0016baaf62463772" + integrity sha512-4vx/aaY6j/j3Lw3fbCHNWP0pPaTCew3F6F3hYyl/tHs/ndmV1q7NW9T5yuJ2XAGwdQrP+6Wu20x06U4APo/iQQ== + async@^2.6.1: version "2.6.3" resolved "https://registry.yarnpkg.com/async/-/async-2.6.3.tgz#d72625e2344a3656e3a3ad4fa749fa83299d82ff" @@ -2011,6 +2078,14 @@ flat@^4.1.0: dependencies: is-buffer "~2.0.3" +fluent-ffmpeg@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz#c952de2240f812ebda0aa8006d7776ee2acf7d74" + integrity sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ= + dependencies: + async ">=0.2.9" + which "^1.1.1" + flush-write-stream@^1.0.2: version "1.1.1" resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8" @@ -4832,6 +4907,11 @@ static-extend@^0.1.1: resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c" integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow= +stream-buffers@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/stream-buffers/-/stream-buffers-3.0.2.tgz#5249005a8d5c2d00b3a32e6e0a6ea209dc4f3521" + integrity sha512-DQi1h8VEBA/lURbSwFtEHnSTb9s2/pwLEaFuNhXwy1Dx3Sa0lOuYT2yNUr4/j2fs8oCAMANtrZ5OrPZtyVs3MQ== + stream-exhaust@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/stream-exhaust/-/stream-exhaust-1.0.2.tgz#acdac8da59ef2bc1e17a2c0ccf6c320d120e555d" @@ -5476,7 +5556,7 @@ which-pm-runs@^1.0.0: resolved "https://registry.yarnpkg.com/which-pm-runs/-/which-pm-runs-1.0.0.tgz#670b3afbc552e0b55df6b7780ca74615f23ad1cb" integrity sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs= -which@1.3.1, which@^1.2.14: +which@1.3.1, which@^1.1.1, which@^1.2.14: version "1.3.1" resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==