Add max file size and video processing

- Add processing for videos to UploadRoute
5 years ago · b33bbd1c14
parent f213503c4e
commit b33bbd1c14
4 changed files with 126 additions and 6 deletions
--- a/config/default.toml
+++ b/config/default.toml
@ -64,6 +64,9 @@ maxQueryComplexity = 5000
 # sets the image format for the site. Values: png or webp
 imageFormat = "png"

+# the max file size for uploading in bytes
+maxFileSize = 5_242_880
+
    # Configuration for the api rate limit
    [api.rateLimit]

--- a/package.json
+++ b/package.json
@ -32,6 +32,7 @@
    "@types/express-graphql": "^0.8.0",
    "@types/express-session": "^1.15.14",
    "@types/express-socket.io-session": "^1.3.2",
+    "@types/fluent-ffmpeg": "^2.1.13",
    "@types/fs-extra": "^8.0.0",
    "@types/graphql-query-complexity": "^0.2.1",
    "@types/http-status": "^0.2.30",
@ -45,6 +46,7 @@
    "@types/sharp": "^0.23.1",
    "@types/socket.io": "^2.1.2",
    "@types/socket.io-redis": "^1.0.25",
+    "@types/stream-buffers": "^3.0.3",
    "@types/uuid": "^3.4.6",
    "@types/validator": "^10.11.3",
    "chai": "^4.2.0",
@ -59,6 +61,7 @@
    "typescript": "^3.7.2"
  },
  "dependencies": {
+    "@ffmpeg-installer/ffmpeg": "^1.0.20",
    "@types/body-parser": "^1.17.1",
    "body-parser": "^1.19.0",
    "compression": "^1.7.4",
@ -72,6 +75,7 @@
    "express-limiter": "^1.6.1",
    "express-session": "^1.16.2",
    "express-socket.io-session": "^1.3.5",
+    "fluent-ffmpeg": "^2.1.2",
    "fs-extra": "^8.1.0",
    "graphql": "^14.4.2",
    "graphql-import": "^0.7.1",
@ -91,6 +95,7 @@
    "socket.io": "^2.2.0",
    "socket.io-redis": "^5.2.0",
    "sqlite3": "^4.1.0",
+    "stream-buffers": "^3.0.2",
    "toml": "^3.0.0",
    "uuid": "^3.3.3",
    "winston": "^3.2.1",
--- a/src/routes/UploadRoute.ts
+++ b/src/routes/UploadRoute.ts
@ -4,14 +4,17 @@ import * as crypto from "crypto";
 import {Router} from "express";
 import * as fileUpload from "express-fileupload";
 import {UploadedFile} from "express-fileupload";
+import * as ffmpeg from "fluent-ffmpeg";
 import * as fsx from "fs-extra";
 import * as status from "http-status";
 import * as path from "path";
 import * as sharp from "sharp";
+import {ReadableStreamBuffer} from "stream-buffers";
 import globals from "../lib/globals";
 import {Group, User} from "../lib/models";
 import Route from "../lib/Route";

+const ffmpegPath = require("@ffmpeg-installer/ffmpeg").path;
 const dataDirName = "data";

 interface IUploadConfirmation {
@ -56,6 +59,7 @@ export class UploadRoute extends Route {
        super();
        this.router = Router();
        this.dataDir = path.join(this.publicPath, dataDirName);
+        ffmpeg.setFfmpegPath(ffmpegPath);
    }

    /**
@ -63,7 +67,9 @@ export class UploadRoute extends Route {
     */
    public async init() {
        await fsx.ensureDir(this.dataDir);
-        this.router.use(fileUpload());
+        this.router.use(fileUpload({
+            limits: config.get<number>("api.maxFileSize"),
+        }));
        this.router.use(bodyParser());
        // Uploads a file to the data directory and returns the filename
        this.router.use(async (req, res) => {
@ -114,7 +120,7 @@ export class UploadRoute extends Route {
            if (user) {
                fileName = await this.processAndStoreImage(profilePic.data);
                if (user.profilePicture) {
-                    await this.deleteWebImage(user.profilePicture);
+                    await this.deleteWebFile(user.profilePicture);
                }
                user.profilePicture = fileName;
                await user.save();
@ -158,7 +164,7 @@ export class UploadRoute extends Route {
                if (isAdmin) {
                    fileName = await this.processAndStoreImage(groupPicture.data);
                    if (group.picture) {
-                        await this.deleteWebImage(group.picture);
+                        await this.deleteWebFile(group.picture);
                    }
                    group.picture = fileName;
                    await group.save();
@ -199,7 +205,7 @@ export class UploadRoute extends Route {
                fit,
            })
            .normalise();
-        if (config.get("api.imageFormat") === "webp") {
+        if (config.get<string>("api.imageFormat") === "webp") {
            image = await image.webp({
                reductionEffort: 6,
                smartSubsample: true,
@ -214,11 +220,37 @@ export class UploadRoute extends Route {
        return `/${dataDirName}/${fileBasename}`;
    }

+    /**
+     * Converts a video into a smaller format and .mp4 and returns the web path
+     * @param data
+     * @param width
+     */
+    private async processAndStoreVideo(data: Buffer, width: number = 720): Promise<string> {
+        return new Promise(async (resolve) => {
+            const fileBasename = UploadRoute.getFileName() + ".mp4";
+            await fsx.ensureDir(this.dataDir);
+            const filePath = path.join(this.dataDir, fileBasename);
+            const videoFileStream = new ReadableStreamBuffer({
+                chunkSize: 2048,
+                frequency: 10,
+            });
+            videoFileStream.put(data);
+            const video = ffmpeg(videoFileStream);
+            video
+                .on("end", () => {
+                    resolve(`/${dataDirName}/${fileBasename}`);
+                })
+                .size(`${width}x?`)
+                .toFormat("libx264")
+                .output(filePath);
+        });
+    }
+
    /**
     * Deletes an image for a provided web path.
     * @param webPath
     */
-    private async deleteWebImage(webPath: string) {
+    private async deleteWebFile(webPath: string) {
        const realPath = path.join(this.dataDir, path.basename(webPath));
        if (await fsx.pathExists(realPath)) {
            await fsx.unlink(realPath);
--- a/yarn.lock
+++ b/yarn.lock
@ -18,6 +18,54 @@
    esutils "^2.0.2"
    js-tokens "^4.0.0"

+"@ffmpeg-installer/darwin-x64@4.1.0":
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/darwin-x64/-/darwin-x64-4.1.0.tgz#48e1706c690e628148482bfb64acb67472089aaa"
+  integrity sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw==
+
+"@ffmpeg-installer/ffmpeg@^1.0.20":
+  version "1.0.20"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/ffmpeg/-/ffmpeg-1.0.20.tgz#d3c9c2bbcd76149468fb0886c2b3fe9e4795490b"
+  integrity sha512-wbgd//6OdwbFXYgV68ZyKrIcozEQpUKlvV66XHaqO2h3sFbX0jYLzx62Q0v8UcFWN21LoxT98NU2P+K0OWsKNA==
+  optionalDependencies:
+    "@ffmpeg-installer/darwin-x64" "4.1.0"
+    "@ffmpeg-installer/linux-arm" "4.1.3"
+    "@ffmpeg-installer/linux-arm64" "4.1.4"
+    "@ffmpeg-installer/linux-ia32" "4.1.0"
+    "@ffmpeg-installer/linux-x64" "4.1.0"
+    "@ffmpeg-installer/win32-ia32" "4.1.0"
+    "@ffmpeg-installer/win32-x64" "4.1.0"
+
+"@ffmpeg-installer/linux-arm64@4.1.4":
+  version "4.1.4"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm64/-/linux-arm64-4.1.4.tgz#7219f3f901bb67f7926cb060b56b6974a6cad29f"
+  integrity sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg==
+
+"@ffmpeg-installer/linux-arm@4.1.3":
+  version "4.1.3"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-arm/-/linux-arm-4.1.3.tgz#c554f105ed5f10475ec25d7bec94926ce18db4c1"
+  integrity sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg==
+
+"@ffmpeg-installer/linux-ia32@4.1.0":
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-ia32/-/linux-ia32-4.1.0.tgz#adad70b0d0d9d8d813983d6e683c5a338a75e442"
+  integrity sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ==
+
+"@ffmpeg-installer/linux-x64@4.1.0":
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/linux-x64/-/linux-x64-4.1.0.tgz#b4a5d89c4e12e6d9306dbcdc573df716ec1c4323"
+  integrity sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A==
+
+"@ffmpeg-installer/win32-ia32@4.1.0":
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-ia32/-/win32-ia32-4.1.0.tgz#6eac4fb691b64c02e7a116c1e2d167f3e9b40638"
+  integrity sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw==
+
+"@ffmpeg-installer/win32-x64@4.1.0":
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/@ffmpeg-installer/win32-x64/-/win32-x64-4.1.0.tgz#17e8699b5798d4c60e36e2d6326a8ebe5e95a2c5"
+  integrity sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg==
+
 "@types/babel-types@*", "@types/babel-types@^7.0.0":
  version "7.0.7"
  resolved "https://registry.yarnpkg.com/@types/babel-types/-/babel-types-7.0.7.tgz#667eb1640e8039436028055737d2b9986ee336e3"
@ -145,6 +193,13 @@
    "@types/express-serve-static-core" "*"
    "@types/serve-static" "*"

+"@types/fluent-ffmpeg@^2.1.13":
+  version "2.1.13"
+  resolved "https://registry.yarnpkg.com/@types/fluent-ffmpeg/-/fluent-ffmpeg-2.1.13.tgz#bfffbcf298b0980924e9ba9aa471aba234626afb"
+  integrity sha512-hg87ZQb9WVcNGQHNhrYwWJM0ARNYbQbLGh1c6CfPl55/I+BH5UTpFJAr5aZWYGbl8BFVY82oF5iG4I+Ra3btiQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/fs-extra@^8.0.0":
  version "8.0.1"
  resolved "https://registry.yarnpkg.com/@types/fs-extra/-/fs-extra-8.0.1.tgz#a2378d6e7e8afea1564e44aafa2e207dadf77686"
@ -265,6 +320,13 @@
  dependencies:
    "@types/node" "*"

+"@types/stream-buffers@^3.0.3":
+  version "3.0.3"
+  resolved "https://registry.yarnpkg.com/@types/stream-buffers/-/stream-buffers-3.0.3.tgz#34e565bf64e3e4bdeee23fd4aa58d4636014a02b"
+  integrity sha512-NeFeX7YfFZDYsCfbuaOmFQ0OjSmHreKBpp7MQ4alWQBHeh2USLsj7qyMyn9t82kjqIX516CR/5SRHnARduRtbQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/uuid@^3.4.6":
  version "3.4.6"
  resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-3.4.6.tgz#d2c4c48eb85a757bf2927f75f939942d521e3016"
@ -601,6 +663,11 @@ async-settle@^1.0.0:
  dependencies:
    async-done "^1.2.2"

+async@>=0.2.9:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/async/-/async-3.1.0.tgz#42b3b12ae1b74927b5217d8c0016baaf62463772"
+  integrity sha512-4vx/aaY6j/j3Lw3fbCHNWP0pPaTCew3F6F3hYyl/tHs/ndmV1q7NW9T5yuJ2XAGwdQrP+6Wu20x06U4APo/iQQ==
+
 async@^2.6.1:
  version "2.6.3"
  resolved "https://registry.yarnpkg.com/async/-/async-2.6.3.tgz#d72625e2344a3656e3a3ad4fa749fa83299d82ff"
@ -2011,6 +2078,14 @@ flat@^4.1.0:
  dependencies:
    is-buffer "~2.0.3"

+fluent-ffmpeg@^2.1.2:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz#c952de2240f812ebda0aa8006d7776ee2acf7d74"
+  integrity sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=
+  dependencies:
+    async ">=0.2.9"
+    which "^1.1.1"
+
 flush-write-stream@^1.0.2:
  version "1.1.1"
  resolved "https://registry.yarnpkg.com/flush-write-stream/-/flush-write-stream-1.1.1.tgz#8dd7d873a1babc207d94ead0c2e0e44276ebf2e8"
@ -4832,6 +4907,11 @@ static-extend@^0.1.1:
  resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.5.0.tgz#161c7dac177659fd9811f43771fa99381478628c"
  integrity sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=

+stream-buffers@^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/stream-buffers/-/stream-buffers-3.0.2.tgz#5249005a8d5c2d00b3a32e6e0a6ea209dc4f3521"
+  integrity sha512-DQi1h8VEBA/lURbSwFtEHnSTb9s2/pwLEaFuNhXwy1Dx3Sa0lOuYT2yNUr4/j2fs8oCAMANtrZ5OrPZtyVs3MQ==
+
 stream-exhaust@^1.0.1:
  version "1.0.2"
  resolved "https://registry.yarnpkg.com/stream-exhaust/-/stream-exhaust-1.0.2.tgz#acdac8da59ef2bc1e17a2c0ccf6c320d120e555d"
@ -5476,7 +5556,7 @@ which-pm-runs@^1.0.0:
  resolved "https://registry.yarnpkg.com/which-pm-runs/-/which-pm-runs-1.0.0.tgz#670b3afbc552e0b55df6b7780ca74615f23ad1cb"
  integrity sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=

-which@1.3.1, which@^1.2.14:
+which@1.3.1, which@^1.1.1, which@^1.2.14:
  version "1.3.1"
  resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"
  integrity sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==