diff --git a/.gitignore b/.gitignore index 91bc7f4..eda6428 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ package-lock bin .idea node_modules +scripts/* +tmp diff --git a/app.js b/app.js index 2ae8aed..6b35600 100644 --- a/app.js +++ b/app.js @@ -4,8 +4,9 @@ const createError = require('http-errors'), cookieParser = require('cookie-parser'), logger = require('morgan'), - indexRouter = require('./routes/index'), - usersRouter = require('./routes/users'); + indexRouter = require('./routes/index'), + usersRouter = require('./routes/users'), + riddleRouter = require('./routes/riddle'); let app = express(); @@ -21,6 +22,7 @@ app.use(express.static(path.join(__dirname, 'public'))); app.use('/', indexRouter); app.use('/users', usersRouter); +app.use(/\/riddle(\/.*)?/, riddleRouter); // catch 404 and forward to error handler app.use(function(req, res, next) { diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..3bb2bb6 --- /dev/null +++ b/install.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +npm i +git clone https://github.com/trivernis/reddit-riddle ./scripts/reddit-riddle +pip3 install -r ./scripts/reddit-riddle/requirements.txt +mkdir tmp diff --git a/package-lock.json b/package-lock.json index 63645dc..0cbd297 100644 --- a/package-lock.json +++ b/package-lock.json @@ -334,11 +334,26 @@ "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" }, + "fs-extra": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-7.0.1.tgz", + "integrity": "sha512-YJDaCJZEnBmcbw13fvdAM9AwNOJwOzrE4pqMqBq5nFiEqXUqHwlK4B+3pUw6JNvfSPtX05xFHtYy/1ni01eGCw==", + "requires": { + "graceful-fs": "^4.1.2", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" }, + "graceful-fs": { + "version": "4.1.15", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", + "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==" + }, "graceful-readlink": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz", @@ -420,6 +435,14 @@ "resolved": "https://registry.npmjs.org/js-stringify/-/js-stringify-1.0.2.tgz", "integrity": "sha1-Fzb939lyTyijaCrcYjCufk6Weds=" }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "requires": { + "graceful-fs": "^4.1.6" + } + }, "jstransformer": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/jstransformer/-/jstransformer-1.0.0.tgz", @@ -819,6 +842,11 @@ "integrity": "sha1-bgkk1r2mta/jSeOabWMoUKD4grc=", "optional": true }, + "universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" + }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/package.json b/package.json index 5ac7627..8a65a6a 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "cookie-parser": "~1.4.4", "debug": "~2.6.9", "express": "~4.16.1", + "fs-extra": "^7.0.1", "http-errors": "~1.6.3", "morgan": "~1.9.1", "pug": "2.0.0-beta11" diff --git a/public/javascripts/riddle-web.js b/public/javascripts/riddle-web.js new file mode 100644 index 0000000..29e1219 --- /dev/null +++ b/public/javascripts/riddle-web.js @@ -0,0 +1,79 @@ +function postLocData(postBody) { + let request = new XMLHttpRequest(); + return new Promise((res, rej) => { + + request.onload = () => { + res({ + status: request.status, + data: request.responseText + }); + }; + + request.onerror = () => { + rej(request.error); + }; + + request.open('POST', '#', true); + request.setRequestHeader("Content-Type", "application/json;charset=UTF-8"); + request.send(JSON.stringify(postBody)); + }); +} + +async function startSubredditDownload(subredditName) { + let data = await postLocData({ + subreddit: subredditName + }); + return JSON.parse(data.data); +} + +async function getDownloadStatus(downloadId) { + let data = await postLocData({ + id: downloadId + }); + return JSON.parse(data.data); +} + +async function refreshDownloadInfo(downloadId) { + let response = await getDownloadStatus(downloadId); + + let dlDiv = document.querySelector(`.download-container[dl-id='${downloadId}']`); + dlDiv.querySelector('.downloadStatus').innerText = response.status; + let subredditName = dlDiv.getAttribute('subreddit-name'); + + if (response.status === 'pending') { + setTimeout(() => refreshDownloadInfo(downloadId), 1000) + } else { + let dlLink = document.createElement('a'); + dlLink.setAttribute('href', response.file); + dlLink.setAttribute('filename', `${subredditName}`); + for (let cNode of dlDiv.childNodes) + dlLink.appendChild(cNode); + dlDiv.appendChild(dlLink); + setTimeout(() => { + dlDiv.remove(); + }, 30000); + } +} + +async function submitDownload() { + let subredditName = document.querySelector('#subreddit-input').value; + let response = await startSubredditDownload(subredditName); + + let dlDiv = document.createElement('div'); + dlDiv.setAttribute('class', 'download-container'); + dlDiv.setAttribute('dl-id', response.id); + dlDiv.setAttribute('subreddit-name', subredditName); + document.querySelector('#download-list').prepend(dlDiv); + + let subnameSpan = document.createElement('span'); + subnameSpan.innerText = subredditName; + subnameSpan.setAttribute('class', 'subredditName'); + dlDiv.appendChild(subnameSpan); + + let dlStatusSpan = document.createElement('span'); + dlStatusSpan.innerText = response.status; + dlStatusSpan.setAttribute('class', 'downloadStatus'); + dlDiv.appendChild(dlStatusSpan); + + await refreshDownloadInfo(response.id); +} diff --git a/routes/riddle.js b/routes/riddle.js new file mode 100644 index 0000000..3ce0092 --- /dev/null +++ b/routes/riddle.js @@ -0,0 +1,99 @@ +const express = require('express'), + router = express.Router(), + cproc = require('child_process'), + fsx = require('fs-extra'); + +const rWordOnly = /^\w+$/; + +let downloads = {}; + +class RedditDownload { + constructor(file) { + this.file = file; + this.status = 'pending'; + this.progress = 'N/A'; + this.process = null; + } +} + +/** + * Generates an id for a subreddit download. + * @param subreddit + * @returns {string} + */ +function generateDownloadId(subreddit) { + return Date.now().toString(16); +} + +/** + * Starts the subreddit download by executing the riddle python file. + * @param subreddit {String} + * @returns {string} + */ +function startDownload(subreddit) { + if (rWordOnly.test(subreddit)) { + let downloadId = generateDownloadId(subreddit); + let dlFilePath = `./public/static/${downloadId}.zip`; + let dlWebPath = `/static/${downloadId}.zip`; + let dl = new RedditDownload(dlWebPath); + + dl.process = cproc.exec(`python -u riddle.py -o ../../public/static/${downloadId} -z --lzma ${subreddit}`, + {cwd: './scripts/reddit-riddle', env: {PYTHONIOENCODING: 'utf-8', PYTHONUNBUFFERED: true}}, + (err, stdout) => { + if (err) { + console.error(err); + } else { + console.log(`riddle.py: ${stdout}`); + } + }); + + dl.process.on('exit', (code) => { + if (code === 0) + dl.status = 'finished'; + else + dl.status = 'failed'; + setTimeout(async () => { + await fsx.remove(dlFilePath); + delete downloads[downloadId]; + }, 300000); // delete the file after 5 minutes + }); + + dl.process.on('message', (msg) => { + console.log(msg) + }); + + downloads[downloadId] = dl; + + return downloadId; + } +} + +router.use('/files', express.static('./tmp')); + +router.get('/', (req, res, next) => { + res.render('riddle'); +}); + +router.post('/', (req, res) => { + if (req.body.subreddit) { + let id = startDownload(req.body.subreddit); + let download = downloads[id]; + + res.send({id: id, status: download.status, file: download.file}); + } else if (req.body.id) { + let id = req.body.id; + let download = downloads[id]; + + if (download) { + res.send({ + id: id, + status: download.status, + file: download.file + }); + } else { + res.send({error: 'Unknown download ID', id: id}); + } + } +}); + +module.exports = router; diff --git a/views/riddle.pug b/views/riddle.pug new file mode 100644 index 0000000..f710a6f --- /dev/null +++ b/views/riddle.pug @@ -0,0 +1,10 @@ +html + head + title= title + link(rel='stylesheet', href='/stylesheets/style.css') + script(type='text/javascript', src='/javascripts/riddle-web.js') + body + h1 Riddle Reddit downloader + input(type='text' placeholder='subreddit' id='subreddit-input') + button(id='submit-download' onclick='submitDownload()') Download + div(id='download-list')