Introducing Caching

SASS, HTML and other Files that are preprocessed will be stored in their 
processed form in the .cache directory for faster access. When a file is 
called more than 10 times in the last minute, it will be loaded into the 
memory and stays there for an hour. This allows the server to respond 
even faster to requests.
pull/4/head
Trivernis 6 years ago
parent a31f91d3ff
commit b40b890c1b

1
.gitignore vendored

@ -29,4 +29,5 @@ node_modules
# Custom additions
.ssh
.log
.cache
package-lock.json

@ -14,6 +14,7 @@ Modules
- jsdom
- winston-daily-rotate-file
- args-parser
- node-sass
**Embedded in the html-file**
- JQuery

@ -35,6 +35,10 @@
".ttf": {
"path": "./res/fonts/",
"mime": "font/opentype"
},
".sass" :{
"path": "./res/sass",
"mime": "style/css"
}
}
}

@ -1,16 +0,0 @@
@font-face {
font-family: ubuntuL;
src: url(/Ubuntu-L.ttf);
}
@font-face {
font-family: ubuntuR;
src: url(/Ubuntu-R.ttf);
}
@font-face {
font-family: ubuntuMono;
src: url(/UbuntuMono-R.ttf);
}
body {
font-family: ubuntuL;
}

@ -0,0 +1,59 @@
@import "./style/colors.sass"
@font-face
font-family: ubuntuL
src: url(/Ubuntu-L.ttf)
@font-face
font-family: ubuntuR
src: url(/Ubuntu-R.ttf)
@font-face
font-family: ubuntuMono
src: url(/UbuntuMono-R.ttf)
// tagnames
body
font-family: ubuntuL
background: $main-color
color: $highlight-color
text-align: center
font-size: 13pt
a
color: $link-color
h1, h2, h3, h4, h5
line-height: 1.5em
// tag-like classes
.title
font-size: 30pt
.content
padding: 0px 10%
margin: auto
line-height: 1.5em
.footer
position: absolute
width: 100%
bottom: 0px
left: 0px
background: $bar-color
.header
padding: 1px 1px
position: static
width: 100%
top: 0px
left: 0px
background: $bar-color
.spacer
height: 50px
// attribute-like classes
.justify
text-align: justify

@ -0,0 +1,4 @@
$main-color: #9289f5
$highlight-color: #FFF
$link-color: #FAA
$bar-color: #2310f7

@ -0,0 +1,76 @@
const fs = require("fs"),
path = require("path"),
config_path = "./config/caching_dump.json",
cache_dump = JSON.parse(fs.readFileSync(config_path)),
cache_dir = "./.cache",
cache = {}; // TODO: Must read from dump again
if (cache_dump != null && cache_dump["last"] != null) cache = cache_dump["last"];
/**
* Returns the data from files that were cached
* @param {String} filename The name of the file that has been cached
* @return {String} The data stored in the file
*/
exports.getCached = function(filename) {
let cf = cache[filename];
let call_passed = (Date.now()-cf.last_call) / 1000; // calculate the time since the last call of the file
if (cf.call_count > 10 && call_passed < 60) {
cf.data = fs.readFileSync(cf.path); // store the file's data into the json
} else if (call_passed > 3600) {
cf.call_count = 0; // reset the counter when an hour has passed since the last call
cf.data = null; // reset the data to free memory
}
cf.last_call = Date.now(); // set the last call to now
cf.call_count += 1; // increase the call count
if (cf.data != null) return cf.data;
return fs.readFileSync(cf.path); // return either the data or read the file
};
/**
* Safes the file with it's corresponding data in the cache directory
* @param {String} filename The name of the file
* @param {String} data The data form the file
*/
exports.cache = function(filename, data) {
if (!fs.existsSync("./.cache")) fs.mkdirSync("./.cache"); // if the cache folder doesn't exist, create it
let cache_fn = filename.replace(/[^\w\.]/g, "__"); // remove invalid path characters
let count = 0;
while (fs.existsSync(filename + count + ".cache")) count++; // check if a file with the same name already exists and increase count
let cache_path = path.join(cache_dir, cache_fn+count+".cache"); // create the final file path. Cachedir + cached filename (without invalid) + count + .cache
fs.writeFile(cache_path, data, (error) => {
cache[filename] = { // create a cache-entry with the file's path when the file is written (so it won't be accessed before)
"path": cache_path, // the last call to the file, the count of calls and an
"last_call": null, // empty data field to store the file's data if the file
"call_count": 0, // was called often
"data": null,
"creation_time": Date.now()
};
}); // write the data asynchronously to the file
};
/**
* Returns if the file is already cached
* @param {String} filename The filename to check
* @return {Boolean} Is it cached or not
* TODO: Use last access or use creation_time property to check if the file might
* be too old. If the function returns false, a new cache-file will be created which
* has a different name from the old. On each startup a function could check if
* there are cache-files that are not listet in the cache_dump and delete them.
*/
exports.isCached = function(filename) {
let cached_entry = cache[filename];
if (cached_entry) {
if (cached_entry.path) {
return fs.existsSync(cached_entry.path);
}
}
return false;
}
/**
* A function that dumps the config into the config file after appending the cache to it.
*/
exports.cleanup = function() {
cache_dump["last"] = cache;
fs.writeFileSync(config_path, JSON.stringify(cache_dump));
}

@ -0,0 +1,59 @@
/**
* Preprocesses html-files
*/
const fs = require("fs"),
{
JSDOM
} = require('jsdom')
// ressources
defaultCss = "/glob/style.sass", // the default style that is embedded in every html
defaultJs = "/glob/script.js"; // the default script that is embedded in every html
/**
* Creates a css DOM element with href as source
* @param {Object} document A html DOM
* @param {String} href the source of the css file
* @return {Object} the Link Element
*/
function createLinkElement(document, href) {
let link = document.createElement('link');
link.setAttribute("rel", "stylesheet");
link.setAttribute("type", "text/css");
link.setAttribute("href", href);
return link;
}
/**
* Creates a javascript Script DOM element with src as source
* @param {Object} document A html DOM
* @param {String} src the source of the javascript file
* @return {Object} the Script Element
*/
function createScriptLinkElement(document, src) {
let script = document.createElement("script");
script.setAttribute("type", "text/javascript");
script.setAttribute("src", src);
return script;
}
/**
* Formats the html string by adding a link to the standard css and to the standard javascript file.
* @param {String} htmlstring A string read from an html file or a html document string itself.
* @return {String} A html-string that represents a document.
*/
exports.formatHtml = function(filename) {
var htmlstring = fs.readFileSync(filename);
try {
let dom = new JSDOM(htmlstring); // creates a dom from the html string
let document = dom.window.document;
let head = document.getElementsByTagName('head')[0]; // gets the documents head
head.prepend(createLinkElement(document, defaultCss)); // prepend the default css to the head
head.prepend(createScriptLinkElement(document, defaultJs)); // prepend the default script to the head
head.prepend(createScriptLinkElement(document, "/glob/jquery.js")); // prepend the JQuery to the head
head.prepend(createScriptLinkElement(document, "/glob/vue.js")); // prepend the Vue to the head
return dom.serialize(); // return a string of the document
} catch (error) {
console.error(error);
return htmlstring;
}
}

@ -0,0 +1,43 @@
const fs = require("fs"),
utils = require("./utils"),
caching = require("./caching"),
// pp (preprocessors)
pp_html = require("./pp_html"),
pp_sass = require("node-sass"),
pp_config = {
".sass" : "sass",
".html": "html",
".htm": "hmtl"
};
/**
* Returns the data for the file. In some cases the data is processed or loaded from cache.
* @param {String} filename The file containing the data
* @return {String} The data that should be send
*/
exports.getProcessed = function(filename) {
try {
var extension = utils.getExtension(filename);
var data = null;
if (caching.isCached(filename)) return caching.getCached(filename)
switch (pp_config[extension]) {
case "sass":
data = Buffer.from(pp_sass.renderSync({
file: filename
}).css).toString("utf-8");
break;
case "html":
data = pp_html.formatHtml(filename);
break;
default:
return fs.readFileSync(filename);
}
caching.cache(filename, data);
return data;
} catch (error) {
console.error(error);
return "Processing Error";
}
}

@ -0,0 +1,20 @@
/**
* A Series of utility functions
*/
/**
* returns the extension of a file for the given filename.
* @param {String} filename The name of the file.
* @return {String} A string that represents the file-extension.
*/
exports.getExtension = function(filename) {
if (!filename) return null;
try {
let exts = filename.match(/\.[a-z]+/g); // get the extension by using regex
if (exts) return exts[exts.length - 1]; // return the found extension
else return null; // return null if no extension could be found
} catch (error) {
logger.warn(error);
return null;
}
}

@ -3,8 +3,41 @@
<title> Landing 1 </title>
</head>
<body>
<h1>
You made it to the Index!
</h1>
<div class="header">
<h1> RCN Landing </h1>
</div>
<div class="content">
<img src="RCN-Logo.png" style="width: auto; height: 200px"/>
<h1 class="title">
Welcome to the Raspberry Communication Network.
</h1>
<p class="justify">
The Raspberry pi Communication-Network is a network of Raspberry-Pi's.
It's currently still in development. <br> The purpose of this server is to
display the status of all raspberry-pi's in the network and the data
of several sensors that are connected to these raspberry pi's. The data
is stored in a database running on the main backend-server. The
frontend-server is running either on a differend device or also on the
backend-server-device. If this is the case, then the data is directly
fetched from several json files instead of using webservices. The json-
files are generated by a script running in the background that get's
data from the database and stores it in these json-files every few
seconds (depending on the type of data).
</p>
<div class="spacer">
</div>
<p class="justify">
</p>
</div>
<div class="footer">
<p>
This webpage and server can also be found on github:
<a href="https://github.com/trivernis/rcn-frontserver">
RCN-Frontserver
</a>
| <i> GNU GPL v3.0 </i>
</p>
</div>
</body>
</html>

@ -2,18 +2,18 @@
const https = require('https'),
fs = require('fs'),
urlparse = require('url'),
{ JSDOM } = require('jsdom'),
perfy = require('perfy'),
winston = require('winston'),
DailyRotateFile = require('winston-daily-rotate-file'),
path = require('path'),
// own modules
utils = require("./lib/utils"),
prepro = require("./lib/preprocessor"),
// args
args = require('args-parser')(process.argv), // create an args parser
// ressources
defaultCss = "/glob/style.css", // the default style that is embedded in every html
defaultJs = "/glob/script.js", // the default script that is embedded in every html
// config file
config = JSON.parse(fs.readFileSync("config.json")),
config = JSON.parse(fs.readFileSync("./config/server.json")),
// logging config using winston
fileLoggingFormat = winston.format.printf(info => {
return `${info.timestamp} ${info.level.toUpperCase()}: ${JSON.stringify(info.message)}`; // the logging format for files
@ -74,7 +74,8 @@ const https = require('https'),
"mime": "text/javascript"
}
},
mounts = config.mounts; // mounts are more important than routes.
mounts = config.mounts, // mounts are more important than routes.
cache = {}; // cache stores filenames for cached processed files
// --- functional declaration part ---
@ -104,28 +105,12 @@ function main() {
} catch (error) {
logger.error(error);
logger.info("Shutting Down...");
caching.cleanup();
winston.end();
return false;
}
}
/**
* returns the extension of a file for the given filename.
* @param {String} filename The name of the file.
* @return {String} A string that represents the file-extension.
*/
function getExtension(filename) {
if (!filename) return null;
try {
let exts = filename.match(/\.[a-z]+/g); // get the extension by using regex
if (exts) return exts[exts.length - 1]; // return the found extension
else return null; // return null if no extension could be found
} catch (error) {
logger.warn(error);
return null;
}
}
/**
* Returns a string that depends on the uri It gets the data from the routes variable.
* @param {String} uriNormally a file-name. Depending on the extension, an other root-uriis choosen.
@ -134,25 +119,24 @@ function getExtension(filename) {
function getResponse(uri) {
if (!uri || uri == "/") uri = "/index.html"; // uri redirects to the index.html if it is not set or if it is root
logger.verbose({'msg':'calculating response', 'path': uri});
let gp = prepro.getProcessed;
try {
// get the file extension
let extension = getExtension(uri);
let extension = utils.getExtension(uri);
// returns the global script or css if the extension is css or js and the root-uriis glob.
if (uri.includes("/glob") && (extension == ".css" || extension == ".js")) {
if (uri.includes("/glob") && (extension == ".sass" || extension == ".js")) {
logger.verbose("Using global uri");
if (extension == ".css") return [fs.readFileSync("." + uri), "text/css"];
else return [fs.readFileSync("." + uri), "text/javascript"];
if (extension == ".sass") return [gp("." + uri), "text/css"];
else return [gp("." + uri), "text/javascript"];
}
let mount = getMount(uri); // get mount for uri it will be uses as path later instead of route
logger.verbose("Mount for uri is "+ mount)
logger.verbose("Mount for uri is "+ mount);
let route = routes[extension]; // get the route from the extension json
logger.verbose("Found route: "+JSON.stringify(route))
logger.verbose("Found route: "+JSON.stringify(route));
if (!route) return ["Not Allowed", "text/plain"]; // return not allowed if no route was found
let rf = fs.readFileSync; // shorten filesync
if (extension == ".html") return [formatHtml(rf(mount || path.join(route["path"]+uri))), route["mime"]]; // format if html and return
return [rf(mount || path.join(route["path"],uri)), route["mime"]]; // return without formatting if it's not an html file. (htm files won't be manipulated)
return [gp(mount || path.join(route["path"],uri)), route["mime"]]; // get processed output (done by preprocessor)
// test the extension for differend file types.
logger.verbose({'msg': 'Error', 'path': uri})
logger.verbose({'msg': 'Error', 'path': uri});
return ["Error with url", "text/plain"]; // return an error if above has not returned
} catch (error) {
logger.error(error);
@ -179,55 +163,6 @@ function getMount(uri) {
return false;
}
/**
* Creates a css DOM element with href as source
* @param {Object} document A html DOM
* @param {String} href the source of the css file
* @return {Object} the Link Element
*/
function createLinkElement(document, href) {
let link = document.createElement('link');
link.setAttribute("rel", "stylesheet");
link.setAttribute("type", "text/css");
link.setAttribute("href", href);
return link;
}
/**
* Creates a javascript Script DOM element with src as source
* @param {Object} document A html DOM
* @param {String} src the source of the javascript file
* @return {Object} the Script Element
*/
function createScriptLinkElement(document, src) {
let script = document.createElement("script");
script.setAttribute("type", "text/javascript");
script.setAttribute("src", src);
return script;
}
/**
* Formats the html string by adding a link to the standard css and to the standard javascript file.
* @param {String} htmlstring A string read from an html file or a html document string itself.
* @return {String} A html-string that represents a document.
*/
function formatHtml(htmlstring) {
logger.debug({'msg': 'Formatting HTML', 'htmlstring': htmlstring});
try {
let dom = new JSDOM(htmlstring); // creates a dom from the html string
let document = dom.window.document;
let head = document.getElementsByTagName('head')[0]; // gets the documents head
head.prepend(createLinkElement(document, defaultCss)); // prepend the default css to the head
head.prepend(createScriptLinkElement(document, defaultJs)); // prepend the default script to the head
head.prepend(createScriptLinkElement(document, "/glob/jquery.js")); // prepend the JQuery to the head
head.prepend(createScriptLinkElement(document, "/glob/vue.js")); // prepend the Vue to the head
return dom.serialize(); // return a string of the document
} catch(error) {
logger.error(error);
return htmlstring;
}
}
// Executing the main function
if (typeof require != 'undefined' && require.main == module) {
logger.exceptions.handle(

Loading…
Cancel
Save