Introducing Caching

SASS, HTML and other Files that are preprocessed will be stored in their 
processed form in the .cache directory for faster access. When a file is 
called more than 10 times in the last minute, it will be loaded into the 
memory and stays there for an hour. This allows the server to respond 
even faster to requests.

.gitignore

@@ -29,4 +29,5 @@ node_modules
 # Custom additions
 .ssh
 .log
+.cache
 package-lock.json

README.md

@@ -14,6 +14,7 @@ Modules
 -   jsdom
 -   winston-daily-rotate-file
 -   args-parser
+-   node-sass
 
 **Embedded in the html-file**
 -   JQuery

config/caching_dump.json

@@ -0,0 +1,3 @@
+{
+
+}

config/server.json

@@ -35,6 +35,10 @@
     ".ttf": {
       "path": "./res/fonts/",
       "mime": "font/opentype"
+    },
+    ".sass" :{
+      "path": "./res/sass",
+      "mime": "style/css"
     }
   }
 }

glob/style.css

@@ -1,16 +0,0 @@
-@font-face {
-  font-family: ubuntuL;
-  src: url(/Ubuntu-L.ttf);
-}
-@font-face {
-  font-family: ubuntuR;
-  src: url(/Ubuntu-R.ttf);
-}
-@font-face {
-  font-family: ubuntuMono;
-  src: url(/UbuntuMono-R.ttf);
-}
-
-body {
-  font-family: ubuntuL;
-}

glob/style.sass

@@ -0,0 +1,59 @@
+@import "./style/colors.sass"
+
+@font-face
+  font-family: ubuntuL
+  src: url(/Ubuntu-L.ttf)
+
+@font-face
+  font-family: ubuntuR
+  src: url(/Ubuntu-R.ttf)
+
+@font-face
+  font-family: ubuntuMono
+  src: url(/UbuntuMono-R.ttf)
+
+// tagnames
+body
+  font-family: ubuntuL
+  background: $main-color
+  color: $highlight-color
+  text-align: center
+  font-size: 13pt
+
+a
+  color: $link-color
+
+h1, h2, h3, h4, h5
+  line-height: 1.5em
+// tag-like classes
+
+.title
+  font-size: 30pt
+
+.content
+  padding: 0px 10%
+  margin: auto
+  line-height: 1.5em
+
+.footer
+  position: absolute
+  width: 100%
+  bottom: 0px
+  left: 0px
+  background: $bar-color
+
+.header
+  padding: 1px 1px
+  position: static
+  width: 100%
+  top: 0px
+  left: 0px
+  background: $bar-color
+
+.spacer
+  height: 50px
+
+// attribute-like classes
+
+.justify
+  text-align: justify

glob/style/colors.sass

@@ -0,0 +1,4 @@
+$main-color: #9289f5
+$highlight-color: #FFF
+$link-color: #FAA
+$bar-color: #2310f7

lib/caching.js

@@ -0,0 +1,76 @@
+const fs = require("fs"),
+  path = require("path"),
+  config_path = "./config/caching_dump.json",
+  cache_dump = JSON.parse(fs.readFileSync(config_path)),
+  cache_dir = "./.cache",
+  cache = {}; // TODO: Must read from dump again
+if (cache_dump != null && cache_dump["last"] != null) cache = cache_dump["last"];
+
+/**
+ * Returns the data from files that were cached
+ * @param  {String} filename The name of the file that has been cached
+ * @return {String}          The data stored in the file
+ */
+exports.getCached = function(filename) {
+  let cf = cache[filename];
+  let call_passed = (Date.now()-cf.last_call) / 1000; // calculate the time since the last call of the file
+  if (cf.call_count > 10 && call_passed < 60) {
+    cf.data = fs.readFileSync(cf.path); // store the file's data into the json
+  } else if (call_passed > 3600) {
+    cf.call_count = 0; // reset the counter when an hour has passed since the last call
+    cf.data = null; // reset the data to free memory
+  }
+  cf.last_call = Date.now(); // set the last call to now
+  cf.call_count += 1; // increase the call count
+  if (cf.data != null)  return cf.data;
+  return fs.readFileSync(cf.path); // return either the data or read the file
+};
+
+/**
+ * Safes the file with it's corresponding data in the cache directory
+ * @param  {String} filename The name of the file
+ * @param  {String} data     The data form the file
+ */
+exports.cache = function(filename, data) {
+  if (!fs.existsSync("./.cache")) fs.mkdirSync("./.cache"); // if the cache folder doesn't exist, create it
+  let cache_fn = filename.replace(/[^\w\.]/g, "__"); // remove invalid path characters
+  let count = 0;
+  while (fs.existsSync(filename + count + ".cache")) count++; // check if a file with the same name already exists and increase count
+  let cache_path = path.join(cache_dir, cache_fn+count+".cache"); // create the final file path. Cachedir + cached filename (without invalid) + count + .cache
+  fs.writeFile(cache_path, data, (error) => {
+    cache[filename] = {   // create a cache-entry with the file's path when the file is written (so it won't be accessed before)
+      "path": cache_path, // the last call to the file, the count of calls and an
+      "last_call": null,  // empty data field to store the file's data if the file
+      "call_count": 0,    // was called often
+      "data": null,
+      "creation_time": Date.now()
+    };
+  }); // write the data asynchronously to the file
+};
+
+/**
+ * Returns if the file is already cached
+ * @param  {String} filename The filename to check
+ * @return {Boolean}          Is it cached or not
+ * TODO: Use last access or use creation_time property to check if the file might
+ * be too old. If the function returns false, a new cache-file will be created which
+ * has a different name from the old. On each startup a function could check if
+ * there are cache-files that are not listet in the cache_dump and delete them.
+ */
+exports.isCached = function(filename) {
+  let cached_entry = cache[filename];
+  if (cached_entry) {
+    if (cached_entry.path) {
+      return fs.existsSync(cached_entry.path);
+    }
+  }
+  return false;
+}
+
+/**
+ * A function that dumps the config into the config file after appending the cache to it.
+ */
+exports.cleanup = function() {
+  cache_dump["last"] = cache;
+  fs.writeFileSync(config_path, JSON.stringify(cache_dump));
+}

lib/pp_html.js

@@ -0,0 +1,59 @@
+/**
+ * Preprocesses html-files
+ */
+const fs = require("fs"),
+  {
+    JSDOM
+  } = require('jsdom')
+// ressources
+  defaultCss = "/glob/style.sass", // the default style that is embedded in every html
+  defaultJs = "/glob/script.js"; // the default script that is embedded in every html
+
+/**
+ * Creates a css DOM element with href as source
+ * @param  {Object} document A html DOM
+ * @param  {String} href      the source of the css file
+ * @return {Object}          the Link Element
+ */
+function createLinkElement(document, href) {
+  let link = document.createElement('link');
+  link.setAttribute("rel", "stylesheet");
+  link.setAttribute("type", "text/css");
+  link.setAttribute("href", href);
+  return link;
+}
+
+/**
+ * Creates a javascript Script DOM element with src as source
+ * @param  {Object} document A html DOM
+ * @param  {String} src      the source of the javascript file
+ * @return {Object}          the Script Element
+ */
+function createScriptLinkElement(document, src) {
+  let script = document.createElement("script");
+  script.setAttribute("type", "text/javascript");
+  script.setAttribute("src", src);
+  return script;
+}
+
+/**
+ * Formats the html string by adding a link to the standard css and to the standard javascript file.
+ * @param  {String} htmlstring A string read from an html file or a html document string itself.
+ * @return {String}            A html-string that represents a document.
+ */
+exports.formatHtml = function(filename) {
+  var htmlstring = fs.readFileSync(filename);
+  try {
+    let dom = new JSDOM(htmlstring); // creates a dom from the html string
+    let document = dom.window.document;
+    let head = document.getElementsByTagName('head')[0]; // gets the documents head
+    head.prepend(createLinkElement(document, defaultCss)); // prepend the default css to the head
+    head.prepend(createScriptLinkElement(document, defaultJs)); // prepend the default script to the head
+    head.prepend(createScriptLinkElement(document, "/glob/jquery.js")); // prepend the JQuery to the head
+    head.prepend(createScriptLinkElement(document, "/glob/vue.js")); // prepend the Vue to the head
+    return dom.serialize(); // return a string of the document
+  } catch (error) {
+    console.error(error);
+    return htmlstring;
+  }
+}

lib/preprocessor.js

@@ -0,0 +1,43 @@
+const fs = require("fs"),
+  utils = require("./utils"),
+  caching = require("./caching"),
+
+// pp (preprocessors)
+  pp_html = require("./pp_html"),
+  pp_sass = require("node-sass"),
+
+  pp_config = {
+    ".sass" : "sass",
+    ".html": "html",
+    ".htm": "hmtl"
+  };
+
+/**
+ * Returns the data for the file. In some cases the data is processed or loaded from cache.
+ * @param  {String} filename The file containing the data
+ * @return {String}          The data that should be send
+ */
+exports.getProcessed = function(filename) {
+  try {
+    var extension = utils.getExtension(filename);
+    var data = null;
+    if (caching.isCached(filename)) return caching.getCached(filename)
+    switch (pp_config[extension]) {
+      case "sass":
+        data = Buffer.from(pp_sass.renderSync({
+          file: filename
+        }).css).toString("utf-8");
+        break;
+      case "html":
+        data = pp_html.formatHtml(filename);
+        break;
+      default:
+        return fs.readFileSync(filename);
+    }
+    caching.cache(filename, data);
+    return data;
+  } catch (error) {
+    console.error(error);
+    return "Processing Error";
+  }
+}

lib/utils.js

@@ -0,0 +1,20 @@
+/**
+ * A Series of utility functions
+ */
+
+/**
+ * returns the extension of a file for the given filename.
+ * @param  {String} filename The name of the file.
+ * @return {String}          A string that represents the file-extension.
+ */
+exports.getExtension = function(filename) {
+  if (!filename) return null;
+  try {
+    let exts = filename.match(/\.[a-z]+/g); // get the extension by using regex
+    if (exts) return exts[exts.length - 1]; // return the found extension
+    else return null; // return null if no extension could be found
+  } catch (error) {
+    logger.warn(error);
+    return null;
+  }
+}

res/html/index.html

@@ -3,8 +3,41 @@
     <title> Landing 1 </title>
   </head>
   <body>
-    <h1>
+    <div class="header">
-      You made it to the Index!
+      <h1> RCN Landing </h1>
-    </h1>
+    </div>
+    <div class="content">
+      <img src="RCN-Logo.png" style="width: auto; height: 200px"/>
+      <h1 class="title">
+        Welcome to the Raspberry Communication Network.
+      </h1>
+      <p class="justify">
+        The Raspberry pi Communication-Network is a network of Raspberry-Pi's.
+        It's currently still in development. <br> The purpose of this server is to
+        display the status of all raspberry-pi's in the network and the data
+        of several sensors that are connected to these raspberry pi's. The data
+        is stored in a database running on the main backend-server. The
+        frontend-server is running either on a differend device or also on the
+        backend-server-device. If this is the case, then the data is directly
+        fetched from several json files instead of using webservices. The json-
+        files are generated by a script running in the background that get's
+        data from the database and stores it in these json-files every few
+        seconds (depending on the type of data).
+      </p>
+      <div class="spacer">
+      </div>
+      <p class="justify">
+
+      </p>
+    </div>
+    <div class="footer">
+      <p>
+        This webpage and server can also be found on github:
+        <a href="https://github.com/trivernis/rcn-frontserver">
+          RCN-Frontserver
+        </a>
+        | <i> GNU GPL v3.0 </i>
+      </p>
+    </div>
   </body>
 </html>

server.js

@@ -2,18 +2,18 @@
 const https = require('https'),
   fs = require('fs'),
   urlparse = require('url'),
-  { JSDOM } = require('jsdom'),
   perfy = require('perfy'),
   winston = require('winston'),
   DailyRotateFile = require('winston-daily-rotate-file'),
   path = require('path'),
+// own modules
+  utils = require("./lib/utils"),
+  prepro = require("./lib/preprocessor"),
 // args
   args = require('args-parser')(process.argv), // create an args parser
-// ressources
-  defaultCss = "/glob/style.css", // the default style that is embedded in every html
-  defaultJs = "/glob/script.js", // the default script that is embedded in every html
 // config file
-  config = JSON.parse(fs.readFileSync("config.json")),
+  config = JSON.parse(fs.readFileSync("./config/server.json")),
+
 // logging config using winston
   fileLoggingFormat = winston.format.printf(info => {
     return `${info.timestamp} ${info.level.toUpperCase()}: ${JSON.stringify(info.message)}`; // the logging format for files
@@ -74,7 +74,8 @@ const https = require('https'),
       "mime": "text/javascript"
     }
   },
-  mounts = config.mounts; // mounts are more important than routes.
+  mounts = config.mounts, // mounts are more important than routes.
+  cache = {}; // cache stores filenames for cached processed files
 
 // --- functional declaration part ---
 
@@ -104,28 +105,12 @@ function main() {
   } catch (error) {
     logger.error(error);
     logger.info("Shutting Down...");
+    caching.cleanup();
     winston.end();
     return false;
   }
 }
 
-/**
- * returns the extension of a file for the given filename.
- * @param  {String} filename The name of the file.
- * @return {String}          A string that represents the file-extension.
- */
-function getExtension(filename) {
-  if (!filename) return null;
-  try {
-    let exts = filename.match(/\.[a-z]+/g); // get the extension by using regex
-    if (exts) return exts[exts.length - 1]; // return the found extension
-    else return null; // return null if no extension could be found
-  } catch (error) {
-    logger.warn(error);
-    return null;
-  }
-}
-
 /**
  * Returns a string that depends on the uri It gets the data from the routes variable.
  * @param  {String} uriNormally a file-name. Depending on the extension, an other root-uriis choosen.
@@ -134,25 +119,24 @@ function getExtension(filename) {
 function getResponse(uri) {
   if (!uri || uri == "/") uri = "/index.html"; // uri redirects to the index.html if it is not set or if it is root
   logger.verbose({'msg':'calculating response', 'path': uri});
+  let gp = prepro.getProcessed;
   try {
     // get the file extension
-    let extension = getExtension(uri);
+    let extension = utils.getExtension(uri);
     // returns the global script or css if the extension is css or js and the root-uriis glob.
-    if (uri.includes("/glob") && (extension  == ".css" || extension == ".js")) {
+    if (uri.includes("/glob") && (extension  == ".sass" || extension == ".js")) {
       logger.verbose("Using global uri");
-      if (extension == ".css") return [fs.readFileSync("." + uri), "text/css"];
+      if (extension == ".sass") return [gp("." + uri), "text/css"];
-      else return [fs.readFileSync("." + uri), "text/javascript"];
+      else return [gp("." + uri), "text/javascript"];
     }
     let mount = getMount(uri); // get mount for uri it will be uses as path later instead of route
-    logger.verbose("Mount for uri is "+ mount)
+    logger.verbose("Mount for uri is "+ mount);
     let route = routes[extension]; // get the route from the extension json
-    logger.verbose("Found route: "+JSON.stringify(route))
+    logger.verbose("Found route: "+JSON.stringify(route));
     if (!route) return ["Not Allowed", "text/plain"]; // return not allowed if no route was found
-    let rf = fs.readFileSync; // shorten filesync
+    return [gp(mount || path.join(route["path"],uri)), route["mime"]]; // get processed output (done by preprocessor)
-    if (extension == ".html") return [formatHtml(rf(mount || path.join(route["path"]+uri))), route["mime"]]; // format if html and return
-    return [rf(mount || path.join(route["path"],uri)), route["mime"]]; // return without formatting if it's not an html file. (htm files won't be manipulated)
     // test the extension for differend file types.
-    logger.verbose({'msg': 'Error', 'path': uri})
+    logger.verbose({'msg': 'Error', 'path': uri});
     return ["Error with url", "text/plain"]; // return an error if above has not returned
   } catch (error) {
     logger.error(error);
@@ -179,55 +163,6 @@ function getMount(uri) {
   return false;
 }
 
-/**
- * Creates a css DOM element with href as source
- * @param  {Object} document A html DOM
- * @param  {String} href      the source of the css file
- * @return {Object}          the Link Element
- */
-function createLinkElement(document, href) {
-  let link = document.createElement('link');
-  link.setAttribute("rel", "stylesheet");
-  link.setAttribute("type", "text/css");
-  link.setAttribute("href", href);
-  return link;
-}
-
-/**
- * Creates a javascript Script DOM element with src as source
- * @param  {Object} document A html DOM
- * @param  {String} src      the source of the javascript file
- * @return {Object}          the Script Element
- */
-function createScriptLinkElement(document, src) {
-  let script = document.createElement("script");
-  script.setAttribute("type", "text/javascript");
-  script.setAttribute("src", src);
-  return script;
-}
-
-/**
- * Formats the html string by adding a link to the standard css and to the standard javascript file.
- * @param  {String} htmlstring A string read from an html file or a html document string itself.
- * @return {String}            A html-string that represents a document.
- */
-function formatHtml(htmlstring) {
-  logger.debug({'msg': 'Formatting HTML', 'htmlstring': htmlstring});
-  try {
-    let dom = new JSDOM(htmlstring);  // creates a dom from the html string
-    let document = dom.window.document;
-    let head = document.getElementsByTagName('head')[0]; // gets the documents head
-    head.prepend(createLinkElement(document, defaultCss)); // prepend the default css to the head
-    head.prepend(createScriptLinkElement(document, defaultJs)); // prepend the default script to the head
-    head.prepend(createScriptLinkElement(document, "/glob/jquery.js")); // prepend the JQuery to the head
-    head.prepend(createScriptLinkElement(document, "/glob/vue.js")); // prepend the Vue to the head
-    return dom.serialize(); // return a string of the document
-  } catch(error) {
-    logger.error(error);
-    return htmlstring;
-  }
-}
-
 // Executing the main function
 if (typeof require != 'undefined' && require.main == module) {
   logger.exceptions.handle(