From 7b94e4e3da213c09da80e0207ff48e3074aa9258 Mon Sep 17 00:00:00 2001 From: Trivernis Date: Sun, 29 Sep 2019 13:03:38 +0200 Subject: [PATCH] Added in-memory caching - added class for caching management - added cache parameter to sql query object - implemented caching for sql querys with cache = true - added loglevel to config - improved some methods of dataaccess classes to enable better caching --- src/default-config.yaml | 3 ++ src/lib/MemoryCache.ts | 73 +++++++++++++++++++++++++++++++ src/lib/QueryHelper.ts | 27 ++++++++++-- src/lib/dataaccess/Chatroom.ts | 14 +++++- src/lib/dataaccess/Profile.ts | 6 ++- src/lib/dataaccess/User.ts | 5 +++ src/lib/globals.ts | 7 ++- src/public/graphql/schema.graphql | 60 +++++++++++++++++++++++-- 8 files changed, 183 insertions(+), 12 deletions(-) create mode 100644 src/lib/MemoryCache.ts diff --git a/src/default-config.yaml b/src/default-config.yaml index e607f15..fda957a 100644 --- a/src/default-config.yaml +++ b/src/default-config.yaml @@ -17,3 +17,6 @@ session: markdown: plugins: - 'markdown-it-emoji' + +logging: + level: info diff --git a/src/lib/MemoryCache.ts b/src/lib/MemoryCache.ts new file mode 100644 index 0000000..e5b8a0b --- /dev/null +++ b/src/lib/MemoryCache.ts @@ -0,0 +1,73 @@ +import {EventEmitter} from "events"; +import * as crypto from "crypto"; + +export class MemoryCache extends EventEmitter { + private cacheItems: any = {}; + private cacheExpires: any = {}; + private expireCheck: NodeJS.Timeout; + + /** + * Creates interval function. + * @param ttl + */ + constructor(private ttl: number = 500) { + super(); + this.expireCheck = setInterval(() => this.checkExpires(), ttl / 2); + } + + /** + * Creates a md5 hash of the given key. + * @param key + */ + public hashKey(key: string): string { + const hash = crypto.createHash("md5"); + const data = hash.update(key, "utf8"); + return data.digest("hex"); + } + + /** + * Sets an entry. + * @param key + * @param value + */ + public set(key: string, value: any) { + this.cacheItems[key] = value; + this.cacheExpires[key] = Date.now() + this.ttl; + this.emit("set", key, value); + } + + /** + * Returns the entry stored with the given key. + * @param key + */ + public get(key: string) { + if (this.cacheItems.hasOwnProperty(key)) { + this.emit("hit", key, this.cacheItems[key]); + return this.cacheItems[key]; + } else { + this.emit("miss", key); + } + } + + /** + * Deletes a cache item. + * @param key + */ + public delete(key: string) { + this.emit("delete", key); + delete this.cacheItems[key]; + } + + /** + * Checks expires and clears items that are over the expire value. + */ + private checkExpires() { + for (const [key, value] of Object.entries(this.cacheExpires)) { + if (value < Date.now()) { + this.emit("delete", key); + delete this.cacheItems[key]; + delete this.cacheExpires[key]; + } + } + } +} diff --git a/src/lib/QueryHelper.ts b/src/lib/QueryHelper.ts index fb1b638..01c98b8 100644 --- a/src/lib/QueryHelper.ts +++ b/src/lib/QueryHelper.ts @@ -11,6 +11,10 @@ import globals from "./globals"; const logger = globals.logger; +export interface IAdvancedQueryConfig extends QueryConfig { + cache?: boolean; +} + /** * Transaction class to wrap SQL transactions. */ @@ -101,7 +105,7 @@ export class QueryHelper { * executes the sql query with values and returns all results. * @param query */ - public async all(query: QueryConfig): Promise { + public async all(query: IAdvancedQueryConfig): Promise { const result = await this.query(query); return result.rows; } @@ -110,7 +114,7 @@ export class QueryHelper { * executes the sql query with values and returns the first result. * @param query */ - public async first(query: QueryConfig): Promise { + public async first(query: IAdvancedQueryConfig): Promise { const result = await this.query(query); if (result.rows && result.rows.length > 0) { return result.rows[0]; @@ -129,9 +133,24 @@ export class QueryHelper { * Queries the database with error handling. * @param query - the sql and values to execute */ - private async query(query: QueryConfig): Promise { + private async query(query: IAdvancedQueryConfig): Promise { try { - return await this.pool.query(query); + query.text = query.text.replace(/[\r\n]/g, " "); + globals.logger.silly(`Executing sql '${JSON.stringify(query)}'`); + + if (query.cache) { + const key = globals.cache.hashKey(JSON.stringify(query)); + const cacheResult = globals.cache.get(key); + if (cacheResult) { + return cacheResult; + } else { + const result = await this.pool.query(query); + globals.cache.set(key, result); + return result; + } + } else { + return await this.pool.query(query); + } } catch (err) { logger.debug(`Error on query "${JSON.stringify(query)}".`); logger.error(`Sql query failed: ${err}`); diff --git a/src/lib/dataaccess/Chatroom.ts b/src/lib/dataaccess/Chatroom.ts index 6b839c6..0787637 100644 --- a/src/lib/dataaccess/Chatroom.ts +++ b/src/lib/dataaccess/Chatroom.ts @@ -1,3 +1,4 @@ +import globals from "../globals"; import {ChatMessage} from "./ChatMessage"; import {queryHelper} from "./index"; import {User} from "./User"; @@ -24,6 +25,7 @@ export class Chatroom { */ public async members(): Promise { const result = await queryHelper.all({ + cache: true, text: `SELECT * FROM chat_members JOIN users ON (chat_members.member = users.id) WHERE chat_members.chat = $1;`, @@ -31,7 +33,8 @@ export class Chatroom { }); const chatMembers = []; for (const row of result) { - chatMembers.push(new User(row.id, row)); + const user = new User(row.id, row); + chatMembers.push(user); } return chatMembers; } @@ -47,13 +50,20 @@ export class Chatroom { const offs = offset || 0; const result = await queryHelper.all({ + cache: true, text: "SELECT * FROM chat_messages WHERE chat = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3", values: [this.id, lim, offs], }); const messages = []; + const users: any = {}; for (const row of result) { - messages.push(new ChatMessage(new User(row.author), this, row.created_at, row.content)); + if (!users[row.author]) { + const user = new User(row.author); + await user.exists(); + users[row.author] = user; + } + messages.push(new ChatMessage(users[row.author], this, row.created_at, row.content)); } if (containing) { return messages.filter((x) => x.content.includes(containing)); diff --git a/src/lib/dataaccess/Profile.ts b/src/lib/dataaccess/Profile.ts index 33eb7eb..957d631 100644 --- a/src/lib/dataaccess/Profile.ts +++ b/src/lib/dataaccess/Profile.ts @@ -6,10 +6,14 @@ export class Profile extends User { /** * Returns all chatrooms (with pagination). + * Skips the query if the user doesn't exist. * @param first * @param offset */ - public async chats({first, offset}: {first: number, offset?: number}) { + public async chats({first, offset}: {first: number, offset?: number}): Promise { + if (!(await this.exists())) { + return []; + } first = first || 10; offset = offset || 0; diff --git a/src/lib/dataaccess/User.ts b/src/lib/dataaccess/User.ts index 6358e59..b3501cc 100644 --- a/src/lib/dataaccess/User.ts +++ b/src/lib/dataaccess/User.ts @@ -1,3 +1,4 @@ +import globals from "../globals"; import {DataObject} from "./DataObject"; import {queryHelper} from "./index"; import {Post} from "./Post"; @@ -47,6 +48,7 @@ export class User extends DataObject { */ public async numberOfPosts(): Promise { const result = await queryHelper.first({ + cache: true, text: "SELECT COUNT(*) count FROM posts WHERE author = $1", values: [this.id], }); @@ -66,6 +68,7 @@ export class User extends DataObject { */ public async friends(): Promise { const result = await queryHelper.all({ + cache: true, text: "SELECT * FROM user_friends WHERE user_id = $1 OR friend_id = $1", values: [this.id], }); @@ -87,6 +90,7 @@ export class User extends DataObject { first = first || 10; offset = offset || 0; const result = await queryHelper.all({ + cache: true, text: "SELECT * FROM posts WHERE author = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3", values: [this.id, first, offset], }); @@ -107,6 +111,7 @@ export class User extends DataObject { result = this.row; } else { result = await queryHelper.first({ + cache: true, text: "SELECT * FROM users WHERE users.id = $1", values: [this.id], }); diff --git a/src/lib/globals.ts b/src/lib/globals.ts index 3457fa5..ea0b8d8 100644 --- a/src/lib/globals.ts +++ b/src/lib/globals.ts @@ -8,6 +8,7 @@ import * as fsx from "fs-extra"; import * as yaml from "js-yaml"; import * as winston from "winston"; +import {MemoryCache} from "./Cache"; const configPath = "config.yaml"; const defaultConfig = __dirname + "/../default-config.yaml"; @@ -26,6 +27,7 @@ if (!(fsx.pathExistsSync(configPath))) { */ namespace globals { export const config = yaml.safeLoad(fsx.readFileSync("config.yaml", "utf-8")); + export const cache = new MemoryCache(1200); export const logger = winston.createLogger({ transports: [ new winston.transports.Console({ @@ -36,10 +38,13 @@ namespace globals { return `${timestamp} ${level}: ${message}`; }), ), - level: "debug", + level: config.logging.level, }), ], }); + cache.on("set", (key) => logger.debug(`Caching '${key}'.`)); + cache.on("miss", (key) => logger.debug(`Cache miss for '${key}'`)); + cache.on("hit", (key) => logger.debug(`Cache hit for '${key}'`)); } export default globals; diff --git a/src/public/graphql/schema.graphql b/src/public/graphql/schema.graphql index fb23b6f..e82d4ec 100644 --- a/src/public/graphql/schema.graphql +++ b/src/public/graphql/schema.graphql @@ -3,7 +3,7 @@ type Query { getUser(userId: ID, handle: String): User "returns the logged in user" - getSelf: User + getSelf: Profile "returns the post object for a post id" getPost(postId: ID!): Post @@ -26,10 +26,10 @@ type Mutation { acceptCookies: Boolean "Login of the user. The passwordHash should be a sha512 hash of the password." - login(email: String, passwordHash: String): User + login(email: String, passwordHash: String): Profile "Registers the user." - register(username: String, email: String, passwordHash: String): User + register(username: String, email: String, passwordHash: String): Profile "Logout of the user." logout: Boolean @@ -62,8 +62,60 @@ type Mutation { createChat(members: [ID!]): ChatRoom } +interface UserData { + "url for the Profile picture of the User" + profilePicture: String + + "name of the User" + name: String! + + "unique identifier name from the User" + handle: String! + + "Id of the User" + id: ID! + + "the total number of posts the user posted" + numberOfPosts: Int + + "returns a given number of posts of a user" + posts(first: Int=10, offset: Int): [Post] + + "creation date of the user account" + joinedAt: String! + + "all friends of the user" + friends: [User] +} + "represents a single user account" -type User { +type User implements UserData{ + "url for the Profile picture of the User" + profilePicture: String + + "name of the User" + name: String! + + "unique identifier name from the User" + handle: String! + + "Id of the User" + id: ID! + + "the total number of posts the user posted" + numberOfPosts: Int + + "returns a given number of posts of a user" + posts(first: Int=10, offset: Int): [Post] + + "creation date of the user account" + joinedAt: String! + + "all friends of the user" + friends: [User] +} + +type Profile implements UserData { "url for the Profile picture of the User" profilePicture: String