From 9fad07910280ecade1c6fb5667d7ef3e6f62156e Mon Sep 17 00:00:00 2001 From: minie4 Date: Thu, 19 May 2022 00:22:55 +0200 Subject: [PATCH] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Refactor=20backend=20and?= =?UTF-8?q?=20add=20comments?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- server/api/index.js | 18 ++++++++++++++++ server/index.js | 36 +++++++++++++++++++++---------- server/logs.js | 6 ++++-- server/parser/dsbmobile.js | 8 ++++--- server/parser/index.js | 43 ++++++++++++++++++++++++++++++-------- 5 files changed, 86 insertions(+), 25 deletions(-) diff --git a/server/api/index.js b/server/api/index.js index 2b3a6b3..d736e76 100644 --- a/server/api/index.js +++ b/server/api/index.js @@ -1,6 +1,8 @@ import Prisma from "@prisma/client"; const prisma = new Prisma.PrismaClient(); +// Get timetable API endpoint (/api/timetable) +// Returns timetable data for requested class if available export async function getTimetable(req, res) { if (!req.query.class) { res.status(400).send({ @@ -30,9 +32,13 @@ export async function getTimetable(req, res) { res.send(timetable.data); } +// Get substitutions API endpoint (/api/substitutions) +// Returns all known substitutions for requested date / class +// If no class is supplied, all substitutions are returned export async function getSubstitutions(req, res) { const requestedClass = (req.query.class || "").toLowerCase(); var from, to, date; + // Check if from or to date is set in request if (req.query.from && req.query.to) { from = new Date(req.query.from).setUTCHours(0, 0, 0, 0); to = new Date(req.query.to).setUTCHours(0, 0, 0, 0); @@ -51,6 +57,7 @@ export async function getSubstitutions(req, res) { if (requestedClass) { prismaOptions.where.class = { has: requestedClass }; } + // Choose which date to use in database query if (from && to) { prismaOptions.where.date = { gte: new Date(from), @@ -59,6 +66,7 @@ export async function getSubstitutions(req, res) { } else if (date) { prismaOptions.where.date = new Date(date); } else { + // Default to all substitutions for today and in the future prismaOptions.where.date = { gte: new Date(new Date().setUTCHours(0, 0, 0, 0)), }; @@ -85,9 +93,13 @@ export async function getSubstitutions(req, res) { res.send(substitutions); } +// Get history API endpoint (/api/history) +// Returns history of changes for all substituions in the date range +// for the requested class if supplied export async function getHistory(req, res) { const requestedClass = (req.query.class || "").toLowerCase(); var from, to, date; + // Check if from or to date is set in request if (req.query.from && req.query.to) { from = new Date(req.query.from).setUTCHours(0, 0, 0, 0); to = new Date(req.query.to).setUTCHours(0, 0, 0, 0); @@ -109,6 +121,7 @@ export async function getHistory(req, res) { if (requestedClass) { prismaOptions.where.substitution.class = { has: requestedClass }; } + // Choose which date to use in database query if (from && to) { prismaOptions.where.substitution.date = { gte: new Date(from), @@ -117,6 +130,7 @@ export async function getHistory(req, res) { } else if (date) { prismaOptions.where.substitution.date = new Date(date); } else { + // Default to history of all substitutions for today and in the future prismaOptions.where.substitution.date = { gte: new Date(new Date().setUTCHours(0, 0, 0, 0)), }; @@ -139,6 +153,9 @@ export async function getHistory(req, res) { res.send(changes); } +// Get classes API endpoints (/api/classes) +// Get all available classes where timetable and +// substitutions can be requested for export async function getClasses(req, res) { const classes = await prisma.class.findMany({ select: { @@ -149,6 +166,7 @@ export async function getClasses(req, res) { name: "asc", }, }); + // Only return the name of the class const classList = classes.map((element) => element.name); res.send(classList); } diff --git a/server/index.js b/server/index.js index 473c3a6..6f54d8a 100644 --- a/server/index.js +++ b/server/index.js @@ -1,4 +1,8 @@ import express from "express"; +import cors from "cors"; +import cookieParser from "cookie-parser"; + +// Import API endpoints import { getTimetable, getSubstitutions, @@ -7,41 +11,51 @@ import { } from "./api/index.js"; import { Auth } from "./api/auth.js"; import { Parser } from "./parser/index.js"; -import cors from "cors"; -import cookieParser from "cookie-parser"; +// Check DSB Mobile credentials are supplied +if (!process.env.DSB_USER || !process.env.DSB_PASSWORD) { + console.error("Error: DSB Auth environment variables missing!"); + process.exit(1); +} + +// Create the Webserver const app = express(); const port = process.env.PORT || 3000; app.use(cors()); app.use(cookieParser()); app.use(express.urlencoded({ extended: true })); -const auth = new Auth(); - -if (!process.env.DSB_USER || !process.env.DSB_PASSWORD) { - console.error("Error: DSB Auth environment variables missing!"); - process.exit(1); -} -const parser = new Parser( +// Initialize the Parser and set it to update the +// substitution plan at the specified update interval +new Parser( process.env.DSB_USER, process.env.DSB_PASSWORD, - process.env.UPDATE_INTERVAL || 1 * 60 * 1000 + process.env.UPDATE_INTERVAL || 1 * 60 * 1000 // Default to 1 minute ); +// Create new Auth class to store sessions +const auth = new Auth(); app.post("/login", auth.login); - +// Check login for every API request app.use("/api", auth.checkLogin); +// Provide check endpoint so the frontend +// can check if the user is logged in app.get("/api/check", (_req, res) => { res.sendStatus(200); }); + +// Register API endpoints app.get("/api/timetable", getTimetable); app.get("/api/substitutions", getSubstitutions); app.get("/api/history", getHistory); app.get("/api/classes", getClasses); +// Respond with 400 for non-existent endpoints app.get("/api/*", (_req, res) => { res.sendStatus(400); }); +// Supply frontend files if any other url +// is requested to make vue router work app.use("/", express.static("../dist")); app.use("/*", express.static("../dist")); diff --git a/server/logs.js b/server/logs.js index b610169..58d735a 100644 --- a/server/logs.js +++ b/server/logs.js @@ -2,8 +2,10 @@ import fs from "fs"; export function log(type, text) { if (!fs.existsSync("logs")) fs.mkdirSync("logs"); - const logName = new Date().toISOString().split("T")[0] + ".log"; - const timestamp = new Date().toISOString().replace("T", " ").split(".")[0]; + const now = new Date().toISOString(); + + const logName = now.split("T")[0] + ".log"; + const timestamp = now.replace("T", " ").split(".")[0]; const logLine = `<${timestamp}> [${type}]: ${text}`; fs.appendFileSync("logs/" + logName, logLine + "\n"); console.log(logLine); diff --git a/server/parser/dsbmobile.js b/server/parser/dsbmobile.js index 1e75476..e428adb 100644 --- a/server/parser/dsbmobile.js +++ b/server/parser/dsbmobile.js @@ -1,9 +1,9 @@ import axios from "axios"; -const baseurl = "https://mobileapi.dsbcontrol.de"; +const baseUrl = "https://mobileapi.dsbcontrol.de"; export async function getAuthtoken(username, password) { const response = await axios.get( - `${baseurl}/authid?user=${username}&password=${password}&bundleid&appversion&osversion&pushid` + `${baseUrl}/authid?user=${username}&password=${password}&bundleid&appversion&osversion&pushid` ); if (response.data == "") throw "Wrong username or password"; return response.data; @@ -11,13 +11,15 @@ export async function getAuthtoken(username, password) { export async function getTimetables(authtoken) { const response = await axios.get( - `${baseurl}/dsbtimetables?authid=${authtoken}` + `${baseUrl}/dsbtimetables?authid=${authtoken}` ); const timetables = response.data; const urls = []; timetables.forEach((timetable) => { const rawTimestamp = timetable.Date; + // Convert the timestamp to the correct + // format so new Date() accepts it const date = rawTimestamp.split(" ")[0].split(".").reverse().join("-"); const time = rawTimestamp.split(" ")[1]; const timestamp = date + " " + time; diff --git a/server/parser/index.js b/server/parser/index.js index de715d9..aa63b50 100644 --- a/server/parser/index.js +++ b/server/parser/index.js @@ -3,11 +3,9 @@ import axios from "axios"; import { log, getLogPath } from "../logs.js"; import { getAuthtoken, getTimetables } from "./dsbmobile.js"; import { parseSubstitutionPlan } from "./untis.js"; -import fs from "fs"; - -const prisma = new Prisma.PrismaClient(); const dsbFiles = ["Schüler_Monitor - subst_001", "Schüler Morgen - subst_001"]; +const prisma = new Prisma.PrismaClient(); export class Parser { dsbUser; @@ -16,22 +14,26 @@ export class Parser { this.dsbUser = dsbUser; this.dsbPassword = dsbPassword; + // Schedule plan updates setInterval(() => this.updatePlan(), interval); + // Do the first update instantly this.updatePlan(); } async updatePlan() { const startedAt = new Date(); try { const data = await this.fetchDSB(); - if (!data) { - throw "DSB request failed!"; - } + if (!data) throw "DSB request failed!"; + const plans = []; for (const entry of data) { + // Download the substitution plan const data = await this.fetchFile(entry.url); + // Parse the substitution plan const parsed = parseSubstitutionPlan(data); plans.push(parsed); } + // Create a new parse event const parseEvent = await prisma.parseEvent.create({ data: { logFile: getLogPath(), @@ -40,10 +42,13 @@ export class Parser { succeeded: true, }, }); + // Insert substitutions of all substitution plans for (const plan of plans) { await this.insertSubstitutions(plan, parseEvent); } } catch (error) { + // If something went wrong, create a failed + // parse event with the error message await prisma.parseEvent.create({ data: { logFile: getLogPath(), @@ -52,13 +57,16 @@ export class Parser { succeeded: false, }, }); + // Log the error log("Parser / Main", "Parse event failed: " + error); } } async fetchDSB() { try { const token = await getAuthtoken(this.dsbUser, this.dsbPassword); + // Fetch available files const response = await getTimetables(token); + // Filter files that should be parsed const timetables = response.filter((e) => dsbFiles.includes(e.title)); return timetables; } catch (error) { @@ -78,7 +86,7 @@ export class Parser { return parseSubstitutionPlan(html); } async insertSubstitutions(parsedData, parseEvent) { - const { updatedAt, date, changes } = parsedData; + const { date, changes } = parsedData; const classList = await prisma.class.findMany(); const knownSubstitutions = await prisma.substitution.findMany({ @@ -88,16 +96,24 @@ export class Parser { }, }); + // Loop through every change of the substitution plan for (const change of changes) { + // Find all classes the substitution belongs to const classes = this.getSubstitutionClasses(classList, change.class); + // If the substitution does not belong to any classes known + // by the server, use the provied class string instead if (classes.length == 0) classes.push(change.class || "unknown"); - // Workaround no currect match possible for subsitutions of this - // type beacuse they don't have a class and a subject attribute + // Workaround: no correct match possible for subsitutions of this + // type beacuse they do not have a class and a subject attribute if (change.type == "Sondereins." && !change.subject) { change.subject = change.notes; } + // Check if a substitution exists in the database that + // it similar enough to the entry in the substitution + // plan to be considered the same substitution + // (Date, Type, Lesson, Classes and Subject need to be the same) const matchingSubstitutionId = knownSubstitutions.findIndex( (substitution) => { return substitution.date == new Date(date).setUTCHours(0, 0, 0, 0) && @@ -113,6 +129,7 @@ export class Parser { const matchingSubstitution = knownSubstitutions[matchingSubstitutionId]; if (!matchingSubstitution) { + // If the substitution is new, create it in the database const newSubstitution = await prisma.substitution.create({ data: { class: classes, @@ -126,6 +143,7 @@ export class Parser { removed: false, }, }); + // Also create a change entry for it const substitutionChange = await prisma.substitutionChange.create({ data: { substitutionId: newSubstitution.id, @@ -150,8 +168,10 @@ export class Parser { `Created new substitution: S:${newSubstitution.id} C:${substitutionChange.id}` ); } else { + // If the entry was updated, find the differences const differences = this.findDifferences(matchingSubstitution, change); if (Object.keys(differences).length > 0) { + // If differences were found, update the entry in the database const prismaOptions = { where: { id: matchingSubstitution.id, @@ -164,6 +184,7 @@ export class Parser { if (differences.notes) prismaOptions.data.notes = change.notes; await prisma.substitution.update(prismaOptions); + // And create a change event for it const substitutionChange = await prisma.substitutionChange.create({ data: { substitutionId: matchingSubstitution.id, @@ -182,9 +203,13 @@ export class Parser { `Substitution unchanged: S:${matchingSubstitution.id}` ); } + // Remove the substitution from the array to later know the + // entries that are not present in the substitution plan knownSubstitutions.splice(matchingSubstitutionId, 1); } } + // Mark all entries as removed that were + // not found in the substitution plan for (const remainingSubstitution of knownSubstitutions) { await prisma.substitution.update({ where: {