♻️ Refactor backend and add comments

This commit is contained in:
2022-05-19 00:22:55 +02:00
parent 73dc67e6b4
commit 9fad079102
5 changed files with 86 additions and 25 deletions

View File

@ -1,6 +1,8 @@
import Prisma from "@prisma/client"; import Prisma from "@prisma/client";
const prisma = new Prisma.PrismaClient(); const prisma = new Prisma.PrismaClient();
// Get timetable API endpoint (/api/timetable)
// Returns timetable data for requested class if available
export async function getTimetable(req, res) { export async function getTimetable(req, res) {
if (!req.query.class) { if (!req.query.class) {
res.status(400).send({ res.status(400).send({
@ -30,9 +32,13 @@ export async function getTimetable(req, res) {
res.send(timetable.data); res.send(timetable.data);
} }
// Get substitutions API endpoint (/api/substitutions)
// Returns all known substitutions for requested date / class
// If no class is supplied, all substitutions are returned
export async function getSubstitutions(req, res) { export async function getSubstitutions(req, res) {
const requestedClass = (req.query.class || "").toLowerCase(); const requestedClass = (req.query.class || "").toLowerCase();
var from, to, date; var from, to, date;
// Check if from or to date is set in request
if (req.query.from && req.query.to) { if (req.query.from && req.query.to) {
from = new Date(req.query.from).setUTCHours(0, 0, 0, 0); from = new Date(req.query.from).setUTCHours(0, 0, 0, 0);
to = new Date(req.query.to).setUTCHours(0, 0, 0, 0); to = new Date(req.query.to).setUTCHours(0, 0, 0, 0);
@ -51,6 +57,7 @@ export async function getSubstitutions(req, res) {
if (requestedClass) { if (requestedClass) {
prismaOptions.where.class = { has: requestedClass }; prismaOptions.where.class = { has: requestedClass };
} }
// Choose which date to use in database query
if (from && to) { if (from && to) {
prismaOptions.where.date = { prismaOptions.where.date = {
gte: new Date(from), gte: new Date(from),
@ -59,6 +66,7 @@ export async function getSubstitutions(req, res) {
} else if (date) { } else if (date) {
prismaOptions.where.date = new Date(date); prismaOptions.where.date = new Date(date);
} else { } else {
// Default to all substitutions for today and in the future
prismaOptions.where.date = { prismaOptions.where.date = {
gte: new Date(new Date().setUTCHours(0, 0, 0, 0)), gte: new Date(new Date().setUTCHours(0, 0, 0, 0)),
}; };
@ -85,9 +93,13 @@ export async function getSubstitutions(req, res) {
res.send(substitutions); res.send(substitutions);
} }
// Get history API endpoint (/api/history)
// Returns history of changes for all substituions in the date range
// for the requested class if supplied
export async function getHistory(req, res) { export async function getHistory(req, res) {
const requestedClass = (req.query.class || "").toLowerCase(); const requestedClass = (req.query.class || "").toLowerCase();
var from, to, date; var from, to, date;
// Check if from or to date is set in request
if (req.query.from && req.query.to) { if (req.query.from && req.query.to) {
from = new Date(req.query.from).setUTCHours(0, 0, 0, 0); from = new Date(req.query.from).setUTCHours(0, 0, 0, 0);
to = new Date(req.query.to).setUTCHours(0, 0, 0, 0); to = new Date(req.query.to).setUTCHours(0, 0, 0, 0);
@ -109,6 +121,7 @@ export async function getHistory(req, res) {
if (requestedClass) { if (requestedClass) {
prismaOptions.where.substitution.class = { has: requestedClass }; prismaOptions.where.substitution.class = { has: requestedClass };
} }
// Choose which date to use in database query
if (from && to) { if (from && to) {
prismaOptions.where.substitution.date = { prismaOptions.where.substitution.date = {
gte: new Date(from), gte: new Date(from),
@ -117,6 +130,7 @@ export async function getHistory(req, res) {
} else if (date) { } else if (date) {
prismaOptions.where.substitution.date = new Date(date); prismaOptions.where.substitution.date = new Date(date);
} else { } else {
// Default to history of all substitutions for today and in the future
prismaOptions.where.substitution.date = { prismaOptions.where.substitution.date = {
gte: new Date(new Date().setUTCHours(0, 0, 0, 0)), gte: new Date(new Date().setUTCHours(0, 0, 0, 0)),
}; };
@ -139,6 +153,9 @@ export async function getHistory(req, res) {
res.send(changes); res.send(changes);
} }
// Get classes API endpoints (/api/classes)
// Get all available classes where timetable and
// substitutions can be requested for
export async function getClasses(req, res) { export async function getClasses(req, res) {
const classes = await prisma.class.findMany({ const classes = await prisma.class.findMany({
select: { select: {
@ -149,6 +166,7 @@ export async function getClasses(req, res) {
name: "asc", name: "asc",
}, },
}); });
// Only return the name of the class
const classList = classes.map((element) => element.name); const classList = classes.map((element) => element.name);
res.send(classList); res.send(classList);
} }

View File

@ -1,4 +1,8 @@
import express from "express"; import express from "express";
import cors from "cors";
import cookieParser from "cookie-parser";
// Import API endpoints
import { import {
getTimetable, getTimetable,
getSubstitutions, getSubstitutions,
@ -7,41 +11,51 @@ import {
} from "./api/index.js"; } from "./api/index.js";
import { Auth } from "./api/auth.js"; import { Auth } from "./api/auth.js";
import { Parser } from "./parser/index.js"; import { Parser } from "./parser/index.js";
import cors from "cors";
import cookieParser from "cookie-parser";
// Check DSB Mobile credentials are supplied
if (!process.env.DSB_USER || !process.env.DSB_PASSWORD) {
console.error("Error: DSB Auth environment variables missing!");
process.exit(1);
}
// Create the Webserver
const app = express(); const app = express();
const port = process.env.PORT || 3000; const port = process.env.PORT || 3000;
app.use(cors()); app.use(cors());
app.use(cookieParser()); app.use(cookieParser());
app.use(express.urlencoded({ extended: true })); app.use(express.urlencoded({ extended: true }));
const auth = new Auth(); // Initialize the Parser and set it to update the
// substitution plan at the specified update interval
if (!process.env.DSB_USER || !process.env.DSB_PASSWORD) { new Parser(
console.error("Error: DSB Auth environment variables missing!");
process.exit(1);
}
const parser = new Parser(
process.env.DSB_USER, process.env.DSB_USER,
process.env.DSB_PASSWORD, process.env.DSB_PASSWORD,
process.env.UPDATE_INTERVAL || 1 * 60 * 1000 process.env.UPDATE_INTERVAL || 1 * 60 * 1000 // Default to 1 minute
); );
// Create new Auth class to store sessions
const auth = new Auth();
app.post("/login", auth.login); app.post("/login", auth.login);
// Check login for every API request
app.use("/api", auth.checkLogin); app.use("/api", auth.checkLogin);
// Provide check endpoint so the frontend
// can check if the user is logged in
app.get("/api/check", (_req, res) => { app.get("/api/check", (_req, res) => {
res.sendStatus(200); res.sendStatus(200);
}); });
// Register API endpoints
app.get("/api/timetable", getTimetable); app.get("/api/timetable", getTimetable);
app.get("/api/substitutions", getSubstitutions); app.get("/api/substitutions", getSubstitutions);
app.get("/api/history", getHistory); app.get("/api/history", getHistory);
app.get("/api/classes", getClasses); app.get("/api/classes", getClasses);
// Respond with 400 for non-existent endpoints
app.get("/api/*", (_req, res) => { app.get("/api/*", (_req, res) => {
res.sendStatus(400); res.sendStatus(400);
}); });
// Supply frontend files if any other url
// is requested to make vue router work
app.use("/", express.static("../dist")); app.use("/", express.static("../dist"));
app.use("/*", express.static("../dist")); app.use("/*", express.static("../dist"));

View File

@ -2,8 +2,10 @@ import fs from "fs";
export function log(type, text) { export function log(type, text) {
if (!fs.existsSync("logs")) fs.mkdirSync("logs"); if (!fs.existsSync("logs")) fs.mkdirSync("logs");
const logName = new Date().toISOString().split("T")[0] + ".log"; const now = new Date().toISOString();
const timestamp = new Date().toISOString().replace("T", " ").split(".")[0];
const logName = now.split("T")[0] + ".log";
const timestamp = now.replace("T", " ").split(".")[0];
const logLine = `<${timestamp}> [${type}]: ${text}`; const logLine = `<${timestamp}> [${type}]: ${text}`;
fs.appendFileSync("logs/" + logName, logLine + "\n"); fs.appendFileSync("logs/" + logName, logLine + "\n");
console.log(logLine); console.log(logLine);

View File

@ -1,9 +1,9 @@
import axios from "axios"; import axios from "axios";
const baseurl = "https://mobileapi.dsbcontrol.de"; const baseUrl = "https://mobileapi.dsbcontrol.de";
export async function getAuthtoken(username, password) { export async function getAuthtoken(username, password) {
const response = await axios.get( const response = await axios.get(
`${baseurl}/authid?user=${username}&password=${password}&bundleid&appversion&osversion&pushid` `${baseUrl}/authid?user=${username}&password=${password}&bundleid&appversion&osversion&pushid`
); );
if (response.data == "") throw "Wrong username or password"; if (response.data == "") throw "Wrong username or password";
return response.data; return response.data;
@ -11,13 +11,15 @@ export async function getAuthtoken(username, password) {
export async function getTimetables(authtoken) { export async function getTimetables(authtoken) {
const response = await axios.get( const response = await axios.get(
`${baseurl}/dsbtimetables?authid=${authtoken}` `${baseUrl}/dsbtimetables?authid=${authtoken}`
); );
const timetables = response.data; const timetables = response.data;
const urls = []; const urls = [];
timetables.forEach((timetable) => { timetables.forEach((timetable) => {
const rawTimestamp = timetable.Date; const rawTimestamp = timetable.Date;
// Convert the timestamp to the correct
// format so new Date() accepts it
const date = rawTimestamp.split(" ")[0].split(".").reverse().join("-"); const date = rawTimestamp.split(" ")[0].split(".").reverse().join("-");
const time = rawTimestamp.split(" ")[1]; const time = rawTimestamp.split(" ")[1];
const timestamp = date + " " + time; const timestamp = date + " " + time;

View File

@ -3,11 +3,9 @@ import axios from "axios";
import { log, getLogPath } from "../logs.js"; import { log, getLogPath } from "../logs.js";
import { getAuthtoken, getTimetables } from "./dsbmobile.js"; import { getAuthtoken, getTimetables } from "./dsbmobile.js";
import { parseSubstitutionPlan } from "./untis.js"; import { parseSubstitutionPlan } from "./untis.js";
import fs from "fs";
const prisma = new Prisma.PrismaClient();
const dsbFiles = ["Schüler_Monitor - subst_001", "Schüler Morgen - subst_001"]; const dsbFiles = ["Schüler_Monitor - subst_001", "Schüler Morgen - subst_001"];
const prisma = new Prisma.PrismaClient();
export class Parser { export class Parser {
dsbUser; dsbUser;
@ -16,22 +14,26 @@ export class Parser {
this.dsbUser = dsbUser; this.dsbUser = dsbUser;
this.dsbPassword = dsbPassword; this.dsbPassword = dsbPassword;
// Schedule plan updates
setInterval(() => this.updatePlan(), interval); setInterval(() => this.updatePlan(), interval);
// Do the first update instantly
this.updatePlan(); this.updatePlan();
} }
async updatePlan() { async updatePlan() {
const startedAt = new Date(); const startedAt = new Date();
try { try {
const data = await this.fetchDSB(); const data = await this.fetchDSB();
if (!data) { if (!data) throw "DSB request failed!";
throw "DSB request failed!";
}
const plans = []; const plans = [];
for (const entry of data) { for (const entry of data) {
// Download the substitution plan
const data = await this.fetchFile(entry.url); const data = await this.fetchFile(entry.url);
// Parse the substitution plan
const parsed = parseSubstitutionPlan(data); const parsed = parseSubstitutionPlan(data);
plans.push(parsed); plans.push(parsed);
} }
// Create a new parse event
const parseEvent = await prisma.parseEvent.create({ const parseEvent = await prisma.parseEvent.create({
data: { data: {
logFile: getLogPath(), logFile: getLogPath(),
@ -40,10 +42,13 @@ export class Parser {
succeeded: true, succeeded: true,
}, },
}); });
// Insert substitutions of all substitution plans
for (const plan of plans) { for (const plan of plans) {
await this.insertSubstitutions(plan, parseEvent); await this.insertSubstitutions(plan, parseEvent);
} }
} catch (error) { } catch (error) {
// If something went wrong, create a failed
// parse event with the error message
await prisma.parseEvent.create({ await prisma.parseEvent.create({
data: { data: {
logFile: getLogPath(), logFile: getLogPath(),
@ -52,13 +57,16 @@ export class Parser {
succeeded: false, succeeded: false,
}, },
}); });
// Log the error
log("Parser / Main", "Parse event failed: " + error); log("Parser / Main", "Parse event failed: " + error);
} }
} }
async fetchDSB() { async fetchDSB() {
try { try {
const token = await getAuthtoken(this.dsbUser, this.dsbPassword); const token = await getAuthtoken(this.dsbUser, this.dsbPassword);
// Fetch available files
const response = await getTimetables(token); const response = await getTimetables(token);
// Filter files that should be parsed
const timetables = response.filter((e) => dsbFiles.includes(e.title)); const timetables = response.filter((e) => dsbFiles.includes(e.title));
return timetables; return timetables;
} catch (error) { } catch (error) {
@ -78,7 +86,7 @@ export class Parser {
return parseSubstitutionPlan(html); return parseSubstitutionPlan(html);
} }
async insertSubstitutions(parsedData, parseEvent) { async insertSubstitutions(parsedData, parseEvent) {
const { updatedAt, date, changes } = parsedData; const { date, changes } = parsedData;
const classList = await prisma.class.findMany(); const classList = await prisma.class.findMany();
const knownSubstitutions = await prisma.substitution.findMany({ const knownSubstitutions = await prisma.substitution.findMany({
@ -88,16 +96,24 @@ export class Parser {
}, },
}); });
// Loop through every change of the substitution plan
for (const change of changes) { for (const change of changes) {
// Find all classes the substitution belongs to
const classes = this.getSubstitutionClasses(classList, change.class); const classes = this.getSubstitutionClasses(classList, change.class);
// If the substitution does not belong to any classes known
// by the server, use the provied class string instead
if (classes.length == 0) classes.push(change.class || "unknown"); if (classes.length == 0) classes.push(change.class || "unknown");
// Workaround no currect match possible for subsitutions of this // Workaround: no correct match possible for subsitutions of this
// type beacuse they don't have a class and a subject attribute // type beacuse they do not have a class and a subject attribute
if (change.type == "Sondereins." && !change.subject) { if (change.type == "Sondereins." && !change.subject) {
change.subject = change.notes; change.subject = change.notes;
} }
// Check if a substitution exists in the database that
// it similar enough to the entry in the substitution
// plan to be considered the same substitution
// (Date, Type, Lesson, Classes and Subject need to be the same)
const matchingSubstitutionId = knownSubstitutions.findIndex( const matchingSubstitutionId = knownSubstitutions.findIndex(
(substitution) => { (substitution) => {
return substitution.date == new Date(date).setUTCHours(0, 0, 0, 0) && return substitution.date == new Date(date).setUTCHours(0, 0, 0, 0) &&
@ -113,6 +129,7 @@ export class Parser {
const matchingSubstitution = knownSubstitutions[matchingSubstitutionId]; const matchingSubstitution = knownSubstitutions[matchingSubstitutionId];
if (!matchingSubstitution) { if (!matchingSubstitution) {
// If the substitution is new, create it in the database
const newSubstitution = await prisma.substitution.create({ const newSubstitution = await prisma.substitution.create({
data: { data: {
class: classes, class: classes,
@ -126,6 +143,7 @@ export class Parser {
removed: false, removed: false,
}, },
}); });
// Also create a change entry for it
const substitutionChange = await prisma.substitutionChange.create({ const substitutionChange = await prisma.substitutionChange.create({
data: { data: {
substitutionId: newSubstitution.id, substitutionId: newSubstitution.id,
@ -150,8 +168,10 @@ export class Parser {
`Created new substitution: S:${newSubstitution.id} C:${substitutionChange.id}` `Created new substitution: S:${newSubstitution.id} C:${substitutionChange.id}`
); );
} else { } else {
// If the entry was updated, find the differences
const differences = this.findDifferences(matchingSubstitution, change); const differences = this.findDifferences(matchingSubstitution, change);
if (Object.keys(differences).length > 0) { if (Object.keys(differences).length > 0) {
// If differences were found, update the entry in the database
const prismaOptions = { const prismaOptions = {
where: { where: {
id: matchingSubstitution.id, id: matchingSubstitution.id,
@ -164,6 +184,7 @@ export class Parser {
if (differences.notes) prismaOptions.data.notes = change.notes; if (differences.notes) prismaOptions.data.notes = change.notes;
await prisma.substitution.update(prismaOptions); await prisma.substitution.update(prismaOptions);
// And create a change event for it
const substitutionChange = await prisma.substitutionChange.create({ const substitutionChange = await prisma.substitutionChange.create({
data: { data: {
substitutionId: matchingSubstitution.id, substitutionId: matchingSubstitution.id,
@ -182,9 +203,13 @@ export class Parser {
`Substitution unchanged: S:${matchingSubstitution.id}` `Substitution unchanged: S:${matchingSubstitution.id}`
); );
} }
// Remove the substitution from the array to later know the
// entries that are not present in the substitution plan
knownSubstitutions.splice(matchingSubstitutionId, 1); knownSubstitutions.splice(matchingSubstitutionId, 1);
} }
} }
// Mark all entries as removed that were
// not found in the substitution plan
for (const remainingSubstitution of knownSubstitutions) { for (const remainingSubstitution of knownSubstitutions) {
await prisma.substitution.update({ await prisma.substitution.update({
where: { where: {