diff --git a/package.json b/package.json index 6682909..825413b 100644 --- a/package.json +++ b/package.json @@ -22,9 +22,11 @@ "node-ts-cache": "^4.2.3", "node-ts-cache-storage-memory": "^4.2.3", "pdfreader": "^1.2.8", + "selenium-webdriver": "^4.0.0-beta.1", "uuid": "^8.3.2" }, "devDependencies": { + "@types/selenium-webdriver": "^4.0.11", "@types/express": "^4.17.11", "@types/needle": "^2.5.1", "@types/node": "^14.14.22", diff --git a/src/handlers/aromav2.ts b/src/handlers/aromav2.ts new file mode 100644 index 0000000..d8491b4 --- /dev/null +++ b/src/handlers/aromav2.ts @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2021 wilmaplus-foodmenu, developed by @developerfromjokela, for Wilma Plus mobile app + */ + +import {Request, Response} from "express"; +import {errorResponse, responseStatus} from "../utils/response_utilities"; +import {Builder, By, ThenableWebDriver} from "selenium-webdriver"; +import {Restaurant} from "../models/Restaurant"; +import {AsyncIterator} from "../utils/iterator"; +import {elementLocated} from "selenium-webdriver/lib/until"; +import {Http} from "../net/http"; +import {parse} from "../parsers/aromiv2"; +import {CacheContainer} from "node-ts-cache"; +import {MemoryStorage} from "node-ts-cache-storage-memory"; +import {HashUtils} from "../crypto/hash"; +import {Day} from "../models/Day"; +import {Diet} from "../models/Diet"; + +const urlRegex = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&/=]*)/; +let httpClient = new Http(); +let userCache = new CacheContainer(new MemoryStorage()); + + +function getRestaurantList(driver: ThenableWebDriver) { + return new Promise((resolve, reject) => { + driver.findElement(By.id("MainContent_RestaurantDropDownList")).then(element => { + let restaurants:Restaurant[] = []; + element.findElements(By.css("option")).then(options => { + new AsyncIterator((item, iterator) => { + item.getAttribute("textContent").then(name => { + if (name != null) { + item.getAttribute("value").then(id => { + if (id != null) { + restaurants.push(new Restaurant(id, name)); + iterator.nextItem(); + } else + iterator.nextItem(); + }).catch(() => { + iterator.nextItem(); + }); + } else + iterator.nextItem(); + }).catch(() => { + iterator.nextItem(); + }); + }, options.splice(1, options.length-1), () => { + resolve(restaurants); + }).start(); + }).catch(error => { + reject(error); + }); + }).catch(error => { + reject(error); + }); + }); + +} + +function selectRestaurant(driver: ThenableWebDriver, id: string) { + return new Promise((resolve, reject) => { + getRestaurantList(driver).then(restaurants => { + let position = -1; + restaurants.forEach((restaurant, index) => { + if (restaurant.id == id) { + position = index + } + }); + if (position === -1) { + reject(new Error("Restaurant not found, check the ID")); + return; + } + driver.findElement(By.id("MainContent_RestaurantDropDownList-button")).then(button => { + button.click().then(() => { + driver.findElement(By.id("MainContent_RestaurantDropDownList-menu")).then(dropDown => { + dropDown.findElements(By.css("li")).then(listItems => { + if (position > listItems.length-1) { + reject(new Error("Restaurant not found, check the ID")); + return; + } + let listItem = listItems[position]; + listItem.click().then(() => { + resolve(); + }).catch(error => { + reject(error); + }); + }).catch(error => { + reject(error); + }); + }).catch(error => { + reject(error); + }); + }).catch(error => { + reject(error); + }); + }).catch(error => { + reject(error); + }); + }).catch(error => { + reject(error); + }); + }); + +} + +function getRestaurantPDFLink(driver: ThenableWebDriver) { + return new Promise((resolve, reject) => { + driver.wait(elementLocated(By.id("MainContent_PdfUrl"))).then(() => { + driver.findElement(By.id("MainContent_PdfUrl")).then(pdfUrl => { + pdfUrl.getAttribute("href").then(url => { + resolve(url); + }).catch(error => { + reject(error); + }); + }).catch(error => { + reject(error); + }); + }).catch(error => { + reject(error); + }); + }); +} + +export function getMenuOptions(req: Request, res: Response) { + if (!req.params.url) { + responseStatus(res, 400, false, {cause: 'URL not specified!'}); + return; + } + let url = req.params.url; + if (!url.match(urlRegex)) { + responseStatus(res, 400, false, {cause: 'Invalid of malformed URL!'}); + return; + } + let hashKey = HashUtils.sha1Digest(url+"_aroma"); + userCache.getItem(hashKey).then(cachedValue => { + if (cachedValue) { + responseStatus(res, 200, true, {restaurants: cachedValue}); + } else { + const driver = new Builder().forBrowser("chrome").build(); + driver.get(url+"/Default.aspx").then(() => { + getRestaurantList(driver).then(restaurants => { + userCache.setItem(hashKey, restaurants, {ttl: 3600}).then(() => { + responseStatus(res, 200, true, {restaurants}); + driver.close(); + }).catch(error => { + responseStatus(res, 500, false, {cause: error.toString()}); + driver.close(); + }); + }).catch(error => { + responseStatus(res, 500, false, {cause: error.toString()}); + driver.close(); + }); + }).catch(error => { + responseStatus(res, 500, false, {cause: error.toString()}); + driver.close(); + }); + } + }).catch(error => { + responseStatus(res, 500, false, {cause: error.toString()}); + }); + +} + +export function getRestaurantPage(req: Request, res: Response) { + if (!req.params.url || !req.params.id) { + responseStatus(res, 400, false, {cause: 'Required parameters not specified!'}); + return; + } + let url = req.params.url; + let id = req.params.id; + if (!url.match(urlRegex)) { + responseStatus(res, 400, false, {cause: 'Invalid of malformed URL!'}); + return; + } + const fetchDocument = (pdfUrl: string) => { + const fetchDate = (date: string, callback: (restaurants: Day[], diets: Diet[]) => void, errorCallback: (error: Error | null) => void) => { + httpClient.get(pdfUrl.replace("%dmd%", date), (error, response) => { + if (error || response == undefined) { + errorCallback(error); + return; + } + parse(response.body, (restaurants, diets) => { + if (restaurants == undefined || diets == undefined) { + errorCallback(new Error("Unable to parse menu!")); + return; + } + callback(restaurants, diets); + }); + }); + }; + const contains = (item: string, items: Diet[]) => { + let found = false; + items.forEach(item2 => { + if (item.toLowerCase() == item2.name.toLowerCase()) + found = true; + }); + return found; + } + fetchDate("1", (restaurants, diets) => { + fetchDate("2", ((restaurants1, diets1) => { + fetchDate("3", ((restaurants2, diets2) => { + restaurants1.forEach(item => restaurants.push(item)); + restaurants2.forEach(item => restaurants.push(item)); + diets1.forEach(dItem => {if (!contains(dItem.name, diets)) {diets.push(dItem)}}); + diets2.forEach(dItem => {if (!contains(dItem.name, diets)) {diets.push(dItem)}}); + responseStatus(res, 200, true, {menu: restaurants, diets: diets}); + }), error => { + errorResponse(res, 500, error); + return; + }); + }), error => { + errorResponse(res, 500, error); + return; + }); + }, error => { + errorResponse(res, 500, error); + return; + }) + } + let hashKey = HashUtils.sha1Digest(url+"_"+id); + userCache.getItem(hashKey).then(value => { + // Check if cached value exists + if (value) + fetchDocument(value as string); + else { + const driver = new Builder().forBrowser("chrome").build(); + driver.get(url+"/Default.aspx").then(() => { + selectRestaurant(driver, id).then(() => { + getRestaurantPDFLink(driver).then(pdfUrl => { + driver.close(); + pdfUrl = pdfUrl.replace("DateMode=0", "DateMode=%dmd%"); + userCache.setItem(hashKey, pdfUrl, {ttl: 3600}).then(() => { + fetchDocument(pdfUrl); + }).catch(error => { + responseStatus(res, 500, false, {cause: error.toString()}); + }); + }).catch(error => { + responseStatus(res, 500, false, {cause: error.toString()}); + driver.close(); + }); + }).catch(error => { + responseStatus(res, 500, false, {cause: error.toString()}); + driver.close(); + }); + }).catch(error => { + responseStatus(res, 500, false, {cause: error.toString()}); + driver.close(); + }); + } + }) + +} \ No newline at end of file diff --git a/src/main.ts b/src/main.ts index 4dce988..b6471be 100644 --- a/src/main.ts +++ b/src/main.ts @@ -11,6 +11,7 @@ const syk = require('./handlers/syk').handleSyk; const steiner = require('./handlers/steiner').handleSteiner; const pyhtaa = require('./handlers/pyhtaa').handlePyhtaa; const kastelli = require('./handlers/kastelli').handleKastelli; +const aromaV2 = require('./handlers/aromav2'); const loviisa = require('./handlers/loviisa_pk'); @@ -25,6 +26,8 @@ app.use('/syk/menu', syk); app.use('/steiner/menu', steiner); app.use('/pyhtaa/menu', pyhtaa); app.use('/kastelli/menu', kastelli); +app.use('/aroma/:url/restaurants/:id', aromaV2.getRestaurantPage); +app.use('/aroma/:url/restaurants', aromaV2.getMenuOptions); app.use('/loviisa/paivakoti/menu', loviisa.handleLoviisaPk); diff --git a/src/models/Restaurant.ts b/src/models/Restaurant.ts new file mode 100644 index 0000000..68c4d22 --- /dev/null +++ b/src/models/Restaurant.ts @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2021 wilmaplus-foodmenu, developed by @developerfromjokela, for Wilma Plus mobile app + */ + +export class Restaurant { + id: string + name: string + + constructor(id: string, name: string) { + this.id = id; + this.name = name; + } +} diff --git a/src/parsers/aromiv2.ts b/src/parsers/aromiv2.ts new file mode 100644 index 0000000..26d2978 --- /dev/null +++ b/src/parsers/aromiv2.ts @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2021 wilmaplus-foodmenu, developed by @developerfromjokela, for Wilma Plus mobile app + */ + +import moment from 'moment'; +import {Day} from "../models/Day"; +import {Menu} from "../models/Menu"; +import {Moment} from "moment/moment"; +import {Meal} from "../models/Meal"; +import {HashUtils} from "../crypto/hash"; +import {Diet} from "../models/Diet"; + +const pdfParser = require("pdfreader"); + +const dateRegex = /[0-9]+\.[0-9]+\.[0-9]{4}/; + +const type = "aromiv2"; + + +export function parse(content: any, callback: (content: Day[]|undefined, diets: Diet[]|undefined) => void) { + let rows: any = {}; // indexed by y-position + let days: Day[] = []; + let diets: Diet[] = []; + new pdfParser.PdfReader().parseBuffer(content, (pdfError: Error, pdf: any) => { + if (pdfError) { + console.error(pdfError); + callback(undefined, undefined); + return; + } + if (!pdf || pdf.page) { + let items = Object.keys(rows).sort((y1, y2) => parseFloat(y1) - parseFloat(y2)); + let lastDate: Moment|null = null; + let tempMenuList:Menu[] = []; + items.forEach(key => { + let item = rows[key]; + if (item.length > 0) { + let firstEntry = item[0]; + if (firstEntry.text.match(dateRegex)) { + // Date found + let regexResult = dateRegex.exec(firstEntry.text); + if (regexResult != null) { + if (tempMenuList.length > 0 && lastDate != null) { + days.push(new Day(lastDate, tempMenuList)); + tempMenuList = []; + } + lastDate = moment(regexResult[0], "DD.MM.YYYY").startOf('day'); + } + } else if (lastDate != null) { + let mealType = "Lounas"; + let items: Meal[] = []; + for (let meal of item) { + if (meal.x < 3 && meal.x > 1) { + mealType = meal.text; + } else if (meal.x > 4) { + items.push(new Meal(HashUtils.sha1Digest(type+mealType+"_"+meal.text), meal.text)); + } + } + tempMenuList.push(new Menu(mealType, items)); + } + } + }); + if (tempMenuList.length > 0 && lastDate != null) { + days.push(new Day(lastDate, tempMenuList)); + tempMenuList = []; + } + try { + if (items.length > 0) { + let dietsText = rows[items[items.length-1]][0].text; + let dietSplit = dietsText.split(", ") + for (let dietItem of dietSplit) { + let parts = dietItem.split(" - "); + if (parts.length > 1) { + diets.push(new Diet(parts[0], parts[1])); + } + } + } + } catch (e) { + console.error(e); + } + if (!pdf) { + days.sort((i1: Day, i2:Day) => { + return i1.date.unix()-i2.date.unix(); + }); + // Formatting date after sorting + let correctedDateDays = days; + correctedDateDays.forEach((item, index) => { + item.date = (item.date.format() as any); + correctedDateDays[index] = item; + }); + callback(correctedDateDays, diets); + } + } else if (pdf.text) { + // accumulate text items into rows object, per line + (rows[pdf.y] = rows[pdf.y] || []).push({text: pdf.text, x: pdf.x}); + } + }); +} \ No newline at end of file diff --git a/src/parsers/loviisa_pk.ts b/src/parsers/loviisa_pk.ts index e3d5a98..3981add 100644 --- a/src/parsers/loviisa_pk.ts +++ b/src/parsers/loviisa_pk.ts @@ -11,7 +11,7 @@ import {Menu} from "../models/Menu"; import {errorResponse} from "../utils/response_utilities"; const pdfParser = require("pdfreader"); -const dateRegex = /[0-9]+.[0-9]+.[0-9]{4}/; +const dateRegex = /[0-9]+\.[0-9]+\.[0-9]{4}/; const whitespace = " "; const type = "loviisa_pk"; diff --git a/src/utils/iterator.ts b/src/utils/iterator.ts new file mode 100644 index 0000000..172c292 --- /dev/null +++ b/src/utils/iterator.ts @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2021 wilmaplus-notifier2, developed by @developerfromjokela, for Wilma Plus mobile app + */ + +export class AsyncIterator { + currentItem = -1 + items: T[] + callback:(item: T, iterator: AsyncIterator) => void; + endCallback:() => void; + + + constructor(callback:(item: T, iterator: AsyncIterator) => void, items:T[], endCallback: () => void) { + this.items = items; + this.callback = callback; + this.endCallback = endCallback; + } + + nextItem() { + if (this.currentItem+1 < this.items.length) { + this.currentItem++; + this.callback(this.items[this.currentItem], this); + } else { + this.endCallback(); + } + } + + start = this.nextItem; +}