From 6bf07c52ec56377c2a29512086e4dc7bf6a563f6 Mon Sep 17 00:00:00 2001 From: makefu Date: Fri, 23 Mar 2018 16:33:35 +0100 Subject: init --- ara2influx.py | 61 ++++++++++++++++++++++++++++++++++ day.py | 104 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ day.sh | 11 +++++++ week.sh | 10 ++++++ 4 files changed, 186 insertions(+) create mode 100644 ara2influx.py create mode 100755 day.py create mode 100644 day.sh create mode 100644 week.sh diff --git a/ara2influx.py b/ara2influx.py new file mode 100644 index 0000000..26d7d0a --- /dev/null +++ b/ara2influx.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +""" usage: fetchday [options] DB + +Options: + --db FILE path to db [Default: db.json] + --cantine CANTINE name of the cantine [Default: thales-deutschland] + --host HOST hostname or ip of influx [Default: omo] +""" +import sys +from docopt import docopt +import logging +from influxdb import InfluxDBClient + + +import json +from datetime import datetime +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger() + + +def db2points(db,cantine): + for day,things in db.items(): + time = datetime.strptime(day,"%Y-%m-%d") + time = time.replace(hour=12) + for thing in things: + ret = { + "time": time.isoformat(), + "measurement":"dish", + "tags": { + "allergies": ",".join( sorted(thing['allergies'].keys()) ), + "additives": ",".join( sorted(thing['additives'].keys()) ), + "title": thing["title"], + "type": thing["type"], + "description": thing["description"], + "cantine": cantine + }, + "fields": { + "price": thing["price"] + } + } + log.debug(ret) + yield ret + +def main(): + args = docopt(__doc__) + dbname = args['DB'] + host = args['--host'] + cantine = args['--cantine'] + client = InfluxDBClient( + host=host, + port=8086, + database='araprice') + with open(dbname,'r+') as f: + db = json.load(f) + l = list(db2points(db,cantine)) + # client.drop_database("araprice") + client.create_database("araprice") + client.write_points(l) + +if __name__ == '__main__': + main() diff --git a/day.py b/day.py new file mode 100755 index 0000000..4364762 --- /dev/null +++ b/day.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +""" usage: fetchday [options] + +Options: + --db FILE path to db [Default: db.json] + --cantine NAME name of the cantine (matches mein.aramark.de/NAME/) [Default: thales-deutschland] +""" +import sys +from docopt import docopt +import requests +from bs4 import BeautifulSoup +import logging + +import json +from datetime import datetime +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger() +from itertools import tee +cantinedb = { + 'thales-deutschland': 'aramark_menu2-2', + 'buerocampus-wangen': 'aramark_menu2-3', + 'ibm-boeblingen': 'aramark_menu2-3', + 'metabo-nuertingen': 'aramark_menu2-3', + 'vodafone-campus': 'aramark_menu2-2', + 'continental-villingen': 'aramark_menu2-3' + } +def get_data(cantine): + headers = { + 'User-Agent': 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:58.0) Gecko/20100101 Firefox/58.0', + 'Referer': f'http://mein.aramark.de/{cantine}/menuplan/', + 'X-Requested-With': 'XMLHttpRequest', + 'Cache-Control': 'max-age=0' + } + data = { + 'action': 'aramark_menu2_content', + 'id': cantinedb[cantine] + } + ret = requests.post(f'http://mein.aramark.de/{cantine}/wp-admin/admin-ajax.php',data=data) + if ret.status_code != 200: + log.error(ret) + raise ValueError('Return code is not 200') + return ret.content.decode() + +def get_data_test(): + return open("./test.html").read() + +def get_things(row): + for item in row.select('.aramark_menu2-counter'): + try: + thing = { + "allergies": {}, + "additives": {}, + "type" : item.select_one('.aramark_menu2-countertitletop').find(text=True), + "description" : item.select_one('.aramark_menu2-menutext > p').find(text=True), + + } + try: + price = float(item.select_one('.aramark_menu2-menuprice > .aramark_menu2-menuprice').find(text=True).replace(',','.').replace('EUR','').strip()) + except: + price = float(thing['description']) + thing['price'] = price + try: + thing['title'] = item.select_one('.aramark_menu2-countermenu h4').find(text=True) + except: + thing['title'] = thing['type'] + + entries = item.select('.aramark_menu2-menuinfoallergenes li span') + for i in range(0,len(entries),2): + thing['allergies'] = { entries[i].find(text=True): entries[i+1].find(text=True) } + + additives = { } + entries = item.select('.aramark_menu2-menuinfoadditives li span') + for i in range(0,len(entries),2): + thing['additives'] = { entries[i].find(text=True): entries[i+1].find(text=True) } + yield thing + except Exception as e: + log.error(f"problem with {e}") + log.debug(item.prettify()) +def main(): + args = docopt(__doc__) + dbname = args['--db'] + cantine = args['--cantine'] + try: + with open(dbname,'r+') as f: + db = json.load(f) + except: + log.info('creating new db') + db = {} + + updated = datetime.now().isoformat().split('T')[0] + soup = BeautifulSoup(get_data(cantine), 'html.parser') + # soup = BeautifulSoup(get_data_test(), 'html.parser') + for row in soup.select('.aramark_menu2-menus > div'): + day = row.get('data-aramark_menu2-day') + things = list(get_things(row)) + db[day] = things + + log.info(f'writing db {dbname}') + with sys.stdout if dbname == '-' else open(dbname,'w+') as f: + json.dump(db,f,indent=2,sort_keys=True) + + +if __name__ == '__main__': + main() diff --git a/day.sh b/day.sh new file mode 100644 index 0000000..6780dd9 --- /dev/null +++ b/day.sh @@ -0,0 +1,11 @@ +#!/bin/sh +OUTDIR=${OUTDIR:-$PWD/db} +# old: thyssenkruppsystemengineering +# empty: metabo-nuertingen +mkdir -p $OUTDIR +for i in thales-deutschland ibm-boeblingen buerocampus-wangen continental-villingen vodafone-campus ;do + db=$OUTDIR/$i.json + python3 day.py --db $db --cantine $i + python3 ara2influx.py $db --cantine $i + # git add $db +done diff --git a/week.sh b/week.sh new file mode 100644 index 0000000..f972e16 --- /dev/null +++ b/week.sh @@ -0,0 +1,10 @@ +NOW=$(date -I) +OUTDIR=${OUTDIR:-$PWD/pdf} +mkdir -p $OUTDIR +# Broken: +# vodafone-campus thyssenkruppsystemengineering +# Gone? metabo-nuertingen +for i in thales-deutschland ibm-boeblingen buerocampus-wangen continental-villingen ;do + db=$OUTDIR/$i-week.pdf + wget http://gast.aramark.de/$i/menu/pdf/woche_de.php -O $db +done -- cgit v1.2.3