From 21899089a86fb9ca2bfac42622ac0c3777059c0b Mon Sep 17 00:00:00 2001 From: makefu Date: Wed, 28 Mar 2018 09:39:20 +0200 Subject: init default.py, setup.py --- ara2influx.py | 61 ----------------------------- arafetch/ara2influx.py | 61 +++++++++++++++++++++++++++++ arafetch/arafetch.py | 104 +++++++++++++++++++++++++++++++++++++++++++++++++ day.py | 104 ------------------------------------------------- day.sh | 11 ------ default.nix | 18 +++++++++ scripts/weekrun | 21 ++++++++++ setup.py | 28 +++++++++++++ week.sh | 10 ----- 9 files changed, 232 insertions(+), 186 deletions(-) delete mode 100644 ara2influx.py create mode 100644 arafetch/ara2influx.py create mode 100755 arafetch/arafetch.py delete mode 100755 day.py delete mode 100644 day.sh create mode 100644 default.nix create mode 100644 scripts/weekrun create mode 100644 setup.py delete mode 100644 week.sh diff --git a/ara2influx.py b/ara2influx.py deleted file mode 100644 index 26d7d0a..0000000 --- a/ara2influx.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -""" usage: fetchday [options] DB - -Options: - --db FILE path to db [Default: db.json] - --cantine CANTINE name of the cantine [Default: thales-deutschland] - --host HOST hostname or ip of influx [Default: omo] -""" -import sys -from docopt import docopt -import logging -from influxdb import InfluxDBClient - - -import json -from datetime import datetime -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger() - - -def db2points(db,cantine): - for day,things in db.items(): - time = datetime.strptime(day,"%Y-%m-%d") - time = time.replace(hour=12) - for thing in things: - ret = { - "time": time.isoformat(), - "measurement":"dish", - "tags": { - "allergies": ",".join( sorted(thing['allergies'].keys()) ), - "additives": ",".join( sorted(thing['additives'].keys()) ), - "title": thing["title"], - "type": thing["type"], - "description": thing["description"], - "cantine": cantine - }, - "fields": { - "price": thing["price"] - } - } - log.debug(ret) - yield ret - -def main(): - args = docopt(__doc__) - dbname = args['DB'] - host = args['--host'] - cantine = args['--cantine'] - client = InfluxDBClient( - host=host, - port=8086, - database='araprice') - with open(dbname,'r+') as f: - db = json.load(f) - l = list(db2points(db,cantine)) - # client.drop_database("araprice") - client.create_database("araprice") - client.write_points(l) - -if __name__ == '__main__': - main() diff --git a/arafetch/ara2influx.py b/arafetch/ara2influx.py new file mode 100644 index 0000000..26d7d0a --- /dev/null +++ b/arafetch/ara2influx.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +""" usage: fetchday [options] DB + +Options: + --db FILE path to db [Default: db.json] + --cantine CANTINE name of the cantine [Default: thales-deutschland] + --host HOST hostname or ip of influx [Default: omo] +""" +import sys +from docopt import docopt +import logging +from influxdb import InfluxDBClient + + +import json +from datetime import datetime +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger() + + +def db2points(db,cantine): + for day,things in db.items(): + time = datetime.strptime(day,"%Y-%m-%d") + time = time.replace(hour=12) + for thing in things: + ret = { + "time": time.isoformat(), + "measurement":"dish", + "tags": { + "allergies": ",".join( sorted(thing['allergies'].keys()) ), + "additives": ",".join( sorted(thing['additives'].keys()) ), + "title": thing["title"], + "type": thing["type"], + "description": thing["description"], + "cantine": cantine + }, + "fields": { + "price": thing["price"] + } + } + log.debug(ret) + yield ret + +def main(): + args = docopt(__doc__) + dbname = args['DB'] + host = args['--host'] + cantine = args['--cantine'] + client = InfluxDBClient( + host=host, + port=8086, + database='araprice') + with open(dbname,'r+') as f: + db = json.load(f) + l = list(db2points(db,cantine)) + # client.drop_database("araprice") + client.create_database("araprice") + client.write_points(l) + +if __name__ == '__main__': + main() diff --git a/arafetch/arafetch.py b/arafetch/arafetch.py new file mode 100755 index 0000000..4364762 --- /dev/null +++ b/arafetch/arafetch.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +""" usage: fetchday [options] + +Options: + --db FILE path to db [Default: db.json] + --cantine NAME name of the cantine (matches mein.aramark.de/NAME/) [Default: thales-deutschland] +""" +import sys +from docopt import docopt +import requests +from bs4 import BeautifulSoup +import logging + +import json +from datetime import datetime +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger() +from itertools import tee +cantinedb = { + 'thales-deutschland': 'aramark_menu2-2', + 'buerocampus-wangen': 'aramark_menu2-3', + 'ibm-boeblingen': 'aramark_menu2-3', + 'metabo-nuertingen': 'aramark_menu2-3', + 'vodafone-campus': 'aramark_menu2-2', + 'continental-villingen': 'aramark_menu2-3' + } +def get_data(cantine): + headers = { + 'User-Agent': 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:58.0) Gecko/20100101 Firefox/58.0', + 'Referer': f'http://mein.aramark.de/{cantine}/menuplan/', + 'X-Requested-With': 'XMLHttpRequest', + 'Cache-Control': 'max-age=0' + } + data = { + 'action': 'aramark_menu2_content', + 'id': cantinedb[cantine] + } + ret = requests.post(f'http://mein.aramark.de/{cantine}/wp-admin/admin-ajax.php',data=data) + if ret.status_code != 200: + log.error(ret) + raise ValueError('Return code is not 200') + return ret.content.decode() + +def get_data_test(): + return open("./test.html").read() + +def get_things(row): + for item in row.select('.aramark_menu2-counter'): + try: + thing = { + "allergies": {}, + "additives": {}, + "type" : item.select_one('.aramark_menu2-countertitletop').find(text=True), + "description" : item.select_one('.aramark_menu2-menutext > p').find(text=True), + + } + try: + price = float(item.select_one('.aramark_menu2-menuprice > .aramark_menu2-menuprice').find(text=True).replace(',','.').replace('EUR','').strip()) + except: + price = float(thing['description']) + thing['price'] = price + try: + thing['title'] = item.select_one('.aramark_menu2-countermenu h4').find(text=True) + except: + thing['title'] = thing['type'] + + entries = item.select('.aramark_menu2-menuinfoallergenes li span') + for i in range(0,len(entries),2): + thing['allergies'] = { entries[i].find(text=True): entries[i+1].find(text=True) } + + additives = { } + entries = item.select('.aramark_menu2-menuinfoadditives li span') + for i in range(0,len(entries),2): + thing['additives'] = { entries[i].find(text=True): entries[i+1].find(text=True) } + yield thing + except Exception as e: + log.error(f"problem with {e}") + log.debug(item.prettify()) +def main(): + args = docopt(__doc__) + dbname = args['--db'] + cantine = args['--cantine'] + try: + with open(dbname,'r+') as f: + db = json.load(f) + except: + log.info('creating new db') + db = {} + + updated = datetime.now().isoformat().split('T')[0] + soup = BeautifulSoup(get_data(cantine), 'html.parser') + # soup = BeautifulSoup(get_data_test(), 'html.parser') + for row in soup.select('.aramark_menu2-menus > div'): + day = row.get('data-aramark_menu2-day') + things = list(get_things(row)) + db[day] = things + + log.info(f'writing db {dbname}') + with sys.stdout if dbname == '-' else open(dbname,'w+') as f: + json.dump(db,f,indent=2,sort_keys=True) + + +if __name__ == '__main__': + main() diff --git a/day.py b/day.py deleted file mode 100755 index 4364762..0000000 --- a/day.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 -""" usage: fetchday [options] - -Options: - --db FILE path to db [Default: db.json] - --cantine NAME name of the cantine (matches mein.aramark.de/NAME/) [Default: thales-deutschland] -""" -import sys -from docopt import docopt -import requests -from bs4 import BeautifulSoup -import logging - -import json -from datetime import datetime -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger() -from itertools import tee -cantinedb = { - 'thales-deutschland': 'aramark_menu2-2', - 'buerocampus-wangen': 'aramark_menu2-3', - 'ibm-boeblingen': 'aramark_menu2-3', - 'metabo-nuertingen': 'aramark_menu2-3', - 'vodafone-campus': 'aramark_menu2-2', - 'continental-villingen': 'aramark_menu2-3' - } -def get_data(cantine): - headers = { - 'User-Agent': 'User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:58.0) Gecko/20100101 Firefox/58.0', - 'Referer': f'http://mein.aramark.de/{cantine}/menuplan/', - 'X-Requested-With': 'XMLHttpRequest', - 'Cache-Control': 'max-age=0' - } - data = { - 'action': 'aramark_menu2_content', - 'id': cantinedb[cantine] - } - ret = requests.post(f'http://mein.aramark.de/{cantine}/wp-admin/admin-ajax.php',data=data) - if ret.status_code != 200: - log.error(ret) - raise ValueError('Return code is not 200') - return ret.content.decode() - -def get_data_test(): - return open("./test.html").read() - -def get_things(row): - for item in row.select('.aramark_menu2-counter'): - try: - thing = { - "allergies": {}, - "additives": {}, - "type" : item.select_one('.aramark_menu2-countertitletop').find(text=True), - "description" : item.select_one('.aramark_menu2-menutext > p').find(text=True), - - } - try: - price = float(item.select_one('.aramark_menu2-menuprice > .aramark_menu2-menuprice').find(text=True).replace(',','.').replace('EUR','').strip()) - except: - price = float(thing['description']) - thing['price'] = price - try: - thing['title'] = item.select_one('.aramark_menu2-countermenu h4').find(text=True) - except: - thing['title'] = thing['type'] - - entries = item.select('.aramark_menu2-menuinfoallergenes li span') - for i in range(0,len(entries),2): - thing['allergies'] = { entries[i].find(text=True): entries[i+1].find(text=True) } - - additives = { } - entries = item.select('.aramark_menu2-menuinfoadditives li span') - for i in range(0,len(entries),2): - thing['additives'] = { entries[i].find(text=True): entries[i+1].find(text=True) } - yield thing - except Exception as e: - log.error(f"problem with {e}") - log.debug(item.prettify()) -def main(): - args = docopt(__doc__) - dbname = args['--db'] - cantine = args['--cantine'] - try: - with open(dbname,'r+') as f: - db = json.load(f) - except: - log.info('creating new db') - db = {} - - updated = datetime.now().isoformat().split('T')[0] - soup = BeautifulSoup(get_data(cantine), 'html.parser') - # soup = BeautifulSoup(get_data_test(), 'html.parser') - for row in soup.select('.aramark_menu2-menus > div'): - day = row.get('data-aramark_menu2-day') - things = list(get_things(row)) - db[day] = things - - log.info(f'writing db {dbname}') - with sys.stdout if dbname == '-' else open(dbname,'w+') as f: - json.dump(db,f,indent=2,sort_keys=True) - - -if __name__ == '__main__': - main() diff --git a/day.sh b/day.sh deleted file mode 100644 index 6780dd9..0000000 --- a/day.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -OUTDIR=${OUTDIR:-$PWD/db} -# old: thyssenkruppsystemengineering -# empty: metabo-nuertingen -mkdir -p $OUTDIR -for i in thales-deutschland ibm-boeblingen buerocampus-wangen continental-villingen vodafone-campus ;do - db=$OUTDIR/$i.json - python3 day.py --db $db --cantine $i - python3 ara2influx.py $db --cantine $i - # git add $db -done diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..556e658 --- /dev/null +++ b/default.nix @@ -0,0 +1,18 @@ +{ pkgs ? import {} }: +with pkgs.python3Packages; +let + inp = [ + python + requests + docopt + influxdb + beautifulsoup4 + pkgs.git + pkgs.wget + ]; +in buildPythonPackage { + name = "arafetch"; + src = ./.; + propagatedBuildInputs = inp; +} + diff --git a/scripts/weekrun b/scripts/weekrun new file mode 100644 index 0000000..316f3ce --- /dev/null +++ b/scripts/weekrun @@ -0,0 +1,21 @@ +#!/bin/sh +# requires: wget git +set -euf +NOW=$(date -I) +OUTDIR=${OUTDIR:-$PWD} +DATADIR=${DATADIR:-$OUTDIR/db} +PDFDIR=${PDFDIR:-$OUTDIR/pdf} + +# old: thyssenkruppsystemengineering +# empty: metabo-nuertingen + +mkdir -p $DATADIR $PDFDIR +for i in thales-deutschland ibm-boeblingen buerocampus-wangen continental-villingen vodafone-campus ;do + db=$DATADIR/$i.json + arafetch --db $db --cantine $i + ara2influx $db --cantine $i + db=$PDFDIR/$i-$NOW.pdf + wget http://gast.aramark.de/$i/menu/pdf/woche_de.php -O $db + git add -A $DATADIR + git commit -m "update $NOW" +done diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..0b6a0d4 --- /dev/null +++ b/setup.py @@ -0,0 +1,28 @@ +from setuptools import setup, find_packages +setup( + name='arafetch', + version='0.1', + description='fetch aramark data', + author='Felix Richter', + license='WTFPL', + author_email='github@syntax-fehler.de', + url='http://cgit.euer.krebsco.de/arafetch', + install_requires = [ + 'docopt', + 'requests', + 'beautifulsoup4', + 'influxdb' + ], + packages=find_packages(), + scripts = [ + "scripts/weekrun" + ], + entry_points = { + 'console_scripts': [ + 'arafetch=arafetch.arafetch:main', + 'ara2influx=arafetch.ara2influx:main' + ], + }, + + keywords=['api'], +) diff --git a/week.sh b/week.sh deleted file mode 100644 index f972e16..0000000 --- a/week.sh +++ /dev/null @@ -1,10 +0,0 @@ -NOW=$(date -I) -OUTDIR=${OUTDIR:-$PWD/pdf} -mkdir -p $OUTDIR -# Broken: -# vodafone-campus thyssenkruppsystemengineering -# Gone? metabo-nuertingen -for i in thales-deutschland ibm-boeblingen buerocampus-wangen continental-villingen ;do - db=$OUTDIR/$i-week.pdf - wget http://gast.aramark.de/$i/menu/pdf/woche_de.php -O $db -done -- cgit v1.2.3