1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
|
import logging
from os.path import expanduser,exists
from datetime import datetime
from dateutil.tz import tzlocal
from .common import html_unescape
import dateutil
import dateutil.parser
import json
log = logging.getLogger("cache")
class Cache():
def __init__(self,path):
self.path = expanduser(path)
self.load(self.path)
# self.reset_init()
def load(self,path):
if not exists(path):
log.info(f"{path} as cache does not exist")
self.store = {}
else:
self.store = json.load(open(path))
log.debug("loaded {len(ret)} entries from {cachefile}")
def reset_seen(self):
""" reset the store 'seen' field to false for this run """
for k in self.store:
self.store[k]['notify']['seen'] = False
def save(self) -> None:
cachefile = self.path
if not exists(cachefile):
log.info(f"{cachefile} does not exist,creating")
with open(cachefile,"w+") as f:
json.dump(self.store,f)
def update(self,ad,searchquery):
now = datetime.now(tzlocal())
ident = ad['id']
creation = ad.get('start-date-time', {}).get('value', '').encode('utf-8')
price = ad.get('price',{ 'amount': {}})['amount'].get('value',0)
d = dateutil.parser.parse(creation)
age_in_h = round((now - d).total_seconds() / 3600,2)
title = ad.get('title', {}).get('value', '').encode('utf-8')
title_unescaped = html_unescape(title)
try:
img = ad['pictures']['picture'][0]['link'][3]['href']
except Exception as e:
img = None
try:
place = ad['locations']['location'][0]['localized-name']['value']
distance = round(float(ad['search-distance']['distance']['value']),2)
except Exception as e:
print(e)
place = None
distance = None
# TODO: fetch detailed article
ad['notify'] = {
'filter': searchquery,
'creation': d.isoformat(),
'filter-name': searchquery['name'],
'first-seen': now.isoformat(),
'title-unescaped': title_unescaped,
'age_in_h': age_in_h,
'place': place,
'distance': distance,
'price': price,
'image': img,
'url': ad['link'][1]['href'],
'seen': True
}
if ident in self.store:
log.debug(f"{ident} already in store, updating")
ad['notify']['first-seen'] = self.store[ident]['notify']['first-seen'] # saving first_seen from cache
else:
log.debug(f"{ident} is new, adding")
self.store[ident] = ad
return ad
|
