diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml index 1c1045c..e0c1f65 100644 --- a/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -24,5 +24,13 @@ + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml index dd1ff53..a1539c0 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -38,6 +38,12 @@ + + + + + + diff --git a/metrik/targets/mongo_target.py b/metrik/targets/mongo_target.py index 0387532..09c0005 100644 --- a/metrik/targets/mongo_target.py +++ b/metrik/targets/mongo_target.py @@ -1,6 +1,7 @@ from luigi import Target from pymongo import MongoClient from metrik.conf import MONGO_HOST, MONGO_PORT, MONGO_DATABASE +from datetime import datetime class MongoTarget(Target): @@ -18,6 +19,7 @@ class MongoTarget(Target): def persist(self, dict_object): id_dict = dict_object id_dict['_id'] = self.id + id_dict['_retrieved'] = datetime.now() return self.collection.insert_one(id_dict).inserted_id def retrieve(self): diff --git a/metrik/tasks/ice.py b/metrik/tasks/ice.py index aeead50..bf3c1ba 100644 --- a/metrik/tasks/ice.py +++ b/metrik/tasks/ice.py @@ -1,14 +1,12 @@ from __future__ import print_function import csv -import pytz import requests from collections import namedtuple from dateutil.parser import parse from io import StringIO -import logging + from luigi.parameter import DateParameter, Parameter -# noinspection PyUnresolvedReferences from six.moves.urllib.parse import quote_plus from metrik.tasks.base import MongoCreateTask