Switch the configuration system

I like this one much better.

metrik/batch.py

@@ -3,7 +3,9 @@ from luigi import build
 from datetime import datetime
 from argparse import ArgumentParser
 from dateutil.parser import parse
+from six import StringIO
 
+from metrik.conf import get_config
 from metrik.flows.rates_flow import LiborFlow
 from metrik.flows.equities_flow import EquitiesFlow
 from metrik import __version__
@@ -49,6 +51,7 @@ def handle_commandline():
     parser.add_argument('-f', '--flow', dest='flow', help='The flow to be run')
     parser.add_argument('-l', '--list-flows', dest='list', action='store_true',
                         help='List all available flows to be run.')
+    parser.add_argument('-o', '--config', action='store_true')
     parser.add_argument('-v', '--version', action='version',
                         version=__version__)
     args = parser.parse_args()
@@ -57,6 +60,11 @@ def handle_commandline():
         print(build_cron_file())
     elif args.list:
         print(list_flows())
+    elif args.config:
+        config = get_config()
+        s = StringIO
+        config.write(s)
+        print(s.getvalue())
     elif args.flow:
         if type(args.present) is datetime:
             run_flow(flows[args.flow], args.present, True)

metrik/conf.py

@@ -1,16 +1,35 @@
 import os
 import sys
+# from six.moves.configparser import RawConfigParser
+from ConfigParser import RawConfigParser
 
-IS_TRAVIS = 'TRAVIS_BUILD_NUMBER' in os.environ
-IS_PYTEST = hasattr(sys, '_called_from_test')
 
-TEST = IS_PYTEST or IS_TRAVIS
+def get_config_locations():
+    return ['/etc/metrik', os.path.expanduser('~/.metrik')]
 
-USER_AGENT = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"
-MONGO_HOST = 'localhost'
-MONGO_PORT = 27017
 
-if TEST:
-    MONGO_DATABASE = 'metrik'
-else:
-    MONGO_DATABASE = 'metrik-test'
+def get_default_conf():
+    cur_file_dir = os.path.dirname(os.path.realpath(__file__))
+    return open(cur_file_dir + '/default.conf', 'r')
+
+
+# Normally it's terrible practice to put static calls into the signature,
+# but this is safe (for now) since the get_config_locations() won't change
+# during a run. I.e. it starts up and that's the only time it's ever needed.
+def get_config(extra_locations=get_config_locations()):
+    config = RawConfigParser()
+
+    config.readfp(get_default_conf())
+
+    conf_files = config.read(extra_locations)
+    for conf_file in conf_files:
+        config.readfp(open(conf_file, 'r'))
+
+    # Not a huge fan of when developers think they're smarter than the
+    # end-users, but I'm calling it a special case for testing
+    is_travis = 'TRAVIS_BUILD_NUMBER' in os.environ
+    is_pytest = hasattr(sys, '_called_from_test')
+    if is_pytest or is_travis:
+        config.set('metrik', 'mongo_database', 'metrik-test')
+
+    return config

metrik/default.conf

@@ -0,0 +1,11 @@
+[metrik]
+user_agent=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0
+mongo_host=localhost
+mongo_port=27017
+mongo_database=metrik
+
+[tradeking]
+consumer_key=
+consumer_secret=
+oauth_token=
+oauth_token_secret=

metrik/targets/mongo.py

@@ -1,13 +1,18 @@
 from luigi import Target
 from pymongo import MongoClient
-from metrik.conf import MONGO_HOST, MONGO_PORT, MONGO_DATABASE
+from metrik.conf import get_config
 from datetime import datetime
 
 
 class MongoTarget(Target):
 
     def __init__(self, collection, id):
-        self.connection = MongoClient(MONGO_HOST, MONGO_PORT)[MONGO_DATABASE]
+        config = get_config()
+        self.connection = MongoClient(
+            host=config.get('metrik', 'mongo_host'),
+            port=config.get('metrik', 'mongo_port'))[
+            config.get('metrik', 'mongo_database')
+        ]
         self.collection = self.connection[collection]
         self.id = id
 

metrik/tasks/base.py

@@ -10,7 +10,7 @@ from pymongo import MongoClient
 
 from metrik.targets.mongo import MongoTarget
 from metrik.targets.noop import NoOpTarget
-from metrik.conf import MONGO_HOST, MONGO_PORT, MONGO_DATABASE
+from metrik.conf import get_config
 
 
 class MongoCreateTask(Task):
@@ -70,52 +70,50 @@ class MongoNoBackCreateTask(MongoCreateTask):
 class MongoRateLimit(object):
     rate_limit_collection = 'rate_limit'
 
-    def __init__(self, service, limit, interval, max_tries=5, backoff=.5):
-        """
-
-        :param present:
-        :type present: datetime.datetime
-        :param service:
-        :param limit:
-        :param interval:
-        :type interval: datetime.timedelta
-        :param max_tries:
-        :param backoff:
-        """
-        self.service = service
-        self.limit = limit
-        self.interval = interval
-        self.max_tries = max_tries
-        self.backoff = backoff
-        self.db = MongoClient(host=MONGO_HOST, port=MONGO_PORT)[MONGO_DATABASE]
+    def __init__(self):
+        config = get_config()
+        self.db = MongoClient(
+            host=config.get('metrik', 'mongo_host'),
+            port=config.getint('metrik', 'mongo_port'))[
+            config.get('metrik', 'mongo_database')
+        ]
 
     def get_present(self):
         return datetime.datetime.now()
 
-    def query_locks(self, present):
+    def query_locks(self, present, interval, service):
         return self.db[self.rate_limit_collection].find(
-            {'_created_at': {'$gt': present - self.interval},
-             'service': self.service}).count()
+            {'_created_at': {'$gt': present - interval},
+             'service': service}).count()
 
-    def save_lock(self, present):
+    def save_lock(self, present, service):
         self.db[self.rate_limit_collection].save({
-            '_created_at': present, 'service': self.service
+            '_created_at': present, 'service': service
         })
 
-    def sleep_until(self, present):
-        future_time = present + self.interval * self.backoff
+    def sleep_until(self, present, interval, backoff):
+        future_time = present + interval * backoff
         return (future_time - present).total_seconds()
 
-    def acquire_lock(self):
+    def acquire_lock(self, service, limit, interval, max_tries=5, backoff=.5):
         num_tries = 0
-        while num_tries < self.max_tries:
+        while num_tries < max_tries:
             num_tries += 1
-            num_locks = self.query_locks(self.get_present())
-            if num_locks < self.limit:
-                self.save_lock(self.get_present())
+            num_locks = self.query_locks(
+                self.get_present(),
+                interval,
+                service
+            )
+
+            if num_locks < limit:
+                self.save_lock(self.get_present(), service)
                 return True
-            elif num_tries < self.max_tries:
-                sleep_amount = self.sleep_until(self.get_present())
+            elif num_tries < max_tries:
+                sleep_amount = self.sleep_until(
+                    self.get_present(),
+                    interval,
+                    backoff
+                )
                 sleep(sleep_amount)
 
         return False