mirror of
https://github.com/bspeice/metrik
synced 2024-11-04 22:48:11 -05:00
Get command-line tools up and running
I now have something that actually gets stuff done.
This commit is contained in:
parent
96c503a106
commit
f2390ba34f
@ -54,7 +54,7 @@
|
|||||||
<ConfirmationsSetting value="0" id="Add" />
|
<ConfirmationsSetting value="0" id="Add" />
|
||||||
<ConfirmationsSetting value="0" id="Remove" />
|
<ConfirmationsSetting value="0" id="Remove" />
|
||||||
</component>
|
</component>
|
||||||
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_3" default="false" assert-keyword="false" jdk-15="false" project-jdk-name="Python 2.7.12 (/usr/bin/python2)" project-jdk-type="Python SDK">
|
<component name="ProjectRootManager" version="2" languageLevel="JDK_1_3" default="false" assert-keyword="false" jdk-15="false" project-jdk-name="Python 2.7.12 virtualenv at ~/Development/metrik/py2-virt" project-jdk-type="Python SDK">
|
||||||
<output url="file://$PROJECT_DIR$/out" />
|
<output url="file://$PROJECT_DIR$/out" />
|
||||||
</component>
|
</component>
|
||||||
<component name="PythonCompatibilityInspectionAdvertiser">
|
<component name="PythonCompatibilityInspectionAdvertiser">
|
||||||
|
@ -0,0 +1,2 @@
|
|||||||
|
__version__ = 0.1
|
||||||
|
__release__ = __version__
|
@ -1,9 +1,64 @@
|
|||||||
|
from __future__ import print_function
|
||||||
from luigi import build
|
from luigi import build
|
||||||
from metrik.flows.libor_flow import LiborFlow
|
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
from argparse import ArgumentParser
|
||||||
|
from dateutil.parser import parse
|
||||||
|
|
||||||
|
from metrik.flows.rates_flow import LiborFlow
|
||||||
|
|
||||||
|
flows = {
|
||||||
|
'LiborFlow': LiborFlow
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def run_flow(flow_class, present):
|
||||||
|
build([flow_class(present=present)])
|
||||||
|
|
||||||
|
|
||||||
|
def build_cron_file():
|
||||||
|
EXEC = 'metrik'
|
||||||
|
FLOW_FLAG = '-f'
|
||||||
|
|
||||||
|
cron_strings = []
|
||||||
|
for flow_name, flow_class in flows.items():
|
||||||
|
cron_string = flow_class.get_schedule().get_cron_string()
|
||||||
|
cron_strings.append(
|
||||||
|
cron_string + ' ' + EXEC + ' ' + FLOW_FLAG + ' ' + flow_name
|
||||||
|
)
|
||||||
|
|
||||||
|
return '\n'.join(cron_strings)
|
||||||
|
|
||||||
|
|
||||||
|
def list_flows():
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def handle_commandline():
|
||||||
|
parser = ArgumentParser(description='Capture ALL THE DATA off the Internet.')
|
||||||
|
parser.add_argument('-c', '--cron', dest='cron', action='store_true',
|
||||||
|
help='Build the cron file used to schedule'
|
||||||
|
'running all flows')
|
||||||
|
parser.add_argument('-d', '--date', dest='present',
|
||||||
|
help='Run a flow as if it was this time '
|
||||||
|
'(default: %(default)s).',
|
||||||
|
default=datetime.now())
|
||||||
|
parser.add_argument('-f', '--flow', dest='flow', help='The flow to be run')
|
||||||
|
parser.add_argument('-l', '--list-flows', dest='list', action='store_true',
|
||||||
|
help='List all available flows to be run.')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.cron:
|
||||||
|
print(build_cron_file())
|
||||||
|
elif args.list:
|
||||||
|
print(list_flows())
|
||||||
|
elif args.flow:
|
||||||
|
if type(args.present) is datetime:
|
||||||
|
run_flow(flows[args.flow], args.present)
|
||||||
|
else:
|
||||||
|
run_flow(flows[args.flow], parse(args.present))
|
||||||
|
else:
|
||||||
|
print("No actions specified, exiting.")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
l = LiborFlow(datetime(2016, 5, 9).date())
|
handle_commandline()
|
||||||
|
|
||||||
build([l], local_scheduler=True)
|
|
||||||
|
50
metrik/flows/base.py
Normal file
50
metrik/flows/base.py
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
from luigi.task import WrapperTask
|
||||||
|
from luigi.parameter import DateMinuteParameter
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
from metrik.trading_days import is_trading_day
|
||||||
|
|
||||||
|
|
||||||
|
class Flow(WrapperTask):
|
||||||
|
present = DateMinuteParameter()
|
||||||
|
|
||||||
|
def __init__(self, force=False, *args, **kwargs):
|
||||||
|
super(Flow, self).__init__(*args, **kwargs)
|
||||||
|
self.force = force
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_schedule():
|
||||||
|
raise NotImplementedError('Your flow should know when it should be run.')
|
||||||
|
|
||||||
|
def _requires(self):
|
||||||
|
raise NotImplementedError('I need to know what tasks should be run!')
|
||||||
|
|
||||||
|
def requires(self):
|
||||||
|
if self.force or self.get_schedule().check_trigger(self.present):
|
||||||
|
return self._requires()
|
||||||
|
|
||||||
|
|
||||||
|
class Schedule(object):
|
||||||
|
def get_cron_string(self):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def check_trigger(self, present):
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class DailyMidnight(Schedule):
|
||||||
|
def get_cron_string(self):
|
||||||
|
return '0 0 * * *'
|
||||||
|
|
||||||
|
|
||||||
|
class WeekdayMidnight(Schedule):
|
||||||
|
def get_cron_string(self):
|
||||||
|
return '0 0 * * 1-5'
|
||||||
|
|
||||||
|
|
||||||
|
class MarketClose(Schedule):
|
||||||
|
def get_cron_string(self):
|
||||||
|
return '5 16 * * 1-5'
|
||||||
|
|
||||||
|
def check_trigger(self, present):
|
||||||
|
return is_trading_day(present)
|
@ -1,13 +0,0 @@
|
|||||||
from luigi import WrapperTask, DateParameter, LocalTarget
|
|
||||||
|
|
||||||
from metrik.tasks.ice import LiborRateTask
|
|
||||||
from metrik.targets.temp_file import TempFileTarget
|
|
||||||
|
|
||||||
|
|
||||||
class LiborFlow(WrapperTask):
|
|
||||||
date = DateParameter()
|
|
||||||
|
|
||||||
def requires(self):
|
|
||||||
currencies = ['USD']
|
|
||||||
return [LiborRateTask(self.date, currency)
|
|
||||||
for currency in currencies]
|
|
14
metrik/flows/rates_flow.py
Normal file
14
metrik/flows/rates_flow.py
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
from metrik.flows.base import Flow, WeekdayMidnight
|
||||||
|
from metrik.tasks.ice import LiborRateTask
|
||||||
|
|
||||||
|
|
||||||
|
class LiborFlow(Flow):
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get_schedule():
|
||||||
|
return WeekdayMidnight()
|
||||||
|
|
||||||
|
def _requires(self):
|
||||||
|
currencies = ['USD']
|
||||||
|
return [LiborRateTask(self.present, currency)
|
||||||
|
for currency in currencies]
|
@ -11,6 +11,7 @@ from luigi.parameter import DateParameter, Parameter
|
|||||||
from six.moves.urllib.parse import quote_plus
|
from six.moves.urllib.parse import quote_plus
|
||||||
|
|
||||||
from metrik.tasks.base import MongoCreateTask
|
from metrik.tasks.base import MongoCreateTask
|
||||||
|
from metrik.trading_days import TradingDay
|
||||||
|
|
||||||
LiborRate = namedtuple('LiborRate', [
|
LiborRate = namedtuple('LiborRate', [
|
||||||
'publication', 'overnight', 'one_week', 'one_month', 'two_month',
|
'publication', 'overnight', 'one_week', 'one_month', 'two_month',
|
||||||
@ -27,7 +28,7 @@ class LiborRateTask(MongoCreateTask):
|
|||||||
return 'libor'
|
return 'libor'
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def retrieve_data(date, currency):
|
def retrieve_historical_libor(date, currency):
|
||||||
url = ('https://www.theice.com/marketdata/reports/icebenchmarkadmin/'
|
url = ('https://www.theice.com/marketdata/reports/icebenchmarkadmin/'
|
||||||
'ICELiborHistoricalRates.shtml?excelExport='
|
'ICELiborHistoricalRates.shtml?excelExport='
|
||||||
'&criteria.reportDate={}&criteria.currencyCode={}').format(
|
'&criteria.reportDate={}&criteria.currencyCode={}').format(
|
||||||
@ -80,3 +81,12 @@ class LiborRateTask(MongoCreateTask):
|
|||||||
'six_month': six_month,
|
'six_month': six_month,
|
||||||
'one_year': one_year
|
'one_year': one_year
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def retrieve_data(date, currency):
|
||||||
|
# ICE publish data a day late, so we actually need to retrieve data
|
||||||
|
# for the trading day prior to this.
|
||||||
|
return LiborRateTask.retrieve_historical_libor(
|
||||||
|
date - TradingDay(1),
|
||||||
|
currency
|
||||||
|
)
|
||||||
|
@ -2,6 +2,7 @@ from pandas.tseries.holiday import AbstractHolidayCalendar, Holiday, \
|
|||||||
nearest_workday, USMartinLutherKingJr, USPresidentsDay, GoodFriday, \
|
nearest_workday, USMartinLutherKingJr, USPresidentsDay, GoodFriday, \
|
||||||
USMemorialDay, USLaborDay, USThanksgivingDay
|
USMemorialDay, USLaborDay, USThanksgivingDay
|
||||||
from pandas.tseries.offsets import CustomBusinessDay
|
from pandas.tseries.offsets import CustomBusinessDay
|
||||||
|
from pandas import date_range
|
||||||
|
|
||||||
|
|
||||||
class USTradingCalendar(AbstractHolidayCalendar):
|
class USTradingCalendar(AbstractHolidayCalendar):
|
||||||
@ -17,5 +18,10 @@ class USTradingCalendar(AbstractHolidayCalendar):
|
|||||||
Holiday('Christmas', month=12, day=25, observance=nearest_workday)
|
Holiday('Christmas', month=12, day=25, observance=nearest_workday)
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def TradingDay(n):
|
def TradingDay(n):
|
||||||
return CustomBusinessDay(n, calendar=USTradingCalendar())
|
return CustomBusinessDay(n, calendar=USTradingCalendar())
|
||||||
|
|
||||||
|
|
||||||
|
def is_trading_day(date):
|
||||||
|
return bool(len(date_range(date, date, freq=TradingDay(1))))
|
||||||
|
@ -5,3 +5,4 @@ requests>=2.9.1
|
|||||||
pymongo>=3.2
|
pymongo>=3.2
|
||||||
python-dateutil>=2.4.2
|
python-dateutil>=2.4.2
|
||||||
pandas>=0.17.1
|
pandas>=0.17.1
|
||||||
|
argparse>=1.1.0
|
8
setup.py
8
setup.py
@ -15,7 +15,8 @@ setup(
|
|||||||
'pymongo >= 3.2',
|
'pymongo >= 3.2',
|
||||||
'pytz >= 2016.6.1',
|
'pytz >= 2016.6.1',
|
||||||
'python-dateutil >= 2.4.2',
|
'python-dateutil >= 2.4.2',
|
||||||
'pandas >= 0.17.1'
|
'pandas >= 0.17.1',
|
||||||
|
'argparse >= 1.1.0'
|
||||||
],
|
],
|
||||||
setup_requires=[
|
setup_requires=[
|
||||||
'pytest_runner'
|
'pytest_runner'
|
||||||
@ -23,5 +24,10 @@ setup(
|
|||||||
tests_require=[
|
tests_require=[
|
||||||
'pytest',
|
'pytest',
|
||||||
'pytest-catchlog'
|
'pytest-catchlog'
|
||||||
|
],
|
||||||
|
entry_points={
|
||||||
|
'console_scripts': [
|
||||||
|
'metrik = metrik.batch:handle_commandline'
|
||||||
]
|
]
|
||||||
|
}
|
||||||
)
|
)
|
@ -11,7 +11,8 @@ class TestICE(TestCase):
|
|||||||
def test_correct_libor_Aug8_2016(self):
|
def test_correct_libor_Aug8_2016(self):
|
||||||
# Validate with:
|
# Validate with:
|
||||||
# https://www.theice.com/marketdata/reports/icebenchmarkadmin/ICELiborHistoricalRates.shtml?excelExport=&criteria.reportDate=8%2F8%2F16&criteria.currencyCode=USD
|
# https://www.theice.com/marketdata/reports/icebenchmarkadmin/ICELiborHistoricalRates.shtml?excelExport=&criteria.reportDate=8%2F8%2F16&criteria.currencyCode=USD
|
||||||
aug8_libor = LiborRateTask.retrieve_data(datetime(2016, 8, 8), 'USD')
|
aug8_libor = LiborRateTask.retrieve_historical_libor(
|
||||||
|
datetime(2016, 8, 8), 'USD')
|
||||||
|
|
||||||
assert aug8_libor['overnight'] == .4189
|
assert aug8_libor['overnight'] == .4189
|
||||||
assert aug8_libor['one_week'] == .4431
|
assert aug8_libor['one_week'] == .4431
|
||||||
@ -23,13 +24,15 @@ class TestICE(TestCase):
|
|||||||
|
|
||||||
london_tz = pytz.timezone('Europe/London')
|
london_tz = pytz.timezone('Europe/London')
|
||||||
actual = london_tz.localize(datetime(2016, 8, 8, 11, 45, 6))
|
actual = london_tz.localize(datetime(2016, 8, 8, 11, 45, 6))
|
||||||
logging.info('Publication date in London time: {}'.format(aug8_libor['publication'].astimezone(london_tz)))
|
logging.info('Publication date in London time: {}'.format(
|
||||||
|
aug8_libor['publication'].astimezone(london_tz)))
|
||||||
assert aug8_libor['publication'] == actual
|
assert aug8_libor['publication'] == actual
|
||||||
|
|
||||||
def test_correct_libor_Aug9_2010(self):
|
def test_correct_libor_Aug9_2010(self):
|
||||||
# Validate with:
|
# Validate with:
|
||||||
# https://www.theice.com/marketdata/reports/icebenchmarkadmin/ICELiborHistoricalRates.shtml?excelExport=&criteria.reportDate=8%2F9%2F10&criteria.currencyCode=USD
|
# https://www.theice.com/marketdata/reports/icebenchmarkadmin/ICELiborHistoricalRates.shtml?excelExport=&criteria.reportDate=8%2F9%2F10&criteria.currencyCode=USD
|
||||||
aug9_libor = LiborRateTask.retrieve_data(datetime(2010, 8, 9), 'USD')
|
aug9_libor = LiborRateTask.retrieve_historical_libor(
|
||||||
|
datetime(2010, 8, 9), 'USD')
|
||||||
|
|
||||||
assert aug9_libor['overnight'] == .23656
|
assert aug9_libor['overnight'] == .23656
|
||||||
assert aug9_libor['one_week'] == .27725
|
assert aug9_libor['one_week'] == .27725
|
||||||
@ -41,7 +44,8 @@ class TestICE(TestCase):
|
|||||||
|
|
||||||
london_tz = pytz.timezone('Europe/London')
|
london_tz = pytz.timezone('Europe/London')
|
||||||
actual = london_tz.localize(datetime(2010, 8, 9, 15, 49, 12))
|
actual = london_tz.localize(datetime(2010, 8, 9, 15, 49, 12))
|
||||||
logging.info('Publication date in London time: {}'.format(aug9_libor['publication'].astimezone(london_tz)))
|
logging.info('Publication date in London time: {}'.format(
|
||||||
|
aug9_libor['publication'].astimezone(london_tz)))
|
||||||
assert aug9_libor['publication'] == actual
|
assert aug9_libor['publication'] == actual
|
||||||
|
|
||||||
def test_correct_date_reasoning(self):
|
def test_correct_date_reasoning(self):
|
||||||
|
@ -1,11 +1,10 @@
|
|||||||
from unittest import TestCase
|
from unittest import TestCase
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
|
||||||
from metrik.trading_days import TradingDay
|
from metrik.trading_days import TradingDay, is_trading_day
|
||||||
|
|
||||||
|
|
||||||
class TradingDayTest(TestCase):
|
class TradingDayTest(TestCase):
|
||||||
|
|
||||||
def test_skip_july4(self):
|
def test_skip_july4(self):
|
||||||
start = datetime(2016, 7, 1) # Friday
|
start = datetime(2016, 7, 1) # Friday
|
||||||
end = start + TradingDay(1)
|
end = start + TradingDay(1)
|
||||||
@ -15,3 +14,11 @@ class TradingDayTest(TestCase):
|
|||||||
end = datetime(2016, 7, 5)
|
end = datetime(2016, 7, 5)
|
||||||
start = end - TradingDay(1)
|
start = end - TradingDay(1)
|
||||||
assert start == datetime(2016, 7, 1)
|
assert start == datetime(2016, 7, 1)
|
||||||
|
|
||||||
|
def test_not_bday(self):
|
||||||
|
for year in range(2000, 2016):
|
||||||
|
date = datetime(year, 7, 4)
|
||||||
|
assert not is_trading_day(date)
|
||||||
|
|
||||||
|
def test_is_bday(self):
|
||||||
|
assert is_trading_day(datetime(2016, 8, 23))
|
||||||
|
Loading…
Reference in New Issue
Block a user