Initial commit for metrik

.gitignore

@@ -0,0 +1,4 @@
+.idea/workspace.xml
+.eggs/
+Metrik.egg-info/
+*.pyc

.idea/compiler.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="CompilerConfiguration">
+    <resourceExtensions />
+    <wildcardResourcePatterns>
+      <entry name="!?*.java" />
+      <entry name="!?*.form" />
+      <entry name="!?*.class" />
+      <entry name="!?*.groovy" />
+      <entry name="!?*.scala" />
+      <entry name="!?*.flex" />
+      <entry name="!?*.kt" />
+      <entry name="!?*.clj" />
+      <entry name="!?*.aj" />
+    </wildcardResourcePatterns>
+    <annotationProcessing>
+      <profile default="true" name="Default" enabled="false">
+        <processorPath useClasspath="true" />
+      </profile>
+    </annotationProcessing>
+  </component>
+</project>

.idea/copyright/profiles_settings.xml

@@ -0,0 +1,3 @@
+<component name="CopyrightManager">
+  <settings default="" />
+</component>

.idea/misc.xml

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ClientPropertiesManager">
+    <properties class="javax.swing.AbstractButton">
+      <property name="hideActionText" class="java.lang.Boolean" />
+    </properties>
+    <properties class="javax.swing.JComponent">
+      <property name="html.disable" class="java.lang.Boolean" />
+    </properties>
+    <properties class="javax.swing.JEditorPane">
+      <property name="JEditorPane.w3cLengthUnits" class="java.lang.Boolean" />
+      <property name="JEditorPane.honorDisplayProperties" class="java.lang.Boolean" />
+      <property name="charset" class="java.lang.String" />
+    </properties>
+    <properties class="javax.swing.JList">
+      <property name="List.isFileList" class="java.lang.Boolean" />
+    </properties>
+    <properties class="javax.swing.JPasswordField">
+      <property name="JPasswordField.cutCopyAllowed" class="java.lang.Boolean" />
+    </properties>
+    <properties class="javax.swing.JSlider">
+      <property name="Slider.paintThumbArrowShape" class="java.lang.Boolean" />
+      <property name="JSlider.isFilled" class="java.lang.Boolean" />
+    </properties>
+    <properties class="javax.swing.JTable">
+      <property name="Table.isFileList" class="java.lang.Boolean" />
+      <property name="JTable.autoStartsEdit" class="java.lang.Boolean" />
+      <property name="terminateEditOnFocusLost" class="java.lang.Boolean" />
+    </properties>
+    <properties class="javax.swing.JToolBar">
+      <property name="JToolBar.isRollover" class="java.lang.Boolean" />
+    </properties>
+    <properties class="javax.swing.JTree">
+      <property name="JTree.lineStyle" class="java.lang.String" />
+    </properties>
+    <properties class="javax.swing.text.JTextComponent">
+      <property name="caretAspectRatio" class="java.lang.Double" />
+      <property name="caretWidth" class="java.lang.Integer" />
+    </properties>
+  </component>
+  <component name="MavenImportPreferences">
+    <option name="generalSettings">
+      <MavenGeneralSettings>
+        <option name="mavenHome" value="Bundled (Maven 3)" />
+      </MavenGeneralSettings>
+    </option>
+  </component>
+  <component name="ProjectLevelVcsManager" settingsEditedManually="false">
+    <OptionsSetting value="true" id="Add" />
+    <OptionsSetting value="true" id="Remove" />
+    <OptionsSetting value="true" id="Checkout" />
+    <OptionsSetting value="true" id="Update" />
+    <OptionsSetting value="true" id="Status" />
+    <OptionsSetting value="true" id="Edit" />
+    <ConfirmationsSetting value="0" id="Add" />
+    <ConfirmationsSetting value="0" id="Remove" />
+  </component>
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_1_3" default="true" assert-keyword="false" jdk-15="false" project-jdk-name="Python 3.5.1+ (/usr/bin/python3)" project-jdk-type="Python SDK">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/metrik.iml" filepath="$PROJECT_DIR$/metrik.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

.travis.yml

@@ -0,0 +1,10 @@
+language: python
+python:
+  - "2.6"
+  - "2.7"
+  - "3.2"
+  - "3.3"
+  - "3.4"
+  - "3.5"
+
+script: python setup.py test

metrik.iml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

metrik/conf.py

@@ -0,0 +1 @@
+USER_AGENT = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0"

metrik/targets/pickle_target.py

@@ -0,0 +1,24 @@
+from luigi.target import Target
+from pickle import dump, dumps, load, loads
+from os.path import exists, join
+from os import unlink
+from tempfile import tempdir
+import base64
+
+
+class PickleTarget(Target):
+    def __init__(self, name):
+        self.name = name
+
+    def full_path(self):
+        return join(tempdir, self.name)
+
+    def exists(self):
+        return exists(self.full_path())
+
+    def write(self, object):
+        with open(self.full_path(), 'w+b') as handle:
+            dump(object, handle)
+
+    def read(self):
+        return load(open(self.full_path(), 'rb'))

metrik/tasks/bloomberg.py

@@ -0,0 +1,21 @@
+from luigi import Task, Parameter
+from pyquery import PyQuery as pq
+import requests
+
+
+class BloombergEquityInfo(Task):
+    bbg_code = Parameter()
+    user_agent = Parameter()
+
+    @staticmethod
+    def retrieve_info(bbg_code, user_agent):
+        html = pq('http://www.bloomberg.com/quote/{}'.format(bbg_code),
+                  {'User-Agent': user_agent})
+
+        sector, industry, sub_industry = (
+            html("div.cell:nth-child(13) > div:nth-child(2)").text(),
+            html("div.cell:nth-child(14) > div:nth-child(2)").text(),
+            html("div.cell:nth-child(15) > div:nth-child(2)").text()
+        )
+
+        return sector, industry, sub_industry

setup.cfg

@@ -0,0 +1,2 @@
+[aliases]
+test=pytest

setup.py

@@ -0,0 +1,20 @@
+from setuptools import setup, find_packages
+
+setup(
+    name='Metrik',
+    description='Data aggregation framework for Python',
+    version='0.1.0',
+    author='Bradlee Speice',
+    author_email='bradlee.speice@gmail.com',
+    packages=find_packages(),
+    install_requires=[
+        'pyquery >= 1.2.13',
+        'luigi >= 2.2.0'
+    ],
+    setup_requires=[
+        'pytest_runner'
+    ],
+    tests_require=[
+        'pytest'
+    ]
+)

test/targets/pickle_target.py

@@ -0,0 +1,40 @@
+from unittest import TestCase
+import luigi
+from metrik.targets.pickle_target import PickleTarget
+
+
+class FibTask(luigi.Task):
+    s = luigi.IntParameter()
+
+    def requires(self):
+        if self.s >= 2:
+            return [FibTask(self.s - 1), FibTask(self.s - 2)]
+        else:
+            return []
+
+    def output(self):
+        return PickleTarget(self.task_id)
+
+    def run(self):
+        if self.s <= 1:
+            val = self.s
+        else:
+            count = 0
+            for input in self.input():
+                count += input.read()
+            val = count
+
+        self.output().write(val)
+
+
+class TestPickleTarget(TestCase):
+    def test_fibonacci(self):
+        f = FibTask(6)
+        luigi.build([f], local_scheduler=True)
+
+        ret = f.output().read()
+        assert ret == 8
+
+        f = FibTask(100)
+        luigi.build([f], local_scheduler=True)
+        assert f.output().read() == 354224848179261915075

test/tasks/test_bloomberg.py

@@ -0,0 +1,22 @@
+from unittest import TestCase
+
+from metrik.conf import USER_AGENT
+from metrik.tasks.bloomberg import BloombergEquityInfo
+
+
+class BloombergTest(TestCase):
+    def test_correct_info_apple(self):
+        sector, industry, sub_industry = \
+            BloombergEquityInfo.retrieve_info("AAPL:US", USER_AGENT)
+
+        assert sector == 'Technology'
+        assert industry == 'Hardware'
+        assert sub_industry == 'Communications Equipment'
+
+    def test_correct_info_kcg(self):
+        sector, industry, sub_industry = \
+            BloombergEquityInfo.retrieve_info("KCG:US", USER_AGENT)
+
+        assert sector == 'Financials'
+        assert industry == 'Institutional Financial Svcs'
+        assert sub_industry == 'Institutional Brokerage'